From ea3e453aa12d1157f0b5a710a963c3d83d5bd269 Mon Sep 17 00:00:00 2001 From: emeric Date: Wed, 3 Jul 2019 13:32:02 +0200 Subject: [PATCH] Improved response time of database queries --- src/api/subsonic/SubsonicResource.cpp | 25 ++++++++++++++----------- src/database/Cluster.cpp | 15 +++++++++++++-- src/database/Cluster.hpp | 7 ++++--- src/database/DatabaseHandler.cpp | 13 +++++++++++++ src/database/DatabaseHandler.hpp | 2 ++ src/database/Release.cpp | 15 ++++++++++++--- src/database/Release.hpp | 3 ++- src/database/Track.cpp | 7 ------- src/database/Track.hpp | 5 ----- src/scanner/MediaScanner.cpp | 18 +++++------------- test/database/DatabaseTest.cpp | 9 +++++++++ 11 files changed, 74 insertions(+), 45 deletions(-) diff --git a/src/api/subsonic/SubsonicResource.cpp b/src/api/subsonic/SubsonicResource.cpp index ad9a9511..94d534d9 100644 --- a/src/api/subsonic/SubsonicResource.cpp +++ b/src/api/subsonic/SubsonicResource.cpp @@ -18,6 +18,7 @@ */ #include "SubsonicResource.hpp" +#include #include #include #include @@ -357,7 +358,11 @@ std::string parameterMapToDebugString(const Wt::Http::ParameterMap& parameterMap void SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Response &response) { - LMS_LOG(API_SUBSONIC, DEBUG) << "Handling request '" << request.path() << "', params = " << parameterMapToDebugString(request.getParameterMap()); + static std::atomic curRequestId{}; + + const std::size_t requestId {curRequestId++}; + + LMS_LOG(API_SUBSONIC, DEBUG) << "Handling request " << requestId << " '" << request.path() << "', params = " << parameterMapToDebugString(request.getParameterMap()); const Wt::Http::ParameterMap& parameters {request.getParameterMap()}; @@ -387,7 +392,7 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp resp.write(response.out(), format); response.setMimeType(ResponseFormatToMimeType(format)); - LMS_LOG(API_SUBSONIC, DEBUG) << "Request '" << request.path() << "' handled!"; + LMS_LOG(API_SUBSONIC, DEBUG) << "Request " << requestId << " '" << request.path() << "' handled!"; return; } @@ -416,7 +421,7 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp continuation->setData(std::move(res.continuationData)); } - LMS_LOG(API_SUBSONIC, DEBUG) << "Request '" << request.path() << "' handled!"; + LMS_LOG(API_SUBSONIC, DEBUG) << "Request " << requestId << " '" << request.path() << "' handled!"; return; } @@ -547,7 +552,7 @@ releaseToResponseNode(const Database::User::pointer& user, const Database::Relea if (id3) { albumNode.setAttribute("name", release->getName()); - albumNode.setAttribute("songCount", std::to_string(release->getTracks().size())); + albumNode.setAttribute("songCount", std::to_string(release->getTracksCount())); albumNode.setAttribute("duration", std::to_string(std::chrono::duration_cast(release->getDuration()).count())); } else @@ -558,8 +563,9 @@ releaseToResponseNode(const Database::User::pointer& user, const Database::Relea albumNode.setAttribute("id", IdToString({Id::Type::Release, release.id()})); albumNode.setAttribute("coverArt", IdToString({Id::Type::Release, release.id()})); - if (release->getReleaseYear()) - albumNode.setAttribute("year", std::to_string(*release->getReleaseYear())); + auto releaseYear {release->getReleaseYear()}; + if (releaseYear) + albumNode.setAttribute("year", std::to_string(*releaseYear)); auto artists {release->getReleaseArtists()}; if (artists.empty()) @@ -632,11 +638,8 @@ clusterToResponseNode(const Database::Cluster::pointer& cluster) Response::Node clusterNode; clusterNode.setValue(cluster->getName()); - clusterNode.setAttribute("songCount", std::to_string(cluster->getTrackIds().size())); - { - auto releases {Database::Release::getByFilter(*cluster.session(), {cluster.id()})}; - clusterNode.setAttribute("albumCount", std::to_string(releases.size())); - } + clusterNode.setAttribute("songCount", std::to_string(cluster->getTracksCount())); + clusterNode.setAttribute("albumCount", std::to_string(cluster->getReleasesCount())); return clusterNode; } diff --git a/src/database/Cluster.cpp b/src/database/Cluster.cpp index c2fd8f7f..3fac783f 100644 --- a/src/database/Cluster.cpp +++ b/src/database/Cluster.cpp @@ -74,7 +74,7 @@ Cluster::addTrack(Wt::Dbo::ptr track) std::vector> Cluster::getTracks(int offset, int limit) const { - Wt::Dbo::collection res = session()->query("select t from track t INNER JOIN cluster c ON c.id = t_c.cluster_id INNER JOIN track_cluster t_c ON t_c.track_id = t.id") + Wt::Dbo::collection res = session()->query("SELECT t FROM track t INNER JOIN cluster c ON c.id = t_c.cluster_id INNER JOIN track_cluster t_c ON t_c.track_id = t.id") .where("c.id = ?").bind(self()->id()) .offset(offset) .limit(limit); @@ -88,13 +88,24 @@ Cluster::getTrackIds() const assert(session()); assert(IdIsValid(self()->id())); - Wt::Dbo::collection res = session()->query("SELECT t_c.track_id from track_cluster t_c INNER JOIN cluster c ON c.id = t_c.cluster_id") + Wt::Dbo::collection res = session()->query("SELECT t_c.track_id FROM track_cluster t_c INNER JOIN cluster c ON c.id = t_c.cluster_id") .where("c.id = ?").bind(self()->id()); return std::set(res.begin(), res.end()); } +std::size_t +Cluster::getReleasesCount() const +{ + assert(session()); + assert(IdIsValid(self()->id())); + + return session()->query("SELECT COUNT(DISTINCT r.id) FROM release r INNER JOIN track t on t.release_id = r.id INNER JOIN cluster c ON c.id = t_c.cluster_id INNER JOIN track_cluster t_c ON t_c.track_id = t.id") + .where("c.id = ?").bind(self()->id()); + +} + ClusterType::ClusterType(std::string name) : _name(name) diff --git a/src/database/Cluster.hpp b/src/database/Cluster.hpp index 09689d33..18e55edc 100644 --- a/src/database/Cluster.hpp +++ b/src/database/Cluster.hpp @@ -51,11 +51,12 @@ class Cluster : public Wt::Dbo::Dbo static pointer create(Wt::Dbo::Session& session, Wt::Dbo::ptr type, std::string name); // Accessors - const std::string& getName() const { return _name; } - Wt::Dbo::ptr getType() const { return _clusterType; } - std::size_t getCount() const { return _tracks.size(); } + const std::string& getName() const { return _name; } + Wt::Dbo::ptr getType() const { return _clusterType; } + std::size_t getTracksCount() const { return _tracks.size(); } std::vector> getTracks(int offset, int limit) const; std::set getTrackIds() const; + std::size_t getReleasesCount() const; void addTrack(Wt::Dbo::ptr track); diff --git a/src/database/DatabaseHandler.cpp b/src/database/DatabaseHandler.cpp index 62beaf5c..b1c5a276 100644 --- a/src/database/DatabaseHandler.cpp +++ b/src/database/DatabaseHandler.cpp @@ -231,15 +231,21 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool) // Indexes _session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)"); _session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)"); _session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)"); _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)"); _session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)"); _session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)"); _session.execute("CREATE INDEX IF NOT EXISTS track_path_idx ON track(file_path)"); _session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)"); _session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)"); _session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)"); _session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)"); _session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)"); _session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)"); @@ -257,6 +263,13 @@ Handler::~Handler() delete _users; } +void +Handler::optimize() +{ + Wt::Dbo::Transaction transaction {_session}; + _session.execute("ANALYZE"); +} + Wt::Auth::AbstractUserDatabase& Handler::getUserDatabase() { diff --git a/src/database/DatabaseHandler.hpp b/src/database/DatabaseHandler.hpp index e9ae76f1..e1fa8078 100644 --- a/src/database/DatabaseHandler.hpp +++ b/src/database/DatabaseHandler.hpp @@ -45,6 +45,8 @@ class Handler Wt::Dbo::Session& getSession() { return _session; } + void optimize(); + Wt::Dbo::ptr getCurrentUser(); // get the current user, may return empty Wt::Dbo::ptr getUser(const std::string& loginName); Wt::Dbo::ptr getUser(const Wt::Auth::User& authUser); diff --git a/src/database/Release.cpp b/src/database/Release.cpp index 41731c2c..f5fd11fc 100644 --- a/src/database/Release.cpp +++ b/src/database/Release.cpp @@ -269,7 +269,10 @@ Release::getArtists(TrackArtistLink::Type linkType) const assert(session()); Wt::Dbo::collection> res = session()->query>( - "SELECT DISTINCT a FROM artist a INNER JOIN track_artist_link t_a_l ON t_a_l.artist_id = a.id INNER JOIN track t ON t.id = t_a_l.track_id INNER JOIN release r ON r.id = t.release_id") + "SELECT DISTINCT a FROM artist a" + " INNER JOIN track_artist_link t_a_l ON t_a_l.artist_id = a.id" + " INNER JOIN track t ON t.id = t_a_l.track_id" + " INNER JOIN release r ON r.id = t.release_id") .where("r.id = ?").bind(self()->id()) .where("t_a_l.type = ?").bind(linkType); @@ -328,6 +331,12 @@ Release::getTracks(const std::set& clusterIds) const return std::vector< Wt::Dbo::ptr > (res.begin(), res.end()); } +std::size_t +Release::getTracksCount() const +{ + return _tracks.size(); +} + std::chrono::milliseconds Release::getDuration() const { @@ -337,8 +346,8 @@ Release::getDuration() const using milli = std::chrono::duration; - Wt::Dbo::Query query {session()->query("SELECT SUM(duration) FROM track t INNER JOIN release r ON t.release_id = r.id WHERE r.id = ?") - .bind(self()->id())}; + Wt::Dbo::Query query {session()->query("SELECT SUM(duration) FROM track t INNER JOIN release r ON t.release_id = r.id") + .where("r.id = ?").bind(self()->id())}; return query.resultValue(); } diff --git a/src/database/Release.hpp b/src/database/Release.hpp index b2675209..2a9d76a9 100644 --- a/src/database/Release.hpp +++ b/src/database/Release.hpp @@ -64,9 +64,10 @@ class Release : public Wt::Dbo::Dbo bool& moreExpected); std::vector> getTracks(const std::set& clusters = std::set()) const; + std::size_t getTracksCount() const; // Get the cluster of the tracks that belong to this release - // Each clusters are grouped by cluster type, sorted by the number of occurence + // Each clusters are grouped by cluster type, sorted by the number of occurence (max to min) // size is the max number of cluster per cluster type std::vector>> getClusterGroups(std::vector> clusterTypes, std::size_t size) const; diff --git a/src/database/Track.cpp b/src/database/Track.cpp index 608f70d2..3a5897e6 100644 --- a/src/database/Track.cpp +++ b/src/database/Track.cpp @@ -103,13 +103,6 @@ Track::getMBIDDuplicates(Wt::Dbo::Session& session) return std::vector(res.begin(), res.end()); } -std::vector -Track::getChecksumDuplicates(Wt::Dbo::Session& session) -{ - Wt::Dbo::collection res = session.query( "SELECT track FROM track WHERE checksum in (SELECT checksum FROM track WHERE Length(checksum) > 0 GROUP BY checksum HAVING COUNT(*) > 1)").orderBy("track.release_id,track.disc_number,track.track_number,track.checksum"); - return std::vector(res.begin(), res.end()); -} - std::vector Track::getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int limit) { diff --git a/src/database/Track.hpp b/src/database/Track.hpp index 9bcaa3aa..92a00216 100644 --- a/src/database/Track.hpp +++ b/src/database/Track.hpp @@ -70,7 +70,6 @@ class Track : public Wt::Dbo::Dbo static std::vector getAllIds(Wt::Dbo::Session& session); // nested transaction static std::vector getAllPaths(Wt::Dbo::Session& session); // nested transaction static std::vector getMBIDDuplicates(Wt::Dbo::Session& session); - static std::vector getChecksumDuplicates(Wt::Dbo::Session& session); static std::vector getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1); static std::vector getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction static std::vector getAllIdsWithFeatures(Wt::Dbo::Session& session, boost::optional limit = {}); // nested transaction @@ -86,7 +85,6 @@ class Track : public Wt::Dbo::Dbo void setDuration(std::chrono::milliseconds duration) { _duration = duration; } void setLastWriteTime(Wt::WDateTime time) { _fileLastWrite = time; } void setAddedTime(Wt::WDateTime time) { _fileAdded = time; } - void setChecksum(const std::vector& checksum) { _fileChecksum = checksum; } void setYear(int year) { _year = year; } void setOriginalYear(int year) { _originalYear = year; } void setHasCover(bool hasCover) { _hasCover = hasCover; } @@ -109,7 +107,6 @@ class Track : public Wt::Dbo::Dbo boost::optional getOriginalYear() const; Wt::WDateTime getLastWriteTime() const { return _fileLastWrite; } Wt::WDateTime getAddedTime() const { return _fileAdded; } - const std::vector& getChecksum() const { return _fileChecksum; } bool hasCover() const { return _hasCover; } const std::string& getMBID() const { return _MBID; } boost::optional getCopyright() const; @@ -136,7 +133,6 @@ class Track : public Wt::Dbo::Dbo Wt::Dbo::field(a, _filePath, "file_path"); Wt::Dbo::field(a, _fileLastWrite, "file_last_write"); Wt::Dbo::field(a, _fileAdded, "file_added"); - Wt::Dbo::field(a, _fileChecksum, "checksum"); Wt::Dbo::field(a, _hasCover, "has_cover"); Wt::Dbo::field(a, _MBID, "mbid"); Wt::Dbo::field(a, _copyright, "copyright"); @@ -165,7 +161,6 @@ class Track : public Wt::Dbo::Dbo int _year = 0; int _originalYear = 0; std::string _filePath; - std::vector _fileChecksum; Wt::WDateTime _fileLastWrite; Wt::WDateTime _fileAdded; bool _hasCover = false; diff --git a/src/scanner/MediaScanner.cpp b/src/scanner/MediaScanner.cpp index fa8a2cf3..6e4746b0 100644 --- a/src/scanner/MediaScanner.cpp +++ b/src/scanner/MediaScanner.cpp @@ -441,6 +441,10 @@ MediaScanner::scan(boost::system::error_code err) _curState = State::NotScheduled; _inProgressStats.reset(); } + + LMS_LOG(DBUPDATER, INFO) << "Optimizing db..."; + _db.optimize(); + LMS_LOG(DBUPDATER, INFO) << "Optimize db done!"; } void @@ -522,14 +526,11 @@ MediaScanner::scanAudioFile(const boost::filesystem::path& file, bool forceScan, stats.scans++; - std::vector checksum ; - computeCrc(file, checksum); - Wt::Dbo::Transaction transaction {_db.getSession()}; Wt::Dbo::ptr track {Track::getByPath(_db.getSession(), file) }; - // We estimate this is a audio file if: + // We estimate this is an audio file if: // - we found a least one audio stream // - the duration is not null if (trackInfo->audioStreams.empty()) @@ -620,7 +621,6 @@ MediaScanner::scanAudioFile(const boost::filesystem::path& file, bool forceScan, track.modify()->addArtistLink(Database::TrackArtistLink::create(_db.getSession(), track, releaseArtist, Database::TrackArtistLink::Type::ReleaseArtist)); track.modify()->setScanVersion(_scanVersion); - track.modify()->setChecksum(checksum); track.modify()->setRelease(release); track.modify()->setClusters(clusters); track.modify()->setLastWriteTime(lastWriteTime); @@ -807,14 +807,6 @@ MediaScanner::checkDuplicatedAudioFiles(Stats& stats) stats.duplicateMBID++; } - tracks = Database::Track::getChecksumDuplicates(_db.getSession()); - for (Track::pointer track : tracks) - { - LMS_LOG(DBUPDATER, INFO) << "Found duplicated checksum [" << bufferToString(track->getChecksum()) << "], file: " << track->getPath().string() << " - " << track->getName(); - stats.duplicateHashes++; - } - - LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!"; } diff --git a/test/database/DatabaseTest.cpp b/test/database/DatabaseTest.cpp index 7f1eb425..e1ab45c4 100644 --- a/test/database/DatabaseTest.cpp +++ b/test/database/DatabaseTest.cpp @@ -427,6 +427,7 @@ testSingleTrackSingleRelease(Wt::Dbo::Session& session) CHECK(release); CHECK(release->getTracks().size() == 1); CHECK(release->getTracks().front().id() == trackId); + CHECK(release->getTracksCount() == 1); } { @@ -547,6 +548,14 @@ testSingleTrackSingleReleaseSingleCluster(Wt::Dbo::Session& session) CHECK(releases.front().id() == releaseId); } + { + Wt::Dbo::Transaction transaction {session}; + + auto cluster {Cluster::getById(session, clusterId)}; + CHECK(cluster->getReleasesCount() == 1); + CHECK(cluster->getTracksCount() == 1); + } + { Wt::Dbo::Transaction transaction {session};