owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved response time of database queries

Browse Source
This commit is contained in:
emeric
2019-07-03 13:32:02 +02:00
parent 78579f2456
commit ea3e453aa1
11 changed files with 74 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/subsonic/SubsonicResource.cpp
+14 -11
View File
@@ -18,6 +18,7 @@
*/
#include "SubsonicResource.hpp"
#include <atomic>
#include <mutex>
#include <numeric>
#include <random>
@@ -357,7 +358,11 @@ std::string parameterMapToDebugString(const Wt::Http::ParameterMap& parameterMap
void
SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Response &response)
{
LMS_LOG(API_SUBSONIC, DEBUG) << "Handling request '" << request.path() << "', params = " << parameterMapToDebugString(request.getParameterMap());
static std::atomic<std::size_t> curRequestId{};
const std::size_t requestId {curRequestId++};
LMS_LOG(API_SUBSONIC, DEBUG) << "Handling request " << requestId << " '" << request.path() << "', params = " << parameterMapToDebugString(request.getParameterMap());
const Wt::Http::ParameterMap& parameters {request.getParameterMap()};
@@ -387,7 +392,7 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp
resp.write(response.out(), format);
response.setMimeType(ResponseFormatToMimeType(format));
LMS_LOG(API_SUBSONIC, DEBUG) << "Request '" << request.path() << "' handled!";
LMS_LOG(API_SUBSONIC, DEBUG) << "Request " << requestId << " '" << request.path() << "' handled!";
return;
}
@@ -416,7 +421,7 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp
continuation->setData(std::move(res.continuationData));
}
LMS_LOG(API_SUBSONIC, DEBUG) << "Request '" << request.path() << "' handled!";
LMS_LOG(API_SUBSONIC, DEBUG) << "Request " << requestId << " '" << request.path() << "' handled!";
return;
}
@@ -547,7 +552,7 @@ releaseToResponseNode(const Database::User::pointer& user, const Database::Relea
if (id3)
{
albumNode.setAttribute("name", release->getName());
albumNode.setAttribute("songCount", std::to_string(release->getTracks().size()));
albumNode.setAttribute("songCount", std::to_string(release->getTracksCount()));
albumNode.setAttribute("duration", std::to_string(std::chrono::duration_cast<std::chrono::seconds>(release->getDuration()).count()));
}
else
@@ -558,8 +563,9 @@ releaseToResponseNode(const Database::User::pointer& user, const Database::Relea
albumNode.setAttribute("id", IdToString({Id::Type::Release, release.id()}));
albumNode.setAttribute("coverArt", IdToString({Id::Type::Release, release.id()}));
if (release->getReleaseYear())
albumNode.setAttribute("year", std::to_string(*release->getReleaseYear()));
auto releaseYear {release->getReleaseYear()};
if (releaseYear)
albumNode.setAttribute("year", std::to_string(*releaseYear));
auto artists {release->getReleaseArtists()};
if (artists.empty())
@@ -632,11 +638,8 @@ clusterToResponseNode(const Database::Cluster::pointer& cluster)
Response::Node clusterNode;
clusterNode.setValue(cluster->getName());
clusterNode.setAttribute("songCount", std::to_string(cluster->getTrackIds().size()));
{
auto releases {Database::Release::getByFilter(*cluster.session(), {cluster.id()})};
clusterNode.setAttribute("albumCount", std::to_string(releases.size()));
}
clusterNode.setAttribute("songCount", std::to_string(cluster->getTracksCount()));
clusterNode.setAttribute("albumCount", std::to_string(cluster->getReleasesCount()));
return clusterNode;
}
src/database/Cluster.cpp
+13 -2
View File
@@ -74,7 +74,7 @@ Cluster::addTrack(Wt::Dbo::ptr<Track> track)
std::vector<Wt::Dbo::ptr<Track>>
Cluster::getTracks(int offset, int limit) const
{
Wt::Dbo::collection<Track::pointer> res = session()->query<Track::pointer>("select t from track t INNER JOIN cluster c ON c.id = t_c.cluster_id INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
Wt::Dbo::collection<Track::pointer> res = session()->query<Track::pointer>("SELECT t FROM track t INNER JOIN cluster c ON c.id = t_c.cluster_id INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.where("c.id = ?").bind(self()->id())
.offset(offset)
.limit(limit);
@@ -88,13 +88,24 @@ Cluster::getTrackIds() const
assert(session());
assert(IdIsValid(self()->id()));
Wt::Dbo::collection<IdType> res = session()->query<IdType>("SELECT t_c.track_id from track_cluster t_c INNER JOIN cluster c ON c.id = t_c.cluster_id")
Wt::Dbo::collection<IdType> res = session()->query<IdType>("SELECT t_c.track_id FROM track_cluster t_c INNER JOIN cluster c ON c.id = t_c.cluster_id")
.where("c.id = ?").bind(self()->id());
return std::set<IdType>(res.begin(), res.end());
}
std::size_t
Cluster::getReleasesCount() const
{
assert(session());
assert(IdIsValid(self()->id()));
return session()->query<int>("SELECT COUNT(DISTINCT r.id) FROM release r INNER JOIN track t on t.release_id = r.id INNER JOIN cluster c ON c.id = t_c.cluster_id INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.where("c.id = ?").bind(self()->id());
}
ClusterType::ClusterType(std::string name)
: _name(name)
src/database/Cluster.hpp
+4 -3
View File
@@ -51,11 +51,12 @@ class Cluster : public Wt::Dbo::Dbo<Cluster>
static pointer create(Wt::Dbo::Session& session, Wt::Dbo::ptr<ClusterType> type, std::string name);
// Accessors
const std::string& getName() const { return _name; }
Wt::Dbo::ptr<ClusterType> getType() const { return _clusterType; }
std::size_t getCount() const { return _tracks.size(); }
const std::string& getName() const { return _name; }
Wt::Dbo::ptr<ClusterType> getType() const { return _clusterType; }
std::size_t getTracksCount() const { return _tracks.size(); }
std::vector<Wt::Dbo::ptr<Track>> getTracks(int offset, int limit) const;
std::set<IdType> getTrackIds() const;
std::size_t getReleasesCount() const;
void addTrack(Wt::Dbo::ptr<Track> track);
src/database/DatabaseHandler.cpp
+13
View File
@@ -231,15 +231,21 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool)
// Indexes
_session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_path_idx ON track(file_path)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)");
_session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)");
@@ -257,6 +263,13 @@ Handler::~Handler()
delete _users;
}
void
Handler::optimize()
{
Wt::Dbo::Transaction transaction {_session};
_session.execute("ANALYZE");
}
Wt::Auth::AbstractUserDatabase&
Handler::getUserDatabase()
{
src/database/DatabaseHandler.hpp
+2
View File
@@ -45,6 +45,8 @@ class Handler
Wt::Dbo::Session& getSession() { return _session; }
void optimize();
Wt::Dbo::ptr<User> getCurrentUser(); // get the current user, may return empty
Wt::Dbo::ptr<User> getUser(const std::string& loginName);
Wt::Dbo::ptr<User> getUser(const Wt::Auth::User& authUser);
src/database/Release.cpp
+12 -3
View File
@@ -269,7 +269,10 @@ Release::getArtists(TrackArtistLink::Type linkType) const
assert(session());
Wt::Dbo::collection<Wt::Dbo::ptr<Artist>> res = session()->query<Wt::Dbo::ptr<Artist>>(
"SELECT DISTINCT a FROM artist a INNER JOIN track_artist_link t_a_l ON t_a_l.artist_id = a.id INNER JOIN track t ON t.id = t_a_l.track_id INNER JOIN release r ON r.id = t.release_id")
"SELECT DISTINCT a FROM artist a"
" INNER JOIN track_artist_link t_a_l ON t_a_l.artist_id = a.id"
" INNER JOIN track t ON t.id = t_a_l.track_id"
" INNER JOIN release r ON r.id = t.release_id")
.where("r.id = ?").bind(self()->id())
.where("t_a_l.type = ?").bind(linkType);
@@ -328,6 +331,12 @@ Release::getTracks(const std::set<IdType>& clusterIds) const
return std::vector< Wt::Dbo::ptr<Track> > (res.begin(), res.end());
}
std::size_t
Release::getTracksCount() const
{
return _tracks.size();
}
std::chrono::milliseconds
Release::getDuration() const
{
@@ -337,8 +346,8 @@ Release::getDuration() const
using milli = std::chrono::duration<int, std::milli>;
Wt::Dbo::Query<milli> query {session()->query<milli>("SELECT SUM(duration) FROM track t INNER JOIN release r ON t.release_id = r.id WHERE r.id = ?")
.bind(self()->id())};
Wt::Dbo::Query<milli> query {session()->query<milli>("SELECT SUM(duration) FROM track t INNER JOIN release r ON t.release_id = r.id")
.where("r.id = ?").bind(self()->id())};
return query.resultValue();
}
src/database/Release.hpp
+2 -1
View File
@@ -64,9 +64,10 @@ class Release : public Wt::Dbo::Dbo<Release>
bool& moreExpected);
std::vector<Wt::Dbo::ptr<Track>> getTracks(const std::set<IdType>& clusters = std::set<IdType>()) const;
std::size_t getTracksCount() const;
// Get the cluster of the tracks that belong to this release
// Each clusters are grouped by cluster type, sorted by the number of occurence
// Each clusters are grouped by cluster type, sorted by the number of occurence (max to min)
// size is the max number of cluster per cluster type
std::vector<std::vector<Wt::Dbo::ptr<Cluster>>> getClusterGroups(std::vector<Wt::Dbo::ptr<ClusterType>> clusterTypes, std::size_t size) const;
src/database/Track.cpp
-7
View File
@@ -103,13 +103,6 @@ Track::getMBIDDuplicates(Wt::Dbo::Session& session)
return std::vector<pointer>(res.begin(), res.end());
}
std::vector<Track::pointer>
Track::getChecksumDuplicates(Wt::Dbo::Session& session)
{
Wt::Dbo::collection<pointer> res = session.query<pointer>( "SELECT track FROM track WHERE checksum in (SELECT checksum FROM track WHERE Length(checksum) > 0 GROUP BY checksum HAVING COUNT(*) > 1)").orderBy("track.release_id,track.disc_number,track.track_number,track.checksum");
return std::vector<pointer>(res.begin(), res.end());
}
std::vector<Track::pointer>
Track::getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int limit)
{
src/database/Track.hpp
-5
View File
@@ -70,7 +70,6 @@ class Track : public Wt::Dbo::Dbo<Track>
static std::vector<IdType> getAllIds(Wt::Dbo::Session& session); // nested transaction
static std::vector<boost::filesystem::path> getAllPaths(Wt::Dbo::Session& session); // nested transaction
static std::vector<pointer> getMBIDDuplicates(Wt::Dbo::Session& session);
static std::vector<pointer> getChecksumDuplicates(Wt::Dbo::Session& session);
static std::vector<pointer> getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1);
static std::vector<pointer> getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction
static std::vector<IdType> getAllIdsWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit = {}); // nested transaction
@@ -86,7 +85,6 @@ class Track : public Wt::Dbo::Dbo<Track>
void setDuration(std::chrono::milliseconds duration) { _duration = duration; }
void setLastWriteTime(Wt::WDateTime time) { _fileLastWrite = time; }
void setAddedTime(Wt::WDateTime time) { _fileAdded = time; }
void setChecksum(const std::vector<unsigned char>& checksum) { _fileChecksum = checksum; }
void setYear(int year) { _year = year; }
void setOriginalYear(int year) { _originalYear = year; }
void setHasCover(bool hasCover) { _hasCover = hasCover; }
@@ -109,7 +107,6 @@ class Track : public Wt::Dbo::Dbo<Track>
boost::optional<int> getOriginalYear() const;
Wt::WDateTime getLastWriteTime() const { return _fileLastWrite; }
Wt::WDateTime getAddedTime() const { return _fileAdded; }
const std::vector<unsigned char>& getChecksum() const { return _fileChecksum; }
bool hasCover() const { return _hasCover; }
const std::string& getMBID() const { return _MBID; }
boost::optional<std::string> getCopyright() const;
@@ -136,7 +133,6 @@ class Track : public Wt::Dbo::Dbo<Track>
Wt::Dbo::field(a, _filePath, "file_path");
Wt::Dbo::field(a, _fileLastWrite, "file_last_write");
Wt::Dbo::field(a, _fileAdded, "file_added");
Wt::Dbo::field(a, _fileChecksum, "checksum");
Wt::Dbo::field(a, _hasCover, "has_cover");
Wt::Dbo::field(a, _MBID, "mbid");
Wt::Dbo::field(a, _copyright, "copyright");
@@ -165,7 +161,6 @@ class Track : public Wt::Dbo::Dbo<Track>
int _year = 0;
int _originalYear = 0;
std::string _filePath;
std::vector<unsigned char> _fileChecksum;
Wt::WDateTime _fileLastWrite;
Wt::WDateTime _fileAdded;
bool _hasCover = false;
src/scanner/MediaScanner.cpp
+5 -13
View File
@@ -441,6 +441,10 @@ MediaScanner::scan(boost::system::error_code err)
_curState = State::NotScheduled;
_inProgressStats.reset();
}
LMS_LOG(DBUPDATER, INFO) << "Optimizing db...";
_db.optimize();
LMS_LOG(DBUPDATER, INFO) << "Optimize db done!";
}
void
@@ -522,14 +526,11 @@ MediaScanner::scanAudioFile(const boost::filesystem::path& file, bool forceScan,
stats.scans++;
std::vector<unsigned char> checksum ;
computeCrc(file, checksum);
Wt::Dbo::Transaction transaction {_db.getSession()};
Wt::Dbo::ptr<Track> track {Track::getByPath(_db.getSession(), file) };
// We estimate this is a audio file if:
// We estimate this is an audio file if:
// - we found a least one audio stream
// - the duration is not null
if (trackInfo->audioStreams.empty())
@@ -620,7 +621,6 @@ MediaScanner::scanAudioFile(const boost::filesystem::path& file, bool forceScan,
track.modify()->addArtistLink(Database::TrackArtistLink::create(_db.getSession(), track, releaseArtist, Database::TrackArtistLink::Type::ReleaseArtist));
track.modify()->setScanVersion(_scanVersion);
track.modify()->setChecksum(checksum);
track.modify()->setRelease(release);
track.modify()->setClusters(clusters);
track.modify()->setLastWriteTime(lastWriteTime);
@@ -807,14 +807,6 @@ MediaScanner::checkDuplicatedAudioFiles(Stats& stats)
stats.duplicateMBID++;
}
tracks = Database::Track::getChecksumDuplicates(_db.getSession());
for (Track::pointer track : tracks)
{
LMS_LOG(DBUPDATER, INFO) << "Found duplicated checksum [" << bufferToString(track->getChecksum()) << "], file: " << track->getPath().string() << " - " << track->getName();
stats.duplicateHashes++;
}
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!";
}