owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Subsonic API: cached cluster stats for track and releases, in order to make the getGenre entrypoint efficient on large databases

Browse Source
This commit is contained in:
emeric
2023-11-08 20:26:24 +01:00
parent c2e1c5d9df
commit 121d27b5cd
20 changed files with 301 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/database/impl/Cluster.cpp
+44 -13
View File
@@ -32,11 +32,12 @@ namespace Database
{
namespace
{
Wt::Dbo::Query<Cluster::ClusterFindResult> createQuery(Session& session, const Cluster::FindParameters& params)
template <typename ResultType>
Wt::Dbo::Query<ResultType> createQuery(Session& session, std::string_view itemToSelect, const Cluster::FindParameters& params)
{
session.checkSharedLocked();
auto query{ session.getDboSession().query<Cluster::ClusterFindResult>("SELECT DISTINCT c.id,c.name FROM cluster c") };
auto query{ session.getDboSession().query<ResultType>("SELECT DISTINCT " + std::string{ itemToSelect } + " FROM cluster c") };
if (params.track.isValid() || params.release.isValid())
{
@@ -54,6 +55,21 @@ namespace Database
return query;
}
template <typename ResultType>
Wt::Dbo::Query<ResultType> createQuery(Session& session, const Cluster::FindParameters& params)
{
std::string_view itemToSelect;
if constexpr (std::is_same_v<ResultType, ClusterId>)
itemToSelect = "c.id";
else if constexpr (std::is_same_v<ResultType, Wt::Dbo::ptr<Cluster>>)
itemToSelect = "c";
else
static_assert("Unhandled type");
return createQuery<ResultType>(session, itemToSelect, params);
}
}
Cluster::Cluster(ObjectPtr<ClusterType> type, std::string_view name)
@@ -74,10 +90,18 @@ namespace Database
return session.getDboSession().query<int>("SELECT COUNT(*) FROM cluster");
}
RangeResults<Cluster::ClusterFindResult> Cluster::find(Session& session, const FindParameters& params)
RangeResults<ClusterId> Cluster::findIds(Session& session, const FindParameters& params)
{
session.checkSharedLocked();
auto query{ createQuery(session, params) };
auto query{ createQuery<ClusterId>(session, params) };
return Utils::execQuery(query, params.range);
}
RangeResults<Cluster::pointer> Cluster::find(Session& session, const FindParameters& params)
{
session.checkSharedLocked();
auto query{ createQuery<Wt::Dbo::ptr<Cluster>>(session, params) };
return Utils::execQuery(query, params.range);
}
@@ -97,6 +121,22 @@ namespace Database
return session.getDboSession().find<Cluster>().where("id = ?").bind(id).resultValue();
}
std::size_t Cluster::computeTrackCount(Session& session, ClusterId id)
{
session.checkSharedLocked();
return session.getDboSession().query<int>("SELECT COUNT(t.id) FROM track t INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.where("t_c.cluster_id = ?").bind(id).resultValue();
}
std::size_t Cluster::computeReleaseCount(Session& session, ClusterId id)
{
session.checkSharedLocked();
return session.getDboSession().query<int>("SELECT COUNT(DISTINCT r.id) FROM release r INNER JOIN track t on t.release_id = r.id INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.where("t_c.cluster_id = ?").bind(id).resultValue();
}
void Cluster::addTrack(ObjectPtr<Track> track)
{
_tracks.insert(getDboPtr(track));
@@ -112,15 +152,6 @@ namespace Database
return Utils::execQuery(query, range);
}
std::size_t Cluster::getReleasesCount() const
{
assert(session());
return session()->query<int>("SELECT COUNT(DISTINCT r.id) FROM release r INNER JOIN track t on t.release_id = r.id INNER JOIN cluster c ON c.id = t_c.cluster_id INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.where("c.id = ?").bind(getId());
}
ClusterType::ClusterType(std::string_view name)
: _name{ name }
{
src/libs/services/database/impl/Migration.cpp
+11
View File
@@ -224,6 +224,16 @@ CREATE TABLE IF NOT EXISTS "track_backup" (
session.getDboSession().execute("UPDATE user SET feedback_backend = scrobbling_backend");
}
static void migrateFromV43(Session& session)
{
// add counts in genre table
session.getDboSession().execute("ALTER TABLE cluster ADD track_count INTEGER");
session.getDboSession().execute("ALTER TABLE cluster ADD release_count INTEGER");
// Just increment the scan version of the settings to make the next scheduled scan rescan everything
ScanSettings::get(session).modify()->incScanVersion();
}
void doDbMigration(Session& session)
{
static const std::string outdatedMsg{ "Outdated database, please rebuild it (delete the .db file and restart)" };
@@ -245,6 +255,7 @@ CREATE TABLE IF NOT EXISTS "track_backup" (
{40, migrateFromV40},
{41, migrateFromV41},
{42, migrateFromV42},
{43, migrateFromV43},
};
{
src/libs/services/database/impl/Migration.hpp
+1 -1
View File
@@ -26,7 +26,7 @@ namespace Database
class Session;
using Version = std::size_t;
static constexpr Version LMS_DATABASE_VERSION{ 43 };
static constexpr Version LMS_DATABASE_VERSION{ 44 };
class VersionInfo
{
public:
src/libs/services/database/include/services/database/Cluster.hpp
+20 -9
View File
@@ -59,26 +59,35 @@ namespace Database {
Cluster() = default;
// Find utility
// As clusters only have a name, this is an optim to directly get the cluster names
using ClusterFindResult = std::tuple<ClusterId, std::string>;
static std::size_t getCount(Session& session);
static RangeResults<ClusterFindResult> find(Session& session, const FindParameters& range);
static RangeResults<ClusterId> findIds(Session& session, const FindParameters& range);
static RangeResults<pointer> find(Session& session, const FindParameters& range);
static void find(Session& session, const FindParameters& range, std::function<void(const pointer& cluster)> _func);
static pointer find(Session& session, ClusterId id);
static RangeResults<ClusterId> findOrphans(Session& session, Range range);
// Accessors
const std::string& getName() const { return _name; }
ObjectPtr<ClusterType> getType() const { return _clusterType; }
std::size_t getTracksCount() const { return _tracks.size(); }
RangeResults<TrackId> getTracks(Range range) const;
std::size_t getReleasesCount() const;
// May be very slow
static std::size_t computeTrackCount(Session& session, ClusterId id);
static std::size_t computeReleaseCount(Session& session, ClusterId id);
// Accessors
std::string_view getName() const { return _name; }
ObjectPtr<ClusterType> getType() const { return _clusterType; }
std::size_t getTracksCount() const { return _trackCount; }
RangeResults<TrackId> getTracks(Range range) const;
std::size_t getReleasesCount() const { return _releaseCount; };
void setReleaseCount(std::size_t releaseCount) { _releaseCount = releaseCount; }
void setTrackCount(std::size_t trackCount) { _trackCount = trackCount; }
void addTrack(ObjectPtr<Track> track);
template<class Action>
void persist(Action& a)
{
Wt::Dbo::field(a, _name, "name");
// cached field since queries are too long
Wt::Dbo::field(a, _trackCount, "track_count");
Wt::Dbo::field(a, _releaseCount, "release_count");
Wt::Dbo::belongsTo(a, _clusterType, "cluster_type", Wt::Dbo::OnDeleteCascade);
Wt::Dbo::hasMany(a, _tracks, Wt::Dbo::ManyToMany, "track_cluster", "", Wt::Dbo::OnDeleteCascade);
@@ -92,6 +101,8 @@ namespace Database {
static const std::size_t _maxNameLength = 128;
std::string _name;
int _trackCount{};
int _releaseCount{};
Wt::Dbo::ptr<ClusterType> _clusterType;
Wt::Dbo::collection< Wt::Dbo::ptr<Track> > _tracks;
src/libs/services/database/test/Cluster.cpp
+23 -11
View File
@@ -49,9 +49,9 @@ TEST_F(DatabaseFixture, Cluster)
EXPECT_EQ(cluster->getType()->getId(), clusterType.getId());
{
const auto clusters{ Cluster::find(session, Cluster::FindParameters {}) };
const auto clusters{ Cluster::findIds(session, Cluster::FindParameters {}) };
ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(std::get<ClusterId>(clusters.results.front()), cluster.getId());
EXPECT_EQ(clusters.results.front(), cluster.getId());
}
{
@@ -106,6 +106,8 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
EXPECT_EQ(clusters.results.size(), 2);
EXPECT_TRUE(track->getClusters().empty());
EXPECT_TRUE(track->getClusterIds().empty());
EXPECT_EQ(Cluster::computeTrackCount(session, cluster1.getId()), 0);
EXPECT_EQ(Cluster::computeTrackCount(session, cluster2.getId()), 0);
}
{
@@ -116,9 +118,11 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
{
auto transaction{ session.createSharedTransaction() };
auto clusters{ Cluster::find(session, Cluster::FindParameters {}.setTrack(track.getId())) };
auto clusters{ Cluster::findIds(session, Cluster::FindParameters {}.setTrack(track.getId())) };
ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(std::get<ClusterId>(clusters.results.front()), cluster1.getId());
EXPECT_EQ(clusters.results.front(), cluster1.getId());
EXPECT_EQ(Cluster::computeTrackCount(session, cluster1.getId()), 1);
EXPECT_EQ(Cluster::computeTrackCount(session, cluster2.getId()), 0);
}
{
@@ -182,6 +186,8 @@ TEST_F(DatabaseFixture, Cluster_singleTrackWithSeveralClusters)
const auto tracks{ Track::findIds(session, Track::FindParameters{}.setClusters(clusterIds)) };
EXPECT_TRUE(tracks.results.empty());
EXPECT_EQ(Cluster::computeTrackCount(session, cluster1.getId()), 1);
EXPECT_EQ(Cluster::computeTrackCount(session, cluster2.getId()), 0);
}
{
@@ -196,6 +202,8 @@ TEST_F(DatabaseFixture, Cluster_singleTrackWithSeveralClusters)
const auto tracks{ Track::findIds(session, Track::FindParameters{}.setClusters(clusterIds)) };
ASSERT_FALSE(tracks.results.empty());
EXPECT_EQ(tracks.results.front(), track.getId());
EXPECT_EQ(Cluster::computeTrackCount(session, cluster1.getId()), 1);
EXPECT_EQ(Cluster::computeTrackCount(session, cluster2.getId()), 1);
}
}
@@ -219,7 +227,7 @@ TEST_F(DatabaseFixture, Cluster_multiTracks)
auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session, Range{}).results.empty());
EXPECT_EQ(cluster->getTracksCount(), tracks.size());
EXPECT_EQ(Cluster::computeTrackCount(session, cluster.getId()), tracks.size());
for (TrackId trackId : cluster->getTracks(Range{}).results)
{
@@ -344,6 +352,8 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
ASSERT_EQ(Cluster::findOrphans(session, Range{}).results.size(), 2);
EXPECT_TRUE(Release::find(session, Release::FindParameters{}.setClusters({ unusedCluster.getId() })).results.empty());
EXPECT_EQ(Release::find(session, Release::FindParameters{}).results.size(), 1);
EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 0);
EXPECT_EQ(Cluster::computeReleaseCount(session, unusedCluster.getId()), 0);
}
{
@@ -361,14 +371,16 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(clusters.results.front(), unusedCluster.getId());
}
EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 1);
EXPECT_EQ(Cluster::computeReleaseCount(session, unusedCluster.getId()), 0);
}
{
auto transaction{ session.createSharedTransaction() };
const auto clusters{ Cluster::find(session, Cluster::FindParameters{}.setRelease(release.getId())) };
const auto clusters{ Cluster::findIds(session, Cluster::FindParameters{}.setRelease(release.getId())) };
ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(std::get<ClusterId>(clusters.results.front()), cluster.getId());
EXPECT_EQ(clusters.results.front(), cluster.getId());
}
{
@@ -389,10 +401,10 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
{
auto transaction{ session.createSharedTransaction() };
EXPECT_EQ(cluster->getReleasesCount(), 1);
EXPECT_EQ(cluster->getTracksCount(), 1);
EXPECT_EQ(unusedCluster->getReleasesCount(), 0);
EXPECT_EQ(unusedCluster->getTracksCount(), 0);
EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 1);
EXPECT_EQ(Cluster::computeTrackCount(session, cluster.getId()), 1);
EXPECT_EQ(Cluster::computeReleaseCount(session, unusedCluster.getId()), 0);
EXPECT_EQ(Cluster::computeTrackCount(session, unusedCluster.getId()), 0);
}
}