Subsonic API: cached cluster stats for track and releases, in order to make the getGenre entrypoint efficient on large databases
This commit is contained in:
@@ -85,6 +85,7 @@
|
||||
<message id="Lms.Admin.ScannerController.status-in-progress">Scanning: step {1}/{2}</message>
|
||||
<message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Checking for duplicate files... {1} files</message>
|
||||
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Checking files... {1}%</message>
|
||||
<message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Computing stats... {1}%</message>
|
||||
<message id="Lms.Admin.ScannerController.step-discovering-files">Discovering files: {1} files</message>
|
||||
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)...</message>
|
||||
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Reloading similarity engine: {1}%...</message>
|
||||
|
||||
@@ -85,6 +85,7 @@
|
||||
<message id="Lms.Admin.ScannerController.status-in-progress">En cours de scan : étape {1}/{2}</message>
|
||||
<message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Vérification des fichiers dupliqués... {1} fichiers</message>
|
||||
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Vérification des fichiers... {1}%</message>
|
||||
<message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calcul des statistiques... {1}%</message>
|
||||
<message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers : {1} fichiers</message>
|
||||
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)...</message>
|
||||
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Rechargement du moteur de recommandation : {1}%...</message>
|
||||
|
||||
@@ -85,6 +85,7 @@
|
||||
<message id="Lms.Admin.ScannerController.status-in-progress">Scansione: passo {1}/{2}</message>
|
||||
|
||||
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Controllo file... {1}%</message>
|
||||
|
||||
<message id="Lms.Admin.ScannerController.step-discovering-files">File trovati: {1} files</message>
|
||||
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)...</message>
|
||||
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Ricarica motore di tracce simili: {1}%...</message>
|
||||
|
||||
@@ -85,6 +85,7 @@
|
||||
<message id="Lms.Admin.ScannerController.status-in-progress">扫描中: 阶段 {1}/{2}</message>
|
||||
|
||||
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">检查文件中... {1}%</message>
|
||||
|
||||
<message id="Lms.Admin.ScannerController.step-discovering-files">检索文件中: {1} 文件</message>
|
||||
<message id="Lms.Admin.ScannerController.step-fetching-track-features">从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)...</message>
|
||||
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">重载相似引擎中 {1}%...</message>
|
||||
|
||||
@@ -32,11 +32,12 @@ namespace Database
|
||||
{
|
||||
namespace
|
||||
{
|
||||
Wt::Dbo::Query<Cluster::ClusterFindResult> createQuery(Session& session, const Cluster::FindParameters& params)
|
||||
template <typename ResultType>
|
||||
Wt::Dbo::Query<ResultType> createQuery(Session& session, std::string_view itemToSelect, const Cluster::FindParameters& params)
|
||||
{
|
||||
session.checkSharedLocked();
|
||||
|
||||
auto query{ session.getDboSession().query<Cluster::ClusterFindResult>("SELECT DISTINCT c.id,c.name FROM cluster c") };
|
||||
auto query{ session.getDboSession().query<ResultType>("SELECT DISTINCT " + std::string{ itemToSelect } + " FROM cluster c") };
|
||||
|
||||
if (params.track.isValid() || params.release.isValid())
|
||||
{
|
||||
@@ -54,6 +55,21 @@ namespace Database
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
template <typename ResultType>
|
||||
Wt::Dbo::Query<ResultType> createQuery(Session& session, const Cluster::FindParameters& params)
|
||||
{
|
||||
std::string_view itemToSelect;
|
||||
|
||||
if constexpr (std::is_same_v<ResultType, ClusterId>)
|
||||
itemToSelect = "c.id";
|
||||
else if constexpr (std::is_same_v<ResultType, Wt::Dbo::ptr<Cluster>>)
|
||||
itemToSelect = "c";
|
||||
else
|
||||
static_assert("Unhandled type");
|
||||
|
||||
return createQuery<ResultType>(session, itemToSelect, params);
|
||||
}
|
||||
}
|
||||
|
||||
Cluster::Cluster(ObjectPtr<ClusterType> type, std::string_view name)
|
||||
@@ -74,10 +90,18 @@ namespace Database
|
||||
return session.getDboSession().query<int>("SELECT COUNT(*) FROM cluster");
|
||||
}
|
||||
|
||||
RangeResults<Cluster::ClusterFindResult> Cluster::find(Session& session, const FindParameters& params)
|
||||
RangeResults<ClusterId> Cluster::findIds(Session& session, const FindParameters& params)
|
||||
{
|
||||
session.checkSharedLocked();
|
||||
auto query{ createQuery(session, params) };
|
||||
auto query{ createQuery<ClusterId>(session, params) };
|
||||
|
||||
return Utils::execQuery(query, params.range);
|
||||
}
|
||||
|
||||
RangeResults<Cluster::pointer> Cluster::find(Session& session, const FindParameters& params)
|
||||
{
|
||||
session.checkSharedLocked();
|
||||
auto query{ createQuery<Wt::Dbo::ptr<Cluster>>(session, params) };
|
||||
|
||||
return Utils::execQuery(query, params.range);
|
||||
}
|
||||
@@ -97,6 +121,22 @@ namespace Database
|
||||
return session.getDboSession().find<Cluster>().where("id = ?").bind(id).resultValue();
|
||||
}
|
||||
|
||||
std::size_t Cluster::computeTrackCount(Session& session, ClusterId id)
|
||||
{
|
||||
session.checkSharedLocked();
|
||||
|
||||
return session.getDboSession().query<int>("SELECT COUNT(t.id) FROM track t INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
|
||||
.where("t_c.cluster_id = ?").bind(id).resultValue();
|
||||
}
|
||||
|
||||
std::size_t Cluster::computeReleaseCount(Session& session, ClusterId id)
|
||||
{
|
||||
session.checkSharedLocked();
|
||||
|
||||
return session.getDboSession().query<int>("SELECT COUNT(DISTINCT r.id) FROM release r INNER JOIN track t on t.release_id = r.id INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
|
||||
.where("t_c.cluster_id = ?").bind(id).resultValue();
|
||||
}
|
||||
|
||||
void Cluster::addTrack(ObjectPtr<Track> track)
|
||||
{
|
||||
_tracks.insert(getDboPtr(track));
|
||||
@@ -112,15 +152,6 @@ namespace Database
|
||||
return Utils::execQuery(query, range);
|
||||
}
|
||||
|
||||
std::size_t Cluster::getReleasesCount() const
|
||||
{
|
||||
assert(session());
|
||||
|
||||
return session()->query<int>("SELECT COUNT(DISTINCT r.id) FROM release r INNER JOIN track t on t.release_id = r.id INNER JOIN cluster c ON c.id = t_c.cluster_id INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
|
||||
.where("c.id = ?").bind(getId());
|
||||
}
|
||||
|
||||
|
||||
ClusterType::ClusterType(std::string_view name)
|
||||
: _name{ name }
|
||||
{
|
||||
|
||||
@@ -224,6 +224,16 @@ CREATE TABLE IF NOT EXISTS "track_backup" (
|
||||
session.getDboSession().execute("UPDATE user SET feedback_backend = scrobbling_backend");
|
||||
}
|
||||
|
||||
static void migrateFromV43(Session& session)
|
||||
{
|
||||
// add counts in genre table
|
||||
session.getDboSession().execute("ALTER TABLE cluster ADD track_count INTEGER");
|
||||
session.getDboSession().execute("ALTER TABLE cluster ADD release_count INTEGER");
|
||||
|
||||
// Just increment the scan version of the settings to make the next scheduled scan rescan everything
|
||||
ScanSettings::get(session).modify()->incScanVersion();
|
||||
}
|
||||
|
||||
void doDbMigration(Session& session)
|
||||
{
|
||||
static const std::string outdatedMsg{ "Outdated database, please rebuild it (delete the .db file and restart)" };
|
||||
@@ -245,6 +255,7 @@ CREATE TABLE IF NOT EXISTS "track_backup" (
|
||||
{40, migrateFromV40},
|
||||
{41, migrateFromV41},
|
||||
{42, migrateFromV42},
|
||||
{43, migrateFromV43},
|
||||
};
|
||||
|
||||
{
|
||||
|
||||
@@ -26,7 +26,7 @@ namespace Database
|
||||
class Session;
|
||||
|
||||
using Version = std::size_t;
|
||||
static constexpr Version LMS_DATABASE_VERSION{ 43 };
|
||||
static constexpr Version LMS_DATABASE_VERSION{ 44 };
|
||||
class VersionInfo
|
||||
{
|
||||
public:
|
||||
|
||||
@@ -59,26 +59,35 @@ namespace Database {
|
||||
Cluster() = default;
|
||||
|
||||
// Find utility
|
||||
// As clusters only have a name, this is an optim to directly get the cluster names
|
||||
using ClusterFindResult = std::tuple<ClusterId, std::string>;
|
||||
static std::size_t getCount(Session& session);
|
||||
static RangeResults<ClusterFindResult> find(Session& session, const FindParameters& range);
|
||||
static RangeResults<ClusterId> findIds(Session& session, const FindParameters& range);
|
||||
static RangeResults<pointer> find(Session& session, const FindParameters& range);
|
||||
static void find(Session& session, const FindParameters& range, std::function<void(const pointer& cluster)> _func);
|
||||
static pointer find(Session& session, ClusterId id);
|
||||
static RangeResults<ClusterId> findOrphans(Session& session, Range range);
|
||||
|
||||
// Accessors
|
||||
const std::string& getName() const { return _name; }
|
||||
ObjectPtr<ClusterType> getType() const { return _clusterType; }
|
||||
std::size_t getTracksCount() const { return _tracks.size(); }
|
||||
RangeResults<TrackId> getTracks(Range range) const;
|
||||
std::size_t getReleasesCount() const;
|
||||
// May be very slow
|
||||
static std::size_t computeTrackCount(Session& session, ClusterId id);
|
||||
static std::size_t computeReleaseCount(Session& session, ClusterId id);
|
||||
|
||||
// Accessors
|
||||
std::string_view getName() const { return _name; }
|
||||
ObjectPtr<ClusterType> getType() const { return _clusterType; }
|
||||
std::size_t getTracksCount() const { return _trackCount; }
|
||||
RangeResults<TrackId> getTracks(Range range) const;
|
||||
std::size_t getReleasesCount() const { return _releaseCount; };
|
||||
|
||||
void setReleaseCount(std::size_t releaseCount) { _releaseCount = releaseCount; }
|
||||
void setTrackCount(std::size_t trackCount) { _trackCount = trackCount; }
|
||||
void addTrack(ObjectPtr<Track> track);
|
||||
|
||||
template<class Action>
|
||||
void persist(Action& a)
|
||||
{
|
||||
Wt::Dbo::field(a, _name, "name");
|
||||
// cached field since queries are too long
|
||||
Wt::Dbo::field(a, _trackCount, "track_count");
|
||||
Wt::Dbo::field(a, _releaseCount, "release_count");
|
||||
|
||||
Wt::Dbo::belongsTo(a, _clusterType, "cluster_type", Wt::Dbo::OnDeleteCascade);
|
||||
Wt::Dbo::hasMany(a, _tracks, Wt::Dbo::ManyToMany, "track_cluster", "", Wt::Dbo::OnDeleteCascade);
|
||||
@@ -92,6 +101,8 @@ namespace Database {
|
||||
static const std::size_t _maxNameLength = 128;
|
||||
|
||||
std::string _name;
|
||||
int _trackCount{};
|
||||
int _releaseCount{};
|
||||
|
||||
Wt::Dbo::ptr<ClusterType> _clusterType;
|
||||
Wt::Dbo::collection< Wt::Dbo::ptr<Track> > _tracks;
|
||||
|
||||
@@ -49,9 +49,9 @@ TEST_F(DatabaseFixture, Cluster)
|
||||
EXPECT_EQ(cluster->getType()->getId(), clusterType.getId());
|
||||
|
||||
{
|
||||
const auto clusters{ Cluster::find(session, Cluster::FindParameters {}) };
|
||||
const auto clusters{ Cluster::findIds(session, Cluster::FindParameters {}) };
|
||||
ASSERT_EQ(clusters.results.size(), 1);
|
||||
EXPECT_EQ(std::get<ClusterId>(clusters.results.front()), cluster.getId());
|
||||
EXPECT_EQ(clusters.results.front(), cluster.getId());
|
||||
}
|
||||
|
||||
{
|
||||
@@ -106,6 +106,8 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
|
||||
EXPECT_EQ(clusters.results.size(), 2);
|
||||
EXPECT_TRUE(track->getClusters().empty());
|
||||
EXPECT_TRUE(track->getClusterIds().empty());
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster1.getId()), 0);
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster2.getId()), 0);
|
||||
}
|
||||
|
||||
{
|
||||
@@ -116,9 +118,11 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
|
||||
|
||||
{
|
||||
auto transaction{ session.createSharedTransaction() };
|
||||
auto clusters{ Cluster::find(session, Cluster::FindParameters {}.setTrack(track.getId())) };
|
||||
auto clusters{ Cluster::findIds(session, Cluster::FindParameters {}.setTrack(track.getId())) };
|
||||
ASSERT_EQ(clusters.results.size(), 1);
|
||||
EXPECT_EQ(std::get<ClusterId>(clusters.results.front()), cluster1.getId());
|
||||
EXPECT_EQ(clusters.results.front(), cluster1.getId());
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster1.getId()), 1);
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster2.getId()), 0);
|
||||
}
|
||||
|
||||
{
|
||||
@@ -182,6 +186,8 @@ TEST_F(DatabaseFixture, Cluster_singleTrackWithSeveralClusters)
|
||||
|
||||
const auto tracks{ Track::findIds(session, Track::FindParameters{}.setClusters(clusterIds)) };
|
||||
EXPECT_TRUE(tracks.results.empty());
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster1.getId()), 1);
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster2.getId()), 0);
|
||||
}
|
||||
|
||||
{
|
||||
@@ -196,6 +202,8 @@ TEST_F(DatabaseFixture, Cluster_singleTrackWithSeveralClusters)
|
||||
const auto tracks{ Track::findIds(session, Track::FindParameters{}.setClusters(clusterIds)) };
|
||||
ASSERT_FALSE(tracks.results.empty());
|
||||
EXPECT_EQ(tracks.results.front(), track.getId());
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster1.getId()), 1);
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster2.getId()), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,7 +227,7 @@ TEST_F(DatabaseFixture, Cluster_multiTracks)
|
||||
auto transaction{ session.createSharedTransaction() };
|
||||
EXPECT_TRUE(Cluster::findOrphans(session, Range{}).results.empty());
|
||||
|
||||
EXPECT_EQ(cluster->getTracksCount(), tracks.size());
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster.getId()), tracks.size());
|
||||
|
||||
for (TrackId trackId : cluster->getTracks(Range{}).results)
|
||||
{
|
||||
@@ -344,6 +352,8 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
|
||||
ASSERT_EQ(Cluster::findOrphans(session, Range{}).results.size(), 2);
|
||||
EXPECT_TRUE(Release::find(session, Release::FindParameters{}.setClusters({ unusedCluster.getId() })).results.empty());
|
||||
EXPECT_EQ(Release::find(session, Release::FindParameters{}).results.size(), 1);
|
||||
EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 0);
|
||||
EXPECT_EQ(Cluster::computeReleaseCount(session, unusedCluster.getId()), 0);
|
||||
}
|
||||
|
||||
{
|
||||
@@ -361,14 +371,16 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
|
||||
ASSERT_EQ(clusters.results.size(), 1);
|
||||
EXPECT_EQ(clusters.results.front(), unusedCluster.getId());
|
||||
}
|
||||
EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 1);
|
||||
EXPECT_EQ(Cluster::computeReleaseCount(session, unusedCluster.getId()), 0);
|
||||
}
|
||||
|
||||
{
|
||||
auto transaction{ session.createSharedTransaction() };
|
||||
|
||||
const auto clusters{ Cluster::find(session, Cluster::FindParameters{}.setRelease(release.getId())) };
|
||||
const auto clusters{ Cluster::findIds(session, Cluster::FindParameters{}.setRelease(release.getId())) };
|
||||
ASSERT_EQ(clusters.results.size(), 1);
|
||||
EXPECT_EQ(std::get<ClusterId>(clusters.results.front()), cluster.getId());
|
||||
EXPECT_EQ(clusters.results.front(), cluster.getId());
|
||||
}
|
||||
|
||||
{
|
||||
@@ -389,10 +401,10 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
|
||||
{
|
||||
auto transaction{ session.createSharedTransaction() };
|
||||
|
||||
EXPECT_EQ(cluster->getReleasesCount(), 1);
|
||||
EXPECT_EQ(cluster->getTracksCount(), 1);
|
||||
EXPECT_EQ(unusedCluster->getReleasesCount(), 0);
|
||||
EXPECT_EQ(unusedCluster->getTracksCount(), 0);
|
||||
EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 1);
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, cluster.getId()), 1);
|
||||
EXPECT_EQ(Cluster::computeReleaseCount(session, unusedCluster.getId()), 0);
|
||||
EXPECT_EQ(Cluster::computeTrackCount(session, unusedCluster.getId()), 0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ add_library(lmsscanner SHARED
|
||||
impl/ScannerService.cpp
|
||||
impl/ScannerStats.cpp
|
||||
impl/ScanStepCheckDuplicatedDbFiles.cpp
|
||||
impl/ScanStepComputeClusterStats.cpp
|
||||
impl/ScanStepDiscoverFiles.cpp
|
||||
impl/ScanStepRemoveOrphanDbFiles.cpp
|
||||
impl/ScanStepScanFiles.cpp
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright (C) 2023 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "ScanStepComputeClusterStats.hpp"
|
||||
#include "services/database/Db.hpp"
|
||||
#include "services/database/Cluster.hpp"
|
||||
#include "services/database/Session.hpp"
|
||||
#include "utils/Logger.hpp"
|
||||
#include "utils/Path.hpp"
|
||||
|
||||
namespace Scanner
|
||||
{
|
||||
void ScanStepComputeClusterStats::process(ScanContext& context)
|
||||
{
|
||||
using namespace Database;
|
||||
|
||||
if (context.stats.nbChanges() == 0)
|
||||
return;
|
||||
|
||||
Session& dbSession{ _db.getTLSSession() };
|
||||
|
||||
const std::size_t clusterCount{ [&] {
|
||||
auto transaction{ dbSession.createSharedTransaction() };
|
||||
return Cluster::getCount(dbSession);
|
||||
}() };
|
||||
|
||||
context.currentStepStats.totalElems = clusterCount;
|
||||
|
||||
foreachSubRange(Range{ 0, clusterCount }, 100, [&](Range range)
|
||||
{
|
||||
const std::vector<ClusterId> clusterIds{ [&]
|
||||
{
|
||||
Cluster::FindParameters params;
|
||||
params.setRange(range);
|
||||
|
||||
{
|
||||
auto transaction{ dbSession.createSharedTransaction() };
|
||||
return std::move(Cluster::findIds(dbSession, params).results);
|
||||
}
|
||||
}() };
|
||||
|
||||
for (const ClusterId clusterId : clusterIds)
|
||||
{
|
||||
std::size_t trackCount;
|
||||
std::size_t releaseCount;
|
||||
|
||||
{
|
||||
auto transaction{ dbSession.createSharedTransaction() };
|
||||
|
||||
trackCount = Cluster::computeTrackCount(dbSession, clusterId);
|
||||
releaseCount = Cluster::computeReleaseCount(dbSession, clusterId);
|
||||
}
|
||||
|
||||
{
|
||||
auto transaction{ dbSession.createUniqueTransaction() };
|
||||
|
||||
auto cluster{ Cluster::find(dbSession, clusterId) };
|
||||
cluster.modify()->setTrackCount(trackCount);
|
||||
cluster.modify()->setReleaseCount(releaseCount);
|
||||
}
|
||||
|
||||
context.currentStepStats.processedElems++;
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG) << "Recomputed stats for " << clusterCount << " clusters!";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (C) 2023 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "ScanStepBase.hpp"
|
||||
|
||||
namespace Scanner
|
||||
{
|
||||
class ScanStepComputeClusterStats : public ScanStepBase
|
||||
{
|
||||
public:
|
||||
using ScanStepBase::ScanStepBase;
|
||||
|
||||
private:
|
||||
ScanStep getStep() const override { return ScanStep::ComputeClusterStats; }
|
||||
std::string_view getStepName() const override { return "Compute cluster stats"; }
|
||||
void process(ScanContext& context) override;
|
||||
};
|
||||
}
|
||||
@@ -36,6 +36,7 @@
|
||||
#include "ScanStepDiscoverFiles.hpp"
|
||||
#include "ScanStepRemoveOrphanDbFiles.hpp"
|
||||
#include "ScanStepScanFiles.hpp"
|
||||
#include "ScanStepComputeClusterStats.hpp"
|
||||
|
||||
using namespace Database;
|
||||
|
||||
@@ -357,6 +358,7 @@ ScannerService::refreshScanSettings()
|
||||
_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
|
||||
_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
|
||||
}
|
||||
|
||||
|
||||
@@ -28,80 +28,81 @@
|
||||
|
||||
namespace Scanner
|
||||
{
|
||||
enum class ScanErrorType
|
||||
{
|
||||
CannotReadFile, // cannot read file
|
||||
CannotParseFile, // cannot parse file
|
||||
NoAudioTrack, // no audio track found
|
||||
BadDuration, // bad duration
|
||||
};
|
||||
enum class ScanErrorType
|
||||
{
|
||||
CannotReadFile, // cannot read file
|
||||
CannotParseFile, // cannot parse file
|
||||
NoAudioTrack, // no audio track found
|
||||
BadDuration, // bad duration
|
||||
};
|
||||
|
||||
enum class DuplicateReason
|
||||
{
|
||||
SameHash,
|
||||
SameTrackMBID,
|
||||
};
|
||||
enum class DuplicateReason
|
||||
{
|
||||
SameHash,
|
||||
SameTrackMBID,
|
||||
};
|
||||
|
||||
struct ScanError
|
||||
{
|
||||
std::filesystem::path file;
|
||||
ScanErrorType error;
|
||||
std::string systemError;
|
||||
struct ScanError
|
||||
{
|
||||
std::filesystem::path file;
|
||||
ScanErrorType error;
|
||||
std::string systemError;
|
||||
|
||||
ScanError(const std::filesystem::path& file, ScanErrorType error, const std::string& systemError = "");
|
||||
};
|
||||
ScanError(const std::filesystem::path& file, ScanErrorType error, const std::string& systemError = "");
|
||||
};
|
||||
|
||||
struct ScanDuplicate
|
||||
{
|
||||
Database::TrackId trackId;
|
||||
DuplicateReason reason;
|
||||
};
|
||||
struct ScanDuplicate
|
||||
{
|
||||
Database::TrackId trackId;
|
||||
DuplicateReason reason;
|
||||
};
|
||||
|
||||
enum class ScanStep
|
||||
{
|
||||
DiscoveringFiles,
|
||||
ScanningFiles,
|
||||
ChekingForMissingFiles,
|
||||
CheckingForDuplicateFiles,
|
||||
FetchingTrackFeatures,
|
||||
ReloadingSimilarityEngine,
|
||||
};
|
||||
static inline constexpr unsigned ScanProgressStepCount {5};
|
||||
enum class ScanStep
|
||||
{
|
||||
DiscoveringFiles,
|
||||
ScanningFiles,
|
||||
ChekingForMissingFiles,
|
||||
CheckingForDuplicateFiles,
|
||||
FetchingTrackFeatures,
|
||||
ReloadingSimilarityEngine,
|
||||
ComputeClusterStats,
|
||||
};
|
||||
static inline constexpr unsigned ScanProgressStepCount{ 6 };
|
||||
|
||||
// reduced scan stats
|
||||
struct ScanStepStats
|
||||
{
|
||||
Wt::WDateTime startTime;
|
||||
// reduced scan stats
|
||||
struct ScanStepStats
|
||||
{
|
||||
Wt::WDateTime startTime;
|
||||
|
||||
ScanStep currentStep;
|
||||
ScanStep currentStep;
|
||||
|
||||
std::size_t totalElems {};
|
||||
std::size_t processedElems {};
|
||||
std::size_t totalElems{};
|
||||
std::size_t processedElems{};
|
||||
|
||||
unsigned progress() const;
|
||||
};
|
||||
unsigned progress() const;
|
||||
};
|
||||
|
||||
struct ScanStats
|
||||
{
|
||||
Wt::WDateTime startTime;
|
||||
Wt::WDateTime stopTime;
|
||||
struct ScanStats
|
||||
{
|
||||
Wt::WDateTime startTime;
|
||||
Wt::WDateTime stopTime;
|
||||
|
||||
std::size_t filesScanned {}; // Total number of files scanned (estimated)
|
||||
std::size_t filesScanned{}; // Total number of files scanned (estimated)
|
||||
|
||||
std::size_t skips {}; // no change since last scan
|
||||
std::size_t scans {}; // actually scanned filed
|
||||
std::size_t skips{}; // no change since last scan
|
||||
std::size_t scans{}; // actually scanned filed
|
||||
|
||||
std::size_t additions {}; // added in DB
|
||||
std::size_t deletions {}; // removed from DB
|
||||
std::size_t updates {}; // updated file in DB
|
||||
std::size_t additions{}; // added in DB
|
||||
std::size_t deletions{}; // removed from DB
|
||||
std::size_t updates{}; // updated file in DB
|
||||
|
||||
std::size_t featuresFetched {}; // features fetched in DB
|
||||
std::size_t featuresFetched{}; // features fetched in DB
|
||||
|
||||
std::vector<ScanError> errors;
|
||||
std::vector<ScanDuplicate> duplicates;
|
||||
std::vector<ScanError> errors;
|
||||
std::vector<ScanDuplicate> duplicates;
|
||||
|
||||
std::size_t nbFiles() const;
|
||||
std::size_t nbChanges() const;
|
||||
};
|
||||
std::size_t nbFiles() const;
|
||||
std::size_t nbChanges() const;
|
||||
};
|
||||
} // namespace Scanner
|
||||
|
||||
|
||||
@@ -168,7 +168,7 @@ namespace API::Subsonic
|
||||
params.setClusterType(clusterType->getId());
|
||||
|
||||
for (const auto& cluster : Cluster::find(context.dbSession, params).results)
|
||||
albumNode.addArrayValue(field, std::get<std::string>(cluster));
|
||||
albumNode.addArrayValue(field, cluster->getName());
|
||||
}
|
||||
} };
|
||||
|
||||
@@ -183,7 +183,7 @@ namespace API::Subsonic
|
||||
params.setClusterType(genreClusterType->getId());
|
||||
|
||||
for (const auto& cluster : Cluster::find(context.dbSession, params).results)
|
||||
albumNode.addArrayChild("genres", createItemGenreNode(std::get<std::string>(cluster)));
|
||||
albumNode.addArrayChild("genres", createItemGenreNode(cluster->getName()));
|
||||
}
|
||||
|
||||
albumNode.createEmptyArrayChild("artists");
|
||||
|
||||
@@ -228,7 +228,7 @@ namespace API::Subsonic
|
||||
params.setClusterType(clusterType->getId());
|
||||
|
||||
for (const auto& cluster : Cluster::find(context.dbSession, params).results)
|
||||
trackResponse.addArrayValue(field, std::get<std::string>(cluster));
|
||||
trackResponse.addArrayValue(field, cluster->getName());
|
||||
}
|
||||
} };
|
||||
|
||||
@@ -243,7 +243,7 @@ namespace API::Subsonic
|
||||
params.setClusterType(genreClusterType->getId());
|
||||
|
||||
for (const auto& cluster : Cluster::find(context.dbSession, params).results)
|
||||
trackResponse.addArrayChild("genres", createItemGenreNode(std::get<std::string>(cluster)));
|
||||
trackResponse.addArrayChild("genres", createItemGenreNode(cluster->getName()));
|
||||
}
|
||||
|
||||
trackResponse.addChild("replayGain", createReplayGainNode(track));
|
||||
|
||||
@@ -107,7 +107,7 @@ namespace UserInterface::Utils
|
||||
} };
|
||||
|
||||
const std::string styleClass{ getStyleClass(cluster) };
|
||||
auto res{ std::make_unique<Wt::WText>(std::string {} + (canDelete ? "<i class=\"fa fa-times-circle\"></i> " : "") + Wt::WString::fromUTF8(cluster->getName()), Wt::TextFormat::UnsafeXHTML) };
|
||||
auto res{ std::make_unique<Wt::WText>(std::string {} + (canDelete ? "<i class=\"fa fa-times-circle\"></i> " : "") + Wt::WString::fromUTF8(std::string{ cluster->getName() }), Wt::TextFormat::UnsafeXHTML) };
|
||||
|
||||
res->setStyleClass("Lms-badge-cluster badge me-1 " + styleClass); // HACK
|
||||
res->setToolTip(cluster->getType()->getName(), Wt::TextFormat::Plain);
|
||||
|
||||
@@ -265,10 +265,15 @@ ScannerController::refreshContents()
|
||||
.arg(status.currentScanStepStats->totalElems)
|
||||
.arg(status.currentScanStepStats->progress()));
|
||||
break;
|
||||
|
||||
case Scanner::ScanStep::ReloadingSimilarityEngine:
|
||||
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine")
|
||||
.arg(status.currentScanStepStats->progress()));
|
||||
break;
|
||||
|
||||
case Scanner::ScanStep::ComputeClusterStats:
|
||||
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats")
|
||||
.arg(status.currentScanStepStats->progress()));
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ Filters::showDialog()
|
||||
for (const Cluster::pointer& cluster : clusterType->getClusters())
|
||||
{
|
||||
if (std::find(std::cbegin(_clusterIds), std::cend(_clusterIds), cluster->getId()) == _clusterIds.end())
|
||||
valueCombo->addItem(Wt::WString::fromUTF8(cluster->getName()));
|
||||
valueCombo->addItem(Wt::WString::fromUTF8(std::string{ cluster->getName() }));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -115,7 +115,7 @@ Filters::showDialog()
|
||||
for (const Cluster::pointer& cluster : clusterType->getClusters())
|
||||
{
|
||||
if (std::find(std::cbegin(_clusterIds), std::cend(_clusterIds), cluster->getId()) == _clusterIds.end())
|
||||
valueCombo->addItem(Wt::WString::fromUTF8(cluster->getName()));
|
||||
valueCombo->addItem(Wt::WString::fromUTF8(std::string{ cluster->getName() }));
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ static void dumpTracksRecommendation(Session session, Recommendation::IRecommend
|
||||
for (auto artist : track->getArtists({ TrackArtistLinkType::Artist }))
|
||||
res += " - " + artist->getName();
|
||||
for (auto cluster : track->getClusters())
|
||||
res += " {" + cluster->getType()->getName() + "-" + cluster->getName() + "}";
|
||||
res += " {" + cluster->getType()->getName() + "-" + std::string{ cluster->getName() } + "}";
|
||||
|
||||
return res;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user