diff --git a/approot/messages.xml b/approot/messages.xml index dad967c1..2bbb4907 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -59,7 +59,7 @@ Cannot read file {1} duplicate files: {1} errors: -Force scan now +Force full rescan now Get report Last scan Not available @@ -73,9 +73,10 @@ Not scheduled Scheduled on {1} Scanning: step {1}/{2} -Checking for missing files... {1}% +Checking files... {1}% Discovering files: {1} files Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)... +Reloading similarity engine: {1}%... Scanning files: {1}/{2} files ({3}%)... diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 955938ae..0b66924a 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -59,7 +59,7 @@ Impossible de lire le fichier {1} fichiers dupliqués : {1} erreurs : -Lancer un scan forcé +Forcer un rescan complet Rapport Dernier scan Non disponible @@ -73,10 +73,11 @@ Non planifié Planifié le {1} En cours de scan : étape {1}/{2} -Vérification des fichiers supprimés... {1}% -Découverte des fichiers: {1} fichiers -Récupération des métadonnées AcousticBrainz: {1}/{2} fichiers ({3}%)... -Scan des fichiers: {1}/{2} fichiers ({3}%)... +Vérification des fichiers... {1}% +Découverte des fichiers : {1} fichiers +Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)... +Rechargement du moteur de recommandation : {1}%... +Scan des fichiers : {1}/{2} fichiers ({3}%)... Ajouter diff --git a/src/libs/recommendation/CMakeLists.txt b/src/libs/recommendation/CMakeLists.txt index 35243f50..b33e36c3 100644 --- a/src/libs/recommendation/CMakeLists.txt +++ b/src/libs/recommendation/CMakeLists.txt @@ -22,5 +22,7 @@ target_link_libraries(lmsrecommendation PRIVATE wt ) +target_include_directories(lmsrecommendation PRIVATE impl) + install(TARGETS lmsrecommendation DESTINATION lib) diff --git a/src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp b/src/libs/recommendation/impl/ClustersClassifierCreator.hpp similarity index 100% rename from src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp rename to src/libs/recommendation/impl/ClustersClassifierCreator.hpp diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp index 26caf31b..397a7cda 100644 --- a/src/libs/recommendation/impl/Engine.cpp +++ b/src/libs/recommendation/impl/Engine.cpp @@ -19,16 +19,39 @@ #include "Engine.hpp" -#include "recommendation/ClustersClassifierCreator.hpp" -#include "recommendation/FeaturesClassifierCreator.hpp" +#include +#include +#include "ClustersClassifierCreator.hpp" +#include "FeaturesClassifierCreator.hpp" + +#include "database/Db.hpp" +#include "database/Session.hpp" #include "database/ScanSettings.hpp" -#include "database/TrackList.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" namespace Recommendation { + +static +std::unique_ptr +createClassifier(ClassifierType type) +{ + switch (type) + { + case ClassifierType::Clusters: + return createClustersClassifier(); + break; + + case ClassifierType::Features: + return createFeaturesClassifier(); + break; + } + + return {}; +} + std::unique_ptr createEngine(Database::Db& db) { @@ -36,65 +59,16 @@ createEngine(Database::Db& db) } Engine::Engine(Database::Db& db) -: _dbSession {db} +: _db {db} { - start(); } -Engine::~Engine() -{ - stop(); -} - -void -Engine::start() -{ - assert(!_running); - _running = true; - _ioService.start(); -} - -void -Engine::stop() -{ - assert(_running); - _running = false; - - cancelPendingClassifiers(); - - _ioService.stop(); -} - -void -Engine::requestLoad() -{ - requestReloadInternal(false); -} - -void -Engine::requestReload() -{ - requestReloadInternal(true); -} - -void -Engine::requestReloadInternal(bool databaseChanged) -{ - LMS_LOG(RECOMMENDATION, DEBUG) << "Reload requested..."; - - _ioService.post([=]() - { - reload(databaseChanged); - }); -} - -std::vector +std::unordered_set Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) { + std::unordered_set res; + std::shared_lock lock {_classifiersMutex}; - - std::vector res; - for (const auto& classifierName : _classifierPriorities) { auto itClassifier {_classifiers.find(classifierName)}; @@ -109,23 +83,23 @@ Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdTy return res; } -std::vector +std::unordered_set Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set& trackIds, std::size_t maxCount) { + std::unordered_set res; + std::shared_lock lock {_classifiersMutex}; - - std::vector res; - - for (const auto& classifierName : _classifierPriorities) + for (ClassifierType classifierType : _classifierPriorities) { - auto itClassifier {_classifiers.find(classifierName)}; + auto itClassifier {_classifiers.find(classifierType)}; if (itClassifier == std::cend(_classifiers)) continue; - res = itClassifier->second->getSimilarTracks(dbSession, trackIds, maxCount); + const IClassifier& classifier {*itClassifier->second}; + res = classifier.getSimilarTracks(dbSession, trackIds, maxCount); if (!res.empty()) { - LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifierName << "'"; + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifier.getName() << "'"; break; } } @@ -133,23 +107,23 @@ Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set< return res; } -std::vector +std::unordered_set Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) { + std::unordered_set res; + std::shared_lock lock {_classifiersMutex}; - - std::vector res; - - for (const auto& classifierName : _classifierPriorities) + for (ClassifierType classifierType : _classifierPriorities) { - auto itClassifier {_classifiers.find(classifierName)}; + auto itClassifier {_classifiers.find(classifierType)}; if (itClassifier == std::cend(_classifiers)) continue; - res = itClassifier->second->getSimilarReleases(dbSession, releaseId, maxCount); + const IClassifier& classifier {*itClassifier->second}; + res = classifier.getSimilarReleases(dbSession, releaseId, maxCount); if (!res.empty()) { - LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifierName << "'"; + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifier.getName() << "'"; break; } } @@ -157,23 +131,23 @@ Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releas return res; } -std::vector +std::unordered_set Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) { + std::unordered_set res; + std::shared_lock lock {_classifiersMutex}; - - std::vector res; - - for (const auto& classifierName : _classifierPriorities) + for (ClassifierType classifierType : _classifierPriorities) { - auto itClassifier {_classifiers.find(classifierName)}; + auto itClassifier {_classifiers.find(classifierType)}; if (itClassifier == std::cend(_classifiers)) continue; - res = itClassifier->second->getSimilarArtists(dbSession, artistId, maxCount); + const IClassifier& classifier {*itClassifier->second}; + res = classifier.getSimilarArtists(dbSession, artistId, maxCount); if (!res.empty()) { - LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifierName << "'"; + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifier.getName() << "'"; return res; } } @@ -181,107 +155,132 @@ Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistI return res; } -void -Engine::reload(bool databaseChanged) +static +Database::ScanSettings::RecommendationEngineType +getRecommendationEngineType(Database::Session& session) { - using namespace Database; + auto transaction {session.createSharedTransaction()}; - LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines..."; - - const ScanSettings::RecommendationEngineType engineType {[&]() - { - auto transaction {_dbSession.createSharedTransaction()}; - - return ScanSettings::get(_dbSession)->getRecommendationEngineType(); - }()}; - - clearClassifiers(); - - switch (engineType) - { - case ScanSettings::RecommendationEngineType::Features: - { - auto clustersClassifier {createClustersClassifier()}; - auto featuresClassifier {createFeaturesClassifier()}; - - setClassifierPriorities({featuresClassifier->getName(), clustersClassifier->getName()}); - - initAndAddClassifier(std::move(clustersClassifier), databaseChanged); // init first since faster - initAndAddClassifier(std::move(featuresClassifier), databaseChanged); - break; - } - - case ScanSettings::RecommendationEngineType::Clusters: - auto clustersClassifier {createClustersClassifier()}; - - setClassifierPriorities({clustersClassifier->getName()}); - - initAndAddClassifier(std::move(clustersClassifier), databaseChanged); - break; - } - - LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines reloaded!"; - - _sigReloaded.emit(); + return Database::ScanSettings::get(session)->getRecommendationEngineType(); } void -Engine::setClassifierPriorities(std::initializer_list classifierPriorities) +Engine::load(bool forceReload, const ProgressCallback& progressCallback) +{ + using namespace Database; + + static const std::unordered_map> classifierMappings + { + {ScanSettings::RecommendationEngineType::Features, {ClassifierType::Clusters, ClassifierType::Features}}, + {ScanSettings::RecommendationEngineType::Clusters, {ClassifierType::Clusters}}, + }; + + LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines..."; + + const ScanSettings::RecommendationEngineType engineType {getRecommendationEngineType(_db.getTLSSession())}; + + assert(_pendingClassifiers.empty()); + clearClassifiers(); + + auto itClassifierTypes {classifierMappings.find(engineType)}; + assert(itClassifierTypes != std::cend(classifierMappings)); + const std::vector& classifierTypes {itClassifierTypes->second}; + + setClassifierPriorities(classifierTypes); + + std::vector> classifiers; + for (ClassifierType type : classifierTypes) + classifiers.emplace_back(createClassifier(type)); + + { + std::scoped_lock lock {_controlMutex}; + + std::transform(std::cbegin(classifiers), std::cend(classifiers), std::inserter(_pendingClassifiers, std::end(_pendingClassifiers)), + [](auto& classifier) { return classifier.get(); }); + } + + for (std::size_t i {}; i < classifiers.size(); ++i) + loadClassifier(std::move(classifiers[i]), classifierTypes[i], forceReload, progressCallback); + + LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines loaded!"; +} + +void +Engine::setClassifierPriorities(const std::vector& classifierPriorities) { std::unique_lock lock {_classifiersMutex}; - _classifierPriorities.clear(); - std::transform(std::cbegin(classifierPriorities), std::cend(classifierPriorities), std::back_inserter(_classifierPriorities), [](std::string_view name) { return std::string {name}; }); + _classifierPriorities = classifierPriorities; } void Engine::clearClassifiers() { - std::unique_lock lock {_classifiersMutex}; + std::unique_lock lock {_classifiersMutex}; _classifiers.clear(); } void -Engine::initAndAddClassifier(std::unique_ptr classifier, bool databaseChanged) +Engine::loadClassifier(std::unique_ptr classifier, + ClassifierType classifierType, + bool forceReload, + const ProgressCallback& progressCallback) { - PendingClassifierHandler pendingClassifier {*this, *classifier.get()}; + IClassifier* rawClassifier {classifier.get()}; - LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'..."; - bool res {classifier->init(_dbSession, databaseChanged)}; - LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE"); + bool res {}; + if (!_loadCancelled) + { + LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'..."; + + auto progress {[&](IClassifier::Progress progress) + { + progressCallback(Progress {progress.processedElems, progress.totalElems}); + }}; + + res = classifier->load(_db.getTLSSession(), forceReload, progressCallback ? progress : IClassifier::ProgressCallback {}); + + LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE"); + } if (res) { - std::unique_lock lock {_classifiersMutex}; + std::unique_lock lock {_classifiersMutex}; - _classifiers.emplace(classifier->getName(), std::move(classifier)); + _classifiers.emplace(classifierType, std::move(classifier)); } + + { + std::scoped_lock lock {_controlMutex}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "About to erase. _pendingClassifiers size = " << _pendingClassifiers.size(); + _pendingClassifiers.erase(rawClassifier); + LMS_LOG(RECOMMENDATION, DEBUG) << "Erased. _pendingClassifiers size = " << _pendingClassifiers.size(); + } + + _pendingClassifiersCondvar.notify_one(); + } void -Engine::cancelPendingClassifiers() +Engine::cancelLoad() { - std::unique_lock lock {_classifiersMutex}; + LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading..."; + + std::unique_lock lock {_controlMutex}; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Still " << _pendingClassifiers.size() << " pending classifiers!"; + + _loadCancelled = true; for (IClassifier* classifier : _pendingClassifiers) - classifier->requestCancelInit(); -} + classifier->requestCancelLoad(); -void -Engine::addPendingClassifier(IClassifier& classifier) -{ - std::unique_lock lock {_classifiersMutex}; + _pendingClassifiersCondvar.wait(lock, [this] {return _pendingClassifiers.empty();}); + _loadCancelled = false; - _pendingClassifiers.insert(&classifier); -} - -void -Engine::removePendingClassifier(IClassifier& classifier) -{ - std::unique_lock lock {_classifiersMutex}; - - _pendingClassifiers.erase(&classifier); + LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading DONE"; } } // ns Similarity diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp index fd2f91fb..dc516863 100644 --- a/src/libs/recommendation/impl/Engine.hpp +++ b/src/libs/recommendation/impl/Engine.hpp @@ -19,23 +19,33 @@ #pragma once -#include +#include #include +#include +#include #include -#include - -#include "database/Session.hpp" #include "recommendation/IEngine.hpp" -#include "recommendation/IClassifier.hpp" +#include "IClassifier.hpp" + +namespace Database +{ + class Db; +} namespace Recommendation { + enum class ClassifierType + { + Clusters, + Features, + }; + class Engine : public IEngine { public: Engine(Database::Db& db); - ~Engine(); + ~Engine() = default; Engine(const Engine&) = delete; Engine(Engine&&) = delete; @@ -43,58 +53,30 @@ namespace Recommendation Engine& operator=(Engine&&) = delete; private: + void load(bool forceReload, const ProgressCallback& progressCallback) override; + void cancelLoad() override; - void start(); - void stop(); + std::unordered_set getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override; + std::unordered_set getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) override; + std::unordered_set getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override; + std::unordered_set getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override; - void requestLoad() override; - void requestReload() override; - Wt::Signal<>& reloaded() override { return _sigReloaded; } - - std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override; - std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) override; - std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override; - std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override; - - - void requestReloadInternal(bool databaseChanged); - void reload(bool databaseChanged); - - void setClassifierPriorities(std::initializer_list classifierNames); + void setClassifierPriorities(const std::vector& classifierTypes); void clearClassifiers(); - void initAndAddClassifier(std::unique_ptr classifier, bool databaseChanged); + void loadClassifier(std::unique_ptr classifier, ClassifierType classifierType, bool forceReload, const ProgressCallback& progressCallback); - class PendingClassifierHandler - { - public: - PendingClassifierHandler(Engine& engine, IClassifier& classifier) : _engine {engine}, _classifier {classifier} - { - _engine.addPendingClassifier(_classifier); - } + Database::Db& _db; - ~PendingClassifierHandler() - { - _engine.removePendingClassifier(_classifier); - } + std::mutex _controlMutex; + bool _loadCancelled {}; + std::condition_variable _pendingClassifiersCondvar; + std::unordered_set _pendingClassifiers; - private: - Engine& _engine; - IClassifier& _classifier; - }; + std::shared_mutex _classifiersMutex; + using ClassifierContainer = std::unordered_map>; + ClassifierContainer _classifiers; + std::vector _classifierPriorities; // ordered by priority - void cancelPendingClassifiers(); - void addPendingClassifier(IClassifier& classifier); - void removePendingClassifier(IClassifier& classifier); - - bool _running {}; - Wt::WIOService _ioService; - Database::Session _dbSession; - Wt::Signal<> _sigReloaded; - - std::shared_mutex _classifiersMutex; - std::map> _classifiers; - std::vector _classifierPriorities; // ordered by priority - std::unordered_set _pendingClassifiers; }; } // ns Recommendation diff --git a/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp b/src/libs/recommendation/impl/FeaturesClassifierCreator.hpp similarity index 95% rename from src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp rename to src/libs/recommendation/impl/FeaturesClassifierCreator.hpp index 28a079d7..7f754768 100644 --- a/src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp +++ b/src/libs/recommendation/impl/FeaturesClassifierCreator.hpp @@ -20,7 +20,7 @@ #pragma once #include -#include "recommendation/IClassifier.hpp" +#include "IClassifier.hpp" namespace Recommendation { diff --git a/src/libs/recommendation/include/recommendation/IClassifier.hpp b/src/libs/recommendation/impl/IClassifier.hpp similarity index 50% rename from src/libs/recommendation/include/recommendation/IClassifier.hpp rename to src/libs/recommendation/impl/IClassifier.hpp index 44fa47d5..4c53df48 100644 --- a/src/libs/recommendation/include/recommendation/IClassifier.hpp +++ b/src/libs/recommendation/impl/IClassifier.hpp @@ -19,9 +19,9 @@ #pragma once +#include #include #include -#include #include "database/Types.hpp" @@ -40,13 +40,19 @@ namespace Recommendation virtual std::string_view getName() const = 0; - virtual bool init(Database::Session& session, bool databaseChanged) = 0; - virtual void requestCancelInit() = 0; + struct Progress + { + std::size_t totalElems {}; + std::size_t processedElems {}; + }; + using ProgressCallback = std::function; + virtual bool load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) = 0; + virtual void requestCancelLoad() = 0; - virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0; - virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const = 0; - virtual std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0; - virtual std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0; + virtual std::unordered_set getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0; + virtual std::unordered_set getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const = 0; + virtual std::unordered_set getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0; + virtual std::unordered_set getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0; }; } // ns Recommendation diff --git a/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp index b433d833..c03f517e 100644 --- a/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp +++ b/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp @@ -27,29 +27,29 @@ #include "database/TrackList.hpp" namespace Recommendation { - + std::unique_ptr createClustersClassifier() { return std::make_unique(); } -std::vector +std::unordered_set ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::unordered_set& trackIds, std::size_t maxCount) const { auto transaction {dbSession.createSharedTransaction()}; - auto tracks {Database::Track::getSimilarTracks(dbSession, trackIds, 0, maxCount)}; - std::vector res; - res.reserve(tracks.size()); + const auto tracks {Database::Track::getSimilarTracks(dbSession, trackIds, 0, maxCount)}; - std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res), [](const auto& track) { return track.id(); }); + std::unordered_set res; + std::transform(std::cbegin(tracks), std::cend(tracks), std::inserter(res, std::end(res)), + [](const auto& track) { return track.id(); }); return res; } -std::vector +std::unordered_set ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const { - std::vector res; + std::unordered_set res; auto transaction {session.createSharedTransaction()}; @@ -57,18 +57,17 @@ ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Dat if (!trackList) return res; - const std::vector tracks {trackList->getSimilarTracks(0, maxCount)}; - res.reserve(tracks.size()); - std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res), + const auto tracks {trackList->getSimilarTracks(0, maxCount)}; + std::transform(std::cbegin(tracks), std::cend(tracks), std::inserter(res, std::end(res)), [](const Database::Track::pointer& track) { return track.id(); }); return res; } -std::vector +std::unordered_set ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) const { - std::vector res; + std::unordered_set res; auto transaction {dbSession.createSharedTransaction()}; @@ -77,16 +76,16 @@ ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::Id return res; const auto releases {release->getSimilarReleases(0, maxCount)}; - res.reserve(releases.size()); - std::transform(std::cbegin(releases), std::cend(releases), std::back_inserter(res), [](const auto& release) { return release.id(); }); + std::transform(std::cbegin(releases), std::cend(releases), std::inserter(res, std::end(res)), + [](const auto& release) { return release.id(); }); return res; } -std::vector +std::unordered_set ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) const { - std::vector res; + std::unordered_set res; auto transaction {dbSession.createSharedTransaction()}; @@ -95,8 +94,8 @@ ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdT return res; const auto artists {artist->getSimilarArtists(0, maxCount)}; - res.reserve(artists.size()); - std::transform(std::cbegin(artists), std::cend(artists), std::back_inserter(res), [](const auto& artist) { return artist.id(); }); + std::transform(std::cbegin(artists), std::cend(artists), std::inserter(res, std::end(res)), + [](const auto& artist) { return artist.id(); }); return res; } diff --git a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp index 53a7bd5f..7f96a828 100644 --- a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp +++ b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp @@ -19,8 +19,7 @@ #pragma once -#include "recommendation/IClassifier.hpp" - +#include "IClassifier.hpp" namespace Recommendation { @@ -38,14 +37,13 @@ namespace Recommendation std::string_view getName() const override { return "Clusters"; } - bool init(Database::Session&, bool) override {return true;} - void requestCancelInit() override {} - - std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; - std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const override; - std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override; - std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override; + bool load(Database::Session&, bool, const ProgressCallback&) override { return true; } + void requestCancelLoad() override {} + std::unordered_set getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; + std::unordered_set getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const override; + std::unordered_set getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override; + std::unordered_set getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override; }; } // namespace Recommendation diff --git a/src/libs/recommendation/impl/features/FeaturesClassifier.cpp b/src/libs/recommendation/impl/features/FeaturesClassifier.cpp index d3b13d5d..119f71de 100644 --- a/src/libs/recommendation/impl/features/FeaturesClassifier.cpp +++ b/src/libs/recommendation/impl/features/FeaturesClassifier.cpp @@ -127,7 +127,7 @@ getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t } bool -FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSettings& trainSettings) +FeaturesClassifier::loadFromTraining(Database::Session& session, const TrainSettings& trainSettings, const ProgressCallback& progressCallback) { LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier..."; @@ -158,7 +158,7 @@ FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSett LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features..."; for (Database::IdType trackId : trackIds) { - if (_initCancelled) + if (_loadCancelled) return false; std::optional featureValuesMap; @@ -201,23 +201,26 @@ FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSett SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)}; network.setDataWeights(weights); - auto progressIndicator{[](const auto& iter) - { - LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount; - }}; + auto somProgressCallback{[&](const SOM::Network::CurrentIteration& iter) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount; + progressCallback(Progress {iter.idIteration, iter.iterationCount}); + }}; LMS_LOG(RECOMMENDATION, DEBUG) << "Training network..."; - network.train(samples, trainSettings.iterationCount, progressIndicator); + network.train(samples, trainSettings.iterationCount, + progressCallback ? somProgressCallback : SOM::Network::ProgressCallback {}, + [this] { return _loadCancelled; }); LMS_LOG(RECOMMENDATION, DEBUG) << "Training network DONE"; - if (_initCancelled) + if (_loadCancelled) return false; LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks..."; ObjectPositions trackPositions; for (std::size_t i {}; i < samples.size(); ++i) { - if (_initCancelled) + if (_loadCancelled) return false; const SOM::Position position {network.getClosestRefVectorPosition(samples[i])}; @@ -227,86 +230,100 @@ FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSett LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks DONE"; - return init(session, std::move(network), std::move(trackPositions)); + return load(session, std::move(network), std::move(trackPositions)); } bool -FeaturesClassifier::initFromCache(Database::Session& session, const FeaturesClassifierCache& cache) +FeaturesClassifier::loadFromCache(Database::Session& session, const FeaturesClassifierCache& cache) { LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier from cache..."; - return init(session, std::move(cache._network), cache._trackPositions); + return load(session, std::move(cache._network), cache._trackPositions); } -std::vector +std::unordered_set FeaturesClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) const { - const std::unordered_set trackIds {[&]() -> std::unordered_set + const std::unordered_set trackIds {[&] { + std::unordered_set res; + auto transaction {session.createSharedTransaction()}; const Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)}; if (trackList) { const std::vector orderedTrackIds {trackList->getTrackIds()}; - return std::unordered_set {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)}; + res = std::unordered_set(std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)); } - return {}; + return res; }()}; return getSimilarTracks(session, trackIds, maxCount); } -std::vector +std::unordered_set FeaturesClassifier::getSimilarTracks(Database::Session& session, const std::unordered_set& tracksIds, std::size_t maxCount) const { - std::vector similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)}; - + auto similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)}; if (!similarTrackIds.empty()) { // Report only existing ids auto transaction {session.createSharedTransaction()}; - similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds), - [&](Database::IdType trackId) { return Database::Track::getById(session, trackId) == Database::Track::pointer {}; }), - std::cend(similarTrackIds)); + for (auto it {std::begin(similarTrackIds)}; it != std::end(similarTrackIds);) + { + const Database::IdType trackId {*it}; + if (!Database::Track::getById(session, trackId)) + it = similarTrackIds.erase(it); + else + it++; + } } return similarTrackIds; } -std::vector +std::unordered_set FeaturesClassifier::getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const { - std::vector similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)}; - + auto similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)}; if (!similarReleaseIds.empty()) { // Report only existing ids auto transaction {session.createSharedTransaction()}; - similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds), - [&](Database::IdType releaseId) { return Database::Release::getById(session, releaseId) == Database::Release::pointer {}; }), - std::cend(similarReleaseIds)); + for (auto it {std::begin(similarReleaseIds)}; it != std::end(similarReleaseIds);) + { + const Database::IdType releaseId {*it}; + if (!Database::Release::getById(session, releaseId)) + it = similarReleaseIds.erase(it); + else + it++; + } } return similarReleaseIds; } -std::vector +std::unordered_set FeaturesClassifier::getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const { - std::vector similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)}; - + auto similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)}; if (!similarArtistIds.empty()) { // Report only existing ids auto transaction {session.createSharedTransaction()}; - similarArtistIds.erase(std::remove_if(std::begin(similarArtistIds), std::end(similarArtistIds), - [&](Database::IdType artistId) { return Database::Artist::getById(session, artistId) == Database::Artist::pointer {}; }), - std::cend(similarArtistIds)); + for (auto it {std::begin(similarArtistIds)}; it != std::end(similarArtistIds);) + { + const Database::IdType artistId {*it}; + if (!Database::Release::getById(session, artistId)) + it = similarArtistIds.erase(it); + else + it++; + } } return similarArtistIds; @@ -319,22 +336,23 @@ FeaturesClassifier::toCache() const } bool -FeaturesClassifier::init(Database::Session& session, bool databaseChanged) +FeaturesClassifier::load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) { - if (databaseChanged) + if (forceReload) { - LMS_LOG(RECOMMENDATION, DEBUG) << "Database changed: invidating cache"; FeaturesClassifierCache::invalidate(); } - - std::optional cache {FeaturesClassifierCache::read()}; - if (cache) - return initFromCache(session, *cache); + else + { + const std::optional cache {FeaturesClassifierCache::read()}; + if (cache) + return loadFromCache(session, *cache); + } TrainSettings trainSettings; trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings(); - bool res {initFromTraining(session, trainSettings)}; + const bool res {loadFromTraining(session, trainSettings, progressCallback)}; if (res) toCache().write(); @@ -342,14 +360,14 @@ FeaturesClassifier::init(Database::Session& session, bool databaseChanged) } void -FeaturesClassifier::requestCancelInit() +FeaturesClassifier::requestCancelLoad() { LMS_LOG(RECOMMENDATION, DEBUG) << "Requesting init cancellation"; - _initCancelled = true; + _loadCancelled = true; } bool -FeaturesClassifier::init(Database::Session& session, +FeaturesClassifier::load(Database::Session& session, SOM::Network network, const ObjectPositions& tracksPosition) { @@ -367,7 +385,7 @@ FeaturesClassifier::init(Database::Session& session, for (auto itTrackCoord : tracksPosition) { - if (_initCancelled) + if (_loadCancelled) return false; auto transaction {session.createSharedTransaction()}; @@ -399,7 +417,7 @@ FeaturesClassifier::init(Database::Session& session, _network = std::make_unique(std::move(network)); - LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully initialized!"; + LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully loaded!"; return true; } @@ -439,13 +457,13 @@ FeaturesClassifier::getObjectsIds(const std::unordered_set& posit return res; } -std::vector +std::unordered_set FeaturesClassifier::getSimilarObjects(const std::unordered_set& ids, const MatrixOfObjects& objectsMap, const ObjectPositions& objectPosition, std::size_t maxCount) const { - std::vector res; + std::unordered_set res; std::unordered_set searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPosition)}; if (searchedRefVectorsPosition.empty()) @@ -459,14 +477,13 @@ FeaturesClassifier::getSimilarObjects(const std::unordered_set for (auto id : ids) closestObjectIds.erase(id); + for (auto it {std::cbegin(closestObjectIds)}; it != std::cend(closestObjectIds); ++it) { - std::vector objectIdsToAdd {std::cbegin(closestObjectIds), std::cend(closestObjectIds)}; - Random::shuffleContainer(objectIdsToAdd ); - std::copy(std::cbegin(objectIdsToAdd), std::cend(objectIdsToAdd), std::back_inserter(res)); - } + if (res.size() == maxCount) + break; - if (res.size() > maxCount) - res.resize(maxCount); + res.insert(*it); + } if (res.size() == maxCount) break; diff --git a/src/libs/recommendation/impl/features/FeaturesClassifier.hpp b/src/libs/recommendation/impl/features/FeaturesClassifier.hpp index f20beec7..2d5b3992 100644 --- a/src/libs/recommendation/impl/features/FeaturesClassifier.hpp +++ b/src/libs/recommendation/impl/features/FeaturesClassifier.hpp @@ -19,15 +19,16 @@ #pragma once +#include #include #include #include -#include "recommendation/IClassifier.hpp" #include "som/DataNormalizer.hpp" #include "som/Network.hpp" #include "FeaturesClassifierCache.hpp" #include "FeaturesDefs.hpp" +#include "IClassifier.hpp" namespace Database { @@ -53,19 +54,20 @@ class FeaturesClassifier : public IClassifier static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; } static const FeatureSettingsMap& getDefaultTrainFeatureSettings(); + private: std::string_view getName() const override { return "Features"; } - bool init(Database::Session& session, bool databaseChanged) override; - void requestCancelInit() override; + bool load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) override; + void requestCancelLoad() override; - std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; - std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const override; - std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override; - std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override; + std::unordered_set getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override; + std::unordered_set getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) const override; + std::unordered_set getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override; + std::unordered_set getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override; - bool initFromCache(Database::Session& session, const FeaturesClassifierCache& cache); + bool loadFromCache(Database::Session& session, const FeaturesClassifierCache& cache); // Use training (may be very slow) struct TrainSettings @@ -74,12 +76,12 @@ class FeaturesClassifier : public IClassifier float sampleCountPerNeuron {4}; FeatureSettingsMap featureSettingsMap; }; - bool initFromTraining(Database::Session& session, const TrainSettings& trainSettings); + bool loadFromTraining(Database::Session& session, const TrainSettings& trainSettings, const ProgressCallback& progressCallback); using ObjectPositions = std::unordered_map>; using MatrixOfObjects = SOM::Matrix>; - bool init(Database::Session& session, + bool load(Database::Session& session, SOM::Network network, const ObjectPositions& tracksPosition); @@ -88,12 +90,12 @@ class FeaturesClassifier : public IClassifier static std::unordered_set getMatchingRefVectorsPosition(const std::unordered_set& ids, const ObjectPositions& objectPositions); static std::unordered_set getObjectsIds(const std::unordered_set& positionSet, const MatrixOfObjects& objectsMap); - std::vector getSimilarObjects(const std::unordered_set& ids, + std::unordered_set getSimilarObjects(const std::unordered_set& ids, const SOM::Matrix>& objectsMap, const ObjectPositions& objectPosition, std::size_t maxCount) const; - bool _initCancelled {}; + bool _loadCancelled {}; std::unique_ptr _network; double _networkRefVectorsDistanceMedian {}; diff --git a/src/libs/recommendation/include/recommendation/IEngine.hpp b/src/libs/recommendation/include/recommendation/IEngine.hpp index 7cf196e8..f1b4ce2b 100644 --- a/src/libs/recommendation/include/recommendation/IEngine.hpp +++ b/src/libs/recommendation/include/recommendation/IEngine.hpp @@ -20,11 +20,8 @@ #pragma once #include -#include #include -#include - #include "database/Types.hpp" namespace Database @@ -40,16 +37,19 @@ namespace Recommendation public: virtual ~IEngine() = default; - virtual void requestLoad() = 0; + struct Progress + { + std::size_t totalElems {}; + std::size_t processedElems {}; + }; + using ProgressCallback = std::function; + virtual void load(bool forceReload, const ProgressCallback& progressCallback = {}) = 0; + virtual void cancelLoad() = 0; - virtual void requestReload() = 0; - virtual Wt::Signal<>& reloaded() = 0; - - // Closest results first - virtual std::vector getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; - virtual std::vector getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) = 0; - virtual std::vector getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) = 0; - virtual std::vector getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0; + virtual std::unordered_set getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0; + virtual std::unordered_set getSimilarTracks(Database::Session& session, const std::unordered_set& tracksId, std::size_t maxCount) = 0; + virtual std::unordered_set getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) = 0; + virtual std::unordered_set getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0; }; std::unique_ptr createEngine(Database::Db& db); diff --git a/src/libs/scanner/CMakeLists.txt b/src/libs/scanner/CMakeLists.txt index 2ec7539e..07f6cf1c 100644 --- a/src/libs/scanner/CMakeLists.txt +++ b/src/libs/scanner/CMakeLists.txt @@ -16,6 +16,7 @@ target_include_directories(lmsscanner PRIVATE target_link_libraries(lmsscanner PRIVATE lmsdatabase lmsmetadata + lmsrecommendation lmsutils ) diff --git a/src/libs/scanner/impl/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp index 7b7d60cb..65fd3399 100644 --- a/src/libs/scanner/impl/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -30,6 +30,7 @@ #include "database/Track.hpp" #include "database/TrackFeatures.hpp" #include "metadata/TagLibParser.hpp" +#include "recommendation/IEngine.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "utils/Path.hpp" @@ -240,13 +241,14 @@ getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames) namespace Scanner { std::unique_ptr -createMediaScanner(Database::Db& db) +createMediaScanner(Database::Db& db, Recommendation::IEngine& recommendationEngine) { - return std::make_unique(db); + return std::make_unique(db, recommendationEngine); } -MediaScanner::MediaScanner(Database::Db& db) -: _dbSession {db} +MediaScanner::MediaScanner(Database::Db& db, Recommendation::IEngine& recommendationEngine) +: _recommendationEngine {recommendationEngine} +, _dbSession {db} { // For now, always use TagLib _metadataParser = std::make_unique(); @@ -260,6 +262,7 @@ MediaScanner::MediaScanner(Database::Db& db) MediaScanner::~MediaScanner() { + LMS_LOG(DBUPDATER, INFO) << "Shutting down MediaScanner..."; stop(); } @@ -268,7 +271,14 @@ MediaScanner::start() { std::scoped_lock lock {_controlMutex}; - scheduleNextScan(); + _ioService.post([this] + { + if (_abortScan) + return; + + _recommendationEngine.load(false); + scheduleNextScan(); + }); _ioService.start(); } @@ -279,8 +289,8 @@ MediaScanner::stop() std::scoped_lock lock {_controlMutex}; _abortScan = true; - _scheduleTimer.cancel(); + _recommendationEngine.cancelLoad(); _ioService.stop(); } @@ -294,6 +304,7 @@ MediaScanner::abortScan() _abortScan = true; _scheduleTimer.cancel(); + _recommendationEngine.cancelLoad(); _ioService.stop(); LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!"; @@ -307,6 +318,9 @@ MediaScanner::requestImmediateScan(bool force) abortScan(); _ioService.post([=]() { + if (_abortScan) + return; + scheduleScan(force); }); } @@ -317,6 +331,9 @@ MediaScanner::requestReload() abortScan(); _ioService.post([=]() { + if (_abortScan) + return; + scheduleNextScan(); }); } @@ -407,12 +424,13 @@ MediaScanner::countAllFiles(ScanStats& stats) if (!ec && isFileSupported(path, _fileExtensions)) { stats.filesScanned++; - stepStats.processedFiles++; + stepStats.processedElems++; notifyInProgressIfNeeded(stepStats); } return true; }); + notifyInProgress(stepStats); } void @@ -476,10 +494,11 @@ MediaScanner::scan(bool forceScan) removeOrphanEntries(); if (!_abortScan) + { checkDuplicatedAudioFiles(stats); - - // Now update all the track features if needed - fetchTrackFeatures(stats); + fetchTrackFeatures(stats); + reloadSimilarityEngine(stats); + } LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size(); @@ -566,7 +585,7 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats) return res; }()}; - stepStats.filesToProcess = tracksToFetch.size(); + stepStats.totalElems = tracksToFetch.size(); notifyInProgress(stepStats); LMS_LOG(DBUPDATER, INFO) << "Found " << tracksToFetch.size() << " track(s) to fetch!"; @@ -579,10 +598,11 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats) if (fetchTrackFeatures(trackToFetch.id, trackToFetch.mbid)) stats.featuresFetched++; - stepStats.processedFiles++; + stepStats.processedElems++; notifyInProgressIfNeeded(stepStats); } + notifyInProgress(stepStats); LMS_LOG(DBUPDATER, INFO) << "Track features fetched!"; } @@ -591,7 +611,7 @@ MediaScanner::refreshScanSettings() { auto transaction {_dbSession.createSharedTransaction()}; - ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; + const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion(); @@ -608,7 +628,7 @@ MediaScanner::refreshScanSettings() _mediaDirectory = scanSettings->getMediaDirectory(); _recommendationEngineType = scanSettings->getRecommendationEngineType(); - auto clusterTypes = scanSettings->getClusterTypes(); + const auto clusterTypes = scanSettings->getClusterTypes(); std::set clusterTypeNames; std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes), @@ -616,7 +636,6 @@ MediaScanner::refreshScanSettings() [](ClusterType::pointer clusterType) { return clusterType->getName(); }); _metadataParser->setClusterTypeNames(clusterTypeNames); - } void @@ -800,7 +819,7 @@ void MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats) { ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles}; - stepStats.filesToProcess = stats.filesScanned; + stepStats.totalElems = stats.filesScanned; notifyInProgress(stepStats); exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) @@ -817,12 +836,14 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo { scanAudioFile(path, forceScan, stats ); - stepStats.processedFiles++; + stepStats.processedElems++; notifyInProgressIfNeeded(stepStats); } return true; }); + + notifyInProgress(stepStats); } // Check if a file exists and is still in a media directory @@ -878,7 +899,7 @@ MediaScanner::removeMissingTracks(ScanStats& stats) } LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked..."; - stepStats.filesToProcess = trackCount; + stepStats.totalElems = trackCount; notifyInProgress(stepStats); std::vector> trackPaths; @@ -902,7 +923,7 @@ MediaScanner::removeMissingTracks(ScanStats& stats) if (!checkFile(trackPath, _mediaDirectory, _fileExtensions)) tracksToRemove.push_back(trackId); - stepStats.processedFiles++; + stepStats.processedElems++; } if (!tracksToRemove.empty()) @@ -992,4 +1013,21 @@ MediaScanner::checkDuplicatedAudioFiles(ScanStats& stats) LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!"; } +void +MediaScanner::reloadSimilarityEngine(ScanStats& stats) +{ + ScanStepStats stepStats {stats.startTime, ScanProgressStep::ReloadingSimilarityEngine}; + + auto progressCallback {[&](const Recommendation::IEngine::Progress& progress) + { + stepStats.totalElems = progress.totalElems; + stepStats.processedElems = progress.processedElems; + notifyInProgressIfNeeded(stepStats); + }}; + + notifyInProgress(stepStats); + _recommendationEngine.load(stats.nbChanges() > 0, progressCallback); + notifyInProgress(stepStats); +} + } // namespace Scanner diff --git a/src/libs/scanner/impl/MediaScanner.hpp b/src/libs/scanner/impl/MediaScanner.hpp index 88cbc27c..6a78b86c 100644 --- a/src/libs/scanner/impl/MediaScanner.hpp +++ b/src/libs/scanner/impl/MediaScanner.hpp @@ -37,12 +37,17 @@ class UUID; +namespace Recommendation +{ + class IEngine; +} + namespace Scanner { class MediaScanner : public IMediaScanner { public: - MediaScanner(Database::Db& db); + MediaScanner(Database::Db& db, Recommendation::IEngine& recommendationEngine); ~MediaScanner(); MediaScanner(const MediaScanner&) = delete; @@ -89,6 +94,9 @@ class MediaScanner : public IMediaScanner Database::IdType doScanAudioFile(const std::filesystem::path& file, ScanStats& stats); void notifyInProgressIfNeeded(const ScanStepStats& stats); void notifyInProgress(const ScanStepStats& stats); + void reloadSimilarityEngine(ScanStats& stats); + + Recommendation::IEngine& _recommendationEngine; std::mutex _controlMutex; std::atomic _abortScan {}; @@ -116,7 +124,6 @@ class MediaScanner : public IMediaScanner std::filesystem::path _mediaDirectory; Database::ScanSettings::RecommendationEngineType _recommendationEngineType; - }; // class MediaScanner } // Scanner diff --git a/src/libs/scanner/impl/MediaScannerStats.cpp b/src/libs/scanner/impl/MediaScannerStats.cpp index 29af4235..17450156 100644 --- a/src/libs/scanner/impl/MediaScannerStats.cpp +++ b/src/libs/scanner/impl/MediaScannerStats.cpp @@ -43,7 +43,7 @@ ScanStats::nbChanges() const unsigned ScanStepStats::progress() const { - return (processedFiles / static_cast(filesToProcess ? filesToProcess : 1)) * 100; + return (processedElems / static_cast(totalElems ? totalElems : 1)) * 100; } } // namespace Scanner diff --git a/src/libs/scanner/include/scanner/IMediaScanner.hpp b/src/libs/scanner/include/scanner/IMediaScanner.hpp index 5a0afbdc..ec8f64d5 100644 --- a/src/libs/scanner/include/scanner/IMediaScanner.hpp +++ b/src/libs/scanner/include/scanner/IMediaScanner.hpp @@ -31,6 +31,11 @@ namespace Database class Db; } +namespace Recommendation +{ + class IEngine; +} + namespace Scanner { class IMediaScanner @@ -73,8 +78,7 @@ class IMediaScanner }; -std::unique_ptr createMediaScanner(Database::Db& db); - +std::unique_ptr createMediaScanner(Database::Db& db, Recommendation::IEngine& recommendationEngine); } // Scanner diff --git a/src/libs/scanner/include/scanner/MediaScannerStats.hpp b/src/libs/scanner/include/scanner/MediaScannerStats.hpp index f5eb67f7..145ea5c3 100644 --- a/src/libs/scanner/include/scanner/MediaScannerStats.hpp +++ b/src/libs/scanner/include/scanner/MediaScannerStats.hpp @@ -62,8 +62,9 @@ namespace Scanner { DiscoveringFiles, ScanningFiles, FetchingTrackFeatures, + ReloadingSimilarityEngine, }; - static inline constexpr unsigned ScanProgressStepCount {4}; + static inline constexpr unsigned ScanProgressStepCount {5}; // reduced scan stats struct ScanStepStats @@ -72,8 +73,8 @@ namespace Scanner { ScanProgressStep currentStep; - std::size_t filesToProcess {}; - std::size_t processedFiles {}; + std::size_t totalElems {}; + std::size_t processedElems {}; unsigned progress() const; }; diff --git a/src/libs/subsonic/impl/Scan.cpp b/src/libs/subsonic/impl/Scan.cpp index 3b9325a6..ff32b960 100644 --- a/src/libs/subsonic/impl/Scan.cpp +++ b/src/libs/subsonic/impl/Scan.cpp @@ -40,7 +40,7 @@ namespace API::Subsonic::Scan std::size_t count{}; if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanProgressStep::ScanningFiles) - count = scanStatus.currentScanStepStats->processedFiles; + count = scanStatus.currentScanStepStats->processedElems; statusResponse.setAttribute("count", count); } diff --git a/src/libs/utils/include/utils/Semaphore.hpp b/src/libs/utils/include/utils/Semaphore.hpp deleted file mode 100644 index 7b82879d..00000000 --- a/src/libs/utils/include/utils/Semaphore.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2020 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include - -class Semaphore -{ - public: - Semaphore() = default; - Semaphore(const Semaphore&) = delete; - Semaphore(Semaphore&&) = delete; - Semaphore& operator=(const Semaphore&) = delete; - Semaphore& operator=(Semaphore&&) = delete; - - void notify() - { - std::unique_lock lock {_mutex}; - - _count++; - _cv.notify_one(); - } - - void wait() - { - std::unique_lock lock(_mutex); - - while (_count == 0) - _cv.wait(lock); - - _count--; - } - - private: - std::mutex _mutex; - std::condition_variable _cv; - unsigned _count {}; -}; - - diff --git a/src/libs/utils/include/utils/Service.hpp b/src/libs/utils/include/utils/Service.hpp index 95a44e37..4c2cf3e7 100644 --- a/src/libs/utils/include/utils/Service.hpp +++ b/src/libs/utils/include/utils/Service.hpp @@ -46,6 +46,11 @@ class Service return Service::get(); } + Class& operator*() const + { + return *Service::get(); + } + static Class* get() { return _service.get(); } private: diff --git a/src/lms/main.cpp b/src/lms/main.cpp index 11649676..c75b9cc1 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -121,7 +121,7 @@ generateWtConfig(std::string execPath) int main(int argc, char* argv[]) { std::filesystem::path configFilePath {"/etc/lms.conf"}; - int res = EXIT_FAILURE; + int res {EXIT_FAILURE}; assert(argc > 0); assert(argv[0] != NULL); @@ -183,22 +183,10 @@ int main(int argc, char* argv[]) config->getULong("cover-max-file-size", 10) * 1000 * 1000, config->getULong("cover-jpeg-quality", 75))}; Service recommendationEngineService {Recommendation::createEngine(database)}; - recommendationEngineService->requestLoad(); - Service mediaScannerService {Scanner::createMediaScanner(database)}; + Service mediaScannerService {Scanner::createMediaScanner(database, *recommendationEngineService)}; mediaScannerService->scanComplete().connect([&]() { - auto status = mediaScannerService->getStatus(); - - if (status.lastCompleteScanStats->nbChanges() > 0 || status.lastCompleteScanStats->featuresFetched > 0) - { - LMS_LOG(MAIN, INFO) << "Scanner changed some files, reloading the recommendation engine..."; - recommendationEngineService->requestReload(); - } - else - { - LMS_LOG(MAIN, INFO) << "Scanner did not change files, not reloading the recommendation engine..."; - } // Flush cover cache even if no changes: // covers may be external files that changed and we don't keep track of them coverArtService->flushCache(); @@ -224,16 +212,18 @@ int main(int argc, char* argv[]) LMS_LOG(MAIN, INFO) << "Stopping server..."; server.stop(); - LMS_LOG(MAIN, INFO) << "Clean stop!"; + LMS_LOG(MAIN, INFO) << "Quitting..."; res = EXIT_SUCCESS; } catch(Wt::WServer::Exception& e) { std::cerr << "Caught a WServer::Exception: " << e.what() << std::endl; + res = EXIT_FAILURE; } catch(std::exception& e) { std::cerr << "Caught std::exception: " << e.what() << std::endl; + res = EXIT_FAILURE; } return res; diff --git a/src/lms/ui/PlayQueue.cpp b/src/lms/ui/PlayQueue.cpp index 0e62b981..ff0e5db1 100644 --- a/src/lms/ui/PlayQueue.cpp +++ b/src/lms/ui/PlayQueue.cpp @@ -501,7 +501,10 @@ PlayQueue::addSome() void PlayQueue::enqueueRadioTracks() { - const std::vector trackToAddIds {Service::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 3)}; + const auto similarTrackIds {Service::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 3)}; + + std::vector trackToAddIds(std::cbegin(similarTrackIds), std::cend(similarTrackIds)); + Random::shuffleContainer(trackToAddIds); enqueueTracks(trackToAddIds); } diff --git a/src/lms/ui/admin/DatabaseSettingsView.cpp b/src/lms/ui/admin/DatabaseSettingsView.cpp index def6885f..fa0fbbc8 100644 --- a/src/lms/ui/admin/DatabaseSettingsView.cpp +++ b/src/lms/ui/admin/DatabaseSettingsView.cpp @@ -47,14 +47,13 @@ class DatabaseSettingsModel : public Wt::WFormModel { public: // Associate each field with a unique string literal. - static const Field MediaDirectoryField; - static const Field UpdatePeriodField; - static const Field UpdateStartTimeField; - static const Field RecommendationEngineTypeField; - static const Field TagsField; + static inline const Field MediaDirectoryField {"media-directory"}; + static inline const Field UpdatePeriodField {"update-period"}; + static inline const Field UpdateStartTimeField {"update-start-time"}; + static inline const Field RecommendationEngineTypeField {"recommendation-engine-type"}; + static inline const Field TagsField {"tags"}; DatabaseSettingsModel() - : Wt::WFormModel() { initializeModels(); @@ -135,7 +134,6 @@ class DatabaseSettingsModel : public Wt::WFormModel } private: - static std::shared_ptr createTagsValidator() { auto v = std::make_shared(); @@ -162,18 +160,11 @@ class DatabaseSettingsModel : public Wt::WFormModel _recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.features"), ScanSettings::RecommendationEngineType::Features); } - std::shared_ptr> _updatePeriodModel; - std::shared_ptr> _updateStartTimeModel; + std::shared_ptr> _updatePeriodModel; + std::shared_ptr> _updateStartTimeModel; std::shared_ptr> _recommendationEngineTypeModel; - }; -const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField = "media-directory"; -const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField = "update-period"; -const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField = "update-start-time"; -const Wt::WFormModel::Field DatabaseSettingsModel::RecommendationEngineTypeField = "recommendation-engine-type"; -const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags"; - DatabaseSettingsView::DatabaseSettingsView() { wApp->internalPathChanged().connect(std::bind([=] @@ -223,7 +214,7 @@ DatabaseSettingsView::refreshView() t->bindNew("scanner-controller"); - saveBtn->clicked().connect([=] () + saveBtn->clicked().connect([=] { t->updateModel(model.get()); @@ -231,7 +222,7 @@ DatabaseSettingsView::refreshView() { model->saveData(); - Service::get()->requestReload(); + Service::get()->requestImmediateScan(false); LmsApp->notifyMsg(MsgType::Success, Wt::WString::tr("Lms.Admin.Database.settings-saved")); } @@ -239,14 +230,14 @@ DatabaseSettingsView::refreshView() t->updateView(model.get()); }); - discardBtn->clicked().connect([=] () + discardBtn->clicked().connect([=] { model->loadData(); model->validate(); t->updateView(model.get()); }); - immScanBtn->clicked().connect([=] () + immScanBtn->clicked().connect([=] { Service::get()->requestImmediateScan(false); }); diff --git a/src/lms/ui/admin/ScannerController.cpp b/src/lms/ui/admin/ScannerController.cpp index f6193b98..2e394e61 100644 --- a/src/lms/ui/admin/ScannerController.cpp +++ b/src/lms/ui/admin/ScannerController.cpp @@ -206,20 +206,24 @@ ScannerController::refreshContents() case Scanner::ScanProgressStep::DiscoveringFiles: bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files") - .arg(status.currentScanStepStats->processedFiles)); + .arg(status.currentScanStepStats->processedElems)); break; case Scanner::ScanProgressStep::ScanningFiles: bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files") - .arg(status.currentScanStepStats->processedFiles) - .arg(status.currentScanStepStats->filesToProcess) + .arg(status.currentScanStepStats->processedElems) + .arg(status.currentScanStepStats->totalElems) .arg(status.currentScanStepStats->progress())); break; case Scanner::ScanProgressStep::FetchingTrackFeatures: bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features") - .arg(status.currentScanStepStats->processedFiles) - .arg(status.currentScanStepStats->filesToProcess) + .arg(status.currentScanStepStats->processedElems) + .arg(status.currentScanStepStats->totalElems) + .arg(status.currentScanStepStats->progress())); + break; + case Scanner::ScanProgressStep::ReloadingSimilarityEngine: + bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine") .arg(status.currentScanStepStats->progress())); break; } diff --git a/src/lms/ui/explore/ArtistView.cpp b/src/lms/ui/explore/ArtistView.cpp index 73487b64..5b16c86f 100644 --- a/src/lms/ui/explore/ArtistView.cpp +++ b/src/lms/ui/explore/ArtistView.cpp @@ -46,7 +46,7 @@ namespace UserInterface { Artist::Artist(Filters* filters) : Wt::WTemplate {Wt::WString::tr("Lms.Explore.Artist.template")} -, _filters(filters) +, _filters {filters} { addFunction("tr", &Wt::WTemplate::Functions::tr); @@ -75,7 +75,7 @@ Artist::refreshView() if (!artistId) throw ArtistNotFoundException {*artistId}; - const std::vector similarArtistIds {Service::get()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)}; + const auto similarArtistIds {Service::get()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; @@ -172,7 +172,7 @@ Artist::refreshView() } void -Artist::refreshSimilarArtists(const std::vector& similarArtistsId) +Artist::refreshSimilarArtists(const std::unordered_set& similarArtistsId) { if (similarArtistsId.empty()) return; diff --git a/src/lms/ui/explore/ArtistView.hpp b/src/lms/ui/explore/ArtistView.hpp index 54639412..5b9fe034 100644 --- a/src/lms/ui/explore/ArtistView.hpp +++ b/src/lms/ui/explore/ArtistView.hpp @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -46,7 +47,7 @@ class Artist : public Wt::WTemplate private: void refreshView(); - void refreshSimilarArtists(const std::vector& similarArtistsId); + void refreshSimilarArtists(const std::unordered_set& similarArtistsId); void refreshLinks(const Wt::Dbo::ptr& artist); std::unique_ptr createRelease(const Wt::Dbo::ptr& artist, const Wt::Dbo::ptr& release); diff --git a/src/lms/ui/explore/ReleaseView.cpp b/src/lms/ui/explore/ReleaseView.cpp index 5b514636..59ed4dc3 100644 --- a/src/lms/ui/explore/ReleaseView.cpp +++ b/src/lms/ui/explore/ReleaseView.cpp @@ -50,11 +50,11 @@ namespace UserInterface { Release::Release(Filters* filters) : Wt::WTemplate {Wt::WString::tr("Lms.Explore.Release.template")} -, _filters(filters) +, _filters {filters} { addFunction("tr", &Wt::WTemplate::Functions::tr); - wApp->internalPathChanged().connect([=]() + wApp->internalPathChanged().connect([=] { refreshView(); }); @@ -79,7 +79,7 @@ Release::refreshView() if (!releaseId) throw ReleaseNotFoundException {*releaseId}; - const std::vector similarReleasesIds {Service::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 6)}; + auto similarReleasesIds {Service::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 6)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; @@ -307,7 +307,7 @@ Release::refreshLinks(const Database::Release::pointer& release) } void -Release::refreshSimilarReleases(const std::vector& similarReleasesId) +Release::refreshSimilarReleases(const std::unordered_set& similarReleasesId) { if (similarReleasesId.empty()) return; diff --git a/src/lms/ui/explore/ReleaseView.hpp b/src/lms/ui/explore/ReleaseView.hpp index 7908c405..64b994f8 100644 --- a/src/lms/ui/explore/ReleaseView.hpp +++ b/src/lms/ui/explore/ReleaseView.hpp @@ -19,6 +19,7 @@ #pragma once +#include #include #include "database/Types.hpp" @@ -45,7 +46,7 @@ namespace UserInterface void refreshView(); void refreshCopyright(const Wt::Dbo::ptr& release); void refreshLinks(const Wt::Dbo::ptr& release); - void refreshSimilarReleases(const std::vector& similarReleasesId); + void refreshSimilarReleases(const std::unordered_set& similarReleasesId); Filters* _filters {}; }; diff --git a/src/tools/recommendation/LmsRecommendation.cpp b/src/tools/recommendation/LmsRecommendation.cpp index 9991a596..5c722c7f 100644 --- a/src/tools/recommendation/LmsRecommendation.cpp +++ b/src/tools/recommendation/LmsRecommendation.cpp @@ -31,7 +31,6 @@ #include "database/Session.hpp" #include "database/Track.hpp" #include "utils/IConfig.hpp" -#include "utils/Semaphore.hpp" #include "utils/Service.hpp" #include "utils/StreamLogger.hpp" #include "recommendation/IEngine.hpp" @@ -164,17 +163,8 @@ int main(int argc, char *argv[]) const auto engine {Recommendation::createEngine(db)}; std::cout << "Recommendation engine created!" << std::endl; - Semaphore sem; - - engine->reloaded().connect([&]() - { - sem.notify(); - }); - - engine->requestLoad(); - - std::cout << "Waiting for the recommendation engine to be loaded..." << std::endl; - sem.wait(); + std::cout << "Loading recommendation engine..." << std::endl; + engine->load(false); std::cout << "Recommendation engine loaded!" << std::endl; if (vm.count("tracks"))