From 6baf0d313d31e75af3b6ce9aee9215e0de0b16f4 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 30 Sep 2021 19:02:49 +0200 Subject: [PATCH] Finished clean on recommendation engines --- src/libs/database/impl/Artist.cpp | 7 + src/libs/database/impl/Release.cpp | 7 + src/libs/database/impl/Track.cpp | 8 + src/libs/database/include/database/Artist.hpp | 3 +- .../database/include/database/Release.hpp | 1 + src/libs/database/include/database/Track.hpp | 7 +- src/libs/database/test/Artist.cpp | 9 + src/libs/database/test/Release.cpp | 9 + src/libs/database/test/Track.cpp | 5 + src/libs/recommendation/CMakeLists.txt | 2 +- ...rCreator.hpp => ClustersEngineCreator.hpp} | 12 +- src/libs/recommendation/impl/Engine.cpp | 463 ++++++++---------- src/libs/recommendation/impl/Engine.hpp | 41 +- .../impl/FeaturesEngineCreator.hpp | 9 +- src/libs/recommendation/impl/IClassifier.hpp | 65 --- .../impl/clusters/ClustersClassifier.cpp | 103 ---- .../impl/clusters/ClustersClassifier.hpp | 53 -- .../impl/clusters/ClustersEngine.cpp | 119 +++++ .../impl/clusters/ClustersEngine.hpp | 51 ++ .../impl/features/FeaturesEngine.cpp | 104 ++-- .../impl/features/FeaturesEngine.hpp | 31 +- .../include/recommendation/IEngine.hpp | 16 +- .../recommendation/IRecommendation.hpp | 38 -- .../listenbrainz/FeedbackSynchronizer.hpp | 92 ++++ src/libs/subsonic/impl/SubsonicResource.cpp | 10 +- src/lms/ui/PlayQueue.cpp | 2 +- src/lms/ui/explore/ArtistView.cpp | 5 +- src/lms/ui/explore/ReleaseView.cpp | 2 +- .../recommendation/LmsRecommendation.cpp | 8 +- 29 files changed, 648 insertions(+), 634 deletions(-) rename src/libs/recommendation/impl/{ClustersClassifierCreator.hpp => ClustersEngineCreator.hpp} (86%) delete mode 100644 src/libs/recommendation/impl/IClassifier.hpp delete mode 100644 src/libs/recommendation/impl/clusters/ClustersClassifier.cpp delete mode 100644 src/libs/recommendation/impl/clusters/ClustersClassifier.hpp create mode 100644 src/libs/recommendation/impl/clusters/ClustersEngine.cpp create mode 100644 src/libs/recommendation/impl/clusters/ClustersEngine.hpp delete mode 100644 src/libs/recommendation/include/recommendation/IRecommendation.hpp create mode 100644 src/libs/scrobbling/impl/listenbrainz/FeedbackSynchronizer.hpp diff --git a/src/libs/database/impl/Artist.cpp b/src/libs/database/impl/Artist.cpp index a519f4c2..9ca631fa 100644 --- a/src/libs/database/impl/Artist.cpp +++ b/src/libs/database/impl/Artist.cpp @@ -66,6 +66,13 @@ Artist::getById(Session& session, ArtistId id) return session.getDboSession().find().where("id = ?").bind(id).resultValue(); } +bool +Artist::exists(Session& session, ArtistId id) +{ + session.checkSharedLocked(); + return session.getDboSession().query("SELECT 1 FROM artist").where("id = ?").bind(id).resultValue() == 1; +} + Artist::pointer Artist::create(Session& session, const std::string& name, const std::optional& MBID) { diff --git a/src/libs/database/impl/Release.cpp b/src/libs/database/impl/Release.cpp index a42d7c91..310de51e 100644 --- a/src/libs/database/impl/Release.cpp +++ b/src/libs/database/impl/Release.cpp @@ -114,6 +114,13 @@ Release::getById(Session& session, ReleaseId id) .resultValue(); } +bool +Release::exists(Session& session, ReleaseId id) +{ + session.checkSharedLocked(); + return session.getDboSession().query("SELECT 1 FROM release").where("id = ?").bind(id).resultValue() == 1; +} + Release::pointer Release::create(Session& session, const std::string& name, const std::optional& MBID) { diff --git a/src/libs/database/impl/Track.cpp b/src/libs/database/impl/Track.cpp index e8f3436b..82dc99cf 100644 --- a/src/libs/database/impl/Track.cpp +++ b/src/libs/database/impl/Track.cpp @@ -156,6 +156,14 @@ Track::getById(Session& session, TrackId id) .resultValue(); } +bool +Track::exists(Session& session, TrackId id) +{ + session.checkSharedLocked(); + + return session.getDboSession().query("SELECT 1 from track").where("id = ?").bind(id).resultValue() == 1; +} + std::vector Track::getByRecordingMBID(Session& session, const UUID& mbid) { diff --git a/src/libs/database/include/database/Artist.hpp b/src/libs/database/include/database/Artist.hpp index d5bab873..23ef7812 100644 --- a/src/libs/database/include/database/Artist.hpp +++ b/src/libs/database/include/database/Artist.hpp @@ -58,8 +58,9 @@ class Artist : public Object // Accessors static pointer getByMBID(Session& session, const UUID& MBID); static pointer getById(Session& session, ArtistId id); + static bool exists(Session& session, ArtistId id); static std::vector getByName(Session& session, const std::string& name); // exact match on name field - static std::vector getByClusters(Session& session, + static std::vector getByClusters(Session& session, const std::vector& clusters, // at least one track that belongs to these clusters SortMethod sortMethod ); diff --git a/src/libs/database/include/database/Release.hpp b/src/libs/database/include/database/Release.hpp index 80c16eeb..ce45a1a8 100644 --- a/src/libs/database/include/database/Release.hpp +++ b/src/libs/database/include/database/Release.hpp @@ -50,6 +50,7 @@ class Release : public Object static pointer getByMBID(Session& session, const UUID& MBID); static std::vector getByName(Session& session, const std::string& name); static pointer getById(Session& session, ReleaseId id); + static bool exists(Session& session, ReleaseId id); static std::vector getAllOrphans(Session& session); // no track related static std::vector getAll(Session& session, std::optional range = std::nullopt); static std::vector getAllIds(Session& session); diff --git a/src/libs/database/include/database/Track.hpp b/src/libs/database/include/database/Track.hpp index 0d78c627..3e2c2ced 100644 --- a/src/libs/database/include/database/Track.hpp +++ b/src/libs/database/include/database/Track.hpp @@ -56,9 +56,10 @@ class Track : public Object Track(const std::filesystem::path& p); // Find utility functions - static std::size_t getCount(Session& session); - static pointer getByPath(Session& session, const std::filesystem::path& p); - static pointer getById(Session& session, TrackId id); + static std::size_t getCount(Session& session); + static pointer getByPath(Session& session, const std::filesystem::path& p); + static pointer getById(Session& session, TrackId id); + static bool exists(Session& session, TrackId id); static std::vector getByRecordingMBID(Session& session, const UUID& MBID); static std::vector getSimilarTracks(Session& session, const std::vector& trackIds, diff --git a/src/libs/database/test/Artist.cpp b/src/libs/database/test/Artist.cpp index 7193b164..dcace44e 100644 --- a/src/libs/database/test/Artist.cpp +++ b/src/libs/database/test/Artist.cpp @@ -23,6 +23,13 @@ using namespace Database; TEST_F(DatabaseFixture, SingleArtist) { + { + auto transaction {session.createSharedTransaction()}; + EXPECT_FALSE(Artist::exists(session, 35)); + EXPECT_FALSE(Artist::exists(session, 0)); + EXPECT_FALSE(Artist::exists(session, 1)); + } + ScopedArtist artist {session, "MyArtist"}; { @@ -31,6 +38,8 @@ TEST_F(DatabaseFixture, SingleArtist) EXPECT_TRUE(artist.get()); EXPECT_FALSE(!artist.get()); EXPECT_EQ(artist.get()->getId(), artist.getId()); + + EXPECT_TRUE(Artist::exists(session, artist.getId())); } { diff --git a/src/libs/database/test/Release.cpp b/src/libs/database/test/Release.cpp index a65db032..801fd9bf 100644 --- a/src/libs/database/test/Release.cpp +++ b/src/libs/database/test/Release.cpp @@ -23,11 +23,20 @@ using namespace Database; TEST_F(DatabaseFixture, SingleRelease) { + { + auto transaction {session.createSharedTransaction()}; + + EXPECT_FALSE(Release::exists(session, 0)); + EXPECT_FALSE(Release::exists(session, 1)); + } + ScopedRelease release {session, "MyRelease"}; { auto transaction {session.createSharedTransaction()}; + EXPECT_TRUE(Release::exists(session, release.getId())); + auto releases {Release::getAllOrphans(session)}; ASSERT_EQ(releases.size(), 1); EXPECT_EQ(releases.front()->getId(), release.getId()); diff --git a/src/libs/database/test/Track.cpp b/src/libs/database/test/Track.cpp index ac919bff..bdc54855 100644 --- a/src/libs/database/test/Track.cpp +++ b/src/libs/database/test/Track.cpp @@ -28,6 +28,7 @@ TEST_F(DatabaseFixture, SingleTrack) { auto transaction {session.createSharedTransaction()}; EXPECT_EQ(Track::getCount(session), 0); + EXPECT_FALSE(Track::exists(session, 0)); } ScopedTrack track {session, "MyTrackFile"}; @@ -37,6 +38,10 @@ TEST_F(DatabaseFixture, SingleTrack) EXPECT_EQ(Track::getAll(session).size(), 1); EXPECT_EQ(Track::getCount(session), 1); + EXPECT_TRUE(Track::exists(session, track.getId())); + auto myTrack {Track::getById(session, track.getId())}; + ASSERT_TRUE(myTrack); + EXPECT_EQ(myTrack->getId(), track.getId()); } } diff --git a/src/libs/recommendation/CMakeLists.txt b/src/libs/recommendation/CMakeLists.txt index 890cf3f9..4682173d 100644 --- a/src/libs/recommendation/CMakeLists.txt +++ b/src/libs/recommendation/CMakeLists.txt @@ -1,6 +1,6 @@ add_library(lmsrecommendation SHARED - impl/clusters/ClustersClassifier.cpp + impl/clusters/ClustersEngine.cpp impl/features/FeaturesEngineCache.cpp impl/features/FeaturesEngine.cpp impl/features/FeaturesDefs.cpp diff --git a/src/libs/recommendation/impl/ClustersClassifierCreator.hpp b/src/libs/recommendation/impl/ClustersEngineCreator.hpp similarity index 86% rename from src/libs/recommendation/impl/ClustersClassifierCreator.hpp rename to src/libs/recommendation/impl/ClustersEngineCreator.hpp index c9527ef8..e0fcc842 100644 --- a/src/libs/recommendation/impl/ClustersClassifierCreator.hpp +++ b/src/libs/recommendation/impl/ClustersEngineCreator.hpp @@ -21,10 +21,14 @@ #include -namespace Recommendation +namespace Database { - class IClassifier; - - std::unique_ptr createClustersClassifier(); + class Db; +} + +namespace Recommendation +{ + class IEngine; + std::unique_ptr createClustersEngine(Database::Db& db); } diff --git a/src/libs/recommendation/impl/Engine.cpp b/src/libs/recommendation/impl/Engine.cpp index 741a9dad..b6808ceb 100644 --- a/src/libs/recommendation/impl/Engine.cpp +++ b/src/libs/recommendation/impl/Engine.cpp @@ -22,7 +22,7 @@ #include #include -#include "ClustersClassifierCreator.hpp" +#include "ClustersEngineCreator.hpp" #include "FeaturesEngineCreator.hpp" #include "database/Db.hpp" @@ -31,260 +31,229 @@ #include "utils/Exception.hpp" #include "utils/Logger.hpp" -namespace Recommendation { - - -static -std::unique_ptr -createClassifier(ClassifierType type) +namespace Recommendation { - switch (type) + + static + std::string_view + engineTypeToString(EngineType engineType) { - case ClassifierType::Clusters: - return createClustersClassifier(); - break; - - case ClassifierType::Features: - return createFeaturesEngine(); - break; - } - - return {}; -} - -std::unique_ptr -createEngine(Database::Db& db) -{ - return std::make_unique(db); -} - -Engine::Engine(Database::Db& db) -: _db {db} -{ -} - -Engine::TrackContainer -Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::TrackListId trackListId, std::size_t maxCount) -{ - TrackContainer res; - - std::shared_lock lock {_classifiersMutex}; - for (const auto& classifierName : _classifierPriorities) - { - auto itClassifier {_classifiers.find(classifierName)}; - if (itClassifier == std::cend(_classifiers)) - continue; - - res = itClassifier->second->getSimilarTracksFromTrackList(session, trackListId, maxCount); - if (!res.empty()) - break; - } - - return res; -} - -Engine::TrackContainer -Engine::getSimilarTracks(Database::Session& dbSession, const std::vector& trackIds, std::size_t maxCount) -{ - TrackContainer res; - - std::shared_lock lock {_classifiersMutex}; - for (ClassifierType classifierType : _classifierPriorities) - { - auto itClassifier {_classifiers.find(classifierType)}; - if (itClassifier == std::cend(_classifiers)) - continue; - - const IClassifier& classifier {*itClassifier->second}; - res = classifier.getSimilarTracks(dbSession, trackIds, maxCount); - if (!res.empty()) + switch (engineType) { - LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifier.getName() << "'"; - break; + case EngineType::Clusters: return "clusters"; + case EngineType::Features: return "features"; + } + + throw LmsException {"Internal error"}; + } + + std::unique_ptr + createEngine(Database::Db& db) + { + return std::make_unique(db); + } + + Engine::Engine(Database::Db& db) + : _db {db} + { + } + + Engine::TrackContainer + Engine::getSimilarTracksFromTrackList(Database::TrackListId trackListId, std::size_t maxCount) const + { + TrackContainer res; + + std::shared_lock lock {_enginesMutex}; + for (const auto& engineType : _enginePriorities) + { + auto itEngine {_engines.find(engineType)}; + if (itEngine == std::cend(_engines)) + continue; + + res = itEngine->second->getSimilarTracksFromTrackList(trackListId, maxCount); + if (!res.empty()) + break; + } + + return res; + } + + Engine::TrackContainer + Engine::getSimilarTracks(const std::vector& trackIds, std::size_t maxCount) const + { + TrackContainer res; + + std::shared_lock lock {_enginesMutex}; + for (EngineType engineType : _enginePriorities) + { + auto itEngine {_engines.find(engineType)}; + if (itEngine == std::cend(_engines)) + continue; + + const IEngine& engine {*itEngine->second}; + res = engine.getSimilarTracks(trackIds, maxCount); + if (!res.empty()) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using engine '" << engineTypeToString(engineType) << "'"; + break; + } + } + + return res; + } + + Engine::ReleaseContainer + Engine::getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const + { + ReleaseContainer res; + + std::shared_lock lock {_enginesMutex}; + for (EngineType engineType : _enginePriorities) + { + auto itEngine {_engines.find(engineType)}; + if (itEngine == std::cend(_engines)) + continue; + + const IEngine& engine {*itEngine->second}; + res = engine.getSimilarReleases(releaseId, maxCount); + if (!res.empty()) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using engine '" << engineTypeToString(engineType) << "'"; + break; + } + + LMS_LOG(RECOMMENDATION, DEBUG) << "No result using engine '" << engineTypeToString(engineType) << "'"; + } + + return res; + } + + Engine::ArtistContainer + Engine::getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const + { + ArtistContainer res; + + std::shared_lock lock {_enginesMutex}; + for (EngineType engineType : _enginePriorities) + { + auto itEngine {_engines.find(engineType)}; + if (itEngine == std::cend(_engines)) + continue; + + const IEngine& engine {*itEngine->second}; + res = engine.getSimilarArtists(artistId, linkTypes, maxCount); + if (!res.empty()) + { + LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using engine '" << engineTypeToString(engineType) << "'"; + return res; + } + } + + return res; + } + + static + Database::ScanSettings::RecommendationEngineType + getRecommendationEngineType(Database::Session& session) + { + auto transaction {session.createSharedTransaction()}; + + return Database::ScanSettings::get(session)->getRecommendationEngineType(); + } + + void + Engine::load(bool forceReload, const ProgressCallback& progressCallback) + { + using namespace Database; + + LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines..."; + + EngineContainer enginesToLoad; + + { + std::unique_lock controlLock {_controlMutex}; + + { + std::unique_lock lock {_enginesMutex}; + _engines.clear(); + } + + switch (getRecommendationEngineType(_db.getTLSSession())) + { + case ScanSettings::RecommendationEngineType::Clusters: + _enginePriorities = {EngineType::Clusters}; + enginesToLoad.try_emplace(EngineType::Clusters, createClustersEngine(_db)); + break; + + case ScanSettings::RecommendationEngineType::Features: + _enginePriorities = {EngineType::Features, EngineType::Clusters}; + + // not same order since clusters is faster to load + enginesToLoad.try_emplace(EngineType::Clusters, createClustersEngine(_db)); + enginesToLoad.try_emplace(EngineType::Features, createFeaturesEngine(_db)); + break; + } + + assert(_pendingEngines.empty()); + for (auto& [engineType, engine] : enginesToLoad) + _pendingEngines.push_back(engine.get()); + } + + for (auto& [engineType, engine] : enginesToLoad) + loadPendingEngine(engineType, std::move(engine), forceReload, progressCallback); + + _pendingEnginesCondvar.notify_all(); + + LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines loaded!"; + } + + void + Engine::loadPendingEngine(EngineType engineType, std::unique_ptr engine, bool forceReload, const ProgressCallback& progressCallback) + { + if (!_loadCancelled) + { + LMS_LOG(RECOMMENDATION, INFO) << "Initializing engine '" << engineTypeToString(engineType) << "'..."; + + auto progress {[&](const IEngine::Progress& progress) + { + progressCallback(progress); + }}; + + engine->load(forceReload, progressCallback ? progress : IEngine::ProgressCallback {}); + + { + std::scoped_lock lock {_controlMutex}; + _pendingEngines.erase(std::find(std::begin(_pendingEngines), std::end(_pendingEngines), engine.get())); + } + LMS_LOG(RECOMMENDATION, INFO) << "Initializing engine '" << engineTypeToString(engineType) << "': " << (_loadCancelled ? "aborted" : "complete"); + } + + if (!_loadCancelled) + { + std::unique_lock lock {_enginesMutex}; + _engines.emplace(engineType, std::move(engine)); } } - return res; -} - -Engine::ReleaseContainer -Engine::getSimilarReleases(Database::Session& dbSession, Database::ReleaseId releaseId, std::size_t maxCount) -{ - ReleaseContainer res; - - std::shared_lock lock {_classifiersMutex}; - for (ClassifierType classifierType : _classifierPriorities) + void + Engine::cancelLoad() { - auto itClassifier {_classifiers.find(classifierType)}; - if (itClassifier == std::cend(_classifiers)) - continue; + LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading..."; - const IClassifier& classifier {*itClassifier->second}; - res = classifier.getSimilarReleases(dbSession, releaseId, maxCount); - if (!res.empty()) - { - LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifier.getName() << "'"; - break; - } + std::unique_lock controlLock {_controlMutex}; + + assert(!_loadCancelled); + _loadCancelled = true; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Still " << _pendingEngines.size() << " pending engines!"; + + for (IEngine* engine : _pendingEngines) + engine->requestCancelLoad(); + + _pendingEnginesCondvar.wait(controlLock, [this] {return _pendingEngines.empty();}); + _loadCancelled = false; + + LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading DONE"; } - return res; -} - -Engine::ArtistContainer -Engine::getSimilarArtists(Database::Session& dbSession, Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) -{ - ArtistContainer res; - - std::shared_lock lock {_classifiersMutex}; - for (ClassifierType classifierType : _classifierPriorities) - { - auto itClassifier {_classifiers.find(classifierType)}; - if (itClassifier == std::cend(_classifiers)) - continue; - - const IClassifier& classifier {*itClassifier->second}; - res = classifier.getSimilarArtists(dbSession, artistId, linkTypes, maxCount); - if (!res.empty()) - { - LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifier.getName() << "'"; - return res; - } - } - - return res; -} - -static -Database::ScanSettings::RecommendationEngineType -getRecommendationEngineType(Database::Session& session) -{ - auto transaction {session.createSharedTransaction()}; - - return Database::ScanSettings::get(session)->getRecommendationEngineType(); -} - -void -Engine::load(bool forceReload, const ProgressCallback& progressCallback) -{ - using namespace Database; - - LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines..."; - struct ClassifierWithType - { - ClassifierType type; - std::unique_ptr classifier; - }; - - std::vector classifiers; - auto addClassifier {[&](ClassifierType type) - { - classifiers.emplace_back(ClassifierWithType {type, createClassifier(type)}); - }}; - - switch (getRecommendationEngineType(_db.getTLSSession())) - { - case ScanSettings::RecommendationEngineType::Clusters: - setClassifierPriorities({ClassifierType::Clusters}); - addClassifier(ClassifierType::Clusters); - break; - case ScanSettings::RecommendationEngineType::Features: - setClassifierPriorities({ClassifierType::Features, ClassifierType::Clusters}); - // not same order since clusters is faster to load - addClassifier(ClassifierType::Clusters); - addClassifier(ClassifierType::Features); - break; - } - - assert(_pendingClassifiers.empty()); - clearClassifiers(); - - { - std::scoped_lock lock {_controlMutex}; - - std::transform(std::cbegin(classifiers), std::cend(classifiers), std::inserter(_pendingClassifiers, std::end(_pendingClassifiers)), - [](auto& classifier) { return classifier.classifier.get(); }); - } - - for (ClassifierWithType& classifier : classifiers) - loadClassifier(std::move(classifier.classifier), classifier.type, forceReload, progressCallback); - - LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines loaded!"; -} - -void -Engine::setClassifierPriorities(const std::vector& classifierPriorities) -{ - std::unique_lock lock {_classifiersMutex}; - - _classifierPriorities = classifierPriorities; -} - -void -Engine::clearClassifiers() -{ - std::unique_lock lock {_classifiersMutex}; - - _classifiers.clear(); -} - -void -Engine::loadClassifier(std::unique_ptr classifier, - ClassifierType classifierType, - bool forceReload, - const ProgressCallback& progressCallback) -{ - IClassifier* rawClassifier {classifier.get()}; - - bool res {}; - if (!_loadCancelled) - { - LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'..."; - - auto progress {[&](IClassifier::Progress progress) - { - progressCallback(Progress {progress.processedElems, progress.totalElems}); - }}; - - res = classifier->load(_db.getTLSSession(), forceReload, progressCallback ? progress : IClassifier::ProgressCallback {}); - - LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE"); - } - - if (res) - { - std::unique_lock lock {_classifiersMutex}; - - _classifiers.emplace(classifierType, std::move(classifier)); - } - - { - std::scoped_lock lock {_controlMutex}; - - _pendingClassifiers.erase(rawClassifier); - } - - _pendingClassifiersCondvar.notify_one(); -} - -void -Engine::cancelLoad() -{ - LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading..."; - - std::unique_lock lock {_controlMutex}; - - LMS_LOG(RECOMMENDATION, DEBUG) << "Still " << _pendingClassifiers.size() << " pending classifiers!"; - - _loadCancelled = true; - - for (IClassifier* classifier : _pendingClassifiers) - classifier->requestCancelLoad(); - - _pendingClassifiersCondvar.wait(lock, [this] {return _pendingClassifiers.empty();}); - _loadCancelled = false; - - LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading DONE"; -} - } // ns Similarity diff --git a/src/libs/recommendation/impl/Engine.hpp b/src/libs/recommendation/impl/Engine.hpp index 39338fbc..ac253b5c 100644 --- a/src/libs/recommendation/impl/Engine.hpp +++ b/src/libs/recommendation/impl/Engine.hpp @@ -20,13 +20,12 @@ #pragma once #include +#include #include #include -#include #include #include "recommendation/IEngine.hpp" -#include "IClassifier.hpp" namespace Database { @@ -35,7 +34,7 @@ namespace Database namespace Recommendation { - enum class ClassifierType + enum class EngineType { Clusters, Features, @@ -57,29 +56,29 @@ namespace Recommendation void cancelLoad() override; void requestCancelLoad() override {}; - ResultContainer getSimilarTracksFromTrackList(Database::Session& session, Database::TrackListId tracklistId, std::size_t maxCount) override; - ResultContainer getSimilarTracks(Database::Session& session, const std::vector& tracksId, std::size_t maxCount) override; - ResultContainer getSimilarReleases(Database::Session& session, Database::ReleaseId releaseId, std::size_t maxCount) override; - ResultContainer getSimilarArtists(Database::Session& session, - Database::ArtistId artistId, - EnumSet linkTypes, - std::size_t maxCount) override; + TrackContainer getSimilarTracksFromTrackList(Database::TrackListId tracklistId, std::size_t maxCount) const override; + TrackContainer getSimilarTracks(const std::vector& tracksId, std::size_t maxCount) const override; + ReleaseContainer getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const override; + ArtistContainer getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const override; - void setClassifierPriorities(const std::vector& classifierTypes); - void clearClassifiers(); - void loadClassifier(std::unique_ptr classifier, ClassifierType classifierType, bool forceReload, const ProgressCallback& progressCallback); + void setEnginePriorities(const std::vector& engineTypes); + void clearEngines(); + void loadPendingEngine(EngineType engineType, std::unique_ptr engine, bool forceReload, const ProgressCallback& progressCallback); Database::Db& _db; - std::mutex _controlMutex; - bool _loadCancelled {}; - std::condition_variable _pendingClassifiersCondvar; - std::unordered_set _pendingClassifiers; + std::mutex _controlMutex; + bool _loadCancelled {}; - std::shared_mutex _classifiersMutex; - using ClassifierContainer = std::unordered_map>; - ClassifierContainer _classifiers; - std::vector _classifierPriorities; // ordered by priority + using EngineContainer = std::unordered_map>; + EngineContainer _engines; + mutable std::shared_mutex _enginesMutex; + + std::vector _pendingEngines; + std::shared_mutex _pendingEnginesMutex; + std::condition_variable _pendingEnginesCondvar; + + std::vector _enginePriorities; // ordered by priority }; } // ns Recommendation diff --git a/src/libs/recommendation/impl/FeaturesEngineCreator.hpp b/src/libs/recommendation/impl/FeaturesEngineCreator.hpp index 5b948e70..1378970f 100644 --- a/src/libs/recommendation/impl/FeaturesEngineCreator.hpp +++ b/src/libs/recommendation/impl/FeaturesEngineCreator.hpp @@ -20,10 +20,15 @@ #pragma once #include -#include "IClassifier.hpp" +#include "recommendation/IEngine.hpp" + +namespace Database +{ + class Db; +} namespace Recommendation { - std::unique_ptr createFeaturesEngine(); + std::unique_ptr createFeaturesEngine(Database::Db& db); } diff --git a/src/libs/recommendation/impl/IClassifier.hpp b/src/libs/recommendation/impl/IClassifier.hpp deleted file mode 100644 index 62483d9e..00000000 --- a/src/libs/recommendation/impl/IClassifier.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2020 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include -#include - -#include "database/Types.hpp" -#include "recommendation/IRecommendation.hpp" -#include "utils/EnumSet.hpp" - -namespace Database -{ - class Session; -} - -namespace Recommendation -{ - - class IClassifier : public IRecommendation - { - public: - virtual ~IClassifier() = default; - - virtual std::string_view getName() const = 0; - - struct Progress - { - std::size_t totalElems {}; - std::size_t processedElems {}; - }; - using ProgressCallback = std::function; - virtual bool load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) = 0; - virtual void requestCancelLoad() = 0; - - template - using ResultContainer = std::vector; - - virtual ResultContainer getSimilarTracksFromTrackList(Database::Session& session, Database::TrackListId tracklistId, std::size_t maxCount) const = 0; - virtual ResultContainer getSimilarTracks(Database::Session& session, const std::vector& tracksId, std::size_t maxCount) const = 0; - virtual ResultContainer getSimilarReleases(Database::Session& session, Database::ReleaseId releaseId, std::size_t maxCount) const = 0; - virtual ResultContainer getSimilarArtists(Database::Session& session, - Database::ArtistId artistId, - EnumSet linkTypes, std::size_t maxCount) const = 0; - }; - -} // ns Recommendation diff --git a/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp deleted file mode 100644 index 44f39a76..00000000 --- a/src/libs/recommendation/impl/clusters/ClustersClassifier.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "ClustersClassifier.hpp" - -#include "database/Artist.hpp" -#include "database/Cluster.hpp" -#include "database/Release.hpp" -#include "database/Session.hpp" -#include "database/Track.hpp" -#include "database/TrackList.hpp" - -namespace Recommendation { - -std::unique_ptr createClustersClassifier() -{ - return std::make_unique(); -} - -IClassifier::ResultContainer -ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::vector& trackIds, std::size_t maxCount) const -{ - ResultContainer res; - - auto transaction {dbSession.createSharedTransaction()}; - - const auto tracks {Database::Track::getSimilarTracks(dbSession, trackIds, 0, maxCount)}; - std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res), [](const auto& track) { return track->getId(); }); - - return res; -} - -IClassifier::ResultContainer -ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::TrackListId tracklistId, std::size_t maxCount) const -{ - ResultContainer res; - - auto transaction {session.createSharedTransaction()}; - - const Database::TrackList::pointer trackList {Database::TrackList::getById(session, tracklistId)}; - if (!trackList) - return res; - - const auto tracks {trackList->getSimilarTracks(0, maxCount)}; - std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res), [](const auto& track) { return track->getId(); }); - - return res; -} - -IClassifier::ResultContainer -ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::ReleaseId releaseId, std::size_t maxCount) const -{ - ResultContainer res; - - auto transaction {dbSession.createSharedTransaction()}; - - auto release {Database::Release::getById(dbSession, releaseId)}; - if (!release) - return res; - - const auto releases {release->getSimilarReleases(0, maxCount)}; - std::transform(std::cbegin(releases), std::cend(releases), std::back_inserter(res), [](const auto& release) { return release->getId(); }); - - return res; -} - -IClassifier::ResultContainer -ClusterClassifier::getSimilarArtists(Database::Session& dbSession, - Database::ArtistId artistId, - EnumSet artistLinkTypes, - std::size_t maxCount) const -{ - ResultContainer res; - - auto transaction {dbSession.createSharedTransaction()}; - - auto artist {Database::Artist::getById(dbSession, artistId)}; - if (!artist) - return res; - - const auto artists {artist->getSimilarArtists(artistLinkTypes, Database::Range {0, maxCount})}; - std::transform(std::cbegin(artists), std::cend(artists), std::back_inserter(res), [](const auto& artist) { return artist->getId(); }); - - return res; -} - -} // namespace Recommendation diff --git a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp b/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp deleted file mode 100644 index 718d6872..00000000 --- a/src/libs/recommendation/impl/clusters/ClustersClassifier.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include "IClassifier.hpp" - -namespace Recommendation -{ - - class ClusterClassifier : public IClassifier - { - public: - ClusterClassifier() = default; - ClusterClassifier(const ClusterClassifier&) = delete; - ClusterClassifier(ClusterClassifier&&) = delete; - ClusterClassifier& operator=(const ClusterClassifier&) = delete; - ClusterClassifier& operator=(ClusterClassifier&&) = delete; - - private: - - std::string_view getName() const override { return "Clusters"; } - - bool load(Database::Session&, bool, const ProgressCallback&) override { return true; } - void requestCancelLoad() override {} - - ResultContainer getSimilarTracksFromTrackList(Database::Session& session, Database::TrackListId tracklistId, std::size_t maxCount) const override; - ResultContainer getSimilarTracks(Database::Session& session, const std::vector& tracksId, std::size_t maxCount) const override; - ResultContainer getSimilarReleases(Database::Session& session, Database::ReleaseId releaseId, std::size_t maxCount) const override; - ResultContainer getSimilarArtists(Database::Session& session, - Database::ArtistId artistId, - EnumSet linkTypes, - std::size_t maxCount) const override; -}; - -} // namespace Recommendation - diff --git a/src/libs/recommendation/impl/clusters/ClustersEngine.cpp b/src/libs/recommendation/impl/clusters/ClustersEngine.cpp new file mode 100644 index 00000000..b0703a0f --- /dev/null +++ b/src/libs/recommendation/impl/clusters/ClustersEngine.cpp @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ClustersEngine.hpp" + +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Db.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "database/TrackList.hpp" + +namespace Recommendation { + +std::unique_ptr createClustersEngine(Database::Db& db) +{ + return std::make_unique(db); +} + +IEngine::TrackContainer +ClusterEngine::getSimilarTracks(const std::vector& trackIds, std::size_t maxCount) const +{ + Database::Session& dbSession {_db.getTLSSession()}; + + TrackContainer res; + + { + auto transaction {dbSession.createSharedTransaction()}; + + const auto tracks {Database::Track::getSimilarTracks(dbSession, trackIds, 0, maxCount)}; + res.reserve(tracks.size()); + std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res), [](const auto& track) { return track->getId(); }); + } + + return res; +} + +IEngine::ResultContainer +ClusterEngine::getSimilarTracksFromTrackList(Database::TrackListId tracklistId, std::size_t maxCount) const +{ + Database::Session& dbSession {_db.getTLSSession()}; + + TrackContainer res; + + { + auto transaction {dbSession.createSharedTransaction()}; + + const Database::TrackList::pointer trackList {Database::TrackList::getById(dbSession, tracklistId)}; + if (!trackList) + return res; + + const auto tracks {trackList->getSimilarTracks(0, maxCount)}; + res.reserve(tracks.size()); + std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res), [](const auto& track) { return track->getId(); }); + } + + return res; +} + +IEngine::ResultContainer +ClusterEngine::getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const +{ + Database::Session& dbSession {_db.getTLSSession()}; + + ReleaseContainer res; + { + auto transaction {dbSession.createSharedTransaction()}; + + auto release {Database::Release::getById(dbSession, releaseId)}; + if (!release) + return res; + + const auto releases {release->getSimilarReleases(0, maxCount)}; + res.reserve(releases.size()); + std::transform(std::cbegin(releases), std::cend(releases), std::back_inserter(res), [](const auto& release) { return release->getId(); }); + } + + return res; +} + +IEngine::ResultContainer +ClusterEngine::getSimilarArtists(Database::ArtistId artistId, EnumSet artistLinkTypes, std::size_t maxCount) const +{ + Database::Session& dbSession {_db.getTLSSession()}; + + ResultContainer res; + { + auto transaction {dbSession.createSharedTransaction()}; + + auto artist {Database::Artist::getById(dbSession, artistId)}; + if (!artist) + return res; + + const auto artists {artist->getSimilarArtists(artistLinkTypes, Database::Range {0, maxCount})}; + res.reserve(artists.size()); + std::transform(std::cbegin(artists), std::cend(artists), std::back_inserter(res), [](const auto& artist) { return artist->getId(); }); + } + + return res; +} + +} // namespace Recommendation diff --git a/src/libs/recommendation/impl/clusters/ClustersEngine.hpp b/src/libs/recommendation/impl/clusters/ClustersEngine.hpp new file mode 100644 index 00000000..47654aac --- /dev/null +++ b/src/libs/recommendation/impl/clusters/ClustersEngine.hpp @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "recommendation/IEngine.hpp" + +namespace Recommendation +{ + + class ClusterEngine : public IEngine + { + public: + ClusterEngine(Database::Db& db) : _db {db} {} + + ClusterEngine(const ClusterEngine&) = delete; + ClusterEngine(ClusterEngine&&) = delete; + ClusterEngine& operator=(const ClusterEngine&) = delete; + ClusterEngine& operator=(ClusterEngine&&) = delete; + + private: + void load(bool, const ProgressCallback&) override {} + void requestCancelLoad() override {} + void cancelLoad() {} + + TrackContainer getSimilarTracksFromTrackList(Database::TrackListId tracklistId, std::size_t maxCount) const override; + TrackContainer getSimilarTracks(const std::vector& tracksId, std::size_t maxCount) const override; + ReleaseContainer getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const override; + ArtistContainer getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const override; + + Database::Db& _db; + }; + +} // namespace Recommendation + diff --git a/src/libs/recommendation/impl/features/FeaturesEngine.cpp b/src/libs/recommendation/impl/features/FeaturesEngine.cpp index 03fb0db6..7b1f39b6 100644 --- a/src/libs/recommendation/impl/features/FeaturesEngine.cpp +++ b/src/libs/recommendation/impl/features/FeaturesEngine.cpp @@ -22,6 +22,7 @@ #include #include "database/Artist.hpp" +#include "database/Db.hpp" #include "database/Release.hpp" #include "database/Session.hpp" #include "database/Track.hpp" @@ -35,9 +36,9 @@ namespace Recommendation { -std::unique_ptr createFeaturesEngine() +std::unique_ptr createFeaturesEngine(Database::Db& db) { - return std::make_unique(); + return std::make_unique(db); } const FeatureSettingsMap& @@ -127,8 +128,8 @@ getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t return weights; } -bool -FeaturesEngine::loadFromTraining(Database::Session& session, const TrainSettings& trainSettings, const ProgressCallback& progressCallback) +void +FeaturesEngine::loadFromTraining(const TrainSettings& trainSettings, const ProgressCallback& progressCallback) { LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier..."; @@ -141,6 +142,8 @@ FeaturesEngine::loadFromTraining(Database::Session& session, const TrainSettings LMS_LOG(RECOMMENDATION, DEBUG) << "Features dimension = " << nbDimensions; + Database::Session& session {_db.getTLSSession()}; + std::vector trackIds; { auto transaction {session.createSharedTransaction()}; @@ -160,7 +163,7 @@ FeaturesEngine::loadFromTraining(Database::Session& session, const TrainSettings for (Database::TrackId trackId : trackIds) { if (_loadCancelled) - return false; + return; std::optional featureValuesMap; @@ -184,7 +187,7 @@ FeaturesEngine::loadFromTraining(Database::Session& session, const TrainSettings if (samples.empty()) { LMS_LOG(RECOMMENDATION, INFO) << "Nothing to classify!"; - return false; + return; } LMS_LOG(RECOMMENDATION, DEBUG) << "Normalizing data..."; @@ -219,15 +222,13 @@ FeaturesEngine::loadFromTraining(Database::Session& session, const TrainSettings [this] { return _loadCancelled; }); LMS_LOG(RECOMMENDATION, DEBUG) << "Training network DONE"; - if (_loadCancelled) - return false; LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks..."; TrackPositions trackPositions; for (std::size_t i {}; i < samples.size(); ++i) { if (_loadCancelled) - return false; + return; const SOM::Position position {network.getClosestRefVectorPosition(samples[i])}; @@ -236,23 +237,25 @@ FeaturesEngine::loadFromTraining(Database::Session& session, const TrainSettings LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks DONE"; - return load(session, std::move(network), std::move(trackPositions)); + load(std::move(network), std::move(trackPositions)); } -bool -FeaturesEngine::loadFromCache(Database::Session& session, const FeaturesEngineCache& cache) +void +FeaturesEngine::loadFromCache(FeaturesEngineCache cache) { LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier from cache..."; - return load(session, std::move(cache._network), cache._trackPositions); + load(std::move(cache._network), cache._trackPositions); } -IClassifier::ResultContainer -FeaturesEngine::getSimilarTracksFromTrackList(Database::Session& session, Database::TrackListId trackListId, std::size_t maxCount) const +IEngine::TrackContainer +FeaturesEngine::getSimilarTracksFromTrackList(Database::TrackListId trackListId, std::size_t maxCount) const { - const std::vector trackIds {[&] + const TrackContainer trackIds {[&] { - std::vector res; + TrackContainer res; + + Database::Session& session {_db.getTLSSession()}; auto transaction {session.createSharedTransaction()}; @@ -263,14 +266,16 @@ FeaturesEngine::getSimilarTracksFromTrackList(Database::Session& session, Databa return res; }()}; - return getSimilarTracks(session, trackIds, maxCount); + return getSimilarTracks(trackIds, maxCount); } -std::vector -FeaturesEngine::getSimilarTracks(Database::Session& session, const std::vector& tracksIds, std::size_t maxCount) const +IEngine::TrackContainer +FeaturesEngine::getSimilarTracks(const std::vector& tracksIds, std::size_t maxCount) const { auto similarTrackIds {getSimilarObjects(tracksIds, _trackMatrix, _trackPositions, maxCount)}; + Database::Session& session {_db.getTLSSession()}; + { // Report only existing ids, as tracks may have been removed a long time ago (refreshing the SOM takes some time) auto transaction {session.createSharedTransaction()}; @@ -278,18 +283,21 @@ FeaturesEngine::getSimilarTracks(Database::Session& session, const std::vector -FeaturesEngine::getSimilarReleases(Database::Session& session, Database::ReleaseId releaseId, std::size_t maxCount) const +IEngine::ReleaseContainer +FeaturesEngine::getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const { - auto similarReleaseIds {getSimilarObjects({releaseId}, _releaseMatrix, _releasePositions, maxCount)}; + auto similarReleaseIds {getSimilarObjects({releaseId}, _releaseMatrix, _releasePositions, maxCount)}; + Database::Session& session {_db.getTLSSession()}; + + if (!similarReleaseIds.empty()) { // Report only existing ids auto transaction {session.createSharedTransaction()}; @@ -297,7 +305,7 @@ FeaturesEngine::getSimilarReleases(Database::Session& session, Database::Release similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds), [&](Database::ReleaseId releaseId) { - return Database::Release::getById(session, releaseId); // TODO exists + return !Database::Release::exists(session, releaseId); }), std::end(similarReleaseIds)); } @@ -305,10 +313,7 @@ FeaturesEngine::getSimilarReleases(Database::Session& session, Database::Release } std::vector -FeaturesEngine::getSimilarArtists(Database::Session& session, - Database::ArtistId artistId, - EnumSet linkTypes, - std::size_t maxCount) const +FeaturesEngine::getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const { auto getSimilarArtistIdsForLinkType {[&] (Database::TrackArtistLinkType linkType) { @@ -333,15 +338,16 @@ FeaturesEngine::getSimilarArtists(Database::Session& session, std::vector res(std::cbegin(similarArtistIds), std::cend(similarArtistIds)); + Database::Session& session {_db.getTLSSession()}; { // Report only existing ids auto transaction {session.createSharedTransaction()}; res.erase(std::remove_if(std::begin(res), std::end(res), - [&](Database::ArtistId artistId) - { - return Database::Artist::getById(session, artistId); // TODO exists - }), std::end(res)); + [&](Database::ArtistId artistId) + { + return !Database::Artist::exists(session, artistId); + }), std::end(res)); } while (res.size() > maxCount) @@ -356,29 +362,25 @@ FeaturesEngine::toCache() const return FeaturesEngineCache {*_network, _trackPositions}; } -bool -FeaturesEngine::load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) +void +FeaturesEngine::load(bool forceReload, const ProgressCallback& progressCallback) { if (forceReload) - { FeaturesEngineCache::invalidate(); } - else + else if (const std::optional cache {FeaturesEngineCache::read()}) { - const std::optional cache {FeaturesEngineCache::read()}; - if (cache) - return loadFromCache(session, *cache); + loadFromCache(*cache); + return; } TrainSettings trainSettings; trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings(); - const bool res {loadFromTraining(session, trainSettings, progressCallback)}; - if (res) + loadFromTraining(trainSettings, progressCallback); + if (!_loadCancelled) toCache().write(); - - return res; } void @@ -388,10 +390,8 @@ FeaturesEngine::requestCancelLoad() _loadCancelled = true; } -bool -FeaturesEngine::load(Database::Session& session, - SOM::Network network, - const TrackPositions& trackPositions) +void +FeaturesEngine::load(const SOM::Network& network, const TrackPositions& trackPositions) { using namespace Database; @@ -406,10 +406,12 @@ FeaturesEngine::load(Database::Session& session, LMS_LOG(RECOMMENDATION, DEBUG) << "Constructing maps..."; + Database::Session& session {_db.getTLSSession()}; + for (const auto& [trackId, positions] : trackPositions) { if (_loadCancelled) - return false; + return; auto transaction {session.createSharedTransaction()}; @@ -445,11 +447,9 @@ FeaturesEngine::load(Database::Session& session, } } - _network = std::make_unique(std::move(network)); + _network = std::make_unique(network); LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully loaded!"; - - return true; } } // ns Recommendation diff --git a/src/libs/recommendation/impl/features/FeaturesEngine.hpp b/src/libs/recommendation/impl/features/FeaturesEngine.hpp index 9ed313a4..d6c1e81b 100644 --- a/src/libs/recommendation/impl/features/FeaturesEngine.hpp +++ b/src/libs/recommendation/impl/features/FeaturesEngine.hpp @@ -26,12 +26,12 @@ #include #include +#include "recommendation/IEngine.hpp" #include "som/DataNormalizer.hpp" #include "som/Network.hpp" #include "utils/Utils.hpp" #include "FeaturesEngineCache.hpp" #include "FeaturesDefs.hpp" -#include "IClassifier.hpp" namespace Database { @@ -42,10 +42,11 @@ namespace Recommendation { using FeatureWeight = double; -class FeaturesEngine : public IClassifier +class FeaturesEngine : public IEngine { public: - FeaturesEngine() = default; + FeaturesEngine(Database::Db& db) : _db {db} {} + FeaturesEngine(const FeaturesEngine&) = delete; FeaturesEngine(FeaturesEngine&&) = delete; FeaturesEngine& operator=(const FeaturesEngine&) = delete; @@ -59,21 +60,16 @@ class FeaturesEngine : public IClassifier static const FeatureSettingsMap& getDefaultTrainFeatureSettings(); private: - - std::string_view getName() const override { return "Features"; } - - bool load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) override; + void load(bool forceReload, const ProgressCallback& progressCallback) override; void requestCancelLoad() override; + void cancelLoad() override {} - ResultContainer getSimilarTracksFromTrackList(Database::Session& session, Database::TrackListId tracklistId, std::size_t maxCount) const override; - ResultContainer getSimilarTracks(Database::Session& session, const std::vector& tracksId, std::size_t maxCount) const override; - ResultContainer getSimilarReleases(Database::Session& session, Database::ReleaseId releaseId, std::size_t maxCount) const override; - ResultContainer getSimilarArtists(Database::Session& session, - Database::ArtistId artistId, - EnumSet linkTypes, - std::size_t maxCount) const override; + TrackContainer getSimilarTracksFromTrackList(Database::TrackListId tracklistId, std::size_t maxCount) const override; + TrackContainer getSimilarTracks(const std::vector& tracksId, std::size_t maxCount) const override; + ReleaseContainer getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const override; + ArtistContainer getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const override; - bool loadFromCache(Database::Session& session, const FeaturesEngineCache& cache); + void loadFromCache(FeaturesEngineCache cache); // Use training (may be very slow) struct TrainSettings @@ -82,7 +78,7 @@ class FeaturesEngine : public IClassifier float sampleCountPerNeuron {4}; FeatureSettingsMap featureSettingsMap; }; - bool loadFromTraining(Database::Session& session, const TrainSettings& trainSettings, const ProgressCallback& progressCallback); + void loadFromTraining(const TrainSettings& trainSettings, const ProgressCallback& progressCallback); template using ObjectPositions = std::unordered_map>; @@ -97,7 +93,7 @@ class FeaturesEngine : public IClassifier using ReleaseMatrix = ObjectMatrix; using TrackMatrix = ObjectMatrix; - bool load(Database::Session& session, SOM::Network network, const TrackPositions& tracksPosition); + void load(const SOM::Network& network, const TrackPositions& tracksPosition); FeaturesEngineCache toCache() const; @@ -113,6 +109,7 @@ class FeaturesEngine : public IClassifier const ObjectPositions& objectPositions, std::size_t maxCount) const; + Database::Db& _db; bool _loadCancelled {}; std::unique_ptr _network; double _networkRefVectorsDistanceMedian {}; diff --git a/src/libs/recommendation/include/recommendation/IEngine.hpp b/src/libs/recommendation/include/recommendation/IEngine.hpp index 6a15187b..26f180bc 100644 --- a/src/libs/recommendation/include/recommendation/IEngine.hpp +++ b/src/libs/recommendation/include/recommendation/IEngine.hpp @@ -21,13 +21,13 @@ #include #include +#include #include "database/Types.hpp" #include "utils/EnumSet.hpp" namespace Database { class Db; - class Session; } namespace Recommendation @@ -44,7 +44,8 @@ namespace Recommendation }; using ProgressCallback = std::function; virtual void load(bool forceReload, const ProgressCallback& progressCallback = {}) = 0; - virtual void cancelLoad() = 0; + virtual void cancelLoad() = 0; // wait for cancel done + virtual void requestCancelLoad() = 0; template using ResultContainer = std::vector; @@ -53,13 +54,10 @@ namespace Recommendation using ReleaseContainer = ResultContainer; using TrackContainer = ResultContainer; - virtual TrackContainer getSimilarTracksFromTrackList(Database::Session& session, Database::TrackListId tracklistId, std::size_t maxCount) = 0; - virtual TrackContainer getSimilarTracks(Database::Session& session, const std::vector& tracksId, std::size_t maxCount) = 0; - virtual ReleaseContainer getSimilarReleases(Database::Session& session, Database::ReleaseId releaseId, std::size_t maxCount) = 0; - virtual ArtistContainer getSimilarArtists(Database::Session& session, Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) = 0; - - protected: - virtual void requestCancelLoad() = 0; + virtual TrackContainer getSimilarTracksFromTrackList(Database::TrackListId tracklistId, std::size_t maxCount) const = 0; + virtual TrackContainer getSimilarTracks(const std::vector& tracksId, std::size_t maxCount) const = 0; + virtual ReleaseContainer getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const = 0; + virtual ArtistContainer getSimilarArtists(Database::ArtistId artistId, EnumSet linkTypes, std::size_t maxCount) const = 0; }; std::unique_ptr createEngine(Database::Db& db); diff --git a/src/libs/recommendation/include/recommendation/IRecommendation.hpp b/src/libs/recommendation/include/recommendation/IRecommendation.hpp deleted file mode 100644 index 834a983e..00000000 --- a/src/libs/recommendation/include/recommendation/IRecommendation.hpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2021 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include "database/Types.hpp" -#include "utils/EnumSet.hpp" - -namespace Database -{ - class Db; - class Session; -} - -namespace Recommendation -{ - class IRecommendation - { - }; - -} // ns Recommendation - diff --git a/src/libs/scrobbling/impl/listenbrainz/FeedbackSynchronizer.hpp b/src/libs/scrobbling/impl/listenbrainz/FeedbackSynchronizer.hpp new file mode 100644 index 00000000..0547921a --- /dev/null +++ b/src/libs/scrobbling/impl/listenbrainz/FeedbackSynchronizer.hpp @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2021 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +#include "SendQueue.hpp" + +namespace Database +{ + class Db; + class Session; + class User; +} + +namespace Scrobbling::ListenBrainz +{ + class ListensSynchronizer + { + public: + FeedbackSynchronizer(boost::asio::io_context& ioContext, Database::Db& db, SendQueue& sendQueue); + + // void updateFeedback(const TimedListen& listen); + + private: + struct UserContext + { + UserContext(Database::UserId id) : userId {id} {} + + UserContext(const UserContext&) = delete; + UserContext(UserContext&&) = delete; + UserContext& operator=(const UserContext&) = delete; + UserContext& operator=(UserContext&&) = delete; + + const Database::UserId userId; + bool fetching {}; + std::optional listenCount {}; + + // resetted at each fetch + std::string listenBrainzUserName; // need to be resolved first + Wt::WDateTime maxDateTime; + std::size_t fetchedListenCount{}; + std::size_t matchedListenCount{}; + std::size_t importedListenCount{}; + }; + + UserContext& getUserContext(Database::UserId userId); + bool isFetching() const; + void scheduleGetListens(std::chrono::seconds fromNow); + void startGetListens(); + void startGetListens(UserContext& context); + void onGetListensEnded(UserContext& context); + void enqueValidateToken(UserContext& context); + void enqueGetListenCount(UserContext& context); + void enqueGetListens(UserContext& context); + std::optional createValidateTokenRequestData(Database::UserId userId); + std::optional createGetListensRequestData(std::string_view listenBrainzUserName, const Wt::WDateTime& maxDateTime); + void processGetListensResponse(std::string_view body, UserContext& context); + + boost::asio::io_context& _ioContext; + boost::asio::io_context::strand _strand {_ioContext}; + Database::Db& _db; + SendQueue& _sendQueue; + boost::asio::steady_timer _getListensTimer {_ioContext}; + + std::unordered_map _userContexts; + + const std::size_t _maxSyncFeedbackCount; + const std::chrono::hours _syncFeedbackPeriod; + }; +} // Scrobbling::ListenBrainz + diff --git a/src/libs/subsonic/impl/SubsonicResource.cpp b/src/libs/subsonic/impl/SubsonicResource.cpp index 403e4ba4..c3c4ba04 100644 --- a/src/libs/subsonic/impl/SubsonicResource.cpp +++ b/src/libs/subsonic/impl/SubsonicResource.cpp @@ -896,10 +896,7 @@ handleGetArtistInfoRequestCommon(RequestContext& context, bool id3) artistInfoNode.createChild("musicBrainzId").setValue(artistMBID->getAsString()); } - auto similarArtistsId {Service::get()->getSimilarArtists(context.dbSession, - id, - {TrackArtistLinkType::Artist, TrackArtistLinkType::ReleaseArtist}, - count)}; + auto similarArtistsId {Service::get()->getSimilarArtists(id, {TrackArtistLinkType::Artist, TrackArtistLinkType::ReleaseArtist}, count)}; { auto transaction {context.dbSession.createSharedTransaction()}; @@ -1135,10 +1132,7 @@ handleGetSimilarSongsRequestCommon(RequestContext& context, bool id3) // Optional params std::size_t count {getParameterAs(context.parameters, "count").value_or(50)}; - const auto similarArtistIds {Service::get()->getSimilarArtists(context.dbSession, - artistId, - {TrackArtistLinkType::Artist, TrackArtistLinkType::ReleaseArtist}, - 5)}; + const auto similarArtistIds {Service::get()->getSimilarArtists(artistId, {TrackArtistLinkType::Artist, TrackArtistLinkType::ReleaseArtist}, 5)}; auto transaction {context.dbSession.createSharedTransaction()}; diff --git a/src/lms/ui/PlayQueue.cpp b/src/lms/ui/PlayQueue.cpp index f07501fe..554b34d2 100644 --- a/src/lms/ui/PlayQueue.cpp +++ b/src/lms/ui/PlayQueue.cpp @@ -520,7 +520,7 @@ PlayQueue::addEntry(const Database::TrackListEntry::pointer& tracklistEntry) void PlayQueue::enqueueRadioTracks() { - const auto similarTrackIds {Service::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 3)}; + const auto similarTrackIds {Service::get()->getSimilarTracksFromTrackList(_tracklistId, 3)}; std::vector trackToAddIds(std::cbegin(similarTrackIds), std::cend(similarTrackIds)); Random::shuffleContainer(trackToAddIds); diff --git a/src/lms/ui/explore/ArtistView.cpp b/src/lms/ui/explore/ArtistView.cpp index 0e803db5..eb6f1d41 100644 --- a/src/lms/ui/explore/ArtistView.cpp +++ b/src/lms/ui/explore/ArtistView.cpp @@ -102,10 +102,7 @@ Artist::refreshView() if (!artistId) throw ArtistNotFoundException {}; - const auto similarArtistIds {Service::get()->getSimilarArtists(LmsApp->getDbSession(), - *artistId, - {TrackArtistLinkType::Artist, TrackArtistLinkType::ReleaseArtist}, - 5)}; + const auto similarArtistIds {Service::get()->getSimilarArtists(*artistId, {TrackArtistLinkType::Artist, TrackArtistLinkType::ReleaseArtist}, 5)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; diff --git a/src/lms/ui/explore/ReleaseView.cpp b/src/lms/ui/explore/ReleaseView.cpp index ceb82d90..15334246 100644 --- a/src/lms/ui/explore/ReleaseView.cpp +++ b/src/lms/ui/explore/ReleaseView.cpp @@ -101,7 +101,7 @@ Release::refreshView() if (!releaseId) throw ReleaseNotFoundException {}; - auto similarReleasesIds {Service::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 6)}; + auto similarReleasesIds {Service::get()->getSimilarReleases(*releaseId, 6)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; diff --git a/src/tools/recommendation/LmsRecommendation.cpp b/src/tools/recommendation/LmsRecommendation.cpp index 0a1f4a3c..c88dcc36 100644 --- a/src/tools/recommendation/LmsRecommendation.cpp +++ b/src/tools/recommendation/LmsRecommendation.cpp @@ -66,7 +66,7 @@ dumpTracksRecommendation(Database::Session session, Recommendation::IEngine& eng }; std::cout << "Processing track '" << trackToString(trackId) << std::endl; - for (Database::TrackId similarTrackId : engine.getSimilarTracks(session, {trackId}, maxSimilarityCount)) + for (Database::TrackId similarTrackId : engine.getSimilarTracks({trackId}, maxSimilarityCount)) std::cout << "\t- Similar track '" << trackToString(similarTrackId) << std::endl; } } @@ -84,7 +84,7 @@ dumpReleasesRecommendation(Database::Session session, Recommendation::IEngine& e std::cout << "*** Releases ***" << std::endl; for (Database::ReleaseId releaseId : releaseIds) { - auto releaseToString = [&](Database::ReleaseId releaseId) + auto releaseToString = [&](Database::ReleaseId releaseId) -> std::string { auto transaction {session.createSharedTransaction()}; @@ -93,7 +93,7 @@ dumpReleasesRecommendation(Database::Session session, Recommendation::IEngine& e }; std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl; - for (Database::ReleaseId similarReleaseId : engine.getSimilarReleases(session, releaseId, maxSimilarityCount)) + for (Database::ReleaseId similarReleaseId : engine.getSimilarReleases(releaseId, maxSimilarityCount)) std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl; } } @@ -120,7 +120,7 @@ dumpArtistsRecommendation(Database::Session session, Recommendation::IEngine& en }; std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl; - for (Database::ArtistId similarArtistId : engine.getSimilarArtists(session, artistId, {Database::TrackArtistLinkType::Artist, Database::TrackArtistLinkType::ReleaseArtist}, maxSimilarityCount)) + for (Database::ArtistId similarArtistId : engine.getSimilarArtists(artistId, {Database::TrackArtistLinkType::Artist, Database::TrackArtistLinkType::ReleaseArtist}, maxSimilarityCount)) { std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl; }