owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made the recommendation engine more usable during initial scan/rescans

Browse Source
This commit is contained in:
emeric
2020-02-21 15:03:42 +01:00
parent 0ad4973a5a
commit d02a7ca86b
8 changed files with 177 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/recommendation/impl/Engine.cpp
+93 -42
View File
@@ -46,7 +46,7 @@ Engine::start()
assert(!_running);
_running = true;
requestReload();
requestReloadInternal(false);
_ioService.start();
}
@@ -64,12 +64,18 @@ Engine::stop()
void
Engine::requestReload()
{
requestReloadInternal(true);
}
void
Engine::requestReloadInternal(bool databaseChanged)
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Reload requested...";
_ioService.post([&]()
_ioService.post([=]()
{
reload();
reload(databaseChanged);
});
}
@@ -80,9 +86,13 @@ Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdTy
std::vector<Database::IdType> res;
for (const auto& [priority, classifier] : _classifiers)
for (const auto& classifierName : _classifierPriorities)
{
res = classifier->getSimilarTracksFromTrackList(session, trackListId, maxCount);
auto itClassifier {_classifiers.find(classifierName)};
if (itClassifier == std::cend(_classifiers))
continue;
res = itClassifier->second->getSimilarTracksFromTrackList(session, trackListId, maxCount);
if (!res.empty())
break;
}
@@ -97,11 +107,18 @@ Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<
std::vector<Database::IdType> res;
for (const auto& [priority, classifier] : _classifiers)
for (const auto& classifierName : _classifierPriorities)
{
res = classifier->getSimilarTracks(dbSession, trackIds, maxCount);
auto itClassifier {_classifiers.find(classifierName)};
if (itClassifier == std::cend(_classifiers))
continue;
res = itClassifier->second->getSimilarTracks(dbSession, trackIds, maxCount);
if (!res.empty())
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifierName << "'";
break;
}
}
return res;
@@ -114,11 +131,18 @@ Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releas
std::vector<Database::IdType> res;
for (const auto& [priority, classifier] : _classifiers)
for (const auto& classifierName : _classifierPriorities)
{
res = classifier->getSimilarReleases(dbSession, releaseId, maxCount);
auto itClassifier {_classifiers.find(classifierName)};
if (itClassifier == std::cend(_classifiers))
continue;
res = itClassifier->second->getSimilarReleases(dbSession, releaseId, maxCount);
if (!res.empty())
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifierName << "'";
break;
}
}
return res;
@@ -131,18 +155,25 @@ Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistI
std::vector<Database::IdType> res;
for (const auto& [priority, classifier] : _classifiers)
for (const auto& classifierName : _classifierPriorities)
{
res = classifier->getSimilarArtists(dbSession, artistId, maxCount);
auto itClassifier {_classifiers.find(classifierName)};
if (itClassifier == std::cend(_classifiers))
continue;
res = itClassifier->second->getSimilarArtists(dbSession, artistId, maxCount);
if (!res.empty())
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifierName << "'";
return res;
}
}
return res;
}
void
Engine::reload()
Engine::reload(bool databaseChanged)
{
using namespace Database;
@@ -155,43 +186,29 @@ Engine::reload()
return ScanSettings::get(_dbSession)->getRecommendationEngineType();
}()};
std::map<ClassifierPriority, std::unique_ptr<IClassifier>> newClassifiers;
// TODO RAII this
auto addClassifier = [&](ClassifierPriority prio, std::unique_ptr<IClassifier> classifier)
{
try
{
addPendingClassifier(*classifier.get());
bool res {classifier->init(_dbSession)};
removePendingClassifier(*classifier.get());
if (res)
newClassifiers.emplace(prio, std::move(classifier));
return res;
}
catch (LmsException& e)
{
removePendingClassifier(*classifier.get());
throw;
}
};
clearClassifiers();
switch (engineType)
{
case ScanSettings::RecommendationEngineType::Features:
addClassifier(0, createFeaturesClassifier()); // higher priority
[[fallthrough]];
{
auto clustersClassifier {createClustersClassifier()};
auto featuresClassifier {createFeaturesClassifier()};
setClassifierPriorities({featuresClassifier->getName(), clustersClassifier->getName()});
initAndAddClassifier(std::move(clustersClassifier), databaseChanged); // init first since faster
initAndAddClassifier(std::move(featuresClassifier), databaseChanged);
break;
}
case ScanSettings::RecommendationEngineType::Clusters:
addClassifier(1, createClustersClassifier()); // lower priority
break;
}
auto clustersClassifier {createClustersClassifier()};
{
std::unique_lock lock {_classifiersMutex};
_classifiers.swap(newClassifiers);
setClassifierPriorities({clustersClassifier->getName()});
initAndAddClassifier(std::move(clustersClassifier), databaseChanged);
break;
}
LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines reloaded!";
@@ -199,6 +216,40 @@ Engine::reload()
_sigReloaded.emit();
}
void
Engine::setClassifierPriorities(std::initializer_list<std::string_view> classifierPriorities)
{
std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
_classifierPriorities.clear();
std::transform(std::cbegin(classifierPriorities), std::cend(classifierPriorities), std::back_inserter(_classifierPriorities), [](std::string_view name) { return std::string {name}; });
}
void
Engine::clearClassifiers()
{
std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
_classifiers.clear();
}
void
Engine::initAndAddClassifier(std::unique_ptr<IClassifier> classifier, bool databaseChanged)
{
PendingClassifierHandler pendingClassifier {*this, *classifier.get()};
LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'...";
bool res {classifier->init(_dbSession, databaseChanged)};
LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE");
if (res)
{
std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
_classifiers.emplace(classifier->getName(), std::move(classifier));
}
}
void
Engine::cancelPendingClassifiers()
{
src/libs/recommendation/impl/Engine.hpp
+28 -4
View File
@@ -21,6 +21,7 @@
#include <map>
#include <shared_mutex>
#include <vector>
#include <Wt/WIOService.h>
@@ -37,8 +38,6 @@ namespace Recommendation
private:
using ClassifierPriority = unsigned;
void start() override;
void stop() override;
@@ -51,7 +50,31 @@ namespace Recommendation
std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override;
void reload();
void requestReloadInternal(bool databaseChanged);
void reload(bool databaseChanged);
void setClassifierPriorities(std::initializer_list<std::string_view> classifierNames);
void clearClassifiers();
void initAndAddClassifier(std::unique_ptr<IClassifier> classifier, bool databaseChanged);
class PendingClassifierHandler
{
public:
PendingClassifierHandler(Engine& engine, IClassifier& classifier) : _engine {engine}, _classifier {classifier}
{
_engine.addPendingClassifier(_classifier);
}
~PendingClassifierHandler()
{
_engine.removePendingClassifier(_classifier);
}
private:
Engine& _engine;
IClassifier& _classifier;
};
void cancelPendingClassifiers();
void addPendingClassifier(IClassifier& classifier);
void removePendingClassifier(IClassifier& classifier);
@@ -62,7 +85,8 @@ namespace Recommendation
Wt::Signal<> _sigReloaded;
std::shared_mutex _classifiersMutex;
std::map<ClassifierPriority, std::unique_ptr<IClassifier>> _classifiers;
std::map<std::string, std::unique_ptr<IClassifier>> _classifiers;
std::vector<std::string> _classifierPriorities; // ordered by priority
std::unordered_set<IClassifier*> _pendingClassifiers;
};
src/libs/recommendation/impl/clusters/ClustersClassifier.hpp
+1 -1
View File
@@ -38,7 +38,7 @@ namespace Recommendation
std::string_view getName() const { return "Clusters"; }
bool init(Database::Session&) override {return true;}
bool init(Database::Session&, bool) override {return true;}
void requestCancelInit() override {}
std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
src/libs/recommendation/impl/features/FeaturesClassifier.cpp
+49 -7
View File
@@ -259,21 +259,57 @@ FeaturesClassifier::getSimilarTracksFromTrackList(Database::Session& session, Da
}
std::vector<Database::IdType>
FeaturesClassifier::getSimilarTracks(Database::Session&, const std::unordered_set<Database::IdType>& tracksIds, std::size_t maxCount) const
FeaturesClassifier::getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksIds, std::size_t maxCount) const
{
return getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount);
std::vector<Database::IdType> similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)};
if (!similarTrackIds.empty())
{
// Report only existing ids
auto transaction {session.createSharedTransaction()};
similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds),
[&](Database::IdType trackId) { return Database::Track::getById(session, trackId) == Database::Track::pointer {}; }),
std::cend(similarTrackIds));
}
return similarTrackIds;
}
std::vector<Database::IdType>
FeaturesClassifier::getSimilarReleases(Database::Session&, Database::IdType releaseId, std::size_t maxCount) const
FeaturesClassifier::getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const
{
return getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount);
std::vector<Database::IdType> similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)};
if (!similarReleaseIds.empty())
{
// Report only existing ids
auto transaction {session.createSharedTransaction()};
similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds),
[&](Database::IdType releaseId) { return Database::Release::getById(session, releaseId) == Database::Release::pointer {}; }),
std::cend(similarReleaseIds));
}
return similarReleaseIds;
}
std::vector<Database::IdType>
FeaturesClassifier::getSimilarArtists(Database::Session&, Database::IdType artistId, std::size_t maxCount) const
FeaturesClassifier::getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const
{
return getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount);
std::vector<Database::IdType> similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)};
if (!similarArtistIds.empty())
{
// Report only existing ids
auto transaction {session.createSharedTransaction()};
similarArtistIds.erase(std::remove_if(std::begin(similarArtistIds), std::end(similarArtistIds),
[&](Database::IdType artistId) { return Database::Artist::getById(session, artistId) == Database::Artist::pointer {}; }),
std::cend(similarArtistIds));
}
return similarArtistIds;
}
FeaturesClassifierCache
@@ -283,8 +319,14 @@ FeaturesClassifier::toCache() const
}
bool
FeaturesClassifier::init(Database::Session& session)
FeaturesClassifier::init(Database::Session& session, bool databaseChanged)
{
if (databaseChanged)
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Database changed: invidating cache";
FeaturesClassifierCache::invalidate();
}
std::optional<FeaturesClassifierCache> cache {FeaturesClassifierCache::read()};
if (cache)
return initFromCache(session, *cache);
src/libs/recommendation/impl/features/FeaturesClassifier.hpp
+1 -1
View File
@@ -57,7 +57,7 @@ class FeaturesClassifier : public IClassifier
std::string_view getName() const { return "Features"; }
bool init(Database::Session& session) override;
bool init(Database::Session& session, bool databaseChanged) override;
void requestCancelInit() override;
std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
src/libs/recommendation/include/recommendation/IClassifier.hpp
+2 -2
View File
@@ -19,7 +19,7 @@
#pragma once
#include <functional>
#include <string_view>
#include <unordered_set>
#include <vector>
@@ -40,7 +40,7 @@ namespace Recommendation
virtual std::string_view getName() const = 0;
virtual bool init(Database::Session& session) = 0;
virtual bool init(Database::Session& session, bool databaseChanged) = 0;
virtual void requestCancelInit() = 0;
virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0;
src/libs/scanner/impl/MediaScanner.cpp
+3 -3
View File
@@ -360,9 +360,7 @@ MediaScanner::countAllFiles(ScanStats& stats)
if (!ec && isFileSupported(path, _fileExtensions))
{
stats.filesToScan++;
if (stats.filesToScan % 250 == 0)
notifyInProgressIfNeeded(stats);
notifyInProgressIfNeeded(stats);
}
return true;
@@ -822,6 +820,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
stats.deletions++;
}
}
notifyInProgressIfNeeded(stats);
}
}