Loading of the recommendation engine now controlled by scanner. Bonus: better control/reporting

approot/messages.xml

@@ -59,7 +59,7 @@
 <message id="Lms.Admin.ScannerController.cannot-read-file">Cannot read file</message>
 <message id="Lms.Admin.ScannerController.duplicates-header">{1} duplicate files:</message>
 <message id="Lms.Admin.ScannerController.errors-header">{1} errors:</message>
-<message id="Lms.Admin.ScannerController.force-scan-now">Force scan now</message>
+<message id="Lms.Admin.ScannerController.force-scan-now">Force full rescan now</message>
 <message id="Lms.Admin.ScannerController.get-report">Get report</message>
 <message id="Lms.Admin.ScannerController.last-scan">Last scan</message>
 <message id="Lms.Admin.ScannerController.last-scan-not-available">Not available</message>
@@ -73,9 +73,10 @@
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Not scheduled</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Scheduled on {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">Scanning: step {1}/{2}</message>
-<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Checking for missing files... {1}%</message>
+<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Checking files... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-discovering-files">Discovering files: {1} files</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Reloading similarity engine: {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-scanning-files">Scanning files: {1}/{2} files ({3}%)...</message>
 
 <!--Users-->

approot/messages_fr.xml

@@ -59,7 +59,7 @@
 <message id="Lms.Admin.ScannerController.cannot-read-file">Impossible de lire le fichier</message>
 <message id="Lms.Admin.ScannerController.duplicates-header">{1} fichiers dupliqués :</message>
 <message id="Lms.Admin.ScannerController.errors-header">{1} erreurs :</message>
-<message id="Lms.Admin.ScannerController.force-scan-now">Lancer un scan forcé</message>
+<message id="Lms.Admin.ScannerController.force-scan-now">Forcer un rescan complet</message>
 <message id="Lms.Admin.ScannerController.get-report">Rapport</message>
 <message id="Lms.Admin.ScannerController.last-scan">Dernier scan</message>
 <message id="Lms.Admin.ScannerController.last-scan-not-available">Non disponible</message>
@@ -73,10 +73,11 @@
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Non planifié</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Planifié le {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">En cours de scan : étape {1}/{2}</message>
-<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Vérification des fichiers supprimés... {1}%</message>
+<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Vérification des fichiers... {1}%</message>
-<message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers: {1} fichiers</message>
+<message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers : {1} fichiers</message>
-<message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz: {1}/{2} fichiers ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)...</message>
-<message id="Lms.Admin.ScannerController.step-scanning-files">Scan des fichiers: {1}/{2} fichiers ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Rechargement du moteur de recommandation : {1}%...</message>
+<message id="Lms.Admin.ScannerController.step-scanning-files">Scan des fichiers : {1}/{2} fichiers ({3}%)...</message>
 
 <!--Users-->
 <message id="Lms.Admin.Users.add">Ajouter</message>

src/libs/recommendation/CMakeLists.txt

@@ -22,5 +22,7 @@ target_link_libraries(lmsrecommendation PRIVATE
 	wt
 	)
 
+target_include_directories(lmsrecommendation PRIVATE impl)
+
 install(TARGETS lmsrecommendation DESTINATION lib)
 

src/libs/recommendation/impl/Engine.cpp

@@ -19,16 +19,39 @@
 
 #include "Engine.hpp"
 
-#include "recommendation/ClustersClassifierCreator.hpp"
+#include <unordered_map>
-#include "recommendation/FeaturesClassifierCreator.hpp"
+#include <vector>
 
+#include "ClustersClassifierCreator.hpp"
+#include "FeaturesClassifierCreator.hpp"
+
+#include "database/Db.hpp"
+#include "database/Session.hpp"
 #include "database/ScanSettings.hpp"
-#include "database/TrackList.hpp"
 #include "utils/Exception.hpp"
 #include "utils/Logger.hpp"
 
 namespace Recommendation {
 
+
+static
+std::unique_ptr<IClassifier>
+createClassifier(ClassifierType type)
+{
+	switch (type)
+	{
+		case ClassifierType::Clusters:
+			return createClustersClassifier();
+			break;
+
+		case ClassifierType::Features:
+			return createFeaturesClassifier();
+			break;
+	}
+
+	return {};
+}
+
 std::unique_ptr<IEngine>
 createEngine(Database::Db& db)
 {
@@ -36,65 +59,16 @@ createEngine(Database::Db& db)
 }
 
 Engine::Engine(Database::Db& db)
-: _dbSession {db}
+: _db {db}
 {
-	start();
 }
 
-Engine::~Engine()
+std::unordered_set<Database::IdType>
-{
-	stop();
-}
-
-void
-Engine::start()
-{
-	assert(!_running);
-	_running = true;
-	_ioService.start();
-}
-
-void
-Engine::stop()
-{
-	assert(_running);
-	_running = false;
-
-	cancelPendingClassifiers();
-
-	_ioService.stop();
-}
-
-void
-Engine::requestLoad()
-{
-	requestReloadInternal(false);
-}
-
-void
-Engine::requestReload()
-{
-	requestReloadInternal(true);
-}
-
-void
-Engine::requestReloadInternal(bool databaseChanged)
-{
-	LMS_LOG(RECOMMENDATION, DEBUG) << "Reload requested...";
-
-	_ioService.post([=]()
-	{
-		reload(databaseChanged);
-	});
-}
-
-std::vector<Database::IdType>
 Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount)
 {
+	std::unordered_set<Database::IdType> res;
+
 	std::shared_lock lock {_classifiersMutex};
-
-	std::vector<Database::IdType> res;
-
 	for (const auto& classifierName : _classifierPriorities)
 	{
 		auto itClassifier {_classifiers.find(classifierName)};
@@ -109,23 +83,23 @@ Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdTy
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount)
 {
+	std::unordered_set<Database::IdType> res;
+
 	std::shared_lock lock {_classifiersMutex};
-
+	for (ClassifierType classifierType : _classifierPriorities)
-	std::vector<Database::IdType> res;
-
-	for (const auto& classifierName : _classifierPriorities)
 	{
-		auto itClassifier {_classifiers.find(classifierName)};
+		auto itClassifier {_classifiers.find(classifierType)};
 		if (itClassifier == std::cend(_classifiers))
 			continue;
 
-		res = itClassifier->second->getSimilarTracks(dbSession, trackIds, maxCount);
+		const IClassifier& classifier {*itClassifier->second};
+		res = classifier.getSimilarTracks(dbSession, trackIds, maxCount);
 		if (!res.empty())
 		{
-			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifierName << "'";
+			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifier.getName() << "'";
 			break;
 		}
 	}
@@ -133,23 +107,23 @@ Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount)
 {
+	std::unordered_set<Database::IdType> res;
+
 	std::shared_lock lock {_classifiersMutex};
-
+	for (ClassifierType classifierType : _classifierPriorities)
-	std::vector<Database::IdType> res;
-
-	for (const auto& classifierName : _classifierPriorities)
 	{
-		auto itClassifier {_classifiers.find(classifierName)};
+		auto itClassifier {_classifiers.find(classifierType)};
 		if (itClassifier == std::cend(_classifiers))
 			continue;
 
-		res = itClassifier->second->getSimilarReleases(dbSession, releaseId, maxCount);
+		const IClassifier& classifier {*itClassifier->second};
+		res = classifier.getSimilarReleases(dbSession, releaseId, maxCount);
 		if (!res.empty())
 		{
-			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifierName << "'";
+			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifier.getName() << "'";
 			break;
 		}
 	}
@@ -157,23 +131,23 @@ Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releas
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount)
 {
+	std::unordered_set<Database::IdType> res;
+
 	std::shared_lock lock {_classifiersMutex};
-
+	for (ClassifierType classifierType : _classifierPriorities)
-	std::vector<Database::IdType> res;
-
-	for (const auto& classifierName : _classifierPriorities)
 	{
-		auto itClassifier {_classifiers.find(classifierName)};
+		auto itClassifier {_classifiers.find(classifierType)};
 		if (itClassifier == std::cend(_classifiers))
 			continue;
 
-		res = itClassifier->second->getSimilarArtists(dbSession, artistId, maxCount);
+		const IClassifier& classifier {*itClassifier->second};
+		res = classifier.getSimilarArtists(dbSession, artistId, maxCount);
 		if (!res.empty())
 		{
-			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifierName << "'";
+			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifier.getName() << "'";
 			return res;
 		}
 	}
@@ -181,107 +155,132 @@ Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistI
 	return res;
 }
 
-void
+static
-Engine::reload(bool databaseChanged)
+Database::ScanSettings::RecommendationEngineType
+getRecommendationEngineType(Database::Session& session)
 {
-	using namespace Database;
+	auto transaction {session.createSharedTransaction()};
 
-	LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines...";
+	return Database::ScanSettings::get(session)->getRecommendationEngineType();
-
-	const ScanSettings::RecommendationEngineType engineType {[&]()
-	{
-		auto transaction {_dbSession.createSharedTransaction()};
-
-		return ScanSettings::get(_dbSession)->getRecommendationEngineType();
-	}()};
-
-	clearClassifiers();
-
-	switch (engineType)
-	{
-		case ScanSettings::RecommendationEngineType::Features:
-		{
-			auto clustersClassifier {createClustersClassifier()};
-			auto featuresClassifier {createFeaturesClassifier()};
-
-			setClassifierPriorities({featuresClassifier->getName(), clustersClassifier->getName()});
-
-			initAndAddClassifier(std::move(clustersClassifier), databaseChanged); // init first since faster
-			initAndAddClassifier(std::move(featuresClassifier), databaseChanged);
-			break;
-		}
-
-		case ScanSettings::RecommendationEngineType::Clusters:
-			auto clustersClassifier {createClustersClassifier()};
-
-			setClassifierPriorities({clustersClassifier->getName()});
-
-			initAndAddClassifier(std::move(clustersClassifier), databaseChanged);
-			break;
-	}
-
-	LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines reloaded!";
-
-	_sigReloaded.emit();
 }
 
 void
-Engine::setClassifierPriorities(std::initializer_list<std::string_view> classifierPriorities)
+Engine::load(bool forceReload, const ProgressCallback& progressCallback)
+{
+	using namespace Database;
+
+	static const std::unordered_map<ScanSettings::RecommendationEngineType, std::vector<ClassifierType>> classifierMappings
+	{
+		{ScanSettings::RecommendationEngineType::Features, {ClassifierType::Clusters, ClassifierType::Features}},
+		{ScanSettings::RecommendationEngineType::Clusters, {ClassifierType::Clusters}},
+	};
+
+	LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines...";
+
+	const ScanSettings::RecommendationEngineType engineType {getRecommendationEngineType(_db.getTLSSession())};
+
+	assert(_pendingClassifiers.empty());
+	clearClassifiers();
+
+	auto itClassifierTypes {classifierMappings.find(engineType)};
+	assert(itClassifierTypes != std::cend(classifierMappings));
+	const std::vector<ClassifierType>& classifierTypes {itClassifierTypes->second};
+
+	setClassifierPriorities(classifierTypes);
+
+	std::vector<std::unique_ptr<IClassifier>> classifiers;
+	for (ClassifierType type : classifierTypes)
+		classifiers.emplace_back(createClassifier(type));
+
+	{
+		std::scoped_lock lock {_controlMutex};
+
+		std::transform(std::cbegin(classifiers), std::cend(classifiers), std::inserter(_pendingClassifiers, std::end(_pendingClassifiers)),
+				[](auto& classifier) { return classifier.get(); });
+	}
+
+	for (std::size_t i {}; i < classifiers.size(); ++i)
+		loadClassifier(std::move(classifiers[i]), classifierTypes[i], forceReload, progressCallback);
+
+	LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines loaded!";
+}
+
+void
+Engine::setClassifierPriorities(const std::vector<ClassifierType>& classifierPriorities)
 {
 	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
 
-	_classifierPriorities.clear();
+	_classifierPriorities = classifierPriorities;
-	std::transform(std::cbegin(classifierPriorities), std::cend(classifierPriorities), std::back_inserter(_classifierPriorities), [](std::string_view name) { return std::string {name}; });
 }
 
 void
 Engine::clearClassifiers()
 {
-	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
+	std::unique_lock lock {_classifiersMutex};
 
 	_classifiers.clear();
 }
 
 void
-Engine::initAndAddClassifier(std::unique_ptr<IClassifier> classifier, bool databaseChanged)
+Engine::loadClassifier(std::unique_ptr<IClassifier> classifier,
+		ClassifierType classifierType,
+		bool forceReload,
+		const ProgressCallback& progressCallback)
 {
-	PendingClassifierHandler pendingClassifier {*this, *classifier.get()};
+	IClassifier* rawClassifier {classifier.get()};
 
-	LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'...";
+	bool res {};
-	bool res {classifier->init(_dbSession, databaseChanged)};
+	if (!_loadCancelled)
-	LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE");
+	{
+		LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'...";
+
+		auto progress {[&](IClassifier::Progress progress)
+		{
+			progressCallback(Progress {progress.processedElems, progress.totalElems});
+		}};
+
+		res = classifier->load(_db.getTLSSession(), forceReload, progressCallback ? progress : IClassifier::ProgressCallback {});
+
+		LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE");
+	}
 
 	if (res)
 	{
-		std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
+		std::unique_lock lock {_classifiersMutex};
 
-		_classifiers.emplace(classifier->getName(), std::move(classifier));
+		_classifiers.emplace(classifierType, std::move(classifier));
 	}
+
+	{
+		std::scoped_lock lock {_controlMutex};
+
+		LMS_LOG(RECOMMENDATION, DEBUG) << "About to erase. _pendingClassifiers size =  " << _pendingClassifiers.size();
+		_pendingClassifiers.erase(rawClassifier);
+		LMS_LOG(RECOMMENDATION, DEBUG) << "Erased. _pendingClassifiers size =  " << _pendingClassifiers.size();
+	}
+
+	_pendingClassifiersCondvar.notify_one();
+
 }
 
 void
-Engine::cancelPendingClassifiers()
+Engine::cancelLoad()
 {
-	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
+	LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading...";
+
+	std::unique_lock lock {_controlMutex};
+
+	LMS_LOG(RECOMMENDATION, DEBUG) << "Still " <<  _pendingClassifiers.size() << " pending classifiers!";
+
+	_loadCancelled = true;
 
 	for (IClassifier* classifier : _pendingClassifiers)
-		classifier->requestCancelInit();
+		classifier->requestCancelLoad();
-}
 
-void
+    _pendingClassifiersCondvar.wait(lock, [this] {return _pendingClassifiers.empty();});
-Engine::addPendingClassifier(IClassifier& classifier)
+	_loadCancelled = false;
-{
-	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
 
-	_pendingClassifiers.insert(&classifier);
+	LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading DONE";
-}
-
-void
-Engine::removePendingClassifier(IClassifier& classifier)
-{
-	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
-
-	_pendingClassifiers.erase(&classifier);
 }
 
 } // ns Similarity

src/libs/recommendation/impl/Engine.hpp

@@ -19,23 +19,33 @@
 
 #pragma once
 
-#include <map>
+#include <condition_variable>
 #include <shared_mutex>
+#include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
-#include <Wt/WIOService.h>
-
-#include "database/Session.hpp"
 #include "recommendation/IEngine.hpp"
-#include "recommendation/IClassifier.hpp"
+#include "IClassifier.hpp"
+
+namespace Database
+{
+	class Db;
+}
 
 namespace Recommendation
 {
+	enum class ClassifierType
+	{
+		Clusters,
+		Features,
+	};
+
 	class Engine : public IEngine
 	{
 		public:
 			Engine(Database::Db& db);
-			~Engine();
+			~Engine() = default;
 
 			Engine(const Engine&) = delete;
 			Engine(Engine&&) = delete;
@@ -43,58 +53,30 @@ namespace Recommendation
 			Engine& operator=(Engine&&) = delete;
 
 		private:
+			void	load(bool forceReload, const ProgressCallback& progressCallback) override;
+			void	cancelLoad() override;
 
-			void start();
+			std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override;
-			void stop();
+			std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) override;
+			std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override;
+			std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override;
 
-			void requestLoad() override;
+			void setClassifierPriorities(const std::vector<ClassifierType>& classifierTypes);
-			void requestReload() override;
-			Wt::Signal<>& reloaded() override { return _sigReloaded; }
-
-			std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override;
-			std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) override;
-			std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override;
-			std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override;
-
-
-			void requestReloadInternal(bool databaseChanged);
-			void reload(bool databaseChanged);
-
-			void setClassifierPriorities(std::initializer_list<std::string_view> classifierNames);
 			void clearClassifiers();
-			void initAndAddClassifier(std::unique_ptr<IClassifier> classifier, bool databaseChanged);
+			void loadClassifier(std::unique_ptr<IClassifier> classifier, ClassifierType classifierType, bool forceReload, const ProgressCallback& progressCallback);
 
-			class PendingClassifierHandler
+			Database::Db&				_db;
-			{
-				public:
-					PendingClassifierHandler(Engine& engine, IClassifier& classifier) : _engine {engine}, _classifier {classifier}
-					{
-						_engine.addPendingClassifier(_classifier);
-					}
 
-					~PendingClassifierHandler()
+			std::mutex							_controlMutex;
-					{
+			bool								_loadCancelled {};
-						_engine.removePendingClassifier(_classifier);
+			std::condition_variable 			_pendingClassifiersCondvar;
-					}
+			std::unordered_set<IClassifier*>	_pendingClassifiers;
 
-				private:
+			std::shared_mutex			_classifiersMutex;
-					Engine& _engine;
+			using ClassifierContainer = std::unordered_map<ClassifierType, std::unique_ptr<IClassifier>>;
-					IClassifier& _classifier;
+			ClassifierContainer			_classifiers;
-			};
+			std::vector<ClassifierType>	_classifierPriorities; // ordered by priority
 
-			void cancelPendingClassifiers();
-			void addPendingClassifier(IClassifier& classifier);
-			void removePendingClassifier(IClassifier& classifier);
-
-			bool			_running {};
-			Wt::WIOService		_ioService;
-			Database::Session	_dbSession;
-			Wt::Signal<>		_sigReloaded;
-
-			std::shared_mutex	_classifiersMutex;
-			std::map<std::string, std::unique_ptr<IClassifier>> _classifiers;
-			std::vector<std::string> _classifierPriorities; // ordered by priority
-			std::unordered_set<IClassifier*> _pendingClassifiers;
 	};
 
 } // ns Recommendation

src/libs/recommendation/impl/FeaturesClassifierCreator.hpp

@@ -20,7 +20,7 @@
 #pragma once
 
 #include <memory>
-#include "recommendation/IClassifier.hpp"
+#include "IClassifier.hpp"
 
 namespace Recommendation
 {

src/libs/recommendation/impl/IClassifier.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
+#include <functional>
 #include <string_view>
 #include <unordered_set>
-#include <vector>
 
 #include "database/Types.hpp"
 
@@ -40,13 +40,19 @@ namespace Recommendation
 
 			virtual std::string_view getName() const = 0;
 
-			virtual bool init(Database::Session& session, bool databaseChanged) = 0;
+			struct Progress
-			virtual void requestCancelInit() = 0;
+			{
+				std::size_t totalElems {};
+				std::size_t	processedElems {};
+			};
+			using ProgressCallback = std::function<void(const Progress&)>;
+			virtual bool load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) = 0;
+			virtual void requestCancelLoad() = 0;
 
-			virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0;
-			virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const = 0;
-			virtual std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0;
-			virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0;
 	};
 
 } // ns Recommendation

src/libs/recommendation/impl/clusters/ClustersClassifier.cpp

@@ -27,29 +27,29 @@
 #include "database/TrackList.hpp"
 
 namespace Recommendation {
-	
+
 std::unique_ptr<IClassifier> createClustersClassifier()
 {
 	return std::make_unique<ClusterClassifier>();
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount) const
 {
 	auto transaction {dbSession.createSharedTransaction()};
 
-	auto tracks {Database::Track::getSimilarTracks(dbSession, trackIds, 0, maxCount)};
+	const auto tracks {Database::Track::getSimilarTracks(dbSession, trackIds, 0, maxCount)};
-	std::vector<Database::IdType> res;
-	res.reserve(tracks.size());
 
-	std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res), [](const auto& track) { return track.id(); });
+	std::unordered_set<Database::IdType> res;
+	std::transform(std::cbegin(tracks), std::cend(tracks), std::inserter(res, std::end(res)),
+			[](const auto& track) { return track.id(); });
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> res;
+	std::unordered_set<Database::IdType> res;
 
 	auto transaction {session.createSharedTransaction()};
 
@@ -57,18 +57,17 @@ ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Dat
 	if (!trackList)
 		return res;
 
-	const std::vector<Database::Track::pointer> tracks {trackList->getSimilarTracks(0, maxCount)};
+	const auto tracks {trackList->getSimilarTracks(0, maxCount)};
-	res.reserve(tracks.size());
+	std::transform(std::cbegin(tracks), std::cend(tracks), std::inserter(res, std::end(res)),
-	std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res),
 			[](const Database::Track::pointer& track) { return track.id(); });
 
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> res;
+	std::unordered_set<Database::IdType> res;
 
 	auto transaction {dbSession.createSharedTransaction()};
 
@@ -77,16 +76,16 @@ ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::Id
 		return res;
 
 	const auto releases {release->getSimilarReleases(0, maxCount)};
-	res.reserve(releases.size());
+	std::transform(std::cbegin(releases), std::cend(releases), std::inserter(res, std::end(res)),
-	std::transform(std::cbegin(releases), std::cend(releases), std::back_inserter(res), [](const auto& release) { return release.id(); });
+			[](const auto& release) { return release.id(); });
 
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> res;
+	std::unordered_set<Database::IdType> res;
 
 	auto transaction {dbSession.createSharedTransaction()};
 
@@ -95,8 +94,8 @@ ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdT
 		return res;
 
 	const auto artists {artist->getSimilarArtists(0, maxCount)};
-	res.reserve(artists.size());
+	std::transform(std::cbegin(artists), std::cend(artists), std::inserter(res, std::end(res)),
-	std::transform(std::cbegin(artists), std::cend(artists), std::back_inserter(res), [](const auto& artist) { return artist.id(); });
+			[](const auto& artist) { return artist.id(); });
 
 	return res;
 }

src/libs/recommendation/impl/clusters/ClustersClassifier.hpp

@@ -19,8 +19,7 @@
 
 #pragma once
 
-#include "recommendation/IClassifier.hpp"
+#include "IClassifier.hpp"
-
 
 namespace Recommendation
 {
@@ -38,14 +37,13 @@ namespace Recommendation
 
 			std::string_view getName() const override { return "Clusters"; }
 
-			bool init(Database::Session&, bool) override {return true;}
+			bool load(Database::Session&, bool, const ProgressCallback&) override { return true; }
-			void requestCancelInit() override {}
+			void requestCancelLoad() override {}
-
-			std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
-			std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
-			std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
-			std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
 
+			std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
+			std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
+			std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
+			std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
 };
 
 } // namespace Recommendation

src/libs/recommendation/impl/features/FeaturesClassifier.cpp

@@ -127,7 +127,7 @@ getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t
 }
 
 bool
-FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSettings& trainSettings)
+FeaturesClassifier::loadFromTraining(Database::Session& session, const TrainSettings& trainSettings, const ProgressCallback& progressCallback)
 {
 	LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier...";
 
@@ -158,7 +158,7 @@ FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSett
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features...";
 	for (Database::IdType trackId : trackIds)
 	{
-		if (_initCancelled)
+		if (_loadCancelled)
 			return false;
 
 		std::optional<FeatureValuesMap> featureValuesMap;
@@ -201,23 +201,26 @@ FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSett
 	SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)};
 	network.setDataWeights(weights);
 
-	auto progressIndicator{[](const auto& iter)
+	auto somProgressCallback{[&](const SOM::Network::CurrentIteration& iter)
-		{
+	{
-			LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount;
+		LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount;
-		}};
+		progressCallback(Progress {iter.idIteration, iter.iterationCount});
+	}};
 
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Training network...";
-	network.train(samples, trainSettings.iterationCount, progressIndicator);
+	network.train(samples, trainSettings.iterationCount,
+			progressCallback ? somProgressCallback : SOM::Network::ProgressCallback {},
+			[this] { return _loadCancelled; });
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Training network DONE";
 
-	if (_initCancelled)
+	if (_loadCancelled)
 		return false;
 
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks...";
 	ObjectPositions trackPositions;
 	for (std::size_t i {}; i < samples.size(); ++i)
 	{
-		if (_initCancelled)
+		if (_loadCancelled)
 			return false;
 
 		const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
@@ -227,86 +230,100 @@ FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSett
 
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks DONE";
 
-	return init(session, std::move(network), std::move(trackPositions));
+	return load(session, std::move(network), std::move(trackPositions));
 }
 
 bool
-FeaturesClassifier::initFromCache(Database::Session& session, const FeaturesClassifierCache& cache)
+FeaturesClassifier::loadFromCache(Database::Session& session, const FeaturesClassifierCache& cache)
 {
 	LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier from cache...";
 
-	return init(session, std::move(cache._network), cache._trackPositions);
+	return load(session, std::move(cache._network), cache._trackPositions);
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) const
 {
-	const std::unordered_set<Database::IdType> trackIds {[&]() -> std::unordered_set<Database::IdType>
+	const std::unordered_set<Database::IdType> trackIds {[&]
 	{
+		std::unordered_set<Database::IdType> res;
+
 		auto transaction {session.createSharedTransaction()};
 
 		const Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)};
 		if (trackList)
 		{
 			const std::vector<Database::IdType> orderedTrackIds {trackList->getTrackIds()};
-			return std::unordered_set<Database::IdType> {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)};
+			res = std::unordered_set<Database::IdType>(std::cbegin(orderedTrackIds), std::cend(orderedTrackIds));
 		}
 
-		return {};
+		return res;
 	}()};
 
 	return getSimilarTracks(session, trackIds, maxCount);
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksIds, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)};
+	auto similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)};
-
 	if (!similarTrackIds.empty())
 	{
 		// Report only existing ids
 		auto transaction {session.createSharedTransaction()};
 
-		similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds),
+		for (auto it {std::begin(similarTrackIds)}; it != std::end(similarTrackIds);)
-					[&](Database::IdType trackId) { return Database::Track::getById(session, trackId) == Database::Track::pointer {}; }),
+		{
-				std::cend(similarTrackIds));
+			const Database::IdType trackId {*it};
+			if (!Database::Track::getById(session, trackId))
+				it = similarTrackIds.erase(it);
+			else
+				it++;
+		}
 	}
 
 	return similarTrackIds;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)};
+	auto similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)};
-
 	if (!similarReleaseIds.empty())
 	{
 		// Report only existing ids
 		auto transaction {session.createSharedTransaction()};
 
-		similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds),
+		for (auto it {std::begin(similarReleaseIds)}; it != std::end(similarReleaseIds);)
-					[&](Database::IdType releaseId) { return Database::Release::getById(session, releaseId) == Database::Release::pointer {}; }),
+		{
-				std::cend(similarReleaseIds));
+			const Database::IdType releaseId {*it};
+			if (!Database::Release::getById(session, releaseId))
+				it = similarReleaseIds.erase(it);
+			else
+				it++;
+		}
 	}
 
 	return similarReleaseIds;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)};
+	auto similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)};
-
 	if (!similarArtistIds.empty())
 	{
 		// Report only existing ids
 		auto transaction {session.createSharedTransaction()};
 
-		similarArtistIds.erase(std::remove_if(std::begin(similarArtistIds), std::end(similarArtistIds),
+		for (auto it {std::begin(similarArtistIds)}; it != std::end(similarArtistIds);)
-					[&](Database::IdType artistId) { return Database::Artist::getById(session, artistId) == Database::Artist::pointer {}; }),
+		{
-				std::cend(similarArtistIds));
+			const Database::IdType artistId {*it};
+			if (!Database::Release::getById(session, artistId))
+				it = similarArtistIds.erase(it);
+			else
+				it++;
+		}
 	}
 
 	return similarArtistIds;
@@ -319,22 +336,23 @@ FeaturesClassifier::toCache() const
 }
 
 bool
-FeaturesClassifier::init(Database::Session& session, bool databaseChanged)
+FeaturesClassifier::load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback)
 {
-	if (databaseChanged)
+	if (forceReload)
 	{
-		LMS_LOG(RECOMMENDATION, DEBUG) << "Database changed: invidating cache";
 		FeaturesClassifierCache::invalidate();
 	}
-
+	else
-	std::optional<FeaturesClassifierCache> cache {FeaturesClassifierCache::read()};
+	{
-	if (cache)
+		const std::optional<FeaturesClassifierCache> cache {FeaturesClassifierCache::read()};
-		return initFromCache(session, *cache);
+		if (cache)
+			return loadFromCache(session, *cache);
+	}
 
 	TrainSettings trainSettings;
 	trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings();
 
-	bool res {initFromTraining(session, trainSettings)};
+	const bool res {loadFromTraining(session, trainSettings, progressCallback)};
 	if (res)
 		toCache().write();
 
@@ -342,14 +360,14 @@ FeaturesClassifier::init(Database::Session& session, bool databaseChanged)
 }
 
 void
-FeaturesClassifier::requestCancelInit()
+FeaturesClassifier::requestCancelLoad()
 {
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Requesting init cancellation";
-	_initCancelled = true;
+	_loadCancelled = true;
 }
 
 bool
-FeaturesClassifier::init(Database::Session& session,
+FeaturesClassifier::load(Database::Session& session,
 			SOM::Network network,
 			const ObjectPositions& tracksPosition)
 {
@@ -367,7 +385,7 @@ FeaturesClassifier::init(Database::Session& session,
 
 	for (auto itTrackCoord : tracksPosition)
 	{
-		if (_initCancelled)
+		if (_loadCancelled)
 			return false;
 
 		auto transaction {session.createSharedTransaction()};
@@ -399,7 +417,7 @@ FeaturesClassifier::init(Database::Session& session,
 
 	_network = std::make_unique<SOM::Network>(std::move(network));
 
-	LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully initialized!";
+	LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully loaded!";
 
 	return true;
 }
@@ -439,13 +457,13 @@ FeaturesClassifier::getObjectsIds(const std::unordered_set<SOM::Position>& posit
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarObjects(const std::unordered_set<Database::IdType>& ids,
 		const MatrixOfObjects& objectsMap,
 		const ObjectPositions& objectPosition,
 		std::size_t maxCount) const
 {
-	std::vector<Database::IdType> res;
+	std::unordered_set<Database::IdType> res;
 
 	std::unordered_set<SOM::Position> searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPosition)};
 	if (searchedRefVectorsPosition.empty())
@@ -459,14 +477,13 @@ FeaturesClassifier::getSimilarObjects(const std::unordered_set<Database::IdType>
 		for (auto id : ids)
 			closestObjectIds.erase(id);
 
+		for (auto it {std::cbegin(closestObjectIds)}; it != std::cend(closestObjectIds); ++it)
 		{
-			std::vector<Database::IdType> objectIdsToAdd {std::cbegin(closestObjectIds), std::cend(closestObjectIds)};
+			if (res.size() == maxCount)
-			Random::shuffleContainer(objectIdsToAdd );
+				break;
-			std::copy(std::cbegin(objectIdsToAdd), std::cend(objectIdsToAdd), std::back_inserter(res));
-		}
 
-		if (res.size() > maxCount)
+			res.insert(*it);
-			res.resize(maxCount);
+		}
 
 		if (res.size() == maxCount)
 			break;

src/libs/recommendation/impl/features/FeaturesClassifier.hpp

@@ -19,15 +19,16 @@
 
 #pragma once
 
+#include <functional>
 #include <unordered_map>
 #include <optional>
 #include <string>
 
-#include "recommendation/IClassifier.hpp"
 #include "som/DataNormalizer.hpp"
 #include "som/Network.hpp"
 #include "FeaturesClassifierCache.hpp"
 #include "FeaturesDefs.hpp"
+#include "IClassifier.hpp"
 
 namespace Database
 {
@@ -53,19 +54,20 @@ class FeaturesClassifier : public IClassifier
 		static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; }
 
 		static const FeatureSettingsMap& getDefaultTrainFeatureSettings();
+
 	private:
 
 		std::string_view getName() const override { return "Features"; }
 
-		bool init(Database::Session& session, bool databaseChanged) override;
+		bool load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) override;
-		void requestCancelInit() override;
+		void requestCancelLoad() override;
 
-		std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
+		std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
-		std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
+		std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
-		std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
+		std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
-		std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
+		std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
 
-		bool initFromCache(Database::Session& session, const FeaturesClassifierCache& cache);
+		bool loadFromCache(Database::Session& session, const FeaturesClassifierCache& cache);
 
 		// Use training (may be very slow)
 		struct TrainSettings
@@ -74,12 +76,12 @@ class FeaturesClassifier : public IClassifier
 			float sampleCountPerNeuron {4};
 			FeatureSettingsMap featureSettingsMap;
 		};
-		bool initFromTraining(Database::Session& session, const TrainSettings& trainSettings);
+		bool loadFromTraining(Database::Session& session, const TrainSettings& trainSettings, const ProgressCallback& progressCallback);
 
 		using ObjectPositions = std::unordered_map<Database::IdType, std::unordered_set<SOM::Position>>;
 		using MatrixOfObjects = SOM::Matrix<std::unordered_set<Database::IdType>>;
 
-		bool init(Database::Session& session,
+		bool load(Database::Session& session,
 				SOM::Network network,
 				const ObjectPositions& tracksPosition);
 
@@ -88,12 +90,12 @@ class FeaturesClassifier : public IClassifier
 		static std::unordered_set<SOM::Position> getMatchingRefVectorsPosition(const std::unordered_set<Database::IdType>& ids, const ObjectPositions& objectPositions);
 		static std::unordered_set<Database::IdType> getObjectsIds(const std::unordered_set<SOM::Position>& positionSet, const MatrixOfObjects& objectsMap);
 
-		std::vector<Database::IdType> getSimilarObjects(const std::unordered_set<Database::IdType>& ids,
+		std::unordered_set<Database::IdType> getSimilarObjects(const std::unordered_set<Database::IdType>& ids,
 				const SOM::Matrix<std::unordered_set<Database::IdType>>& objectsMap,
 				const ObjectPositions& objectPosition,
 				std::size_t maxCount) const;
 
-		bool				_initCancelled {};
+		bool				_loadCancelled {};
 		std::unique_ptr<SOM::Network>	_network;
 		double				_networkRefVectorsDistanceMedian {};
 

src/libs/recommendation/include/recommendation/IEngine.hpp

@@ -20,11 +20,8 @@
 #pragma once
 
 #include <functional>
-#include <vector>
 #include <unordered_set>
 
-#include <Wt/WSignal.h>
-
 #include "database/Types.hpp"
 
 namespace Database
@@ -40,16 +37,19 @@ namespace Recommendation
 		public:
 			virtual ~IEngine() = default;
 
-			virtual void requestLoad() = 0;
+			struct Progress
+			{
+				std::size_t totalElems {};
+				std::size_t	processedElems {};
+			};
+			using ProgressCallback = std::function<void(const Progress&)>;
+			virtual void load(bool forceReload, const ProgressCallback& progressCallback = {}) = 0;
+			virtual void cancelLoad() = 0;
 
-			virtual void requestReload() = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0;
-			virtual Wt::Signal<>& reloaded() = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) = 0;
-
+			virtual std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) = 0;
-			// Closest results first
+			virtual std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0;
-			virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0;
-			virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) = 0;
-			virtual std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) = 0;
-			virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0;
 	};
 
 	std::unique_ptr<IEngine> createEngine(Database::Db& db);

src/libs/scanner/CMakeLists.txt

@@ -16,6 +16,7 @@ target_include_directories(lmsscanner PRIVATE
 target_link_libraries(lmsscanner PRIVATE
 	lmsdatabase
 	lmsmetadata
+	lmsrecommendation
 	lmsutils
 	)
 

src/libs/scanner/impl/MediaScanner.cpp

@@ -30,6 +30,7 @@
 #include "database/Track.hpp"
 #include "database/TrackFeatures.hpp"
 #include "metadata/TagLibParser.hpp"
+#include "recommendation/IEngine.hpp"
 #include "utils/Exception.hpp"
 #include "utils/Logger.hpp"
 #include "utils/Path.hpp"
@@ -240,13 +241,14 @@ getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames)
 namespace Scanner {
 
 std::unique_ptr<IMediaScanner>
-createMediaScanner(Database::Db& db)
+createMediaScanner(Database::Db& db, Recommendation::IEngine& recommendationEngine)
 {
-	return std::make_unique<MediaScanner>(db);
+	return std::make_unique<MediaScanner>(db, recommendationEngine);
 }
 
-MediaScanner::MediaScanner(Database::Db& db)
+MediaScanner::MediaScanner(Database::Db& db, Recommendation::IEngine& recommendationEngine)
-: _dbSession {db}
+: _recommendationEngine {recommendationEngine}
+, _dbSession {db}
 {
 	// For now, always use TagLib
 	_metadataParser = std::make_unique<MetaData::TagLibParser>();
@@ -260,6 +262,7 @@ MediaScanner::MediaScanner(Database::Db& db)
 
 MediaScanner::~MediaScanner()
 {
+	LMS_LOG(DBUPDATER, INFO) << "Shutting down MediaScanner...";
 	stop();
 }
 
@@ -268,7 +271,14 @@ MediaScanner::start()
 {
 	std::scoped_lock lock {_controlMutex};
 
-	scheduleNextScan();
+	_ioService.post([this]
+	{
+		if (_abortScan)
+			return;
+
+		_recommendationEngine.load(false);
+		scheduleNextScan();
+	});
 
 	_ioService.start();
 }
@@ -279,8 +289,8 @@ MediaScanner::stop()
 	std::scoped_lock lock {_controlMutex};
 
 	_abortScan = true;
-
 	_scheduleTimer.cancel();
+	_recommendationEngine.cancelLoad();
 	_ioService.stop();
 }
 
@@ -294,6 +304,7 @@ MediaScanner::abortScan()
 
 	_abortScan = true;
 	_scheduleTimer.cancel();
+	_recommendationEngine.cancelLoad();
 	_ioService.stop();
 	LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!";
 
@@ -307,6 +318,9 @@ MediaScanner::requestImmediateScan(bool force)
 	abortScan();
 	_ioService.post([=]()
 	{
+		if (_abortScan)
+			return;
+
 		scheduleScan(force);
 	});
 }
@@ -317,6 +331,9 @@ MediaScanner::requestReload()
 	abortScan();
 	_ioService.post([=]()
 	{
+		if (_abortScan)
+			return;
+
 		scheduleNextScan();
 	});
 }
@@ -407,12 +424,13 @@ MediaScanner::countAllFiles(ScanStats& stats)
 		if (!ec && isFileSupported(path, _fileExtensions))
 		{
 			stats.filesScanned++;
-			stepStats.processedFiles++;
+			stepStats.processedElems++;
 			notifyInProgressIfNeeded(stepStats);
 		}
 
 		return true;
 	});
+	notifyInProgress(stepStats);
 }
 
 void
@@ -476,10 +494,11 @@ MediaScanner::scan(bool forceScan)
 	removeOrphanEntries();
 
 	if (!_abortScan)
+	{
 		checkDuplicatedAudioFiles(stats);
-
+		fetchTrackFeatures(stats);
-	// Now update all the track features if needed
+		reloadSimilarityEngine(stats);
-	fetchTrackFeatures(stats);
+	}
 
 	LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ",  duplicates = " << stats.duplicates.size();
 
@@ -566,7 +585,7 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats)
 		return res;
 	}()};
 
-	stepStats.filesToProcess = tracksToFetch.size();
+	stepStats.totalElems = tracksToFetch.size();
 	notifyInProgress(stepStats);
 
 	LMS_LOG(DBUPDATER, INFO) << "Found " << tracksToFetch.size() << " track(s) to fetch!";
@@ -579,10 +598,11 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats)
 		if (fetchTrackFeatures(trackToFetch.id, trackToFetch.mbid))
 			stats.featuresFetched++;
 
-		stepStats.processedFiles++;
+		stepStats.processedElems++;
 		notifyInProgressIfNeeded(stepStats);
 	}
 
+	notifyInProgress(stepStats);
 	LMS_LOG(DBUPDATER, INFO) << "Track features fetched!";
 }
 
@@ -591,7 +611,7 @@ MediaScanner::refreshScanSettings()
 {
 	auto transaction {_dbSession.createSharedTransaction()};
 
-	ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
+	const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
 
 	LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion();
 
@@ -608,7 +628,7 @@ MediaScanner::refreshScanSettings()
 	_mediaDirectory = scanSettings->getMediaDirectory();
 	_recommendationEngineType = scanSettings->getRecommendationEngineType();
 
-	auto clusterTypes = scanSettings->getClusterTypes();
+	const auto clusterTypes = scanSettings->getClusterTypes();
 	std::set<std::string> clusterTypeNames;
 
 	std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
@@ -616,7 +636,6 @@ MediaScanner::refreshScanSettings()
 			[](ClusterType::pointer clusterType) { return clusterType->getName(); });
 
 	_metadataParser->setClusterTypeNames(clusterTypeNames);
-
 }
 
 void
@@ -800,7 +819,7 @@ void
 MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
 {
 	ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles};
-	stepStats.filesToProcess = stats.filesScanned;
+	stepStats.totalElems = stats.filesScanned;
 	notifyInProgress(stepStats);
 
 	exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
@@ -817,12 +836,14 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo
 		{
 			scanAudioFile(path, forceScan, stats );
 
-			stepStats.processedFiles++;
+			stepStats.processedElems++;
 			notifyInProgressIfNeeded(stepStats);
 		}
 
 		return true;
 	});
+
+	notifyInProgress(stepStats);
 }
 
 // Check if a file exists and is still in a media directory
@@ -878,7 +899,7 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
 	}
 	LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked...";
 
-	stepStats.filesToProcess = trackCount;
+	stepStats.totalElems = trackCount;
 	notifyInProgress(stepStats);
 
 	std::vector<std::pair<Database::IdType, std::filesystem::path>> trackPaths;
@@ -902,7 +923,7 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
 			if (!checkFile(trackPath, _mediaDirectory, _fileExtensions))
 				tracksToRemove.push_back(trackId);
 
-			stepStats.processedFiles++;
+			stepStats.processedElems++;
 		}
 
 		if (!tracksToRemove.empty())
@@ -992,4 +1013,21 @@ MediaScanner::checkDuplicatedAudioFiles(ScanStats& stats)
 	LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!";
 }
 
+void
+MediaScanner::reloadSimilarityEngine(ScanStats& stats)
+{
+	ScanStepStats stepStats {stats.startTime, ScanProgressStep::ReloadingSimilarityEngine};
+
+	auto progressCallback {[&](const Recommendation::IEngine::Progress& progress)
+	{
+		stepStats.totalElems = progress.totalElems;
+		stepStats.processedElems = progress.processedElems;
+		notifyInProgressIfNeeded(stepStats);
+	}};
+
+	notifyInProgress(stepStats);
+	_recommendationEngine.load(stats.nbChanges() > 0, progressCallback);
+	notifyInProgress(stepStats);
+}
+
 } // namespace Scanner

src/libs/scanner/impl/MediaScanner.hpp

@@ -37,12 +37,17 @@
 
 class UUID;
 
+namespace Recommendation
+{
+	class IEngine;
+}
+
 namespace Scanner {
 
 class MediaScanner : public IMediaScanner
 {
 	public:
-		MediaScanner(Database::Db& db);
+		MediaScanner(Database::Db& db, Recommendation::IEngine& recommendationEngine);
 		~MediaScanner();
 
 		MediaScanner(const MediaScanner&) = delete;
@@ -89,6 +94,9 @@ class MediaScanner : public IMediaScanner
 		Database::IdType doScanAudioFile(const std::filesystem::path& file, ScanStats& stats);
 		void notifyInProgressIfNeeded(const ScanStepStats& stats);
 		void notifyInProgress(const ScanStepStats& stats);
+		void reloadSimilarityEngine(ScanStats& stats);
+
+		Recommendation::IEngine&				_recommendationEngine;
 
 		std::mutex								_controlMutex;
 		std::atomic<bool>						_abortScan {};
@@ -116,7 +124,6 @@ class MediaScanner : public IMediaScanner
 		std::filesystem::path			_mediaDirectory;
 		Database::ScanSettings::RecommendationEngineType _recommendationEngineType;
 
-
 }; // class MediaScanner
 
 } // Scanner

src/libs/scanner/impl/MediaScannerStats.cpp

@@ -43,7 +43,7 @@ ScanStats::nbChanges() const
 unsigned
 ScanStepStats::progress() const
 {
-	return (processedFiles / static_cast<float>(filesToProcess ? filesToProcess : 1)) * 100;
+	return (processedElems / static_cast<float>(totalElems ? totalElems : 1)) * 100;
 }
 
 } // namespace Scanner

src/libs/scanner/include/scanner/IMediaScanner.hpp

@@ -31,6 +31,11 @@ namespace Database
 	class Db;
 }
 
+namespace Recommendation
+{
+	class IEngine;
+}
+
 namespace Scanner {
 
 class IMediaScanner
@@ -73,8 +78,7 @@ class IMediaScanner
 
 };
 
-std::unique_ptr<IMediaScanner> createMediaScanner(Database::Db& db);
+std::unique_ptr<IMediaScanner> createMediaScanner(Database::Db& db, Recommendation::IEngine& recommendationEngine);
-
 
 } // Scanner
 

src/libs/scanner/include/scanner/MediaScannerStats.hpp

@@ -62,8 +62,9 @@ namespace Scanner {
 		DiscoveringFiles,
 		ScanningFiles,
 		FetchingTrackFeatures,
+		ReloadingSimilarityEngine,
 	};
-	static inline constexpr unsigned ScanProgressStepCount {4};
+	static inline constexpr unsigned ScanProgressStepCount {5};
 
 	// reduced scan stats
 	struct ScanStepStats
@@ -72,8 +73,8 @@ namespace Scanner {
 
 		ScanProgressStep currentStep;
 
-		std::size_t     filesToProcess {};
+		std::size_t	totalElems {};
-		std::size_t		processedFiles {};
+		std::size_t	processedElems {};
 
 		unsigned		progress() const;
 	};

src/libs/subsonic/impl/Scan.cpp

@@ -40,7 +40,7 @@ namespace API::Subsonic::Scan
 			std::size_t count{};
 
 			if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanProgressStep::ScanningFiles)
-				count = scanStatus.currentScanStepStats->processedFiles;
+				count = scanStatus.currentScanStepStats->processedElems;
 
 			statusResponse.setAttribute("count", count);
 		}

src/libs/utils/include/utils/Semaphore.hpp

@@ -1,58 +0,0 @@
-/*
- * Copyright (C) 2020 Emeric Poupon
- *
- * This file is part of LMS.
- *
- * LMS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LMS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#pragma once
-
-#include <mutex>
-#include <condition_variable>
-
-class Semaphore
-{
-	public:
-		Semaphore() = default;
-		Semaphore(const Semaphore&) = delete;
-		Semaphore(Semaphore&&) = delete;
-		Semaphore& operator=(const Semaphore&) = delete;
-		Semaphore& operator=(Semaphore&&) = delete;
-
-		void notify()
-		{
-			std::unique_lock<std::mutex> lock {_mutex};
-
-			_count++;
-			_cv.notify_one();
-		}
-
-		void wait()
-		{
-			std::unique_lock<std::mutex> lock(_mutex);
-
-			while (_count == 0)
-				_cv.wait(lock);
-
-			_count--;
-		}
-
-	private:
-		std::mutex			_mutex;
-		std::condition_variable		_cv;
-		unsigned			_count {};
-};
-
-

src/libs/utils/include/utils/Service.hpp

@@ -46,6 +46,11 @@ class Service
 			return Service<Class>::get();
 		}
 
+		Class& operator*() const
+		{
+			return *Service<Class>::get();
+		}
+
 		static Class* get() { return _service.get(); }
 
 	private:

src/lms/main.cpp

@@ -121,7 +121,7 @@ generateWtConfig(std::string execPath)
 int main(int argc, char* argv[])
 {
 	std::filesystem::path configFilePath {"/etc/lms.conf"};
-	int res = EXIT_FAILURE;
+	int res {EXIT_FAILURE};
 
 	assert(argc > 0);
 	assert(argv[0] != NULL);
@@ -183,22 +183,10 @@ int main(int argc, char* argv[])
 				config->getULong("cover-max-file-size", 10) * 1000 * 1000,
 				config->getULong("cover-jpeg-quality", 75))};
 		Service<Recommendation::IEngine> recommendationEngineService {Recommendation::createEngine(database)};
-		recommendationEngineService->requestLoad();
+		Service<Scanner::IMediaScanner> mediaScannerService {Scanner::createMediaScanner(database, *recommendationEngineService)};
-		Service<Scanner::IMediaScanner> mediaScannerService {Scanner::createMediaScanner(database)};
 
 		mediaScannerService->scanComplete().connect([&]()
 		{
-			auto status = mediaScannerService->getStatus();
-
-			if (status.lastCompleteScanStats->nbChanges() > 0 || status.lastCompleteScanStats->featuresFetched > 0)
-			{
-				LMS_LOG(MAIN, INFO) << "Scanner changed some files, reloading the recommendation engine...";
-				recommendationEngineService->requestReload();
-			}
-			else
-			{
-				LMS_LOG(MAIN, INFO) << "Scanner did not change files, not reloading the recommendation engine...";
-			}
 			// Flush cover cache even if no changes:
 			// covers may be external files that changed and we don't keep track of them
 			coverArtService->flushCache();
@@ -224,16 +212,18 @@ int main(int argc, char* argv[])
 		LMS_LOG(MAIN, INFO) << "Stopping server...";
 		server.stop();
 
-		LMS_LOG(MAIN, INFO) << "Clean stop!";
+		LMS_LOG(MAIN, INFO) << "Quitting...";
 		res = EXIT_SUCCESS;
 	}
 	catch(Wt::WServer::Exception& e)
 	{
 		std::cerr << "Caught a WServer::Exception: " << e.what() << std::endl;
+		res = EXIT_FAILURE;
 	}
 	catch(std::exception& e)
 	{
 		std::cerr << "Caught std::exception: " << e.what() << std::endl;
+		res = EXIT_FAILURE;
 	}
 
 	return res;

src/lms/ui/PlayQueue.cpp

@@ -501,7 +501,10 @@ PlayQueue::addSome()
 void
 PlayQueue::enqueueRadioTracks()
 {
-	const std::vector<Database::IdType> trackToAddIds {Service<Recommendation::IEngine>::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 3)};
+	const auto similarTrackIds {Service<Recommendation::IEngine>::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 3)};
+
+	std::vector<Database::IdType> trackToAddIds(std::cbegin(similarTrackIds), std::cend(similarTrackIds));
+	Random::shuffleContainer(trackToAddIds);
 	enqueueTracks(trackToAddIds);
 }
 

src/lms/ui/admin/DatabaseSettingsView.cpp

@@ -47,14 +47,13 @@ class DatabaseSettingsModel : public Wt::WFormModel
 {
 	public:
 		// Associate each field with a unique string literal.
-		static const Field MediaDirectoryField;
+		static inline const Field MediaDirectoryField {"media-directory"};
-		static const Field UpdatePeriodField;
+		static inline const Field UpdatePeriodField {"update-period"};
-		static const Field UpdateStartTimeField;
+		static inline const Field UpdateStartTimeField {"update-start-time"};
-		static const Field RecommendationEngineTypeField;
+		static inline const Field RecommendationEngineTypeField {"recommendation-engine-type"};
-		static const Field TagsField;
+		static inline const Field TagsField {"tags"};
 
 		DatabaseSettingsModel()
-			: Wt::WFormModel()
 		{
 			initializeModels();
 
@@ -135,7 +134,6 @@ class DatabaseSettingsModel : public Wt::WFormModel
 		}
 
 	private:
-
 		static std::shared_ptr<Wt::WValidator> createTagsValidator()
 		{
 			auto v = std::make_shared<Wt::WValidator>();
@@ -162,18 +160,11 @@ class DatabaseSettingsModel : public Wt::WFormModel
 			_recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.features"), ScanSettings::RecommendationEngineType::Features);
 		}
 
-		std::shared_ptr<ValueStringModel<ScanSettings::UpdatePeriod>>		_updatePeriodModel;
+		std::shared_ptr<ValueStringModel<ScanSettings::UpdatePeriod>>				_updatePeriodModel;
-		std::shared_ptr<ValueStringModel<Wt::WTime>>				_updateStartTimeModel;
+		std::shared_ptr<ValueStringModel<Wt::WTime>>								_updateStartTimeModel;
 		std::shared_ptr<ValueStringModel<ScanSettings::RecommendationEngineType>>	_recommendationEngineTypeModel;
-
 };
 
-const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField			= "media-directory";
-const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField			= "update-period";
-const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField			= "update-start-time";
-const Wt::WFormModel::Field DatabaseSettingsModel::RecommendationEngineTypeField	= "recommendation-engine-type";
-const Wt::WFormModel::Field DatabaseSettingsModel::TagsField				= "tags";
-
 DatabaseSettingsView::DatabaseSettingsView()
 {
 	wApp->internalPathChanged().connect(std::bind([=]
@@ -223,7 +214,7 @@ DatabaseSettingsView::refreshView()
 
 	t->bindNew<ScannerController>("scanner-controller");
 
-	saveBtn->clicked().connect([=] ()
+	saveBtn->clicked().connect([=]
 	{
 		t->updateModel(model.get());
 
@@ -231,7 +222,7 @@ DatabaseSettingsView::refreshView()
 		{
 			model->saveData();
 
-			Service<Scanner::IMediaScanner>::get()->requestReload();
+			Service<Scanner::IMediaScanner>::get()->requestImmediateScan(false);
 			LmsApp->notifyMsg(MsgType::Success, Wt::WString::tr("Lms.Admin.Database.settings-saved"));
 		}
 
@@ -239,14 +230,14 @@ DatabaseSettingsView::refreshView()
 		t->updateView(model.get());
 	});
 
-	discardBtn->clicked().connect([=] ()
+	discardBtn->clicked().connect([=]
 	{
 		model->loadData();
 		model->validate();
 		t->updateView(model.get());
 	});
 
-	immScanBtn->clicked().connect([=] ()
+	immScanBtn->clicked().connect([=]
 	{
 		Service<Scanner::IMediaScanner>::get()->requestImmediateScan(false);
 	});

src/lms/ui/admin/ScannerController.cpp

@@ -206,20 +206,24 @@ ScannerController::refreshContents()
 
 				case Scanner::ScanProgressStep::DiscoveringFiles:
 					bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
-						.arg(status.currentScanStepStats->processedFiles));
+						.arg(status.currentScanStepStats->processedElems));
 						break;
 
 				case Scanner::ScanProgressStep::ScanningFiles:
 					bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
-						.arg(status.currentScanStepStats->processedFiles)
+						.arg(status.currentScanStepStats->processedElems)
-						.arg(status.currentScanStepStats->filesToProcess)
+						.arg(status.currentScanStepStats->totalElems)
 						.arg(status.currentScanStepStats->progress()));
 					break;
 
 				case Scanner::ScanProgressStep::FetchingTrackFeatures:
 					bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
-						.arg(status.currentScanStepStats->processedFiles)
+						.arg(status.currentScanStepStats->processedElems)
-						.arg(status.currentScanStepStats->filesToProcess)
+						.arg(status.currentScanStepStats->totalElems)
+						.arg(status.currentScanStepStats->progress()));
+					break;
+				case Scanner::ScanProgressStep::ReloadingSimilarityEngine:
+					bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine")
 						.arg(status.currentScanStepStats->progress()));
 					break;
 			}

src/lms/ui/explore/ArtistView.cpp

@@ -46,7 +46,7 @@ namespace UserInterface {
 
 Artist::Artist(Filters* filters)
 : Wt::WTemplate {Wt::WString::tr("Lms.Explore.Artist.template")}
-, _filters(filters)
+, _filters {filters}
 {
 	addFunction("tr", &Wt::WTemplate::Functions::tr);
 
@@ -75,7 +75,7 @@ Artist::refreshView()
 	if (!artistId)
 		throw ArtistNotFoundException {*artistId};
 
-	const std::vector<Database::IdType> similarArtistIds {Service<Recommendation::IEngine>::get()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)};
+	const auto similarArtistIds {Service<Recommendation::IEngine>::get()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)};
 
     auto transaction {LmsApp->getDbSession().createSharedTransaction()};
 
@@ -172,7 +172,7 @@ Artist::refreshView()
 }
 
 void
-Artist::refreshSimilarArtists(const std::vector<Database::IdType>& similarArtistsId)
+Artist::refreshSimilarArtists(const std::unordered_set<Database::IdType>& similarArtistsId)
 {
 	if (similarArtistsId.empty())
 		return;

src/lms/ui/explore/ArtistView.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include <memory>
+#include <unordered_set>
 
 #include <Wt/WSignal.h>
 #include <Wt/WTemplate.h>
@@ -46,7 +47,7 @@ class Artist : public Wt::WTemplate
 
 	private:
 		void refreshView();
-		void refreshSimilarArtists(const std::vector<Database::IdType>& similarArtistsId);
+		void refreshSimilarArtists(const std::unordered_set<Database::IdType>& similarArtistsId);
 		void refreshLinks(const Wt::Dbo::ptr<Database::Artist>& artist);
 
 		std::unique_ptr<Wt::WTemplate> createRelease(const Wt::Dbo::ptr<Database::Artist>& artist, const Wt::Dbo::ptr<Database::Release>& release);

src/lms/ui/explore/ReleaseView.cpp

@@ -50,11 +50,11 @@ namespace UserInterface {
 
 Release::Release(Filters* filters)
 : Wt::WTemplate {Wt::WString::tr("Lms.Explore.Release.template")}
-, _filters(filters)
+, _filters {filters}
 {
 	addFunction("tr", &Wt::WTemplate::Functions::tr);
 
-	wApp->internalPathChanged().connect([=]()
+	wApp->internalPathChanged().connect([=]
 	{
 		refreshView();
 	});
@@ -79,7 +79,7 @@ Release::refreshView()
 	if (!releaseId)
 		throw ReleaseNotFoundException {*releaseId};
 
-	const std::vector<Database::IdType> similarReleasesIds {Service<Recommendation::IEngine>::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 6)};
+	auto similarReleasesIds {Service<Recommendation::IEngine>::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 6)};
 
     auto transaction {LmsApp->getDbSession().createSharedTransaction()};
 
@@ -307,7 +307,7 @@ Release::refreshLinks(const Database::Release::pointer& release)
 }
 
 void
-Release::refreshSimilarReleases(const std::vector<Database::IdType>& similarReleasesId)
+Release::refreshSimilarReleases(const std::unordered_set<Database::IdType>& similarReleasesId)
 {
 	if (similarReleasesId.empty())
 		return;

src/lms/ui/explore/ReleaseView.hpp

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include <unordered_set>
 #include <Wt/WTemplate.h>
 
 #include "database/Types.hpp"
@@ -45,7 +46,7 @@ namespace UserInterface
 			void refreshView();
 			void refreshCopyright(const Wt::Dbo::ptr<Database::Release>& release);
 			void refreshLinks(const Wt::Dbo::ptr<Database::Release>& release);
-			void refreshSimilarReleases(const std::vector<Database::IdType>& similarReleasesId);
+			void refreshSimilarReleases(const std::unordered_set<Database::IdType>& similarReleasesId);
 
 			Filters* _filters {};
 	};

src/tools/recommendation/LmsRecommendation.cpp

@@ -31,7 +31,6 @@
 #include "database/Session.hpp"
 #include "database/Track.hpp"
 #include "utils/IConfig.hpp"
-#include "utils/Semaphore.hpp"
 #include "utils/Service.hpp"
 #include "utils/StreamLogger.hpp"
 #include "recommendation/IEngine.hpp"
@@ -164,17 +163,8 @@ int main(int argc, char *argv[])
 		const auto engine {Recommendation::createEngine(db)};
 		std::cout << "Recommendation engine created!" << std::endl;
 
-		Semaphore sem;
+		std::cout << "Loading recommendation engine..." << std::endl;
-
+		engine->load(false);
-		engine->reloaded().connect([&]()
-		{
-			sem.notify();
-		});
-
-		engine->requestLoad();
-
-		std::cout << "Waiting for the recommendation engine to be loaded..." << std::endl;
-		sem.wait();
 		std::cout << "Recommendation engine loaded!" << std::endl;
 
 		if (vm.count("tracks"))