Loading of the recommendation engine now controlled by scanner. Bonus: better control/reporting

src/libs/recommendation/CMakeLists.txt

@@ -22,5 +22,7 @@ target_link_libraries(lmsrecommendation PRIVATE
 	wt
 	)
 
+target_include_directories(lmsrecommendation PRIVATE impl)
+
 install(TARGETS lmsrecommendation DESTINATION lib)
 

src/libs/recommendation/impl/Engine.cpp

@@ -19,16 +19,39 @@
 
 #include "Engine.hpp"
 
-#include "recommendation/ClustersClassifierCreator.hpp"
-#include "recommendation/FeaturesClassifierCreator.hpp"
+#include <unordered_map>
+#include <vector>
 
+#include "ClustersClassifierCreator.hpp"
+#include "FeaturesClassifierCreator.hpp"
+
+#include "database/Db.hpp"
+#include "database/Session.hpp"
 #include "database/ScanSettings.hpp"
-#include "database/TrackList.hpp"
 #include "utils/Exception.hpp"
 #include "utils/Logger.hpp"
 
 namespace Recommendation {
 
+
+static
+std::unique_ptr<IClassifier>
+createClassifier(ClassifierType type)
+{
+	switch (type)
+	{
+		case ClassifierType::Clusters:
+			return createClustersClassifier();
+			break;
+
+		case ClassifierType::Features:
+			return createFeaturesClassifier();
+			break;
+	}
+
+	return {};
+}
+
 std::unique_ptr<IEngine>
 createEngine(Database::Db& db)
 {
@@ -36,65 +59,16 @@ createEngine(Database::Db& db)
 }
 
 Engine::Engine(Database::Db& db)
-: _dbSession {db}
+: _db {db}
 {
-	start();
 }
 
-Engine::~Engine()
-{
-	stop();
-}
-
-void
-Engine::start()
-{
-	assert(!_running);
-	_running = true;
-	_ioService.start();
-}
-
-void
-Engine::stop()
-{
-	assert(_running);
-	_running = false;
-
-	cancelPendingClassifiers();
-
-	_ioService.stop();
-}
-
-void
-Engine::requestLoad()
-{
-	requestReloadInternal(false);
-}
-
-void
-Engine::requestReload()
-{
-	requestReloadInternal(true);
-}
-
-void
-Engine::requestReloadInternal(bool databaseChanged)
-{
-	LMS_LOG(RECOMMENDATION, DEBUG) << "Reload requested...";
-
-	_ioService.post([=]()
-	{
-		reload(databaseChanged);
-	});
-}
-
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount)
 {
+	std::unordered_set<Database::IdType> res;
+
 	std::shared_lock lock {_classifiersMutex};
-
-	std::vector<Database::IdType> res;
-
 	for (const auto& classifierName : _classifierPriorities)
 	{
 		auto itClassifier {_classifiers.find(classifierName)};
@@ -109,23 +83,23 @@ Engine::getSimilarTracksFromTrackList(Database::Session& session, Database::IdTy
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount)
 {
+	std::unordered_set<Database::IdType> res;
+
 	std::shared_lock lock {_classifiersMutex};
-
-	std::vector<Database::IdType> res;
-
-	for (const auto& classifierName : _classifierPriorities)
+	for (ClassifierType classifierType : _classifierPriorities)
 	{
-		auto itClassifier {_classifiers.find(classifierName)};
+		auto itClassifier {_classifiers.find(classifierType)};
 		if (itClassifier == std::cend(_classifiers))
 			continue;
 
-		res = itClassifier->second->getSimilarTracks(dbSession, trackIds, maxCount);
+		const IClassifier& classifier {*itClassifier->second};
+		res = classifier.getSimilarTracks(dbSession, trackIds, maxCount);
 		if (!res.empty())
 		{
-			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifierName << "'";
+			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar tracks using classifier '" << classifier.getName() << "'";
 			break;
 		}
 	}
@@ -133,23 +107,23 @@ Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount)
 {
+	std::unordered_set<Database::IdType> res;
+
 	std::shared_lock lock {_classifiersMutex};
-
-	std::vector<Database::IdType> res;
-
-	for (const auto& classifierName : _classifierPriorities)
+	for (ClassifierType classifierType : _classifierPriorities)
 	{
-		auto itClassifier {_classifiers.find(classifierName)};
+		auto itClassifier {_classifiers.find(classifierType)};
 		if (itClassifier == std::cend(_classifiers))
 			continue;
 
-		res = itClassifier->second->getSimilarReleases(dbSession, releaseId, maxCount);
+		const IClassifier& classifier {*itClassifier->second};
+		res = classifier.getSimilarReleases(dbSession, releaseId, maxCount);
 		if (!res.empty())
 		{
-			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifierName << "'";
+			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar releases using classifier '" << classifier.getName() << "'";
 			break;
 		}
 	}
@@ -157,23 +131,23 @@ Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releas
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount)
 {
+	std::unordered_set<Database::IdType> res;
+
 	std::shared_lock lock {_classifiersMutex};
-
-	std::vector<Database::IdType> res;
-
-	for (const auto& classifierName : _classifierPriorities)
+	for (ClassifierType classifierType : _classifierPriorities)
 	{
-		auto itClassifier {_classifiers.find(classifierName)};
+		auto itClassifier {_classifiers.find(classifierType)};
 		if (itClassifier == std::cend(_classifiers))
 			continue;
 
-		res = itClassifier->second->getSimilarArtists(dbSession, artistId, maxCount);
+		const IClassifier& classifier {*itClassifier->second};
+		res = classifier.getSimilarArtists(dbSession, artistId, maxCount);
 		if (!res.empty())
 		{
-			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifierName << "'";
+			LMS_LOG(RECOMMENDATION, DEBUG) << "Got " << res.size() << " similar artists using classifier '" << classifier.getName() << "'";
 			return res;
 		}
 	}
@@ -181,107 +155,132 @@ Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistI
 	return res;
 }
 
-void
-Engine::reload(bool databaseChanged)
+static
+Database::ScanSettings::RecommendationEngineType
+getRecommendationEngineType(Database::Session& session)
 {
-	using namespace Database;
+	auto transaction {session.createSharedTransaction()};
 
-	LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines...";
-
-	const ScanSettings::RecommendationEngineType engineType {[&]()
-	{
-		auto transaction {_dbSession.createSharedTransaction()};
-
-		return ScanSettings::get(_dbSession)->getRecommendationEngineType();
-	}()};
-
-	clearClassifiers();
-
-	switch (engineType)
-	{
-		case ScanSettings::RecommendationEngineType::Features:
-		{
-			auto clustersClassifier {createClustersClassifier()};
-			auto featuresClassifier {createFeaturesClassifier()};
-
-			setClassifierPriorities({featuresClassifier->getName(), clustersClassifier->getName()});
-
-			initAndAddClassifier(std::move(clustersClassifier), databaseChanged); // init first since faster
-			initAndAddClassifier(std::move(featuresClassifier), databaseChanged);
-			break;
-		}
-
-		case ScanSettings::RecommendationEngineType::Clusters:
-			auto clustersClassifier {createClustersClassifier()};
-
-			setClassifierPriorities({clustersClassifier->getName()});
-
-			initAndAddClassifier(std::move(clustersClassifier), databaseChanged);
-			break;
-	}
-
-	LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines reloaded!";
-
-	_sigReloaded.emit();
+	return Database::ScanSettings::get(session)->getRecommendationEngineType();
 }
 
 void
-Engine::setClassifierPriorities(std::initializer_list<std::string_view> classifierPriorities)
+Engine::load(bool forceReload, const ProgressCallback& progressCallback)
+{
+	using namespace Database;
+
+	static const std::unordered_map<ScanSettings::RecommendationEngineType, std::vector<ClassifierType>> classifierMappings
+	{
+		{ScanSettings::RecommendationEngineType::Features, {ClassifierType::Clusters, ClassifierType::Features}},
+		{ScanSettings::RecommendationEngineType::Clusters, {ClassifierType::Clusters}},
+	};
+
+	LMS_LOG(RECOMMENDATION, INFO) << "Reloading recommendation engines...";
+
+	const ScanSettings::RecommendationEngineType engineType {getRecommendationEngineType(_db.getTLSSession())};
+
+	assert(_pendingClassifiers.empty());
+	clearClassifiers();
+
+	auto itClassifierTypes {classifierMappings.find(engineType)};
+	assert(itClassifierTypes != std::cend(classifierMappings));
+	const std::vector<ClassifierType>& classifierTypes {itClassifierTypes->second};
+
+	setClassifierPriorities(classifierTypes);
+
+	std::vector<std::unique_ptr<IClassifier>> classifiers;
+	for (ClassifierType type : classifierTypes)
+		classifiers.emplace_back(createClassifier(type));
+
+	{
+		std::scoped_lock lock {_controlMutex};
+
+		std::transform(std::cbegin(classifiers), std::cend(classifiers), std::inserter(_pendingClassifiers, std::end(_pendingClassifiers)),
+				[](auto& classifier) { return classifier.get(); });
+	}
+
+	for (std::size_t i {}; i < classifiers.size(); ++i)
+		loadClassifier(std::move(classifiers[i]), classifierTypes[i], forceReload, progressCallback);
+
+	LMS_LOG(RECOMMENDATION, INFO) << "Recommendation engines loaded!";
+}
+
+void
+Engine::setClassifierPriorities(const std::vector<ClassifierType>& classifierPriorities)
 {
 	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
 
-	_classifierPriorities.clear();
-	std::transform(std::cbegin(classifierPriorities), std::cend(classifierPriorities), std::back_inserter(_classifierPriorities), [](std::string_view name) { return std::string {name}; });
+	_classifierPriorities = classifierPriorities;
 }
 
 void
 Engine::clearClassifiers()
 {
-	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
+	std::unique_lock lock {_classifiersMutex};
 
 	_classifiers.clear();
 }
 
 void
-Engine::initAndAddClassifier(std::unique_ptr<IClassifier> classifier, bool databaseChanged)
+Engine::loadClassifier(std::unique_ptr<IClassifier> classifier,
+		ClassifierType classifierType,
+		bool forceReload,
+		const ProgressCallback& progressCallback)
 {
-	PendingClassifierHandler pendingClassifier {*this, *classifier.get()};
+	IClassifier* rawClassifier {classifier.get()};
 
-	LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'...";
-	bool res {classifier->init(_dbSession, databaseChanged)};
-	LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE");
+	bool res {};
+	if (!_loadCancelled)
+	{
+		LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "'...";
+
+		auto progress {[&](IClassifier::Progress progress)
+		{
+			progressCallback(Progress {progress.processedElems, progress.totalElems});
+		}};
+
+		res = classifier->load(_db.getTLSSession(), forceReload, progressCallback ? progress : IClassifier::ProgressCallback {});
+
+		LMS_LOG(RECOMMENDATION, INFO) << "Initializing classifier '" << classifier->getName() << "': " << (res ? "SUCCESS" : "FAILURE");
+	}
 
 	if (res)
 	{
-		std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
+		std::unique_lock lock {_classifiersMutex};
 
-		_classifiers.emplace(classifier->getName(), std::move(classifier));
+		_classifiers.emplace(classifierType, std::move(classifier));
 	}
+
+	{
+		std::scoped_lock lock {_controlMutex};
+
+		LMS_LOG(RECOMMENDATION, DEBUG) << "About to erase. _pendingClassifiers size =  " << _pendingClassifiers.size();
+		_pendingClassifiers.erase(rawClassifier);
+		LMS_LOG(RECOMMENDATION, DEBUG) << "Erased. _pendingClassifiers size =  " << _pendingClassifiers.size();
+	}
+
+	_pendingClassifiersCondvar.notify_one();
+
 }
 
 void
-Engine::cancelPendingClassifiers()
+Engine::cancelLoad()
 {
-	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
+	LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading...";
+
+	std::unique_lock lock {_controlMutex};
+
+	LMS_LOG(RECOMMENDATION, DEBUG) << "Still " <<  _pendingClassifiers.size() << " pending classifiers!";
+
+	_loadCancelled = true;
 
 	for (IClassifier* classifier : _pendingClassifiers)
-		classifier->requestCancelInit();
-}
+		classifier->requestCancelLoad();
 
-void
-Engine::addPendingClassifier(IClassifier& classifier)
-{
-	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
+    _pendingClassifiersCondvar.wait(lock, [this] {return _pendingClassifiers.empty();});
+	_loadCancelled = false;
 
-	_pendingClassifiers.insert(&classifier);
-}
-
-void
-Engine::removePendingClassifier(IClassifier& classifier)
-{
-	std::unique_lock<std::shared_mutex> lock {_classifiersMutex};
-
-	_pendingClassifiers.erase(&classifier);
+	LMS_LOG(RECOMMENDATION, DEBUG) << "Cancelling loading DONE";
 }
 
 } // ns Similarity

src/libs/recommendation/impl/Engine.hpp

@@ -19,23 +19,33 @@
 
 #pragma once
 
-#include <map>
+#include <condition_variable>
 #include <shared_mutex>
+#include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
-#include <Wt/WIOService.h>
-
-#include "database/Session.hpp"
 #include "recommendation/IEngine.hpp"
-#include "recommendation/IClassifier.hpp"
+#include "IClassifier.hpp"
+
+namespace Database
+{
+	class Db;
+}
 
 namespace Recommendation
 {
+	enum class ClassifierType
+	{
+		Clusters,
+		Features,
+	};
+
 	class Engine : public IEngine
 	{
 		public:
 			Engine(Database::Db& db);
-			~Engine();
+			~Engine() = default;
 
 			Engine(const Engine&) = delete;
 			Engine(Engine&&) = delete;
@@ -43,58 +53,30 @@ namespace Recommendation
 			Engine& operator=(Engine&&) = delete;
 
 		private:
+			void	load(bool forceReload, const ProgressCallback& progressCallback) override;
+			void	cancelLoad() override;
 
-			void start();
-			void stop();
+			std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override;
+			std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) override;
+			std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override;
+			std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override;
 
-			void requestLoad() override;
-			void requestReload() override;
-			Wt::Signal<>& reloaded() override { return _sigReloaded; }
-
-			std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override;
-			std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) override;
-			std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override;
-			std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override;
-
-
-			void requestReloadInternal(bool databaseChanged);
-			void reload(bool databaseChanged);
-
-			void setClassifierPriorities(std::initializer_list<std::string_view> classifierNames);
+			void setClassifierPriorities(const std::vector<ClassifierType>& classifierTypes);
 			void clearClassifiers();
-			void initAndAddClassifier(std::unique_ptr<IClassifier> classifier, bool databaseChanged);
+			void loadClassifier(std::unique_ptr<IClassifier> classifier, ClassifierType classifierType, bool forceReload, const ProgressCallback& progressCallback);
 
-			class PendingClassifierHandler
-			{
-				public:
-					PendingClassifierHandler(Engine& engine, IClassifier& classifier) : _engine {engine}, _classifier {classifier}
-					{
-						_engine.addPendingClassifier(_classifier);
-					}
+			Database::Db&				_db;
 
-					~PendingClassifierHandler()
-					{
-						_engine.removePendingClassifier(_classifier);
-					}
+			std::mutex							_controlMutex;
+			bool								_loadCancelled {};
+			std::condition_variable 			_pendingClassifiersCondvar;
+			std::unordered_set<IClassifier*>	_pendingClassifiers;
 
-				private:
-					Engine& _engine;
-					IClassifier& _classifier;
-			};
+			std::shared_mutex			_classifiersMutex;
+			using ClassifierContainer = std::unordered_map<ClassifierType, std::unique_ptr<IClassifier>>;
+			ClassifierContainer			_classifiers;
+			std::vector<ClassifierType>	_classifierPriorities; // ordered by priority
 
-			void cancelPendingClassifiers();
-			void addPendingClassifier(IClassifier& classifier);
-			void removePendingClassifier(IClassifier& classifier);
-
-			bool			_running {};
-			Wt::WIOService		_ioService;
-			Database::Session	_dbSession;
-			Wt::Signal<>		_sigReloaded;
-
-			std::shared_mutex	_classifiersMutex;
-			std::map<std::string, std::unique_ptr<IClassifier>> _classifiers;
-			std::vector<std::string> _classifierPriorities; // ordered by priority
-			std::unordered_set<IClassifier*> _pendingClassifiers;
 	};
 
 } // ns Recommendation

src/libs/recommendation/impl/FeaturesClassifierCreator.hpp

@@ -20,7 +20,7 @@
 #pragma once
 
 #include <memory>
-#include "recommendation/IClassifier.hpp"
+#include "IClassifier.hpp"
 
 namespace Recommendation
 {

src/libs/recommendation/impl/IClassifier.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
+#include <functional>
 #include <string_view>
 #include <unordered_set>
-#include <vector>
 
 #include "database/Types.hpp"
 
@@ -40,13 +40,19 @@ namespace Recommendation
 
 			virtual std::string_view getName() const = 0;
 
-			virtual bool init(Database::Session& session, bool databaseChanged) = 0;
-			virtual void requestCancelInit() = 0;
+			struct Progress
+			{
+				std::size_t totalElems {};
+				std::size_t	processedElems {};
+			};
+			using ProgressCallback = std::function<void(const Progress&)>;
+			virtual bool load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) = 0;
+			virtual void requestCancelLoad() = 0;
 
-			virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0;
-			virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const = 0;
-			virtual std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0;
-			virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0;
 	};
 
 } // ns Recommendation

src/libs/recommendation/impl/clusters/ClustersClassifier.cpp

@@ -27,29 +27,29 @@
 #include "database/TrackList.hpp"
 
 namespace Recommendation {
-	
+
 std::unique_ptr<IClassifier> createClustersClassifier()
 {
 	return std::make_unique<ClusterClassifier>();
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount) const
 {
 	auto transaction {dbSession.createSharedTransaction()};
 
-	auto tracks {Database::Track::getSimilarTracks(dbSession, trackIds, 0, maxCount)};
-	std::vector<Database::IdType> res;
-	res.reserve(tracks.size());
+	const auto tracks {Database::Track::getSimilarTracks(dbSession, trackIds, 0, maxCount)};
 
-	std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res), [](const auto& track) { return track.id(); });
+	std::unordered_set<Database::IdType> res;
+	std::transform(std::cbegin(tracks), std::cend(tracks), std::inserter(res, std::end(res)),
+			[](const auto& track) { return track.id(); });
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> res;
+	std::unordered_set<Database::IdType> res;
 
 	auto transaction {session.createSharedTransaction()};
 
@@ -57,18 +57,17 @@ ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Dat
 	if (!trackList)
 		return res;
 
-	const std::vector<Database::Track::pointer> tracks {trackList->getSimilarTracks(0, maxCount)};
-	res.reserve(tracks.size());
-	std::transform(std::cbegin(tracks), std::cend(tracks), std::back_inserter(res),
+	const auto tracks {trackList->getSimilarTracks(0, maxCount)};
+	std::transform(std::cbegin(tracks), std::cend(tracks), std::inserter(res, std::end(res)),
 			[](const Database::Track::pointer& track) { return track.id(); });
 
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> res;
+	std::unordered_set<Database::IdType> res;
 
 	auto transaction {dbSession.createSharedTransaction()};
 
@@ -77,16 +76,16 @@ ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::Id
 		return res;
 
 	const auto releases {release->getSimilarReleases(0, maxCount)};
-	res.reserve(releases.size());
-	std::transform(std::cbegin(releases), std::cend(releases), std::back_inserter(res), [](const auto& release) { return release.id(); });
+	std::transform(std::cbegin(releases), std::cend(releases), std::inserter(res, std::end(res)),
+			[](const auto& release) { return release.id(); });
 
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> res;
+	std::unordered_set<Database::IdType> res;
 
 	auto transaction {dbSession.createSharedTransaction()};
 
@@ -95,8 +94,8 @@ ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdT
 		return res;
 
 	const auto artists {artist->getSimilarArtists(0, maxCount)};
-	res.reserve(artists.size());
-	std::transform(std::cbegin(artists), std::cend(artists), std::back_inserter(res), [](const auto& artist) { return artist.id(); });
+	std::transform(std::cbegin(artists), std::cend(artists), std::inserter(res, std::end(res)),
+			[](const auto& artist) { return artist.id(); });
 
 	return res;
 }

src/libs/recommendation/impl/clusters/ClustersClassifier.hpp

@@ -19,8 +19,7 @@
 
 #pragma once
 
-#include "recommendation/IClassifier.hpp"
-
+#include "IClassifier.hpp"
 
 namespace Recommendation
 {
@@ -38,14 +37,13 @@ namespace Recommendation
 
 			std::string_view getName() const override { return "Clusters"; }
 
-			bool init(Database::Session&, bool) override {return true;}
-			void requestCancelInit() override {}
-
-			std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
-			std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
-			std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
-			std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
+			bool load(Database::Session&, bool, const ProgressCallback&) override { return true; }
+			void requestCancelLoad() override {}
 
+			std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
+			std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
+			std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
+			std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
 };
 
 } // namespace Recommendation

src/libs/recommendation/impl/features/FeaturesClassifier.cpp

@@ -127,7 +127,7 @@ getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t
 }
 
 bool
-FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSettings& trainSettings)
+FeaturesClassifier::loadFromTraining(Database::Session& session, const TrainSettings& trainSettings, const ProgressCallback& progressCallback)
 {
 	LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier...";
 
@@ -158,7 +158,7 @@ FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSett
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features...";
 	for (Database::IdType trackId : trackIds)
 	{
-		if (_initCancelled)
+		if (_loadCancelled)
 			return false;
 
 		std::optional<FeatureValuesMap> featureValuesMap;
@@ -201,23 +201,26 @@ FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSett
 	SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)};
 	network.setDataWeights(weights);
 
-	auto progressIndicator{[](const auto& iter)
-		{
-			LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount;
-		}};
+	auto somProgressCallback{[&](const SOM::Network::CurrentIteration& iter)
+	{
+		LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount;
+		progressCallback(Progress {iter.idIteration, iter.iterationCount});
+	}};
 
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Training network...";
-	network.train(samples, trainSettings.iterationCount, progressIndicator);
+	network.train(samples, trainSettings.iterationCount,
+			progressCallback ? somProgressCallback : SOM::Network::ProgressCallback {},
+			[this] { return _loadCancelled; });
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Training network DONE";
 
-	if (_initCancelled)
+	if (_loadCancelled)
 		return false;
 
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks...";
 	ObjectPositions trackPositions;
 	for (std::size_t i {}; i < samples.size(); ++i)
 	{
-		if (_initCancelled)
+		if (_loadCancelled)
 			return false;
 
 		const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
@@ -227,86 +230,100 @@ FeaturesClassifier::initFromTraining(Database::Session& session, const TrainSett
 
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks DONE";
 
-	return init(session, std::move(network), std::move(trackPositions));
+	return load(session, std::move(network), std::move(trackPositions));
 }
 
 bool
-FeaturesClassifier::initFromCache(Database::Session& session, const FeaturesClassifierCache& cache)
+FeaturesClassifier::loadFromCache(Database::Session& session, const FeaturesClassifierCache& cache)
 {
 	LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier from cache...";
 
-	return init(session, std::move(cache._network), cache._trackPositions);
+	return load(session, std::move(cache._network), cache._trackPositions);
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType trackListId, std::size_t maxCount) const
 {
-	const std::unordered_set<Database::IdType> trackIds {[&]() -> std::unordered_set<Database::IdType>
+	const std::unordered_set<Database::IdType> trackIds {[&]
 	{
+		std::unordered_set<Database::IdType> res;
+
 		auto transaction {session.createSharedTransaction()};
 
 		const Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)};
 		if (trackList)
 		{
 			const std::vector<Database::IdType> orderedTrackIds {trackList->getTrackIds()};
-			return std::unordered_set<Database::IdType> {std::cbegin(orderedTrackIds), std::cend(orderedTrackIds)};
+			res = std::unordered_set<Database::IdType>(std::cbegin(orderedTrackIds), std::cend(orderedTrackIds));
 		}
 
-		return {};
+		return res;
 	}()};
 
 	return getSimilarTracks(session, trackIds, maxCount);
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksIds, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)};
-
+	auto similarTrackIds {getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount)};
 	if (!similarTrackIds.empty())
 	{
 		// Report only existing ids
 		auto transaction {session.createSharedTransaction()};
 
-		similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds),
-					[&](Database::IdType trackId) { return Database::Track::getById(session, trackId) == Database::Track::pointer {}; }),
-				std::cend(similarTrackIds));
+		for (auto it {std::begin(similarTrackIds)}; it != std::end(similarTrackIds);)
+		{
+			const Database::IdType trackId {*it};
+			if (!Database::Track::getById(session, trackId))
+				it = similarTrackIds.erase(it);
+			else
+				it++;
+		}
 	}
 
 	return similarTrackIds;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)};
-
+	auto similarReleaseIds {getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount)};
 	if (!similarReleaseIds.empty())
 	{
 		// Report only existing ids
 		auto transaction {session.createSharedTransaction()};
 
-		similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds),
-					[&](Database::IdType releaseId) { return Database::Release::getById(session, releaseId) == Database::Release::pointer {}; }),
-				std::cend(similarReleaseIds));
+		for (auto it {std::begin(similarReleaseIds)}; it != std::end(similarReleaseIds);)
+		{
+			const Database::IdType releaseId {*it};
+			if (!Database::Release::getById(session, releaseId))
+				it = similarReleaseIds.erase(it);
+			else
+				it++;
+		}
 	}
 
 	return similarReleaseIds;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const
 {
-	std::vector<Database::IdType> similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)};
-
+	auto similarArtistIds {getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount)};
 	if (!similarArtistIds.empty())
 	{
 		// Report only existing ids
 		auto transaction {session.createSharedTransaction()};
 
-		similarArtistIds.erase(std::remove_if(std::begin(similarArtistIds), std::end(similarArtistIds),
-					[&](Database::IdType artistId) { return Database::Artist::getById(session, artistId) == Database::Artist::pointer {}; }),
-				std::cend(similarArtistIds));
+		for (auto it {std::begin(similarArtistIds)}; it != std::end(similarArtistIds);)
+		{
+			const Database::IdType artistId {*it};
+			if (!Database::Release::getById(session, artistId))
+				it = similarArtistIds.erase(it);
+			else
+				it++;
+		}
 	}
 
 	return similarArtistIds;
@@ -319,22 +336,23 @@ FeaturesClassifier::toCache() const
 }
 
 bool
-FeaturesClassifier::init(Database::Session& session, bool databaseChanged)
+FeaturesClassifier::load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback)
 {
-	if (databaseChanged)
+	if (forceReload)
 	{
-		LMS_LOG(RECOMMENDATION, DEBUG) << "Database changed: invidating cache";
 		FeaturesClassifierCache::invalidate();
 	}
-
-	std::optional<FeaturesClassifierCache> cache {FeaturesClassifierCache::read()};
-	if (cache)
-		return initFromCache(session, *cache);
+	else
+	{
+		const std::optional<FeaturesClassifierCache> cache {FeaturesClassifierCache::read()};
+		if (cache)
+			return loadFromCache(session, *cache);
+	}
 
 	TrainSettings trainSettings;
 	trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings();
 
-	bool res {initFromTraining(session, trainSettings)};
+	const bool res {loadFromTraining(session, trainSettings, progressCallback)};
 	if (res)
 		toCache().write();
 
@@ -342,14 +360,14 @@ FeaturesClassifier::init(Database::Session& session, bool databaseChanged)
 }
 
 void
-FeaturesClassifier::requestCancelInit()
+FeaturesClassifier::requestCancelLoad()
 {
 	LMS_LOG(RECOMMENDATION, DEBUG) << "Requesting init cancellation";
-	_initCancelled = true;
+	_loadCancelled = true;
 }
 
 bool
-FeaturesClassifier::init(Database::Session& session,
+FeaturesClassifier::load(Database::Session& session,
 			SOM::Network network,
 			const ObjectPositions& tracksPosition)
 {
@@ -367,7 +385,7 @@ FeaturesClassifier::init(Database::Session& session,
 
 	for (auto itTrackCoord : tracksPosition)
 	{
-		if (_initCancelled)
+		if (_loadCancelled)
 			return false;
 
 		auto transaction {session.createSharedTransaction()};
@@ -399,7 +417,7 @@ FeaturesClassifier::init(Database::Session& session,
 
 	_network = std::make_unique<SOM::Network>(std::move(network));
 
-	LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully initialized!";
+	LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully loaded!";
 
 	return true;
 }
@@ -439,13 +457,13 @@ FeaturesClassifier::getObjectsIds(const std::unordered_set<SOM::Position>& posit
 	return res;
 }
 
-std::vector<Database::IdType>
+std::unordered_set<Database::IdType>
 FeaturesClassifier::getSimilarObjects(const std::unordered_set<Database::IdType>& ids,
 		const MatrixOfObjects& objectsMap,
 		const ObjectPositions& objectPosition,
 		std::size_t maxCount) const
 {
-	std::vector<Database::IdType> res;
+	std::unordered_set<Database::IdType> res;
 
 	std::unordered_set<SOM::Position> searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPosition)};
 	if (searchedRefVectorsPosition.empty())
@@ -459,14 +477,13 @@ FeaturesClassifier::getSimilarObjects(const std::unordered_set<Database::IdType>
 		for (auto id : ids)
 			closestObjectIds.erase(id);
 
+		for (auto it {std::cbegin(closestObjectIds)}; it != std::cend(closestObjectIds); ++it)
 		{
-			std::vector<Database::IdType> objectIdsToAdd {std::cbegin(closestObjectIds), std::cend(closestObjectIds)};
-			Random::shuffleContainer(objectIdsToAdd );
-			std::copy(std::cbegin(objectIdsToAdd), std::cend(objectIdsToAdd), std::back_inserter(res));
-		}
+			if (res.size() == maxCount)
+				break;
 
-		if (res.size() > maxCount)
-			res.resize(maxCount);
+			res.insert(*it);
+		}
 
 		if (res.size() == maxCount)
 			break;

src/libs/recommendation/impl/features/FeaturesClassifier.hpp

@@ -19,15 +19,16 @@
 
 #pragma once
 
+#include <functional>
 #include <unordered_map>
 #include <optional>
 #include <string>
 
-#include "recommendation/IClassifier.hpp"
 #include "som/DataNormalizer.hpp"
 #include "som/Network.hpp"
 #include "FeaturesClassifierCache.hpp"
 #include "FeaturesDefs.hpp"
+#include "IClassifier.hpp"
 
 namespace Database
 {
@@ -53,19 +54,20 @@ class FeaturesClassifier : public IClassifier
 		static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; }
 
 		static const FeatureSettingsMap& getDefaultTrainFeatureSettings();
+
 	private:
 
 		std::string_view getName() const override { return "Features"; }
 
-		bool init(Database::Session& session, bool databaseChanged) override;
-		void requestCancelInit() override;
+		bool load(Database::Session& session, bool forceReload, const ProgressCallback& progressCallback) override;
+		void requestCancelLoad() override;
 
-		std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
-		std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
-		std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
-		std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
+		std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
+		std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
+		std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
+		std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
 
-		bool initFromCache(Database::Session& session, const FeaturesClassifierCache& cache);
+		bool loadFromCache(Database::Session& session, const FeaturesClassifierCache& cache);
 
 		// Use training (may be very slow)
 		struct TrainSettings
@@ -74,12 +76,12 @@ class FeaturesClassifier : public IClassifier
 			float sampleCountPerNeuron {4};
 			FeatureSettingsMap featureSettingsMap;
 		};
-		bool initFromTraining(Database::Session& session, const TrainSettings& trainSettings);
+		bool loadFromTraining(Database::Session& session, const TrainSettings& trainSettings, const ProgressCallback& progressCallback);
 
 		using ObjectPositions = std::unordered_map<Database::IdType, std::unordered_set<SOM::Position>>;
 		using MatrixOfObjects = SOM::Matrix<std::unordered_set<Database::IdType>>;
 
-		bool init(Database::Session& session,
+		bool load(Database::Session& session,
 				SOM::Network network,
 				const ObjectPositions& tracksPosition);
 
@@ -88,12 +90,12 @@ class FeaturesClassifier : public IClassifier
 		static std::unordered_set<SOM::Position> getMatchingRefVectorsPosition(const std::unordered_set<Database::IdType>& ids, const ObjectPositions& objectPositions);
 		static std::unordered_set<Database::IdType> getObjectsIds(const std::unordered_set<SOM::Position>& positionSet, const MatrixOfObjects& objectsMap);
 
-		std::vector<Database::IdType> getSimilarObjects(const std::unordered_set<Database::IdType>& ids,
+		std::unordered_set<Database::IdType> getSimilarObjects(const std::unordered_set<Database::IdType>& ids,
 				const SOM::Matrix<std::unordered_set<Database::IdType>>& objectsMap,
 				const ObjectPositions& objectPosition,
 				std::size_t maxCount) const;
 
-		bool				_initCancelled {};
+		bool				_loadCancelled {};
 		std::unique_ptr<SOM::Network>	_network;
 		double				_networkRefVectorsDistanceMedian {};
 

src/libs/recommendation/include/recommendation/IEngine.hpp

@@ -20,11 +20,8 @@
 #pragma once
 
 #include <functional>
-#include <vector>
 #include <unordered_set>
 
-#include <Wt/WSignal.h>
-
 #include "database/Types.hpp"
 
 namespace Database
@@ -40,16 +37,19 @@ namespace Recommendation
 		public:
 			virtual ~IEngine() = default;
 
-			virtual void requestLoad() = 0;
+			struct Progress
+			{
+				std::size_t totalElems {};
+				std::size_t	processedElems {};
+			};
+			using ProgressCallback = std::function<void(const Progress&)>;
+			virtual void load(bool forceReload, const ProgressCallback& progressCallback = {}) = 0;
+			virtual void cancelLoad() = 0;
 
-			virtual void requestReload() = 0;
-			virtual Wt::Signal<>& reloaded() = 0;
-
-			// Closest results first
-			virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0;
-			virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) = 0;
-			virtual std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) = 0;
-			virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) = 0;
+			virtual std::unordered_set<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0;
 	};
 
 	std::unique_ptr<IEngine> createEngine(Database::Db& db);