Started to restore clusters based recommendation engine

README.md

@@ -17,7 +17,7 @@ A [demo](http://lms.demo.poupon.io) instance is available, with the following li
 * Audio transcode for maximum interoperability and low bandwith requirements
 * Persistent play queue across sessions
 * Subsonic API
-* Album artist
+* Compilation support
 * Multi-value tags: artists, genres, ...
 * Custom tags (ex: _mood_, _genre_, _albummood_, _albumgrouping_, ...)
 * MusicBrainzID support to handle duplicated artist and release names

approot/messages.xml

@@ -39,9 +39,9 @@
 <message id="Lms.Admin.Database.monthly">Monthly</message>
 <message id="Lms.Admin.Database.never">Never</message>
 <message id="Lms.Admin.Database.path">Media root directory</message>
-<message id="Lms.Admin.Database.similarity-engine-type">Recommendation engine</message>
+<message id="Lms.Admin.Database.recommendation-engine-type">Recommendation engine</message>
-<message id="Lms.Admin.Database.similarity-engine-type.clusters">Tags based</message>
+<message id="Lms.Admin.Database.recommendation-engine-type.clusters">Tags based</message>
-<message id="Lms.Admin.Database.similarity-engine-type.features">Audio analysis based</message>
+<message id="Lms.Admin.Database.recommendation-engine-type.features">Audio analysis based</message>
 <message id="Lms.Admin.Database.scan-complete">Scan complete: {1} total files, {2} additions, {3} updates, {4} deletions, {5} duplicates, {6} errors</message>
 <message id="Lms.Admin.Database.scan-launched">Scan launched!</message>
 <message id="Lms.Admin.Database.scan-options">Scan options</message>

approot/messages_fr.xml

@@ -39,9 +39,9 @@
 <message id="Lms.Admin.Database.monthly">Tous les mois</message>
 <message id="Lms.Admin.Database.never">Jamais</message>
 <message id="Lms.Admin.Database.path">Dossier racine des fichiers de musique</message>
-<message id="Lms.Admin.Database.similarity-engine-type">Moteur de recommandation</message>
+<message id="Lms.Admin.Database.recommendation-engine-type">Moteur de recommandation</message>
-<message id="Lms.Admin.Database.similarity-engine-type.clusters">Basé sur les tags</message>
+<message id="Lms.Admin.Database.recommendation-engine-type.clusters">Basé sur les tags</message>
-<message id="Lms.Admin.Database.similarity-engine-type.features">Basé sur l'analyse audio</message>
+<message id="Lms.Admin.Database.recommendation-engine-type.features">Basé sur l'analyse audio</message>
 <message id="Lms.Admin.Database.scan-complete">Scan terminé : {1} fichiers, {2} ajouts, {3} mises à jour, {4} suppressions, {5} duplicatas, {6} erreurs</message>
 <message id="Lms.Admin.Database.scan-launched">Scan lancé !</message>
 <message id="Lms.Admin.Database.scan-options">Options </message>

src/libs/database/impl/Artist.cpp

@@ -104,6 +104,21 @@ Artist::getAllOrphans(Session& session)
 	return std::vector<pointer>(res.begin(), res.end());
 }
 
+std::vector<IdType>
+Artist::getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit)
+{
+	session.checkSharedLocked();
+
+	Wt::Dbo::collection<IdType> res = session.getDboSession().query<IdType>
+		("SELECT DISTINCT a.id FROM artist a"
+		 	" INNER JOIN track t ON t.id = t_a_l.track_id INNER JOIN track_artist_link t_a_l ON t_a_l.artist_id = a.id"
+		 	" INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
+		.limit(limit ? static_cast<int>(*limit) : -1);
+
+	return std::vector<IdType>(res.begin(), res.end());
+}
+
+
 static
 Wt::Dbo::Query<Artist::pointer>
 getQuery(Session& session,

src/libs/database/impl/Release.cpp

@@ -252,6 +252,21 @@ Release::getByFilter(Session& session,
 	return res;
 }
 
+std::vector<IdType>
+Release::getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit)
+{
+	session.checkSharedLocked();
+
+	Wt::Dbo::collection<IdType> res = session.getDboSession().query<IdType>
+		("SELECT DISTINCT r.id FROM release r"
+		 	" INNER JOIN track t ON t.release_id = r.id"
+		 	" INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
+		.limit(limit ? static_cast<int>(*limit) : -1);
+
+	return std::vector<IdType>(res.begin(), res.end());
+}
+
+
 std::optional<std::size_t>
 Release::getTotalTrackNumber(void) const
 {

src/libs/database/impl/Session.cpp

@@ -118,7 +118,7 @@ Session::doDatabaseMigrationIfNeeded()
 		{
 			_session.execute("DROP TABLE similarity_settings");
 			_session.execute("DROP TABLE similarity_settings_feature");
-			_session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast<int>(ScanSettings::SimilarityEngineType::Clusters)) + ")");
+			_session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast<int>(ScanSettings::RecommendationEngineType::Clusters)) + ")");
 		}
 		else if (version == 8)
 		{

src/libs/database/impl/Track.cpp

@@ -163,6 +163,19 @@ Track::getAllIdsWithFeatures(Session& session, std::optional<std::size_t> limit)
 	return std::vector<IdType>(res.begin(), res.end());
 }
 
+std::vector<IdType>
+Track::getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit)
+{
+	session.checkSharedLocked();
+
+	Wt::Dbo::collection<IdType> res = session.getDboSession().query<IdType>
+		("SELECT DISTINCT t.id FROM track t"
+		 	" INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
+		.limit(limit ? static_cast<int>(*limit) : -1);
+
+	return std::vector<IdType>(res.begin(), res.end());
+}
+
 std::vector<Cluster::pointer>
 Track::getClusters(void) const
 {
@@ -263,7 +276,7 @@ Track::getByFilter(Session& session,
 
 std::vector<Track::pointer>
 Track::getSimilarTracks(Session& session,
-				const std::set<IdType>& tracks,
+				const std::unordered_set<IdType>& tracks,
 				std::optional<std::size_t> offset,
 				std::optional<std::size_t> size)
 {

src/libs/database/include/database/Artist.hpp

@@ -68,6 +68,7 @@ class Artist : public Wt::Dbo::Dbo<Artist>
 		static std::vector<IdType>	getAllIds(Session& session);
 		static std::vector<pointer>	getAllOrphans(Session& session); // No track related
 		static std::vector<pointer>	getLastAdded(Session& session, Wt::WDateTime after, std::optional<std::size_t> size = {});
+		static std::vector<IdType>	getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit = {});
 
 		// Accessors
 		const std::string& getName(void) const { return _name; }

src/libs/database/include/database/Release.hpp

@@ -66,6 +66,7 @@ class Release : public Wt::Dbo::Dbo<Release>
 							std::optional<std::size_t> offset,
 							std::optional<std::size_t> size,
 							bool& moreExpected);
+		static std::vector<IdType>	getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit = {});
 
 		std::vector<Wt::Dbo::ptr<Track>> getTracks(const std::set<IdType>& clusters = std::set<IdType>()) const;
 		std::size_t			getTracksCount() const;

src/libs/database/include/database/ScanSettings.hpp

@@ -43,7 +43,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
 		};
 
 		// Do not modify values (just add)
-		enum class SimilarityEngineType
+		enum class RecommendationEngineType
 		{
 			Clusters = 0,
 			Features,
@@ -60,7 +60,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
 		UpdatePeriod getUpdatePeriod() const { return _updatePeriod; }
 		std::vector<Wt::Dbo::ptr<ClusterType>> getClusterTypes() const;
 		std::set<std::filesystem::path> getAudioFileExtensions() const;
-		SimilarityEngineType getSimilarityEngineType() const { return _similarityEngineType; }
+		RecommendationEngineType getRecommendationEngineType() const { return _recommendationEngineType; }
 
 		// Setters
 		void addAudioFileExtension(const std::filesystem::path& ext);
@@ -68,7 +68,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
 		void setUpdateStartTime(Wt::WTime t) { _startTime = t; }
 		void setUpdatePeriod(UpdatePeriod p) { _updatePeriod = p; }
 		void setClusterTypes(Session& session, const std::set<std::string>& clusterTypeNames);
-		void setSimilarityEngineType(SimilarityEngineType type) { _similarityEngineType = type; }
+		void setRecommendationEngineType(RecommendationEngineType type) { _recommendationEngineType = type; }
 		void incScanVersion();
 
 		template<class Action>
@@ -79,7 +79,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
 			Wt::Dbo::field(a, _startTime,		"start_time");
 			Wt::Dbo::field(a, _updatePeriod,	"update_period");
 			Wt::Dbo::field(a, _audioFileExtensions,	"audio_file_extensions");
-			Wt::Dbo::field(a, _similarityEngineType,"similarity_engine_type");
+			Wt::Dbo::field(a, _recommendationEngineType,"similarity_engine_type");
 			Wt::Dbo::hasMany(a, _clusterTypes, Wt::Dbo::ManyToOne, "scan_settings");
 		}
 
@@ -89,7 +89,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
 		std::string	_mediaDirectory;
 		Wt::WTime	_startTime = Wt::WTime {0,0,0};
 		UpdatePeriod	_updatePeriod {UpdatePeriod::Never};
-		SimilarityEngineType _similarityEngineType {SimilarityEngineType::Clusters};
+		RecommendationEngineType _recommendationEngineType {RecommendationEngineType::Clusters};
 		std::string	_audioFileExtensions {".alac .mp3 .ogg .oga .aac .m4a .m4b .flac .wav .wma .aif .aiff .ape .mpc .shn .opus"};
 		Wt::Dbo::collection<Wt::Dbo::ptr<ClusterType>>	_clusterTypes;
 };

src/libs/database/include/database/Track.hpp

@@ -22,8 +22,9 @@
 #include <chrono>
 #include <filesystem>
 #include <optional>
-#include <vector>
 #include <string>
+#include <unordered_set>
+#include <vector>
 
 #include <Wt/Dbo/Dbo.h>
 #include <Wt/WDateTime.h>
@@ -58,7 +59,7 @@ class Track : public Wt::Dbo::Dbo<Track>
 		static pointer getById(Session& session, IdType id);
 		static pointer getByMBID(Session& session, const UUID& MBID);
 		static std::vector<pointer>	getSimilarTracks(Session& session,
-							const std::set<IdType>& trackIds,
+							const std::unordered_set<IdType>& trackIds,
 							std::optional<std::size_t> offset = {},
 							std::optional<std::size_t> size = {});
 		static std::vector<pointer>	getByClusters(Session& session,
@@ -78,6 +79,7 @@ class Track : public Wt::Dbo::Dbo<Track>
 		static std::vector<pointer>	getLastAdded(Session& session, const Wt::WDateTime& after, std::optional<std::size_t> size = 1);
 		static std::vector<pointer>	getAllWithMBIDAndMissingFeatures(Session& session);
 		static std::vector<IdType>	getAllIdsWithFeatures(Session& session, std::optional<std::size_t> limit = {});
+		static std::vector<IdType>	getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit = {});
 
 		// Create utility
 		static pointer	create(Session& session, const std::filesystem::path& p);

src/libs/recommendation/CMakeLists.txt

@@ -1,5 +1,6 @@
 
 add_library(lmsrecommendation SHARED
+	impl/clusters/ClustersClassifier.cpp
 	impl/Engine.cpp
 	impl/ClassifierCreator.cpp
 	)

src/libs/recommendation/impl/ClassifierCreator.cpp

@@ -17,22 +17,17 @@
  * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "recommendation/ClustersClassifierCreator.hpp"
 #include "recommendation/FeaturesClassifierCreator.hpp"
-#include "recommendation/Classifier.hpp"
+#include "recommendation/IClassifier.hpp"
 
 namespace Recommendation
 {
 
-	std::unique_ptr<Classifier> createClustersClassifier()
+	std::unique_ptr<IClassifier> createFeaturesClassifier()
 	{
 		return {};
 	}
 	
-	std::unique_ptr<Classifier> createFeaturesClassifier()
-	{
-		return {};
-	}
 
 }
 

src/libs/recommendation/impl/Engine.cpp

@@ -19,30 +19,49 @@
 
 #include "Engine.hpp"
 
-//#include "features/SimilarityFeaturesScannerAddon.hpp"
+#include "recommendation/ClustersClassifierCreator.hpp"
-//#include "cluster/SimilarityClusterSearcher.hpp"
+#include "recommendation/FeaturesClassifierCreator.hpp"
 
 #include "database/ScanSettings.hpp"
+#include "database/Session.hpp"
 #include "database/TrackList.hpp"
 
 namespace Recommendation {
 
 std::unique_ptr<IEngine>
-createEngine()
+createEngine(Database::Session& session)
 {
-	return std::make_unique<Engine>();
+	return std::make_unique<Engine>(session);
+}
+
+Engine::Engine(Database::Session& session)
+{
+	reloadSettings(session);
 }
 
 void
-Engine::clearClassifiers()
+Engine::reloadSettings(Database::Session& session)
 {
-	_classifiers.clear();
+	using namespace Database;
-}
 
-void
+	const ScanSettings::RecommendationEngineType engineType {[&]()
-Engine::addClassifier(std::unique_ptr<Classifier> classifier, unsigned priority)
+	{
-{
+		auto transaction {session.createSharedTransaction()};
-	_classifiers.emplace(priority, std::move(classifier));
+		return ScanSettings::get(session)->getRecommendationEngineType();
+	}()};
+
+	clearClassifiers();
+
+	switch (engineType)
+	{
+		case ScanSettings::RecommendationEngineType::Features:
+//			_classifiers.emplace(0, createFeaturesClassifier()); // higher priority
+//			[[fallthrough]];
+
+		case ScanSettings::RecommendationEngineType::Clusters:
+			_classifiers.emplace(1, createClustersClassifier(session)); // lower priority
+			break;
+	}
 }
 
 std::vector<Database::IdType>
@@ -79,60 +98,52 @@ Engine::getSimilarTracksFromTrackList(Database::Session& /*session*/, Database::
 }
 
 std::vector<Database::IdType>
-Engine::getSimilarTracks(Database::Session& /*dbSession*/, const std::unordered_set<Database::IdType>& /*trackIds*/, std::size_t /*maxCount*/)
+Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount)
 {
-#if 0
+	for (const auto& [priority, classifier] : _classifiers)
-	auto engineType {getEngineType(dbSession)};
-	auto somSearcher {_somAddon.getSearcher()};
-
-	if (engineType == Database::ScanSettings::SimilarityEngineType::Features
-		&& somSearcher
-		&& std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return somSearcher->isTrackClassified(trackId); } ))
 	{
-		return somSearcher->getSimilarTracks(trackIds, maxCount);
+		if (std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return classifier->isTrackClassified(trackId); } ))
+			return classifier->getSimilarTracks(dbSession, trackIds, maxCount);
 	}
-	else
+
-		return ClusterEngine::getSimilarTracks(dbSession, trackIds, maxCount);
-#endif
 	return {};
 }
 
 std::vector<Database::IdType>
-Engine::getSimilarReleases(Database::Session& /*dbSession*/, Database::IdType /*releaseId*/, std::size_t /*maxCount*/)
+Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount)
 {
-#if 0
+	for (const auto& [priority, classifier] : _classifiers)
-	auto engineType {getEngineType(dbSession)};
-	auto somSearcher {_somAddon.getSearcher()};
-
-	if (engineType == Database::ScanSettings::SimilarityEngineType::Features
-		&& somSearcher
-		&& somSearcher->isReleaseClassified(releaseId))
 	{
-		return somSearcher->getSimilarReleases(releaseId, maxCount);
+		if (classifier->isReleaseClassified(releaseId))
+			return classifier->getSimilarReleases(dbSession, releaseId, maxCount);
 	}
-	else
+
-		return ClusterEngine::getSimilarReleases(dbSession, releaseId, maxCount);
-#endif
 	return {};
 }
 
 std::vector<Database::IdType>
-Engine::getSimilarArtists(Database::Session& /*dbSession*/, Database::IdType /*artistId*/, std::size_t /*maxCount*/)
+Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount)
 {
-#if 0
+	for (const auto& [priority, classifier] : _classifiers)
-	auto engineType {getEngineType(dbSession)};
-	auto somSearcher {_somAddon.getSearcher()};
-
-	if (engineType == Database::ScanSettings::SimilarityEngineType::Features
-		&& somSearcher
-		&& somSearcher->isArtistClassified(artistId))
 	{
-		return somSearcher->getSimilarArtists(artistId, maxCount);
+		if (classifier->isArtistClassified(artistId))
+			return classifier->getSimilarArtists(dbSession, artistId, maxCount);
 	}
-	else
+
-		return ClusterEngine::getSimilarArtists(dbSession, artistId, maxCount);
-#endif
 	return {};
 }
 
+void
+Engine::clearClassifiers()
+{
+	_classifiers.clear();
+}
+
+void
+Engine::addClassifier(std::unique_ptr<IClassifier> classifier, unsigned priority)
+{
+	_classifiers.emplace(priority, std::move(classifier));
+}
+
+
 } // ns Similarity

src/libs/recommendation/impl/Engine.hpp

@@ -22,7 +22,7 @@
 #include <map>
 
 #include "recommendation/IEngine.hpp"
-#include "recommendation/Classifier.hpp"
+#include "recommendation/IClassifier.hpp"
 
 namespace Database
 {
@@ -34,17 +34,22 @@ namespace Recommendation
 	class Engine : public IEngine
 	{
 		public:
+			Engine(Database::Session& session);
+
+		private:
+
+			void reloadSettings(Database::Session& session) override;
+
 			// Closest results first
 			std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override;
 			std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) override;
 			std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override;
 			std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override;
 
-		private:
 			void clearClassifiers();
-			void addClassifier(std::unique_ptr<Classifier> classifier, unsigned priority);
+			void addClassifier(std::unique_ptr<IClassifier> classifier, unsigned priority);
 
-			std::map<unsigned, std::unique_ptr<Classifier>> _classifiers;
+			std::map<unsigned, std::unique_ptr<IClassifier>> _classifiers;
 	};
 
 } // ns Recommendation

src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.hpp

@@ -1,40 +0,0 @@
-/*
- * Copyright (C) 2018 Emeric Poupon
- *
- * This file is part of LMS.
- *
- * LMS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LMS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#pragma once
-
-#include <set>
-
-#include "database/Types.hpp"
-
-namespace Database {
-	class Session;
-}
-
-namespace Similarity {
-
-namespace ClusterSearcher
-{
-	std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount);
-	std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::set<Database::IdType>& tracksId, std::size_t maxCount);
-	std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount);
-	std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount);
-};
-
-} // namespace Similarity

src/libs/recommendation/impl/clusters/ClustersClassifier.cpp

@@ -17,7 +17,7 @@
  * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "SimilarityClusterSearcher.hpp"
+#include "ClustersClassifier.hpp"
 
 #include "database/Artist.hpp"
 #include "database/Cluster.hpp"
@@ -26,11 +26,60 @@
 #include "database/Track.hpp"
 #include "database/TrackList.hpp"
 
-namespace Similarity {
+namespace Recommendation {
-namespace ClusterSearcher {
+	
+std::unique_ptr<IClassifier> createClustersClassifier(Database::Session& session)
+{
+	return std::make_unique<ClusterClassifier>(session);
+}
+
+ClusterClassifier::ClusterClassifier(Database::Session& session)
+{
+	classify(session);
+}
+
+void
+ClusterClassifier::classify(Database::Session& session)
+{
+	auto transaction {session.createSharedTransaction()};
+
+	{
+		std::vector<Database::IdType> trackIds {Database::Track::getAllIdsWithClusters(session)};
+		_classifiedTracks = std::unordered_set<Database::IdType>(std::cbegin(trackIds), std::cend(trackIds));
+	}
+
+	{
+		std::vector<Database::IdType> releaseIds {Database::Release::getAllIdsWithClusters(session)};
+		_classifiedReleases = std::unordered_set<Database::IdType>(std::cbegin(releaseIds), std::cend(releaseIds));
+	}
+
+	{
+		std::vector<Database::IdType> artistIds {Database::Artist::getAllIdsWithClusters(session)};
+		_classifiedArtists = std::unordered_set<Database::IdType>(std::cbegin(artistIds), std::cend(artistIds));
+	}
+}
+
+bool
+ClusterClassifier::isTrackClassified(Database::IdType trackId) const
+{
+	return _classifiedTracks.find(trackId) != std::cend(_classifiedTracks);
+}
+
+bool
+ClusterClassifier::isReleaseClassified(Database::IdType releaseId) const
+{
+	return _classifiedReleases.find(releaseId) != std::cend(_classifiedReleases);
+}
+
+bool
+ClusterClassifier::isArtistClassified(Database::IdType artistId) const
+{
+	return _classifiedArtists.find(artistId) != std::cend(_classifiedArtists);
+}
+
 
 std::vector<Database::IdType>
-getSimilarTracks(Database::Session& dbSession, const std::set<Database::IdType>& trackIds, std::size_t maxCount)
+ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount) const
 {
 	auto transaction {dbSession.createSharedTransaction()};
 
@@ -43,7 +92,7 @@ getSimilarTracks(Database::Session& dbSession, const std::set<Database::IdType>&
 }
 
 std::vector<Database::IdType>
-getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount)
+ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const
 {
 	std::vector<Database::IdType> res;
 
@@ -62,7 +111,7 @@ getSimilarTracksFromTrackList(Database::Session& session, Database::IdType track
 }
 
 std::vector<Database::IdType>
-getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount)
+ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) const
 {
 	std::vector<Database::IdType> res;
 
@@ -80,7 +129,7 @@ getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std
 }
 
 std::vector<Database::IdType>
-getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount)
+ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) const
 {
 	std::vector<Database::IdType> res;
 
@@ -97,5 +146,4 @@ getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::
 	return res;
 }
 
-} // namespace ClusterSearcher
+} // namespace Recommendation
-} // namespace Similarity

src/libs/recommendation/impl/clusters/ClustersClassifier.hpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "recommendation/IClassifier.hpp"
+
+
+namespace Recommendation
+{
+
+	class ClusterClassifier : public IClassifier
+	{
+		public:
+			ClusterClassifier(Database::Session& session);
+			ClusterClassifier(const ClusterClassifier&) = delete;
+			ClusterClassifier(ClusterClassifier&&) = delete;
+			ClusterClassifier& operator=(const ClusterClassifier&) = delete;
+			ClusterClassifier& operator=(ClusterClassifier&&) = delete;
+
+		private:
+
+			bool isTrackClassified(Database::IdType trackId) const override;
+			bool isReleaseClassified(Database::IdType releaseId) const override;
+			bool isArtistClassified(Database::IdType artistId) const override;
+
+			std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
+			std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
+			std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
+			std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
+
+			void classify(Database::Session& session);
+
+			std::unordered_set<Database::IdType>	_classifiedArtists;
+			std::unordered_set<Database::IdType>	_classifiedReleases;
+			std::unordered_set<Database::IdType>	_classifiedTracks;
+};
+
+} // namespace Recommendation
+

src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp

@@ -21,15 +21,10 @@
 
 #include <memory>
 
-namespace Database
-{
-	class Session;
-}
-
 namespace Recommendation
 {
-	class Classifier;
+	class IClassifier;
 
-	std::unique_ptr<Classifier> createClustersClassifier();
+	std::unique_ptr<IClassifier> createClustersClassifier(Database::Session& session);
 }
 

src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp

@@ -21,15 +21,10 @@
 
 #include <memory>
 
-namespace Database
-{
-	class Session;
-}
-
 namespace Recommendation
 {
-	class Classifier;
+	class IClassifier;
 
-	std::unique_ptr<Classifier> createFeaturesClassifier();
+	std::unique_ptr<IClassifier> createFeaturesClassifier();
 }
 

src/libs/recommendation/include/recommendation/IClassifier.hpp

@@ -32,21 +32,19 @@ namespace Database
 namespace Recommendation
 {
 
-	class Classifier
+	class IClassifier
 	{
 		public:
-			virtual ~Classifier() = default;
+			virtual ~IClassifier() = default;
-
-			virtual void classify() = 0;
 
 			virtual bool isTrackClassified(Database::IdType trackId) const = 0;
 			virtual bool isReleaseClassified(Database::IdType releaseId) const = 0;
 			virtual bool isArtistClassified(Database::IdType artistId) const = 0;
 
-			virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0;
+			virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0;
 			virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const = 0;
-			virtual std::vector<Database::IdType> getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const = 0;
+			virtual std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0;
-			virtual std::vector<Database::IdType> getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const = 0;
+			virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0;
 	};
 
 } // ns Recommendation

src/libs/recommendation/include/recommendation/IEngine.hpp

@@ -23,7 +23,6 @@
 #include <unordered_set>
 
 #include "database/Types.hpp"
-#include "Classifier.hpp"
 
 namespace Database
 {
@@ -32,12 +31,13 @@ namespace Database
 
 namespace Recommendation
 {
-	class Classifier;
 	class IEngine
 	{
 		public:
 			virtual ~IEngine() = default;
 
+			virtual void reloadSettings(Database::Session& session) = 0;
+
 			// Closest results first
 			virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0;
 			virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) = 0;
@@ -45,7 +45,7 @@ namespace Recommendation
 			virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0;
 	};
 
-	std::unique_ptr<IEngine> createEngine();
+	std::unique_ptr<IEngine> createEngine(Database::Session& session);
 
 } // ns Recommendation
 

src/lms/main.cpp

@@ -143,7 +143,10 @@ int main(int argc, char* argv[])
 		ServiceProvider<Auth::IPasswordService>::assign(Auth::createPasswordService(ServiceProvider<IConfig>::get()->getULong("login-throttler-max-entriees", 10000)));
 		Scanner::IMediaScanner& mediaScanner {ServiceProvider<Scanner::IMediaScanner>::assign(Scanner::createMediaScanner(database))};
 
-		ServiceProvider<Recommendation::IEngine>::assign(Recommendation::createEngine());
+		{
+			Database::Session session {database};
+			ServiceProvider<Recommendation::IEngine>::assign(Recommendation::createEngine(session));
+		}
 
 		CoverArt::IGrabber& coverArtGrabber {ServiceProvider<CoverArt::IGrabber>::assign(CoverArt::createGrabber(argv[0]))};
 		coverArtGrabber.setDefaultCover(server.appRoot() + "/images/unknown-cover.jpg");

src/lms/ui/admin/DatabaseSettingsView.cpp

@@ -49,7 +49,7 @@ class DatabaseSettingsModel : public Wt::WFormModel
 		static const Field MediaDirectoryField;
 		static const Field UpdatePeriodField;
 		static const Field UpdateStartTimeField;
-		static const Field SimilarityEngineTypeField;
+		static const Field RecommendationEngineTypeField;
 		static const Field TagsField;
 
 		DatabaseSettingsModel()
@@ -60,7 +60,7 @@ class DatabaseSettingsModel : public Wt::WFormModel
 			addField(MediaDirectoryField);
 			addField(UpdatePeriodField);
 			addField(UpdateStartTimeField);
-			addField(SimilarityEngineTypeField);
+			addField(RecommendationEngineTypeField);
 			addField(TagsField);
 
 			auto dirValidator {std::make_shared<DirectoryValidator>()};
@@ -69,7 +69,7 @@ class DatabaseSettingsModel : public Wt::WFormModel
 
 			setValidator(UpdatePeriodField, createMandatoryValidator());
 			setValidator(UpdateStartTimeField, createMandatoryValidator());
-			setValidator(SimilarityEngineTypeField, createMandatoryValidator());
+			setValidator(RecommendationEngineTypeField, createMandatoryValidator());
 			setValidator(TagsField, createTagsValidator());
 
 			// populate the model with initial data
@@ -78,7 +78,7 @@ class DatabaseSettingsModel : public Wt::WFormModel
 
 		std::shared_ptr<Wt::WAbstractItemModel> updatePeriodModel() { return _updatePeriodModel; }
 		std::shared_ptr<Wt::WAbstractItemModel> updateStartTimeModel() { return _updateStartTimeModel; }
-		std::shared_ptr<Wt::WAbstractItemModel> similarityEngineTypeModel() { return _similarityEngineTypeModel; }
+		std::shared_ptr<Wt::WAbstractItemModel> recommendationEngineTypeModel() { return _recommendationEngineTypeModel; }
 
 		void loadData()
 		{
@@ -96,9 +96,9 @@ class DatabaseSettingsModel : public Wt::WFormModel
 			if (startTimeRow)
 				setValue(UpdateStartTimeField, _updateStartTimeModel->getString(*startTimeRow));
 
-			auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromValue(scanSettings->getSimilarityEngineType())};
+			auto recommendationEngineTypeRow {_recommendationEngineTypeModel->getRowFromValue(scanSettings->getRecommendationEngineType())};
-			if (similarityEngineTypeRow)
+			if (recommendationEngineTypeRow)
-				setValue(SimilarityEngineTypeField, _similarityEngineTypeModel->getString(*similarityEngineTypeRow));
+				setValue(RecommendationEngineTypeField, _recommendationEngineTypeModel->getString(*recommendationEngineTypeRow));
 
 			auto clusterTypes {scanSettings->getClusterTypes()};
 			if (!clusterTypes.empty())
@@ -125,9 +125,9 @@ class DatabaseSettingsModel : public Wt::WFormModel
 			if (startTimeRow)
 				scanSettings.modify()->setUpdateStartTime(_updateStartTimeModel->getValue(*startTimeRow));
 
-			auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromString(valueText(SimilarityEngineTypeField))};
+			auto recommendationEngineTypeRow {_recommendationEngineTypeModel->getRowFromString(valueText(RecommendationEngineTypeField))};
-			if (similarityEngineTypeRow)
+			if (recommendationEngineTypeRow)
-				scanSettings.modify()->setSimilarityEngineType(_similarityEngineTypeModel->getValue(*similarityEngineTypeRow));
+				scanSettings.modify()->setRecommendationEngineType(_recommendationEngineTypeModel->getValue(*recommendationEngineTypeRow));
 
 			auto clusterTypes {StringUtils::splitString(valueText(TagsField).toUTF8(), " ")};
 			scanSettings.modify()->setClusterTypes(LmsApp->getDbSession(), std::set<std::string>(clusterTypes.begin(), clusterTypes.end()));
@@ -156,22 +156,22 @@ class DatabaseSettingsModel : public Wt::WFormModel
 				_updateStartTimeModel->add(time.toString(), time);
 			}
 
-			_similarityEngineTypeModel = std::make_shared<ValueStringModel<ScanSettings::SimilarityEngineType>>();
+			_recommendationEngineTypeModel = std::make_shared<ValueStringModel<ScanSettings::RecommendationEngineType>>();
-			_similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.clusters"), ScanSettings::SimilarityEngineType::Clusters);
+			_recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.clusters"), ScanSettings::RecommendationEngineType::Clusters);
-			_similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.features"), ScanSettings::SimilarityEngineType::Features);
+			_recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.features"), ScanSettings::RecommendationEngineType::Features);
 		}
 
 		std::shared_ptr<ValueStringModel<ScanSettings::UpdatePeriod>>		_updatePeriodModel;
 		std::shared_ptr<ValueStringModel<Wt::WTime>>				_updateStartTimeModel;
-		std::shared_ptr<ValueStringModel<ScanSettings::SimilarityEngineType>>	_similarityEngineTypeModel;
+		std::shared_ptr<ValueStringModel<ScanSettings::RecommendationEngineType>>	_recommendationEngineTypeModel;
 
 };
 
-const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField		= "media-directory";
+const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField			= "media-directory";
-const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField		= "update-period";
+const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField			= "update-period";
-const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField		= "update-start-time";
+const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField			= "update-start-time";
-const Wt::WFormModel::Field DatabaseSettingsModel::SimilarityEngineTypeField	= "similarity-engine-type";
+const Wt::WFormModel::Field DatabaseSettingsModel::RecommendationEngineTypeField	= "recommendation-engine-type";
-const Wt::WFormModel::Field DatabaseSettingsModel::TagsField			= "tags";
+const Wt::WFormModel::Field DatabaseSettingsModel::TagsField				= "tags";
 
 DatabaseSettingsView::DatabaseSettingsView()
 {
@@ -207,10 +207,10 @@ DatabaseSettingsView::refreshView()
 	updateStartTime->setModel(model->updateStartTimeModel());
 	t->setFormWidget(DatabaseSettingsModel::UpdateStartTimeField, std::move(updateStartTime));
 
-	// Similarity engine type
+	// recommendation engine type
-	auto similarityEngineType {std::make_unique<Wt::WComboBox>()};
+	auto recommendationEngineType {std::make_unique<Wt::WComboBox>()};
-	similarityEngineType->setModel(model->similarityEngineTypeModel());
+	recommendationEngineType->setModel(model->recommendationEngineTypeModel());
-	t->setFormWidget(DatabaseSettingsModel::SimilarityEngineTypeField, std::move(similarityEngineType));
+	t->setFormWidget(DatabaseSettingsModel::RecommendationEngineTypeField, std::move(recommendationEngineType));
 
 	// Tags
 	t->setFormWidget(DatabaseSettingsModel::TagsField, std::make_unique<Wt::WLineEdit>());

src/test/database/DatabaseTest.cpp

@@ -453,12 +453,24 @@ testSingleTrackSingleCluster(Session& session)
 		CHECK(track->getClusterIds().empty());
 	}
 
+	{
+		auto transaction {session.createSharedTransaction()};
+		CHECK(Track::getAllIdsWithClusters(session).empty());
+	}
+
 	{
 		auto transaction {session.createUniqueTransaction()};
 
 		cluster1.get().modify()->addTrack(track.get());
 	}
 
+	{
+		auto transaction {session.createSharedTransaction()};
+		auto tracks {Track::getAllIdsWithClusters(session)};
+		CHECK(tracks.size() == 1);
+		CHECK(tracks.front() == track.getId());
+	}
+
 	{
 		auto transaction {session.createSharedTransaction()};
 		auto clusters {Cluster::getAllOrphans(session)};
@@ -611,6 +623,11 @@ testSingleTrackSingleReleaseSingleCluster(Session& session)
 	ScopedClusterType clusterType {session, "MyClusterType"};
 	ScopedCluster cluster {session, clusterType .lockAndGet(), "MyCluster"};
 
+	{
+		auto transaction {session.createSharedTransaction()};
+		CHECK(Release::getAllIdsWithClusters(session).empty());
+	}
+
 	{
 		auto transaction {session.createUniqueTransaction()};
 
@@ -618,6 +635,13 @@ testSingleTrackSingleReleaseSingleCluster(Session& session)
 		cluster.get().modify()->addTrack(track.get());
 	}
 
+	{
+		auto transaction {session.createSharedTransaction()};
+		auto releases {Release::getAllIdsWithClusters(session)};
+		CHECK(releases.size() == 1);
+		CHECK(releases.front() == release.getId());
+	}
+
 	{
 		auto transaction {session.createSharedTransaction()};
 
@@ -858,6 +882,11 @@ testSingleTrackSingleReleaseSingleArtistSingleCluster(Session& session)
 	ScopedClusterType clusterType {session, "MyType"};
 	ScopedCluster cluster {session, clusterType.lockAndGet(), "MyCluster"};
 
+	{
+		auto transaction {session.createSharedTransaction()};
+		CHECK(Artist::getAllIdsWithClusters(session).empty());
+	}
+
 	{
 		auto transaction {session.createUniqueTransaction()};
 
@@ -875,6 +904,13 @@ testSingleTrackSingleReleaseSingleArtistSingleCluster(Session& session)
 		CHECK(Release::getAllOrphans(session).empty());
 	}
 
+	{
+		auto transaction {session.createSharedTransaction()};
+		auto artists {Artist::getAllIdsWithClusters(session)};
+		CHECK(artists.size() == 1);
+		CHECK(artists.front() == artist.getId());
+	}
+
 	{
 		auto transaction {session.createSharedTransaction()};
 

src/tools/recommendation-features/LmsRecommendationFeatures.cpp

@@ -32,6 +32,7 @@
 #include "utils/Service.hpp"
 #include "utils/StreamLogger.hpp"
 #include "recommendation/IEngine.hpp"
+#include "recommendation/IClassifier.hpp"
 #include "recommendation/FeaturesClassifierCreator.hpp"
 
 int main(int argc, char *argv[])
@@ -50,11 +51,9 @@ int main(int argc, char *argv[])
 		Database::Db db {ServiceProvider<IConfig>::get()->getPath("working-dir") / "lms.db"};
 		Database::Session session {db};
 
-		auto classifier {Recommendation::createFeaturesClassifier()};
-
-		std::cout << "Classifying tracks..." << std::endl;
 		// may be long...
-		classifier->classify();
+		std::cout << "Classifying tracks..." << std::endl;
+		auto classifier {Recommendation::createFeaturesClassifier()};
 		std::cout << "Classifying tracks DONE" << std::endl;
 
 		const std::vector<Database::IdType> trackIds = std::invoke([&]()
@@ -106,7 +105,7 @@ int main(int argc, char *argv[])
 			};
 
 			std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl;
-			for (Database::IdType similarReleaseId : classifier->getSimilarReleases({releaseId}, 3))
+			for (Database::IdType similarReleaseId : classifier->getSimilarReleases(session, {releaseId}, 3))
 				std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl;
 		}
 
@@ -128,7 +127,7 @@ int main(int argc, char *argv[])
 			};
 
 			std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl;
-			for (Database::IdType similarArtistId : classifier->getSimilarArtists({artistId}, 3))
+			for (Database::IdType similarArtistId : classifier->getSimilarArtists(session, {artistId}, 3))
 				std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl;
 		}