[WIP] Added Artist/ReleaseInfo

Makefile.am

@@ -28,6 +28,8 @@ lms_approot_DATA = \
 		approot/admin-users.xml		\
 		approot/admin-initwizard.xml	\
 		approot/artist.xml	\
+		approot/artistinfo.xml	\
+		approot/artistlink.xml	\
 		approot/artists.xml	\
 		approot/artistsinfo.xml	\
 		approot/explore.xml	\
@@ -38,6 +40,8 @@ lms_approot_DATA = \
 		approot/playhistory.xml	\
 		approot/playqueue.xml	\
 		approot/release.xml	\
+		approot/releaseinfo.xml	\
+		approot/releaselink.xml	\
 		approot/releases.xml	\
 		approot/releasesinfo.xml \
 		approot/settings.xml	\

approot/artistinfo.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<messages xmlns:if="Wt.WTemplate.conditions">
+<!--FORMS message blocks-->
+
+<message id="Lms.Explore.ArtistInfo.template">
+	<h3>${tr:Lms.Explore.ArtistInfo.similar-artists}</h3>
+	${similar-artists}
+</message>
+
+</messages>

approot/artistlink.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<messages xmlns:if="Wt.WTemplate.conditions">
+<!--FORMS message blocks-->
+
+<message id="Lms.Explore.ArtistLink.template">
+	<div class="row Lms-explore-artistlink Lms-vertical-align">
+		<div class="col-xs-12">
+			${name}
+		</div>
+	</div>
+</message>
+
+</messages>

approot/artistsinfo.xml

@@ -9,12 +9,4 @@
 	${recently-added}
 </message>
 
-<message id="Lms.Explore.ArtistsInfo.template.entry">
-	<div class="row Lms-explore-artistsinfo-entry Lms-vertical-align">
-		<div class="col-xs-12">
-			${name}
-		</div>
-	</div>
-</message>
-
 </messages>

approot/explore.xml

@@ -3,12 +3,12 @@
 
 <message id="Lms.Explore.template">
 	<div class="row">
-		<div class="col-lg-4 col-lg-push-8">
+		<div class="col-lg-8">
-			${filters class="Lms-explore-filters"}
+			${contents class="Lms-explore-contents"}
-			${info class="Lms-explore-info hidden-md hidden-xs"}
 		</div>
-		<div class="col-lg-8 col-lg-pull-4">
+		<div class="col-lg-4">
-			${contents}
+			${filters class="Lms-explore-filters"}
+			${info class="Lms-explore-info"}
 		</div>
 	</div>
 </message>

approot/messages.xml

@@ -76,11 +76,15 @@
 <message id="Lms.Explore.value">Value</message>
 <message id="Lms.Explore.various-artists">Various artists</message>
 
+<!--Explore:Artist-->
+<message id="Lms.Explore.ArtistInfo.similar-artists">Similar artists</message>
+
 <!--Explore:Artists-->
 <message id="Lms.Explore.ArtistsInfo.recently-added">Recently added</message>
 <message id="Lms.Explore.ArtistsInfo.most-played">Top artists</message>
 
 <!--Explore:Release-->
+<message id="Lms.Explore.ReleaseInfo.similar-releases">Similar albums</message>
 <message id="Lms.Explore.Release.copyright">Copyright</message>
 
 <!--Explore:Releases-->

approot/messages_fr.xml

@@ -70,17 +70,21 @@
 <message id="Lms.Explore.filters">Filtres</message>
 <message id="Lms.Explore.releases">Albums</message>
 <message id="Lms.Explore.search-placeholder">Rechercher...</message>
-<message id="Lms.Explore.show-more">Montrer plus</message>
+<message id="Lms.Explore.show-more">Voir plus</message>
 <message id="Lms.Explore.tracks">Pistes</message>
 <message id="Lms.Explore.type">Type</message>
 <message id="Lms.Explore.value">Valeur</message>
 <message id="Lms.Explore.various-artists">Artistes divers</message>
 
+<!--Explore:Artist-->
+<message id="Lms.Explore.ArtistInfo.similar-artists">Artists similaires</message>
+
 <!--Explore:Artists-->
 <message id="Lms.Explore.ArtistsInfo.recently-added">Ajouts récents</message>
 <message id="Lms.Explore.ArtistsInfo.most-played">Artistes populaires</message>
 
 <!--Explore:Release-->
+<message id="Lms.Explore.ReleaseInfo.similar-releases">Albums similaires</message>
 <message id="Lms.Explore.Release.copyright">Copyright</message>
 
 <!--Explore:Releases-->

approot/releaseinfo.xml

@@ -0,0 +1,10 @@
+
+<?xml version="1.0" encoding="UTF-8" ?>
+<messages xmlns:if="Wt.WTemplate.conditions">
+
+<message id="Lms.Explore.ReleaseInfo.template">
+	<h3>${tr:Lms.Explore.ReleaseInfo.similar-releases}</h3>
+	${similar-releases}
+</message>
+
+</messages>

approot/releaselink.xml

@@ -0,0 +1,23 @@
+
+<?xml version="1.0" encoding="UTF-8" ?>
+<messages xmlns:if="Wt.WTemplate.conditions">
+
+<message id="Lms.Explore.ReleaseLink.template">
+	<div class="row Lms-explore-releaselink">
+		<div class="col-xs-12">
+			<div class="media">
+				<div class="media-left media-middle">
+					${cover class="media-object"}
+				</div>
+				<div class="media-body media-middle">
+					<div class="media-heading">
+						${release-name}
+					</div>
+					${<if-has-artist>}<small><i>${artist-name}</i></small>${</if-has-artist>}
+				</div>
+			</div>
+		</div>
+	</div>
+</message>
+
+</messages>

approot/releasesinfo.xml

@@ -9,22 +9,4 @@
 	${recently-added}
 </message>
 
-<message id="Lms.Explore.ReleasesInfo.template.entry">
-	<div class="row Lms-explore-releasesinfo-entry">
-		<div class="col-xs-12">
-			<div class="media">
-				<div class="media-left media-middle">
-					${cover class="media-object"}
-				</div>
-				<div class="media-body media-middle">
-					<div class="media-heading">
-						${release-name}
-					</div>
-					${<if-has-artist>}<small><i>${artist-name}</i></small>${</if-has-artist>}
-				</div>
-			</div>
-		</div>
-	</div>
-</message>
-
 </messages>

conf/lms.conf

@@ -22,3 +22,7 @@ approot = "/usr/share/lms/approot";
 
 # Turn on this option to allow the demo account creation/use
 #demo = false;
+
+# Acoustic brainz's root API
+acousticbrainz-api-url = "https://acousticbrainz.org/api/v1/"
+

configure.ac

@@ -85,8 +85,8 @@ AC_CONFIG_FILES([Makefile
 		 src/Makefile
 		 test/Makefile
 		 tools/Makefile
-		 tools/metadata/Makefile
+		 tools/feature-extractor/Makefile
-		 tools/clusterer/Makefile])
+		 tools/metadata/Makefile])
 
 AC_OUTPUT
 

docroot/css/lms.css

@@ -18,6 +18,10 @@ a:hover {
 	justify-content: center;
 }
 
+.Lms-contents {
+	margin-bottom: 78px;
+}
+
 .Lms-show-more {
 	margin-top: 8px;
 	margin-bottom: 8px;
@@ -105,8 +109,8 @@ a:hover {
 	margin-bottom: 4px;
 }
 
-.Lms-contents {
+.Lms-explore-contents {
-	margin-bottom: 78px;
+	margin-bottom: 24px;
 }
 
 .Lms-explore-filters {
@@ -131,16 +135,6 @@ a:hover {
 	background-color: lightgrey;
 }
 
-.Lms-explore-releasesinfo-entry {
-	margin: 0px;
-	padding-top: 4px;
-	padding-bottom: 4px;
-}
-
-.Lms-explore-releasesinfo-entry:hover {
-	background-color: lightgrey;
-}
-
 .Lms-explore-release-cover {
 	max-width: 100%;
 	border-radius: 8px;
@@ -165,6 +159,17 @@ a:hover {
 	font-weight: bold;
 }
 
+.Lms-explore-releaselink {
+	margin: 0px;
+	padding-top: 4px;
+	padding-bottom: 4px;
+}
+
+.Lms-explore-releaselink:hover {
+	background-color: lightgrey;
+}
+
+
 .Lms-explore-artists-entry {
 	min-height: 48px;
 }
@@ -173,15 +178,6 @@ a:hover {
 	background-color: lightgrey;
 }
 
-.Lms-explore-artistsinfo-entry {
-	min-height: 28px;
-	margin: 0px;
-}
-
-.Lms-explore-artistsinfo-entry:hover {
-	background-color: lightgrey;
-}
-
 .Lms-explore-artist-entry {
 	padding-top: 8px;
 	padding-bottom: 8px;
@@ -191,6 +187,16 @@ a:hover {
 	background-color: lightgrey;
 }
 
+.Lms-explore-artistlink {
+	min-height: 28px;
+	margin: 0px;
+}
+
+.Lms-explore-artistlink:hover {
+	background-color: lightgrey;
+}
+
+
 .Lms-explore-tracks-entry {
 	min-height: 32px;
 	padding-top: 2px;

src/Makefile.am

@@ -1,23 +1,33 @@
 bin_PROGRAMS = lms
 
 lms_SOURCES = \
-	$(srcdir)/main/main.cpp					\
 	$(srcdir)/av/AvInfo.cpp					\
 	$(srcdir)/av/AvTranscoder.cpp				\
 	$(srcdir)/cover/CoverArtGrabber.cpp			\
 	$(srcdir)/database/Artist.cpp				\
 	$(srcdir)/database/Cluster.cpp				\
 	$(srcdir)/database/DatabaseHandler.cpp			\
-	$(srcdir)/database/TrackList.cpp				\
+	$(srcdir)/database/TrackFeature.cpp			\
+	$(srcdir)/database/TrackList.cpp			\
 	$(srcdir)/database/Release.cpp				\
 	$(srcdir)/database/ScanSettings.cpp			\
+	$(srcdir)/database/SimilaritySettings.cpp		\
 	$(srcdir)/database/SqlQuery.cpp				\
 	$(srcdir)/database/Track.cpp				\
 	$(srcdir)/database/User.cpp				\
 	$(srcdir)/image/Image.cpp				\
+	$(srcdir)/main/main.cpp					\
+	$(srcdir)/main/Services.cpp				\
 	$(srcdir)/metadata/AvFormat.cpp				\
 	$(srcdir)/metadata/TagLibParser.cpp			\
 	$(srcdir)/scanner/MediaScanner.cpp			\
+	$(srcdir)/similarity/SimilaritySearcher.cpp		\
+	$(srcdir)/similarity/cluster/SimilarityClusterSearcher.cpp \
+	$(srcdir)/similarity/som/AcousticBrainzUtils.cpp	\
+	$(srcdir)/similarity/som/DataNormalizer.cpp		\
+	$(srcdir)/similarity/som/Network.cpp			\
+	$(srcdir)/similarity/som/SimilaritySOMScannerAddon.cpp	\
+	$(srcdir)/similarity/som/SimilaritySOMSearcher.cpp	\
 	$(srcdir)/ui/Auth.cpp					\
 	$(srcdir)/ui/LmsApplication.cpp				\
 	$(srcdir)/ui/LmsApplicationGroup.cpp			\
@@ -30,11 +40,15 @@ lms_SOURCES = \
 	$(srcdir)/ui/admin/UserView.cpp				\
 	$(srcdir)/ui/admin/UsersView.cpp			\
 	$(srcdir)/ui/common/Validators.cpp			\
-	$(srcdir)/ui/explore/ArtistView.cpp			\
+	$(srcdir)/ui/explore/ArtistInfoView.cpp			\
+	$(srcdir)/ui/explore/ArtistLink.cpp			\
 	$(srcdir)/ui/explore/ArtistsInfoView.cpp		\
+	$(srcdir)/ui/explore/ArtistView.cpp			\
 	$(srcdir)/ui/explore/ArtistsView.cpp			\
 	$(srcdir)/ui/explore/Explore.cpp			\
 	$(srcdir)/ui/explore/Filters.cpp			\
+	$(srcdir)/ui/explore/ReleaseInfoView.cpp		\
+	$(srcdir)/ui/explore/ReleaseLink.cpp			\
 	$(srcdir)/ui/explore/ReleasesInfoView.cpp		\
 	$(srcdir)/ui/explore/ReleasesView.cpp			\
 	$(srcdir)/ui/explore/ReleaseView.cpp			\

src/clusterer/Clusterer.hpp

@@ -1,181 +0,0 @@
-/*
- * Copyright (C) 2018 Emeric Poupon
- *
- * This file is part of LMS.
- *
- * LMS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LMS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#pragma once
-
-#include <cmath>
-#include "SOM.hpp"
-#include "DataNormalizer.hpp"
-
-/*
- * For each InputVector, associate vector<T> values
- */
-template<typename T>
-class Clusterer
-{
-	public:
-		using SampleType = std::pair<SOM::InputVector /* key */, T /* value*/ >;
-		using Cluster = std::vector<T>;
-
-		Clusterer(const std::vector<SampleType>& samples, std::size_t inputDimCount, std::size_t iterationCount);
-
-		const Cluster& getCluster(const SOM::InputVector& data) const;
-
-		// Sorted results (best first)
-		std::vector<Cluster> getClusters(const SOM::InputVector& data, std::size_t nbClusters) const;
-
-		const std::vector<Cluster>& getAllClusters() const;
-
-		void dump(std::ostream& os) const;
-
-	private:
-
-		void train(const std::vector<std::pair<SOM::InputVector, T>>& samples, std::size_t iterationCount);
-
-		std::vector<T>& getValues(SOM::Coords coords);
-		const std::vector<T>& getValues(SOM::Coords coords) const;
-
-		std::size_t _width;
-		std::size_t _height;
-		std::vector<std::vector<T>> _values; // Map of T vectors
-		SOM::DataNormalizer _dataNormalizer;
-		SOM::Network _network;
-};
-
-
-template<typename T>
-Clusterer<T>::Clusterer(const std::vector<SampleType>& samples, std::size_t inputDimCount, std::size_t iterationCount)
-:
-_width(std::sqrt(samples.size()/20)),
-_height(std::sqrt(samples.size()/20)),
-_dataNormalizer(inputDimCount),
-_network(_width, _height, inputDimCount)
-{
-	_values.resize(_width * _height);
-	train(samples, iterationCount);
-}
-
-template<typename T>
-std::vector<T>&
-Clusterer<T>::getValues(SOM::Coords coords)
-{
-	return _values[ coords.x + coords.y*_width ];
-}
-
-template<typename T>
-const std::vector<T>&
-Clusterer<T>::getValues(SOM::Coords coords) const
-{
-	return _values[ coords.x + coords.y*_width ];
-}
-
-
-template<typename T>
-void
-Clusterer<T>::train(const std::vector<std::pair<SOM::InputVector, T>>& samples, std::size_t iterationCount)
-{
-	// Train
-	{
-		std::vector<SOM::InputVector> inputVectors;
-		inputVectors.reserve(samples.size());
-
-		for (const auto& sample : samples)
-		{
-			inputVectors.push_back(sample.first);
-		}
-
-		_dataNormalizer.computeNormalizationFactors(inputVectors);
-
-		for (auto& inputVector : inputVectors)
-			_dataNormalizer.normalizeData(inputVector);
-
-		_network.train(inputVectors, iterationCount);
-	}
-
-	// Classify data
-	for (const auto& sample : samples)
-	{
-		auto inputVector = sample.first;
-		const auto& value = sample.second;
-
-		_dataNormalizer.normalizeData(inputVector);
-		auto coords = _network.classify(inputVector);
-		auto& values = getValues(coords);
-
-		values.push_back(value);
-	}
-}
-
-template<typename T>
-const typename Clusterer<T>::Cluster&
-Clusterer<T>::getCluster(const SOM::InputVector& inputVector) const
-{
-	auto inputVectorNormalized = inputVector;
-	_dataNormalizer.normalizeData(inputVectorNormalized);
-
-	return getValues(_network.classify(inputVectorNormalized));
-}
-
-template<typename T>
-std::vector<typename Clusterer<T>::Cluster>
-Clusterer<T>::getClusters(const SOM::InputVector& inputVector, std::size_t nbClusters) const
-{
-	auto inputVectorNormalized = inputVector;
-	_dataNormalizer.normalizeData(inputVectorNormalized);
-
-	std::vector<typename Clusterer<T>::Cluster> res;
-	for (auto& cluster : _network.classify(inputVectorNormalized, nbClusters))
-	{
-		res.push_back(getValues(cluster));
-	}
-
-	return res;
-}
-
-template<typename T>
-const std::vector<typename Clusterer<T>::Cluster>&
-Clusterer<T>::getAllClusters() const
-{
-	return _values;
-}
-
-template<typename T>
-void
-Clusterer<T>::dump(std::ostream& os) const
-{
-	os << "Normalizer:" << std::endl;
-	_dataNormalizer.dump(os);
-	os << std::endl;
-	os << "Internal network:" << std::endl;
-	_network.dump(os);
-	os << "Values: " << std::endl;
-	for (std::size_t y = 0; y < _height; ++y)
-	{
-		for (std::size_t x = 0; x < _width; ++x)
-		{
-			os << "[";
-			for (const auto& value : getValues({x, y}))
-				os << value << " ";
-			os << "] ";
-		}
-		os << std::endl;
-	}
-
-}
-

src/cover/CoverArtGrabber.cpp

@@ -50,15 +50,13 @@ namespace CoverArt {
 
 Grabber::Grabber()
 {
-	if (!_defaultCover.load( Wt::WApplication::instance()->docRoot() + "/images/unknown-cover.jpg"))
-		throw LmsException("Cannot read default cover file");
 }
 
-Grabber&
+void
-Grabber::instance()
+Grabber::setDefaultCover(boost::filesystem::path p)
 {
-	static Grabber instance;
+	if (!_defaultCover.load(p))
-	return instance;
+		throw LmsException("Cannot read default cover file '" + p.string() + "'");
 }
 
 Image::Image

src/cover/CoverArtGrabber.hpp

@@ -32,18 +32,19 @@ namespace CoverArt {
 class Grabber
 {
 	public:
+		Grabber();
 		Grabber(const Grabber&) = delete;
 		Grabber& operator=(const Grabber&) = delete;
+		Grabber(Grabber&&) = delete;
+		Grabber& operator=(Grabber&&) = delete;
 
-		static Grabber& instance();
+		void			setDefaultCover(boost::filesystem::path defaultCoverPath);
 
 		std::vector<uint8_t>	getFromTrack(Wt::Dbo::Session& session, Database::IdType trackId, Image::Format format, std::size_t size);
 		std::vector<uint8_t>	getFromRelease(Wt::Dbo::Session& session, Database::IdType releaseId, Image::Format format, std::size_t size);
 
 	private:
 
-		Grabber();
-
 		Image::Image		getFromTrack(Wt::Dbo::Session& session, Database::IdType trackId, std::size_t size);
 		Image::Image		getFromRelease(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t size);
 

src/database/Artist.cpp

@@ -199,6 +199,12 @@ Artist::getReleases(const std::set<IdType>& clusterIds) const
 	return std::vector< Wt::Dbo::ptr<Release> > (res.begin(), res.end());
 }
 
+std::vector<Wt::Dbo::ptr<Track>>
+Artist::getTracks() const
+{
+	return std::vector<Wt::Dbo::ptr<Track>>(_tracks.begin(), _tracks.end());
+}
+
 std::vector<std::vector<Wt::Dbo::ptr<Cluster>>>
 Artist::getClusterGroups(std::vector<ClusterType::pointer> clusterTypes, std::size_t size) const
 {

src/database/Artist.hpp

@@ -65,6 +65,7 @@ class Artist : public Wt::Dbo::Dbo<Artist>
 
 		// Get the releases that have at least one track for this artist that belongs to optional cluster filters
 		std::vector<Wt::Dbo::ptr<Release>>	getReleases(const std::set<IdType>& clusterIds = std::set<IdType>()) const;
+		std::vector<Wt::Dbo::ptr<Track>>	getTracks() const;
 
 		// Get the cluster of the tracks made by this artist
 		// Each clusters are grouped by cluster type, sorted by the number of occurence

src/database/Cluster.hpp

@@ -37,7 +37,7 @@ class ScanSettings;
 class Cluster : public Wt::Dbo::Dbo<Cluster>
 {
 	public:
-		typedef Wt::Dbo::ptr<Cluster> pointer;
+		using pointer = Wt::Dbo::ptr<Cluster>;
 
 		Cluster();
 		Cluster(Wt::Dbo::ptr<ClusterType> type, std::string name);
@@ -50,7 +50,7 @@ class Cluster : public Wt::Dbo::Dbo<Cluster>
 		static pointer create(Wt::Dbo::Session& session, Wt::Dbo::ptr<ClusterType> type, std::string name);
 
 		// Accessors
-		const std::string& getName(void) const { return _name; }
+		const std::string& getName() const { return _name; }
 		Wt::Dbo::ptr<ClusterType> getType() const { return _clusterType; }
 		std::size_t getCount() const { return _tracks.size(); }
 		std::vector<Wt::Dbo::ptr<Track>> getTracks(int offset, int limit) const;

src/database/DatabaseHandler.cpp

@@ -35,10 +35,12 @@
 
 #include "Artist.hpp"
 #include "Cluster.hpp"
-#include "TrackList.hpp"
 #include "Release.hpp"
 #include "ScanSettings.hpp"
+#include "SimilaritySettings.hpp"
 #include "Track.hpp"
+#include "TrackList.hpp"
+#include "TrackFeature.hpp"
 
 namespace Database {
 
@@ -104,8 +106,11 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool)
 	_session.mapClass<TrackListEntry>("tracklist_entry");
 	_session.mapClass<Release>("release");
 	_session.mapClass<Track>("track");
+	_session.mapClass<TrackFeature>("track_feature");
+	_session.mapClass<TrackFeatureType>("track_feature_type");
 
 	_session.mapClass<ScanSettings>("scan_settings");
+	_session.mapClass<SimilaritySettings>("similarity_settings");
 
 	_session.mapClass<AuthInfo>("auth_info");
 	_session.mapClass<AuthInfo::AuthIdentityType>("auth_identity");
@@ -129,13 +134,18 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool)
 
 		// Indexes
 		_session.execute("CREATE INDEX IF NOT EXISTS track_path_idx ON track(file_path)");
+		_session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)");
 		_session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
 		_session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
 		_session.execute("CREATE INDEX IF NOT EXISTS track_artist_idx ON track(artist_id)");
 		_session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)");
 		_session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)");
 		_session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)");
-		_session.execute("CREATE INDEX IF NOT EXISTS tracklist_name ON tracklist(name)");
+		_session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)");
+		_session.execute("CREATE INDEX IF NOT EXISTS track_feature_type_name_idx ON track_feature_type(name)");
+		_session.execute("CREATE INDEX IF NOT EXISTS track_feature_type_idx ON track_feature(type_id)");
+		_session.execute("CREATE INDEX IF NOT EXISTS track_feature_track_idx ON track_feature(track_id)");
+		_session.execute("CREATE INDEX IF NOT EXISTS track_feature_type_track_idx ON track_feature(type_id, track_id)");
 	}
 
 	_users = new UserDatabase(_session);

src/database/ScanSettings.cpp

@@ -45,7 +45,7 @@ namespace Database {
 ScanSettings::pointer
 ScanSettings::get(Wt::Dbo::Session& session)
 {
-	ScanSettings::pointer settings = session.find<ScanSettings>();
+	pointer settings = session.find<ScanSettings>();
 	if (!settings)
 	{
 		settings = session.add(std::make_unique<ScanSettings>());
@@ -81,16 +81,17 @@ ScanSettings::setClusterTypes(const std::set<std::string>& clusterTypeNames)
 	assert(session());
 
 	// Create any missing cluster type
-	for (auto clusterTypeName : clusterTypeNames)
+	for (const auto& clusterTypeName : clusterTypeNames)
 	{
 		auto clusterType = ClusterType::getByName(*session(), clusterTypeName);
 		if (!clusterType)
 		{
 			LMS_LOG(DB, INFO) << "Creating cluster type " << clusterTypeName;
 			clusterType = ClusterType::create(*session(), clusterTypeName);
+			_clusterTypes.insert(clusterType);
+
 			needRescan = true;
 		}
-		_clusterTypes.insert(clusterType);
 	}
 
 	// Delete no longer existing cluster types
@@ -101,7 +102,6 @@ ScanSettings::setClusterTypes(const std::set<std::string>& clusterTypeNames)
 		{
 			LMS_LOG(DB, INFO) << "Deleting cluster type " << clusterType->getName();
 			clusterType.remove();
-			needRescan = true;
 		}
 	}
 
@@ -109,5 +109,6 @@ ScanSettings::setClusterTypes(const std::set<std::string>& clusterTypeNames)
 		_scanVersion += 1;
 }
 
+
 } // namespace Database
 

src/database/ScanSettings.hpp

@@ -27,6 +27,7 @@
 namespace Database {
 
 class ClusterType;
+
 class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
 {
 	public:
@@ -39,8 +40,6 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
 			Monthly
 		};
 
-		ScanSettings() {}
-
 		static pointer get(Wt::Dbo::Session& session);
 
 		// Getters

src/database/SimilaritySettings.cpp

@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SimilaritySettings.hpp"
+
+#include "utils/Logger.hpp"
+#include "utils/Utils.hpp"
+
+#include "TrackFeature.hpp"
+
+namespace {
+
+std::set<std::string> defaultTrackFeaturesNames =
+{
+	"lowlevel.average_loudness",
+	"lowlevel.barkbands_flatness_db.mean",
+	"lowlevel.dissonance.mean",
+	"lowlevel.dynamic_complexity",
+	"lowlevel.hfc.mean",				// GOOD
+	"lowlevel.melbands_crest.mean",
+	"lowlevel.melbands_kurtosis.mean",
+	"lowlevel.melbands_skewness.mean",
+	"lowlevel.melbands_spread.mean",
+	"lowlevel.pitch_salience.mean",
+	"lowlevel.pitch_salience.var",
+	"lowlevel.silence_rate_30dB.mean",
+	"lowlevel.silence_rate_60dB.mean",
+	"lowlevel.spectral_centroid.mean",
+	"lowlevel.spectral_complexity.mean",
+	"lowlevel.spectral_decrease.mean",
+	"lowlevel.spectral_energy.mean",
+	"lowlevel.spectral_energyband_high.mean",
+	"lowlevel.spectral_energyband_low.mean",
+	"lowlevel.spectral_energyband_middle_high.mean",
+	"lowlevel.spectral_energyband_middle_low.mean",
+	"lowlevel.spectral_entropy.mean",
+	"lowlevel.spectral_flux.mean",
+	"lowlevel.spectral_kurtosis.mean",
+	"lowlevel.spectral_rms.mean",
+	"lowlevel.spectral_skewness.mean",
+	"lowlevel.spectral_spread.mean",
+	"lowlevel.spectral_strongpeak.mean",
+	"lowlevel.zerocrossingrate.mean",
+	"rhythm.beats_loudness.mean",		// BAD
+	"rhythm.bpm",
+	"tonal.chords_changes_rate",		// OK
+	"tonal.chords_number_rate",		// BAD
+	"tonal.chords_strength.mean",		// OK
+	"tonal.hpcp_entropy.mean",		// GOOD
+};
+
+} // namespace
+
+namespace Database {
+
+
+SimilaritySettings::pointer
+SimilaritySettings::get(Wt::Dbo::Session& session)
+{
+	pointer settings = session.find<SimilaritySettings>();
+	if (!settings)
+	{
+		settings = session.add(std::make_unique<SimilaritySettings>());
+		settings.modify()->setTrackFeatureTypes(defaultTrackFeaturesNames);
+	}
+
+	return settings;
+}
+
+std::vector<Wt::Dbo::ptr<TrackFeatureType>>
+SimilaritySettings::getTrackFeatureTypes() const
+{
+	return std::vector<Wt::Dbo::ptr<TrackFeatureType>>(_trackFeatureTypes.begin(), _trackFeatureTypes.end());
+}
+
+void
+SimilaritySettings::setTrackFeatureTypes(const std::set<std::string>& featuresNames)
+{
+	bool needRescan = false;
+	assert(session());
+
+	// Create any missing feature type
+	for (const auto& featureName : featuresNames)
+	{
+		auto featureType = TrackFeatureType::getByName(*session(), featureName);
+		if (!featureType)
+		{
+			LMS_LOG(DB, INFO) << "Creating feature type " << featureName;
+			featureType = TrackFeatureType::create(*session(), featureName);
+			_trackFeatureTypes.insert(featureType);
+
+			needRescan = true;
+		}
+	}
+
+	// Delete no longer existing feature type
+	for (auto trackFeatureType : _trackFeatureTypes)
+	{
+		if (std::none_of(featuresNames.begin(), featuresNames.end(),
+			[trackFeatureType](const std::string& name) { return name == trackFeatureType->getName(); }))
+		{
+			LMS_LOG(DB, INFO) << "Deleting track feature type " << trackFeatureType->getName();
+			trackFeatureType.remove();
+		}
+	}
+
+	if (needRescan)
+		_scanVersion += 1;
+}
+
+
+
+} // namespace Database
+

src/database/SimilaritySettings.hpp

@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/Dbo/Dbo.h>
+
+namespace Database {
+
+class TrackFeature;
+class TrackFeatureType;
+
+class SimilaritySettings : public Wt::Dbo::Dbo<SimilaritySettings>
+{
+	public:
+		using pointer = Wt::Dbo::ptr<SimilaritySettings>;
+
+		static pointer get(Wt::Dbo::Session& session);
+
+		std::size_t getVersion() const { return _scanVersion; }
+
+		std::vector<Wt::Dbo::ptr<TrackFeatureType>> getTrackFeatureTypes() const;
+		void setTrackFeatureTypes(const std::set<std::string>& featureTypeNames);
+
+		void setNetworkData(std::string data) { _refFeaturesData = data; }
+		const std::string& getNetworkData() const { return _refFeaturesData; }
+
+		void setNormalizationData(std::string data) { _normalizationData = data; }
+		const std::string& getNormalizationData() const { return _normalizationData; }
+
+		template<class Action>
+		void persist(Action& a)
+		{
+			Wt::Dbo::field(a, _scanVersion,		"settings_version");
+			Wt::Dbo::field(a, _normalizationData,	"normalization_data");
+			Wt::Dbo::field(a, _refFeaturesData,	"ref_features_data");
+			Wt::Dbo::hasMany(a, _trackFeatureTypes, Wt::Dbo::ManyToOne, "similarity_settings");
+		}
+
+	private:
+
+		int		_scanVersion = 0;
+		std::string	_normalizationData;
+		std::string	_refFeaturesData;
+		Wt::Dbo::collection<Wt::Dbo::ptr<TrackFeatureType>>	_trackFeatureTypes;
+};
+
+
+} // namespace Database
+

src/database/Track.cpp

@@ -26,6 +26,7 @@
 #include "Artist.hpp"
 #include "Cluster.hpp"
 #include "Release.hpp"
+#include "TrackFeature.hpp"
 #include "SqlQuery.hpp"
 
 namespace Database {
@@ -107,6 +108,25 @@ Track::getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int limit)
 	return std::vector<pointer>(res.begin(), res.end());
 }
 
+std::vector<Track::pointer>
+Track::getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session)
+{
+	Wt::Dbo::collection<pointer> res = session.query<pointer>
+		("SELECT t FROM track t")
+		.where("LENGTH(t.mbid) > 0")
+		.where("NOT EXISTS (SELECT * FROM track_feature t_f WHERE t_f.track_id = t.id)");
+	return std::vector<pointer>(res.begin(), res.end());
+}
+
+std::vector<Track::pointer>
+Track::getAllWithFeatures(Wt::Dbo::Session& session)
+{
+	Wt::Dbo::collection<pointer> res = session.query<pointer>
+		("SELECT t FROM track t")
+		.where("EXISTS (SELECT * from track_feature t_f WHERE t_f.track_id = t.id)");
+	return std::vector<pointer>(res.begin(), res.end());
+}
+
 std::vector<Cluster::pointer>
 Track::getClusters(void) const
 {
@@ -115,6 +135,12 @@ Track::getClusters(void) const
 	return clusters;
 }
 
+bool
+Track::hasTrackFeatures() const
+{
+	return !_trackFeatures.empty();
+}
+
 static
 Wt::Dbo::Query< Track::pointer >
 getQuery(Wt::Dbo::Session& session,
@@ -236,6 +262,18 @@ Track::getCopyrightURL() const
 	return _copyrightURL != "" ? boost::make_optional<std::string>(_copyrightURL) : boost::none;
 }
 
+Wt::Dbo::ptr<TrackFeature>
+Track::getTrackFeature(Wt::Dbo::ptr<TrackFeatureType> type) const
+{
+	assert(self());
+	assert(IdIsValid(self()->id()));
+	assert(session());
+
+	return session()->find<TrackFeature>()
+		.where("type_id = ?").bind(type.id())
+		.where("track_id = ?").bind(self()->id());
+}
+
 std::vector<std::vector<Cluster::pointer>>
 Track::getClusterGroups(std::vector<ClusterType::pointer> clusterTypes, std::size_t size) const
 {

src/database/Track.hpp

@@ -36,15 +36,17 @@ namespace Database {
 class Artist;
 class Cluster;
 class ClusterType;
-class TrackListEntry;
 class Release;
+class TrackFeature;
+class TrackFeatureType;
+class TrackListEntry;
 class TrackStats;
 
 class Track : public Wt::Dbo::Dbo<Track>
 {
 	public:
 
-		typedef Wt::Dbo::ptr<Track> pointer;
+		using pointer = Wt::Dbo::ptr<Track>;
 
 		Track() {}
 		Track(const boost::filesystem::path& p);
@@ -68,6 +70,8 @@ class Track : public Wt::Dbo::Dbo<Track>
 		static std::vector<pointer> getMBIDDuplicates(Wt::Dbo::Session& session);
 		static std::vector<pointer> getChecksumDuplicates(Wt::Dbo::Session& session);
 		static std::vector<pointer> getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1);
+		static std::vector<pointer> getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction
+		static std::vector<pointer> getAllWithFeatures(Wt::Dbo::Session& session); // nested transaction
 
 		// Create utility
 		static pointer	create(Wt::Dbo::Session& session, const boost::filesystem::path& p);
@@ -76,7 +80,8 @@ class Track : public Wt::Dbo::Dbo<Track>
 		static void removeClusters(std::string type);
 
 		// Accessors
-		void setScanVersion(std::size_t version)			{_scanVersion = version; }
+		void setScanVersion(std::size_t version)			{ _scanVersion = version; }
+		void setSimilarityScanVersion(std::size_t version)		{ _similarityScanVersion = version; }
 		void setTrackNumber(int num)					{ _trackNumber = num; }
 		void setTotalTrackNumber(int num)				{ _totalTrackNumber = num; }
 		void setDiscNumber(int num)					{ _discNumber = num; }
@@ -96,8 +101,10 @@ class Track : public Wt::Dbo::Dbo<Track>
 		void setArtist(Wt::Dbo::ptr<Artist> artist)			{ _artist = artist; }
 		void setRelease(Wt::Dbo::ptr<Release> release)			{ _release = release; }
 		void eraseClusters()						{ _clusters.clear(); }
+		void eraseFeatures()						{ _trackFeatures.clear(); }
 
 		std::size_t 			getScanVersion() const		{ return _scanVersion; }
+		std::size_t 			getSimilarityScanVersion() const	{ return _similarityScanVersion; }
 		boost::optional<std::size_t>	getTrackNumber() const;
 		boost::optional<std::size_t>	getTotalTrackNumber() const;
 		boost::optional<std::size_t>	getDiscNumber() const;
@@ -117,6 +124,9 @@ class Track : public Wt::Dbo::Dbo<Track>
 		Wt::Dbo::ptr<Artist>		getArtist() const		{ return _artist; }
 		Wt::Dbo::ptr<Release>		getRelease() const		{ return _release; }
 		std::vector<Wt::Dbo::ptr<Cluster>>	getClusters() const;
+		std::vector<Wt::Dbo::ptr<TrackFeature>>	getTrackFeatures() const;	// ordered by feature's name
+		bool				hasTrackFeatures() const;
+		Wt::Dbo::ptr<TrackFeature>	getTrackFeature(Wt::Dbo::ptr<TrackFeatureType> type) const;
 
 		std::vector<std::vector<Wt::Dbo::ptr<Cluster>>> getClusterGroups(std::vector<Wt::Dbo::ptr<ClusterType>> clusterTypes, std::size_t size) const;
 
@@ -124,6 +134,7 @@ class Track : public Wt::Dbo::Dbo<Track>
 			void persist(Action& a)
 			{
 				Wt::Dbo::field(a, _scanVersion,		"scan_version");
+				Wt::Dbo::field(a, _similarityScanVersion,	"similarity_version");
 				Wt::Dbo::field(a, _trackNumber,		"track_number");
 				Wt::Dbo::field(a, _totalTrackNumber,	"total_track_number");
 				Wt::Dbo::field(a, _discNumber,		"disc_number");
@@ -145,6 +156,7 @@ class Track : public Wt::Dbo::Dbo<Track>
 				Wt::Dbo::belongsTo(a, _artist, "artist", Wt::Dbo::OnDeleteCascade);
 				Wt::Dbo::hasMany(a, _clusters, Wt::Dbo::ManyToMany, "track_cluster", "", Wt::Dbo::OnDeleteCascade);
 				Wt::Dbo::hasMany(a, _playlistEntries, Wt::Dbo::ManyToOne, "track");
+				Wt::Dbo::hasMany(a, _trackFeatures, Wt::Dbo::ManyToOne, "track");
 			}
 
 	private:
@@ -154,6 +166,7 @@ class Track : public Wt::Dbo::Dbo<Track>
 		static const std::size_t _maxCopyrightURLLength = 128;
 
 		int					_scanVersion = 0;
+		int					_similarityScanVersion = 0;
 		int					_trackNumber = 0;
 		int					_totalTrackNumber = 0;
 		int					_discNumber = 0;
@@ -178,6 +191,7 @@ class Track : public Wt::Dbo::Dbo<Track>
 		Wt::Dbo::ptr<Release>			_release;
 		Wt::Dbo::collection<Wt::Dbo::ptr<Cluster>> _clusters;
 		Wt::Dbo::collection<Wt::Dbo::ptr<TrackListEntry>> _playlistEntries;
+		Wt::Dbo::collection<Wt::Dbo::ptr<TrackFeature>> _trackFeatures;
 
 };
 

src/database/TrackFeature.cpp

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "TrackFeature.hpp"
+
+#include "SimilaritySettings.hpp"
+#include "Track.hpp"
+
+namespace Database {
+
+TrackFeatureType::TrackFeatureType(std::string name)
+	: _name(name)
+{
+}
+
+TrackFeatureType::pointer
+TrackFeatureType::getByName(Wt::Dbo::Session& session, std::string name)
+{
+	return session.find<TrackFeatureType>().where("name = ?").bind(name);
+}
+
+TrackFeatureType::pointer
+TrackFeatureType::create(Wt::Dbo::Session& session, std::string name)
+{
+	return session.add(std::make_unique<TrackFeatureType>(name));
+}
+
+
+TrackFeature::TrackFeature(Wt::Dbo::ptr<TrackFeatureType> type, Wt::Dbo::ptr<Track> track, double value)
+: _type(type),
+_track(track),
+_value(value)
+{
+}
+
+TrackFeature::pointer
+TrackFeature::create(Wt::Dbo::Session& session, Wt::Dbo::ptr<TrackFeatureType> type, Wt::Dbo::ptr<Track> track, double value)
+{
+	return session.add(std::make_unique<TrackFeature>(type, track, value));
+}
+
+} // namespace Database

src/database/TrackFeature.hpp

@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <string>
+
+#include <Wt/Dbo/Dbo.h>
+
+#include "Types.hpp"
+
+namespace Database {
+
+class Track;
+class SimilaritySettings;
+
+class TrackFeatureType : public Wt::Dbo::Dbo<TrackFeatureType>
+{
+	public:
+		using pointer = Wt::Dbo::ptr<TrackFeatureType>;
+
+		TrackFeatureType() = default;
+		TrackFeatureType(std::string name);
+
+		// Find utility
+		static pointer getByName(Wt::Dbo::Session& session, std::string name);
+
+		// Create utility
+		static pointer create(Wt::Dbo::Session& session, std::string name);
+
+		// Accessors
+		const std::string& getName() const { return _name; }
+
+		template<class Action>
+		void persist(Action& a)
+		{
+			Wt::Dbo::field(a, _name,	"name");
+			Wt::Dbo::belongsTo(a, _similaritySettings, "similarity_settings", Wt::Dbo::OnDeleteCascade);
+		}
+
+	private:
+
+		std::string _name;
+
+		Wt::Dbo::ptr<SimilaritySettings> _similaritySettings;
+};
+
+class TrackFeature : public Wt::Dbo::Dbo<TrackFeature>
+{
+	public:
+
+		using pointer = Wt::Dbo::ptr<TrackFeature>;
+
+		TrackFeature() = default;
+		TrackFeature(Wt::Dbo::ptr<TrackFeatureType> type, Wt::Dbo::ptr<Track> track, double value);
+
+		// Create utility
+		static pointer create(Wt::Dbo::Session& session, Wt::Dbo::ptr<TrackFeatureType> type, Wt::Dbo::ptr<Track> track, double value);
+
+		Wt::Dbo::ptr<TrackFeatureType> getType() const { return _type; }
+		double getValue() const { return _value; }
+
+		template<class Action>
+		void persist(Action& a)
+		{
+			Wt::Dbo::field(a, _value,	"value");
+			Wt::Dbo::belongsTo(a, _type, "type", Wt::Dbo::OnDeleteCascade);
+			Wt::Dbo::belongsTo(a, _track, "track", Wt::Dbo::OnDeleteCascade);
+		}
+
+	private:
+
+		Wt::Dbo::ptr<TrackFeatureType> _type;
+		Wt::Dbo::ptr<Track> _track;
+		double	_value = 0.;
+};
+
+
+} // namespace database
+
+

src/main/Services.cpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "Services.hpp"
+
+#include "cover/CoverArtGrabber.hpp"
+#include "scanner/MediaScanner.hpp"
+#include "similarity/SimilaritySearcher.hpp"
+
+Services& getServices()
+{
+	static Services services;
+	return services;
+}
+

src/main/Services.hpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <memory>
+
+namespace CoverArt
+{
+	class Grabber;
+}
+
+namespace Scanner {
+	class MediaScanner;
+}
+
+namespace Similarity {
+	class Searcher;
+}
+
+
+struct Services
+{
+	std::unique_ptr<CoverArt::Grabber>	coverArtGrabber;
+	std::unique_ptr<Scanner::MediaScanner>	mediaScanner;
+	std::unique_ptr<Similarity::Searcher>	similaritySearcher;
+};
+
+Services& getServices();
+

src/main/main.cpp

@@ -23,15 +23,17 @@
 #include <Wt/WServer.h>
 #include <Wt/WApplication.h>
 
-#include "utils/Config.hpp"
-#include "utils/Logger.hpp"
 #include "av/AvInfo.hpp"
 #include "av/AvTranscoder.hpp"
+#include "cover/CoverArtGrabber.hpp"
 #include "image/Image.hpp"
-
 #include "scanner/MediaScanner.hpp"
-
+#include "similarity/som/SimilaritySOMScannerAddon.hpp"
+#include "similarity/SimilaritySearcher.hpp"
 #include "ui/LmsApplication.hpp"
+#include "utils/Config.hpp"
+#include "utils/Logger.hpp"
+#include "Services.hpp"
 
 std::vector<std::string> generateWtConfig(std::string execPath)
 {
@@ -123,16 +125,24 @@ int main(int argc, char* argv[])
 		auto connectionPool = Database::Handler::createConnectionPool(Config::instance().getPath("working-dir") / "lms.db");
 
 		UserInterface::LmsApplicationGroupContainer appGroups;
-		Scanner::MediaScanner scanner(*connectionPool);
+
+		// Service initialization order is important
+		getServices().mediaScanner = std::make_unique<Scanner::MediaScanner>(*connectionPool);
+
+		Similarity::SOMScannerAddon similaritySOMScannerAddon(*connectionPool);
+
+		getServices().mediaScanner->setAddon(similaritySOMScannerAddon);
+		getServices().coverArtGrabber = std::make_unique<CoverArt::Grabber>();
+		getServices().similaritySearcher = std::make_unique<Similarity::Searcher>(similaritySOMScannerAddon);
 
 		// bind entry point
 		server.addEntryPoint(Wt::EntryPointType::Application,
 				std::bind(UserInterface::LmsApplication::create,
-					std::placeholders::_1, std::ref(*connectionPool), std::ref(appGroups), std::ref(scanner)));
+					std::placeholders::_1, std::ref(*connectionPool), std::ref(appGroups)));
 
 		// Start
-		LMS_LOG(MAIN, INFO) << "Starting Media scanner...";
+		LMS_LOG(MAIN, INFO) << "Starting media scanner...";
-		scanner.start();
+		getServices().mediaScanner->start();
 
 		LMS_LOG(MAIN, INFO) << "Starting server...";
 		server.start();
@@ -145,8 +155,8 @@ int main(int argc, char* argv[])
 		LMS_LOG(MAIN, INFO) << "Stopping server...";
 		server.stop();
 
-		LMS_LOG(MAIN, INFO) << "Stopping database updater...";
+		LMS_LOG(MAIN, INFO) << "Stopping media scanner...";
-		scanner.stop();
+		getServices().mediaScanner->stop();
 
 		LMS_LOG(MAIN, INFO) << "Clean stop!";
 		res = EXIT_SUCCESS;

src/scanner/MediaScanner.cpp

@@ -27,13 +27,11 @@
 #include <Wt/WLocalDateTime.h>
 
 #include "cover/CoverArtGrabber.hpp"
-
 #include "database/Artist.hpp"
 #include "database/Cluster.hpp"
 #include "database/Release.hpp"
 #include "database/ScanSettings.hpp"
 #include "database/Track.hpp"
-
 #include "utils/Logger.hpp"
 #include "utils/Path.hpp"
 #include "utils/Utils.hpp"
@@ -198,6 +196,12 @@ _db(connectionPool)
 	refreshScanSettings();
 }
 
+void
+MediaScanner::setAddon(MediaScannerAddon& addon)
+{
+	_addons.push_back(&addon);
+}
+
 void
 MediaScanner::restart(void)
 {
@@ -332,9 +336,11 @@ MediaScanner::scan(boost::system::error_code err)
 
 	LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.scanErrors << ", not imported = " << stats.incompleteScans << "), duplicates = " << stats.nbDuplicates() << " (hash = " << stats.duplicateHashes << ", mbid = " << stats.duplicateMBID << ")";
 
-	// Save the last scan only if it has been completed
 	if (_running)
 	{
+		for (auto& addon : _addons)
+			addon->preScanComplete();
+
 		scheduleScan();
 
 		scanComplete().emit(stats);
@@ -365,6 +371,11 @@ MediaScanner::refreshScanSettings()
 			[](ClusterType::pointer clusterType) -> std::string { return clusterType->getName(); });
 
 	_metadataParser.setClusterTypeNames(clusterTypeNames);
+
+	transaction.commit();
+
+	for (auto& addon : _addons)
+		addon->refreshSettings();
 }
 
 void
@@ -492,12 +503,14 @@ MediaScanner::scanAudioFile(const boost::filesystem::path& file, bool forceScan,
 
 	// If file already exist, update data
 	// Otherwise, create it
+	bool trackAdded = false;
 	if (!track)
 	{
 		// Create a new song
 		track = Track::create(_db.getSession(), file);
 		LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'";
 		stats.additions++;
+		trackAdded = true;
 	}
 	else
 	{
@@ -583,6 +596,14 @@ MediaScanner::scanAudioFile(const boost::filesystem::path& file, bool forceScan,
 	}
 
 	transaction.commit();
+
+	for (auto& addon : _addons)
+	{
+		if (trackAdded)
+			addon->trackAdded(track.id());
+		else
+			addon->trackUpdated(track.id());
+	}
 }
 
 void
@@ -753,5 +774,4 @@ MediaScanner::checkDuplicatedAudioFiles(Stats& stats)
 	LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!";
 }
 
-
 } // namespace Scanner

src/scanner/MediaScanner.hpp

@@ -26,10 +26,11 @@
 
 #include <boost/asio/system_timer.hpp>
 
-#include "metadata/TagLibParser.hpp"
-
 #include "database/ScanSettings.hpp"
 #include "database/DatabaseHandler.hpp"
+#include "metadata/TagLibParser.hpp"
+
+#include "MediaScannerAddon.hpp"
 
 namespace Scanner {
 
@@ -39,6 +40,8 @@ class MediaScanner
 
 		MediaScanner(Wt::Dbo::SqlConnectionPool& connectionPool);
 
+		void setAddon(MediaScannerAddon& addon);
+
 		void start();
 		void stop();
 		void restart();
@@ -83,10 +86,10 @@ class MediaScanner
 		// Helpers
 		void refreshScanSettings();
 
-		void removeMissingTracks( Stats& stats );
+		void removeMissingTracks(Stats& stats);
 		void removeOrphanEntries();
-		void checkDuplicatedAudioFiles( Stats& stats );
+		void checkDuplicatedAudioFiles(Stats& stats);
-		void scanAudioFile( const boost::filesystem::path& file, bool forceScan, Stats& stats);
+		void scanAudioFile(const boost::filesystem::path& file, bool forceScan, Stats& stats);
 
 		bool			_running;
 		Wt::WIOService		_ioService;
@@ -106,6 +109,8 @@ class MediaScanner
 
 		MetaData::TagLibParser 	_metadataParser;
 
+		std::vector<MediaScannerAddon*> _addons;
+
 }; // class MediaScanner
 
 } // Scanner

src/scanner/MediaScannerAddon.hpp

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "database/Types.hpp"
+
+namespace Scanner {
+
+class MediaScannerAddon
+{
+	public:
+
+		virtual void refreshSettings() = 0;
+
+		virtual void trackAdded(Database::IdType trackId) = 0;
+		virtual void trackToRemove(Database::IdType trackId) = 0;
+		virtual void trackUpdated(Database::IdType trackId) = 0;
+		virtual void preScanComplete() = 0;
+};
+
+} // ns Scanner
+

src/similarity/SimilaritySearcher.cpp

@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SimilaritySearcher.hpp"
+
+namespace Similarity {
+
+Searcher::Searcher(SOMScannerAddon& somAddon)
+: _somAddon(somAddon)
+{}
+
+std::vector<Database::IdType>
+Searcher::getSimilarTracks(const std::vector<Database::IdType>& tracksId, std::size_t maxCount)
+{
+	auto somSearcher = _somAddon.getSearcher();
+	if (!somSearcher)
+		return {};
+
+	return somSearcher->getSimilarTracks(tracksId, maxCount);
+}
+
+std::vector<Database::IdType>
+Searcher::getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount)
+{
+	auto somSearcher = _somAddon.getSearcher();
+	if (!somSearcher)
+		return {};
+
+	return somSearcher->getSimilarReleases(session, releaseId, maxCount);
+}
+
+std::vector<Database::IdType>
+Searcher::getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount)
+{
+	auto somSearcher = _somAddon.getSearcher();
+	if (!somSearcher)
+		return {};
+
+	return somSearcher->getSimilarArtists(session, artistId, maxCount);
+}
+
+} // ns Similarity

src/similarity/SimilaritySearcher.hpp

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/Dbo/Session.h>
+
+#include "database/Types.hpp"
+#include "som/SimilaritySOMScannerAddon.hpp"
+
+namespace Similarity {
+
+class Searcher
+{
+	public:
+		Searcher(SOMScannerAddon& somAddon);
+
+		std::vector<Database::IdType> getSimilarTracks(const std::vector<Database::IdType>& tracksId, std::size_t maxCount);
+		std::vector<Database::IdType> getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount);
+		std::vector<Database::IdType> getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount);
+
+	private:
+
+		SOMScannerAddon& _somAddon;
+};
+
+} // ns Similarity

src/similarity/cluster/SimilarityClusterSearcher.cpp

@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SimilarityClusterSearcher.hpp"
+
+#include <random>
+#include <chrono>
+
+#include "database/Cluster.hpp"
+#include "database/Track.hpp"
+#include "utils/Utils.hpp"
+
+namespace Similarity {
+
+std::vector<Database::IdType>
+ClusterSearcher::getSimilarTracks(Wt::Dbo::Session& session, const std::vector<Database::IdType>& tracksId, std::size_t maxCount)
+{
+	Wt::Dbo::Transaction transaction(session);
+
+	std::vector<Database::IdType> clusterIds;
+	for (auto trackId : tracksId)
+	{
+		auto track = Database::Track::getById(session, trackId);
+		if (!track)
+			continue;
+
+		auto clusters = track->getClusters();
+		if (clusters.empty())
+			continue;
+
+		for (const auto& cluster : clusters)
+			clusterIds.push_back(cluster.id());
+	}
+
+	std::vector<Database::IdType> sortedClusterIds;
+	uniqueAndSortedByOccurence(clusterIds.begin(), clusterIds.end(), std::back_inserter(sortedClusterIds));
+
+
+#if 0
+	auto now = std::chrono::system_clock::now();
+	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
+
+
+	std::set<Database::IdType> trackIds;
+	{
+		auto ids = tracklist->getTrackIds();
+		trackIds = std::set<Database::IdType>(ids.begin(), ids.end());
+	}
+
+	// Get all the tracks of the tracklist, get the cluster that is mostly used
+	// and reuse it to get the next track
+	auto clusters = tracklist->getClusters();
+	if (clusters.empty())
+		return;
+
+	for (auto cluster : clusters)
+	{
+		std::set<Database::IdType> clusterTrackIds = cluster->getTrackIds();
+
+		std::set<Database::IdType> candidateTrackIds;
+		std::set_difference(clusterTrackIds.begin(), clusterTrackIds.end(),
+				trackIds.begin(), trackIds.end(),
+				std::inserter(candidateTrackIds, candidateTrackIds.end()));
+
+		if (candidateTrackIds.empty())
+			continue;
+
+		std::uniform_int_distribution<int> dist(0, candidateTrackIds.size() - 1);
+
+		auto trackToAdd = Database::Track::getById(LmsApp->getDboSession(), *std::next(candidateTrackIds.begin(), dist(randGenerator)));
+		enqueueTrack(trackToAdd);
+
+		return;
+	}
+
+	LMS_LOG(UI, INFO) << "No more track to be added!";
+#endif
+	return {};
+}
+
+std::vector<Database::IdType>
+ClusterSearcher::getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount)
+{
+
+	return {};
+}
+
+std::vector<Database::IdType>
+ClusterSearcher::getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount)
+{
+	return {};
+}
+
+} // namespace Similarity

src/similarity/cluster/SimilarityClusterSearcher.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <vector>
+
+#include "database/Types.hpp"
+
+namespace Similarity {
+
+class ClusterSearcher
+{
+	public:
+		std::vector<Database::IdType> getSimilarTracks(Wt::Dbo::Session& session, const std::vector<Database::IdType>& tracksId, std::size_t maxCount);
+		std::vector<Database::IdType> getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount);
+		std::vector<Database::IdType> getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount);
+};
+
+} // namespace Similarity

src/similarity/som/AcousticBrainzUtils.cpp

@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "AcousticBrainzUtils.hpp"
+
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/json_parser.hpp>
+#include <curl/curl.h>
+
+#include "utils/Config.hpp"
+#include "utils/Logger.hpp"
+
+
+namespace AcousticBrainz
+{
+
+
+static size_t writeToOStringStream(void *buffer, size_t size, size_t nmemb, void* ctx)
+{
+	std::ostringstream& oss = *reinterpret_cast<std::ostringstream*>(ctx);
+
+	oss.write(reinterpret_cast<char*>(buffer), size * nmemb);
+
+	return size * nmemb;
+}
+
+static bool
+getFeaturesFromJsonData(const std::string& jsonData, const std::set<std::string>& featuresName, std::map<std::string, double>& features)
+{
+	try
+	{
+		boost::property_tree::ptree root;
+
+		std::istringstream iss(jsonData);
+
+		boost::property_tree::read_json(iss, root);
+
+		for (const auto& featureName : featuresName)
+		{
+			features[featureName] = root.get<double>(featureName);
+		}
+
+		return true;
+	}
+	catch (std::exception& e)
+	{
+		LMS_LOG(DBUPDATER, ERROR) << "Cannot extract feature: " << e.what();
+		return false;
+	}
+}
+
+static std::string
+getJsonData(const std::string& mbid)
+{
+	static const std::string defaultAPIURL = "https://acousticbrainz.org/api/v1/";
+
+	std::string data;
+	std::string url = Config::instance().getString("acousticbrainz-api-url", defaultAPIURL) + mbid + "/low-level";
+
+	CURL *curl;
+	CURLcode res;
+
+	curl = curl_easy_init();
+	if (!curl)
+	{
+		LMS_LOG(DBUPDATER, ERROR) << "CURL init failed";
+		return data;
+	}
+
+	std::ostringstream oss;
+
+	curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeToOStringStream);
+	curl_easy_setopt(curl, CURLOPT_WRITEDATA, &oss);
+
+	res = curl_easy_perform(curl);
+	if (res != CURLE_OK)
+	{
+		LMS_LOG(DBUPDATER, ERROR) << "CURL perform failed: " << curl_easy_strerror(res);
+		return data;
+	}
+
+	curl_easy_cleanup(curl);
+
+	data = std::move(oss.str());
+
+	return data;
+}
+
+bool
+extractFeatures(const std::string& mbid, const std::set<std::string>& featuresName, std::map<std::string, double>& features)
+{
+	return getFeaturesFromJsonData(getJsonData(mbid), featuresName, features);
+}
+
+} // namespace Scanner::AcousticBrainz

src/similarity/som/AcousticBrainzUtils.hpp

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <map>
+#include <set>
+#include <string>
+
+namespace AcousticBrainz
+{
+
+	bool extractFeatures(const std::string& MBID, const std::set<std::string>& featuresName, std::map<std::string, double>& features);
+}
+

src/similarity/som/DataNormalizer.cpp

@@ -21,6 +21,7 @@
 
 #include <algorithm>
 #include <numeric>
+#include <sstream>
 
 namespace SOM
 {
@@ -49,6 +50,11 @@ DataNormalizer::DataNormalizer(std::size_t inputDimCount)
 {
 }
 
+DataNormalizer::DataNormalizer(const std::string& data)
+{
+	serializeFrom(data);
+}
+
 void
 DataNormalizer::computeNormalizationFactors(const std::vector<InputVector>& inputVectors)
 {
@@ -97,6 +103,34 @@ DataNormalizer::normalizeData(InputVector& a) const
 	}
 }
 
+std::string
+DataNormalizer::serializeTo() const
+{
+	std::ostringstream oss;
+
+	oss << _inputDimCount << " ";
+	for (std::size_t i = 0; i < _inputDimCount; ++i)
+		oss << _minmax[i].min << " " << _minmax[i].max;
+
+	return oss.str();
+}
+
+void
+DataNormalizer::serializeFrom(const std::string& data)
+{
+	std::istringstream iss(data);
+
+	iss >> _inputDimCount;
+	_minmax.resize(_inputDimCount);
+
+	for (std::size_t i = 0; i < _inputDimCount; ++i)
+	{
+		iss >> _minmax[i].min;
+		iss >> _minmax[i].max;
+	}
+
+}
+
 void
 DataNormalizer::dump(std::ostream& os) const
 {

src/similarity/som/DataNormalizer.hpp

@@ -22,7 +22,7 @@
 #include <vector>
 #include <ostream>
 
-#include "SOM.hpp"
+#include "Network.hpp"
 
 namespace SOM
 {
@@ -30,15 +30,20 @@ namespace SOM
 class DataNormalizer
 {
 	public:
+
 		DataNormalizer(std::size_t inputDimCount);
+		DataNormalizer(const std::string& data);
 
 		void computeNormalizationFactors(const std::vector<InputVector>& dataSamples);
 
 		void normalizeData(InputVector& data) const;
 
+		std::string serializeTo() const;
+
 		void dump(std::ostream& os) const;
 
 	private:
+		void serializeFrom(const std::string& data);
 		InputVector::value_type normalizeValue(InputVector::value_type value, std::size_t dimensionId) const;
 
 		std::size_t _inputDimCount;

src/similarity/som/Matrix.hpp

@@ -0,0 +1,104 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <cassert>
+#include <sstream>
+#include <vector>
+
+namespace SOM
+{
+
+struct Coords
+{
+	std::size_t x;
+	std::size_t y;
+
+	bool operator<(const Coords& other) const
+	{
+		return x < other.x && y < other.y;
+	}
+};
+
+template <typename T>
+class Matrix
+{
+	public:
+
+		Matrix(std::size_t width, std::size_t height)
+		: _width(width),
+		_height(height)
+		{
+			_values.resize(_width*_height);
+		}
+
+		Matrix(std::size_t width, std::size_t height, std::vector<T> values)
+		: _width(width),
+		_height(height),
+		_values(std::move(values))
+		{
+			assert(_values.size() == _width * _height);
+		}
+
+		void clear()
+		{
+			std::vector<T> values(_width*_height);
+			_values.swap(values);
+		}
+
+		std::size_t getHeight() const { return _height; }
+		std::size_t getWidth() const { return _width; }
+
+		T& get(Coords coords)
+		{
+			assert(coords.x < _width);
+			assert(coords.y < _height);
+
+			return _values[coords.x + _width*coords.y];
+		}
+
+		const T& get(Coords coords) const
+		{
+			assert(coords.x < _width);
+			assert(coords.y < _height);
+			return _values[coords.x + _width*coords.y];
+		}
+
+		T& operator[](Coords coords) { return get(coords); }
+		const T& operator[](Coords coords) const { return get(coords); }
+
+		template <typename Func>
+		Coords getCoordsMinElement(Func func) const
+		{
+			auto it = std::min_element(_values.begin(), _values.end(), func);
+			auto index = std::distance(_values.begin(), it);
+
+			return {index % _height, index / _height};
+		}
+
+	private:
+
+		std::size_t _width;
+		std::size_t _height;
+		std::vector<T> _values;
+};
+
+} // ns SOM

src/similarity/som/Network.cpp

@@ -17,12 +17,15 @@
  * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "SOM.hpp"
+#include "Network.hpp"
 
-#include <chrono>
-#include <random>
-#include <cmath>
 #include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <random>
+#include <sstream>
+
+#include "utils/Logger.hpp"
 
 namespace SOM
 {
@@ -41,7 +44,7 @@ checkSameDimensions(const InputVector& a, std::size_t inputDimCount)
 		throw SOMException("Bad data dimension count");
 }
 
-InputVector::value_type
+static InputVector::value_type
 defaultLearningFactor(Network::Progress progress)
 {
 	constexpr InputVector::value_type initialValue = 1;
@@ -49,7 +52,7 @@ defaultLearningFactor(Network::Progress progress)
 	return initialValue * exp(-((progress.idIteration + 1) / static_cast<InputVector::value_type>(progress.iterationCount)));
 }
 
-InputVector::value_type
+static InputVector::value_type
 euclidianSquareDistance(const InputVector& a, const InputVector& b, const InputVector& weights)
 {
 	checkSameDimensions(a, b);
@@ -74,6 +77,7 @@ sigmaFunc(Network::Progress progress)
 	return sigma0 * exp(- ((progress.idIteration + 1) / static_cast<InputVector::value_type>(progress.iterationCount)));
 }
 
+static
 InputVector::value_type
 defaultNeighborhoodFunc(InputVector::value_type norm, Network::Progress progress)
 {
@@ -97,7 +101,7 @@ operator<<(std::ostream& os, const InputVector& a)
 }
 
 
-//static
+static
 InputVector::value_type
 norm(const InputVector& a)
 {
@@ -111,7 +115,7 @@ norm(const InputVector& a)
 	return sqrt(res);
 }
 
-//static
+static
 InputVector
 operator+(const InputVector& a, const InputVector& b)
 {
@@ -143,7 +147,7 @@ operator-(const InputVector& a, const InputVector& b)
 	return res;
 }
 
-//static
+static
 InputVector
 operator*(const InputVector& a, InputVector::value_type factor)
 {
@@ -157,33 +161,42 @@ operator*(const InputVector& a, InputVector::value_type factor)
 	return res;
 }
 
-
 Network::Network(std::size_t width, std::size_t height, std::size_t inputDimCount)
-: _width(width),
+:
-_height(height),
 _inputDimCount(inputDimCount),
 _weights(inputDimCount, static_cast<InputVector::value_type>(1)),
+_refVectors(width, height),
 _distanceFunc(euclidianSquareDistance),
 _learningFactorFunc(defaultLearningFactor),
 _neighborhoodFunc(defaultNeighborhoodFunc)
 {
-	_refVectors.resize(width * height);
-
 	auto now = std::chrono::system_clock::now();
 	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
 
 	// init each vector with a random normalized value
 	std::uniform_real_distribution<InputVector::value_type> dist(0, 1);
 
-	for (auto& refVector : _refVectors)
+	for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
 	{
-		refVector.resize(inputDimCount);
+		for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
-
+		{
-		for (auto& val : refVector)
+			auto& refVector = _refVectors.get({x,y});
-			val = dist(randGenerator);
+			refVector.resize(_inputDimCount);
+			for (auto& val : refVector)
+				val = dist(randGenerator);
+		}
 	}
 }
 
+Network::Network(const std::string& data)
+: _refVectors(0, 0),
+_distanceFunc(euclidianSquareDistance),
+_learningFactorFunc(defaultLearningFactor),
+_neighborhoodFunc(defaultNeighborhoodFunc)
+{
+	serializeFrom(data);
+}
+
 void
 Network::setDataWeights(const InputVector& weights)
 {
@@ -192,28 +205,16 @@ Network::setDataWeights(const InputVector& weights)
 	_weights = weights;
 }
 
-InputVector&
-Network::getRefVector(std::size_t x, std::size_t y)
-{
-	return _refVectors[x + y*_width];
-}
-
-const InputVector&
-Network::getRefVector(std::size_t x, std::size_t y) const
-{
-	return _refVectors[x + y*_width];
-}
-
 void
 Network::dump(std::ostream& os) const
 {
-	os << "Width: " << _width << ", Height: " << _height << std::endl;;
+	os << "Width: " << _refVectors.getWidth() << ", Height: " << _refVectors.getHeight() << std::endl;;
 
-	for (std::size_t y = 0; y < _height; ++y)
+	for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
 	{
-		for (std::size_t x = 0; x < _width; ++x)
+		for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
 		{
-			os << getRefVector(x, y) << " ";
+			os << _refVectors.get({x, y}) << " ";
 		}
 
 		os << std::endl;
@@ -224,15 +225,10 @@ Network::dump(std::ostream& os) const
 Coords
 Network::getClosestRefVector(const InputVector& data) const
 {
-	auto it = std::min_element(_refVectors.begin(), _refVectors.end(),
+	return _refVectors.getCoordsMinElement([&](const auto& a, const auto& b)
-			[&](const auto& a, const auto& b)
 			{
 				return (_distanceFunc(a, data, _weights) < _distanceFunc(b, data, _weights));
 			});
-
-	auto index = std::distance(_refVectors.begin(), it);
-
-	return {index % _height, index / _height};
 }
 
 Coords
@@ -251,15 +247,15 @@ Network::classify(const InputVector& data, std::size_t size) const
 	};
 	std::vector<Entry> sortedEntries;
 
-	for (std::size_t x = 0; x < _width; ++x)
+	for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
 	{
-		for (std::size_t y = 0; y < _height; ++y)
+		for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
 		{
-			sortedEntries.push_back( Entry{{x, y}, getRefVector(x, y)} );
+			sortedEntries.push_back( Entry{{x, y}, _refVectors.get({x, y})} );
 		}
 	}
 
-	const InputVector& closestRefVector = getRefVector(getClosestRefVector(data));
+	const InputVector& closestRefVector = _refVectors.get(getClosestRefVector(data));
 
 	std::sort(sortedEntries.begin(), sortedEntries.end(),
 		[&](const Entry& a, const Entry& b)
@@ -292,11 +288,11 @@ computeCoordsNorm(Coords c1, Coords c2)
 void
 Network::updateRefVectors(Coords closestRefVectorCoords, const InputVector& input, Progress progress)
 {
-	for (std::size_t y = 0; y < _height; ++y)
+	for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
 	{
-		for (std::size_t x = 0; x < _width; ++x)
+		for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
 		{
-			auto& refVector = getRefVector(x, y);
+			auto& refVector = _refVectors.get({x, y});
 
 			auto delta = input - refVector;
 			auto n = computeCoordsNorm({x, y}, closestRefVectorCoords);
@@ -335,6 +331,68 @@ Network::train(const std::vector<InputVector>& inputData, std::size_t nbIteratio
 	}
 }
 
+std::string
+Network::serializeTo() const
+{
+	std::ostringstream oss;
+
+	oss << _inputDimCount << " ";
+
+	for (auto weight : _weights)
+		oss << weight << " ";
+
+	// Matrix
+	oss << _refVectors.getWidth() << " " << _refVectors.getHeight() << " ";
+	for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
+	{
+		for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
+		{
+			for (auto val : _refVectors.get({x,y}))
+				oss << val << " ";
+		}
+	}
+
+	return oss.str();
+}
+
+void
+Network::serializeFrom(const std::string& data)
+{
+	std::istringstream iss(data);
+
+	LMS_LOG(SIMILARITY, DEBUG) << "data = '" << data << "'";
+	iss >> _inputDimCount;
+	LMS_LOG(SIMILARITY, DEBUG) << "Input dim count = " << _inputDimCount;
+
+	for (std::size_t i = 0; i < _inputDimCount; ++i)
+	{
+		InputVector::value_type val;
+		iss >> val;
+		_weights.push_back(val);
+	}
+
+	LMS_LOG(SIMILARITY, DEBUG) << "Reading matrix...";
+	std::size_t width, height;
+	iss >> width >> height;
+	_refVectors = Matrix<SOM::InputVector>(width, height);
+
+	for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
+	{
+		for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
+		{
+			InputVector refVector;
+			refVector.reserve(_inputDimCount);
+			for (std::size_t i = 0; i < _inputDimCount; ++i)
+			{
+				InputVector::value_type val;
+				iss >> val;
+				refVector.push_back(val);
+			}
+			_refVectors.get({x, y}) = refVector;
+		}
+	}
+}
+
 } // namespace SOM
 
 

src/similarity/som/Network.hpp

@@ -23,6 +23,8 @@
 #include <ostream>
 #include <functional>
 
+#include "Matrix.hpp"
+
 #include "utils/Exception.hpp"
 
 namespace SOM
@@ -39,19 +41,20 @@ class SOMException : public LmsException
 		SOMException(const std::string& msg) : LmsException(msg) {}
 };
 
-// Top Left is (0,0)
-struct Coords
-{
-	std::size_t x;
-	std::size_t y;
-};
 
 class Network
 {
 	public:
 
+		// Init a network with random values
 		Network(std::size_t width, std::size_t height, std::size_t inputDimCount);
 
+		// Init a network with serialized values
+		Network(const std::string& data);
+
+		std::size_t getWidth() const { return _refVectors.getWidth(); }
+		std::size_t getHeight() const { return _refVectors.getHeight(); }
+		std::size_t getInputDimCount() const {return _inputDimCount;}
 		// Set weight for each dimension (default is 1 for each weight)
 		void setDataWeights(const InputVector& weights);
 
@@ -85,21 +88,19 @@ class Network
 		using NeighborhoodFunc = std::function<InputVector::value_type(InputVector::value_type /* norm(Coords - CoordMatchingRefVector) */, Progress)>;
 		void setNeighborhoodFunc(NeighborhoodFunc neighborhoodFunc);
 
+		std::string serializeTo() const;
+
 	private:
 
-		InputVector& getRefVector(std::size_t x, std::size_t y);
+		void serializeFrom(const std::string& data);
-		const InputVector& getRefVector(std::size_t x, std::size_t y) const;
+
-		const InputVector& getRefVector(Coords coords) const { return getRefVector(coords.x, coords.y); }
 		Coords getClosestRefVector(const InputVector& data) const;
 
 		void updateRefVectors(Coords closestRefVectorCoords, const InputVector& input, Progress progress);
 
-		std::size_t _width;
-		std::size_t _height;
 		std::size_t _inputDimCount;
-
+		InputVector _weights;	// weight for each dimension
-		InputVector _weights;
+		Matrix<InputVector> _refVectors;
-		std::vector<InputVector> _refVectors; // reference vectors
 
 		DistanceFunc _distanceFunc;
 		LearningFactorFunc _learningFactorFunc;

src/similarity/som/SimilaritySOMScannerAddon.cpp

@@ -0,0 +1,272 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SimilaritySOMScannerAddon.hpp"
+
+#include <cmath>
+
+#include "database/Track.hpp"
+#include "database/SimilaritySettings.hpp"
+#include "database/TrackFeature.hpp"
+#include "utils/Logger.hpp"
+
+#include "AcousticBrainzUtils.hpp"
+#include "DataNormalizer.hpp"
+#include "Network.hpp"
+
+
+namespace Similarity {
+
+namespace {
+
+struct TrackInfo
+{
+	Database::IdType id;
+	std::string mbid;
+};
+
+std::vector<TrackInfo>
+getTracksWithMBIDAndMissingFeatures(Wt::Dbo::Session& session)
+{
+	std::vector<TrackInfo> res;
+
+	Wt::Dbo::Transaction transaction(session);
+
+	auto tracks = Database::Track::getAllWithMBIDAndMissingFeatures(session);
+	for (auto track : tracks)
+		res.push_back({track.id(), track->getMBID()});
+
+	return res;
+}
+
+std::vector<Database::TrackFeatureType::pointer>
+getTrackFeatureTypes(Wt::Dbo::Session& session, const std::set<std::string>& featureNames)
+{
+	std::vector<Database::TrackFeatureType::pointer> res;
+
+	for (const auto& featureName : featureNames)
+	{
+		auto trackFeatureType = Database::TrackFeatureType::getByName(session, featureName);
+		if (!trackFeatureType)
+		{
+			LMS_LOG(DBUPDATER, ERROR) << "Missing feature type '" << featureName << "'";
+			res.clear();
+			return res;
+		}
+
+		res.push_back(trackFeatureType);
+	}
+
+	return res;
+}
+
+
+bool
+extractFeatures(const Database::Track::pointer& track, const std::vector<Database::TrackFeatureType::pointer>& trackFeatureTypes, std::vector<double>& features)
+{
+	features.reserve(trackFeatureTypes.size());
+
+	for (const auto& trackFeatureType : trackFeatureTypes)
+	{
+		auto feature = track->getTrackFeature(trackFeatureType);
+		if (!feature)
+		{
+			LMS_LOG(DBUPDATER, ERROR) << "Missing feature " << trackFeatureType->getName() << " for track '" << track->getPath().string() << "'";
+			return false;
+		}
+
+		features.emplace_back(feature->getValue());
+	}
+
+	return true;
+}
+
+
+} // namespace
+
+SOMScannerAddon::SOMScannerAddon(Wt::Dbo::SqlConnectionPool& connectionPool)
+: _db(connectionPool)
+{
+	refreshSettings();
+	clusterize();
+}
+
+std::shared_ptr<Similarity::SOMSearcher>
+SOMScannerAddon::getSearcher()
+{
+	return std::atomic_load(&_finder);
+}
+
+void
+SOMScannerAddon::trackUpdated(Database::IdType trackId)
+{
+	Wt::Dbo::Transaction transaction(_db.getSession());
+
+	auto track = Database::Track::getById(_db.getSession(), trackId);
+	if (!track)
+		return;
+
+	track.modify()->eraseFeatures();
+}
+
+void
+SOMScannerAddon::preScanComplete()
+{
+	auto tracksInfo = getTracksWithMBIDAndMissingFeatures(_db.getSession());
+	for (const auto& trackInfo : tracksInfo)
+		fetchFeatures(trackInfo.id, trackInfo.mbid);
+
+	LMS_LOG(DBUPDATER, INFO) << "Clustering tracks...";
+	clusterize();
+	LMS_LOG(DBUPDATER, INFO) << "Clusterization complete!";
+}
+
+void
+SOMScannerAddon::clusterize()
+{
+	Wt::Dbo::Transaction transaction(_db.getSession());
+
+	auto trackFeatureTypes = getTrackFeatureTypes(_db.getSession(), _featuresName);
+
+	LMS_LOG(DBUPDATER, DEBUG) << "Getting feature types DONE...";
+
+	LMS_LOG(DBUPDATER, DEBUG) << "Getting Tracks with features...";
+	auto tracks = Database::Track::getAllWithFeatures(_db.getSession());
+	LMS_LOG(DBUPDATER, DEBUG) << "Getting Tracks with features DONE";
+
+	std::vector<SOM::InputVector> samples;
+	std::vector<Database::IdType> tracksIds;
+
+	LMS_LOG(DBUPDATER, DEBUG) << "Extracting features...";
+	for (auto track : tracks)
+	{
+		SOM::InputVector sample;
+
+		if (!extractFeatures(track, trackFeatureTypes, sample))
+			continue;
+
+		samples.emplace_back(std::move(sample));
+		tracksIds.emplace_back(track.id());
+	}
+	LMS_LOG(DBUPDATER, DEBUG) << "Extracting features DONE";
+
+	transaction.commit();
+
+	if (tracksIds.empty())
+	{
+		LMS_LOG(DBUPDATER, INFO) << "Nothing to classify!";
+		std::atomic_store(&_finder, std::shared_ptr<SOMSearcher>());
+		return;
+	}
+
+	LMS_LOG(DBUPDATER, DEBUG) << "Normalizing data...";
+	SOM::DataNormalizer normalizer(_featuresName.size());
+
+	normalizer.computeNormalizationFactors(samples);
+	for (auto& sample : samples)
+		normalizer.normalizeData(sample);
+
+	std::size_t size = std::sqrt(samples.size()/5);
+	LMS_LOG(DBUPDATER, DEBUG) << "Found " << samples.size() << " tracks, Constructing a " << size << "*" << size << " network";
+	SOM::Network network(size, size, _featuresName.size());
+
+	LMS_LOG(DBUPDATER, DEBUG) << "Training network...";
+	network.train(samples, 20);
+	LMS_LOG(DBUPDATER, DEBUG) << "Training network DONE";
+
+	// Now classify all the tracks
+	LMS_LOG(DBUPDATER, DEBUG) << "Classifying tracks...";
+	SOM::Matrix<std::vector<Database::IdType>> tracksMap(network.getWidth(), network.getHeight());
+	std::map<Database::IdType, SOM::Coords> trackIdsCoords;
+
+	for (std::size_t i = 0; i < samples.size(); ++i)
+	{
+		const auto& sample = samples[i];
+		auto trackId = tracksIds[i];
+
+		auto coords = network.classify(sample);
+		tracksMap[coords].push_back(trackId);
+		trackIdsCoords[trackId] = coords;
+	}
+
+	Similarity::SOMSearcher::ConstructionParams params{std::move(network), std::move(normalizer), std::move(tracksMap), std::move(trackIdsCoords)};
+
+	auto finder = std::make_shared<Similarity::SOMSearcher>(std::move(params));
+
+	std::atomic_store(&_finder, finder);
+
+	LMS_LOG(DBUPDATER, DEBUG) << "Classifying tracks DONE";
+
+	LMS_LOG(DBUPDATER, DEBUG) << "Dumping classifier:";
+
+	std::ofstream ofs("/tmp/output");
+	finder->dump(_db.getSession(), ofs);
+
+	LMS_LOG(DBUPDATER, DEBUG) << "Dumping classifier DONE";
+
+}
+
+void
+SOMScannerAddon::refreshSettings()
+{
+	Wt::Dbo::Transaction transaction(_db.getSession());
+
+	auto settings = Database::SimilaritySettings::get(_db.getSession());
+
+	_settingsVersion = settings->getVersion();
+
+	for (auto trackFeatureType : settings->getTrackFeatureTypes())
+	{
+		_featuresName.insert(trackFeatureType->getName());
+	}
+}
+
+bool
+SOMScannerAddon::fetchFeatures(Database::IdType trackId, const std::string& MBID)
+{
+	std::map<std::string, double> features;
+
+	if (!AcousticBrainz::extractFeatures(MBID, _featuresName, features))
+	{
+		LMS_LOG(DBUPDATER, ERROR) << "Cannot extract features using AcousticBrainz!";
+		return false;
+	}
+
+	Wt::Dbo::Transaction transaction(_db.getSession());
+
+	Wt::Dbo::ptr<Database::Track> track = Database::Track::getById(_db.getSession(), trackId);
+	if (!track)
+		return false;
+
+	LMS_LOG(DBUPDATER, DEBUG) << "Successfully extracted AcousticBrainz lowlevel features for track '" << track->getPath().string() << "'";
+
+	for (const auto& feature : features)
+	{
+		auto featureType = Database::TrackFeatureType::getByName(_db.getSession(), feature.first);
+		if (!featureType)
+			return false;
+
+		Database::TrackFeature::create(_db.getSession(), featureType, track, feature.second);
+	}
+
+	return true;
+}
+
+} // namespace Similarity
+

src/similarity/som/SimilaritySOMScannerAddon.hpp

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/Dbo/SqlConnectionPool.h>
+
+#include "database/DatabaseHandler.hpp"
+#include "scanner/MediaScannerAddon.hpp"
+
+#include "SimilaritySOMSearcher.hpp"
+
+namespace Similarity {
+
+class SOMScannerAddon : public Scanner::MediaScannerAddon
+{
+	public:
+
+		SOMScannerAddon(Wt::Dbo::SqlConnectionPool& connectionPool);
+
+		std::shared_ptr<SOMSearcher> getSearcher();
+
+	private:
+		void refreshSettings() override;
+		void trackAdded(Database::IdType trackId) override {}
+		void trackToRemove(Database::IdType trackId) override {}
+		void trackUpdated(Database::IdType trackId) override;
+		void preScanComplete() override;
+
+		bool fetchFeatures(Database::IdType trackId, const std::string& MBID);
+
+		void clusterize();
+
+		std::size_t 		_settingsVersion;
+		std::set<std::string>	_featuresName;
+		Database::Handler	_db;
+
+		std::shared_ptr<SOMSearcher> _finder;
+};
+
+SOMScannerAddon* setSOMScannerAddon(SOMScannerAddon addon);
+SOMScannerAddon* getSOMScannerAddon();
+
+} // namespace Similarity
+

src/similarity/som/SimilaritySOMSearcher.cpp

@@ -0,0 +1,259 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SimilaritySOMSearcher.hpp"
+
+#include <random>
+
+#include "database/Artist.hpp"
+#include "database/SimilaritySettings.hpp"
+#include "database/Release.hpp"
+#include "database/Track.hpp"
+#include "utils/Logger.hpp"
+#include "utils/Utils.hpp"
+
+namespace Similarity {
+
+SOMSearcher::SOMSearcher(ConstructionParams params)
+: _network(std::move(params.network)),
+_normalizer(std::move(params.normalizer)),
+_tracksMap(std::move(params.tracksMap)),
+_trackIdsCoords(std::move(params.trackIdsCoords))
+{
+}
+
+std::vector<Database::IdType>
+SOMSearcher::getSimilarTracks(const std::vector<Database::IdType>& tracksIds, std::size_t maxCount)
+{
+	std::vector<Database::IdType> res;
+
+	auto bestCoords = getBestMatchingCoords(tracksIds);
+	if (!bestCoords)
+		return res;
+
+	auto tracks = _tracksMap[*bestCoords];
+
+	auto now = std::chrono::system_clock::now();
+	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
+
+	std::shuffle(tracks.begin(), tracks.end(), randGenerator);
+
+	if (tracks.size() > maxCount)
+		tracks.resize(maxCount);
+
+	return tracks;
+}
+
+std::vector<Database::IdType>
+SOMSearcher::getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount)
+{
+	std::vector<Database::IdType> res;
+
+	Wt::Dbo::Transaction transaction(session);
+
+	auto release = Database::Release::getById(session, releaseId);
+	if (!release)
+		return res;
+
+	auto tracks = release->getTracks();
+
+	std::vector<Database::IdType> tracksIds;
+	for (auto track : tracks)
+		tracksIds.push_back(track.id());
+
+	auto matchingCoords = getMatchingCoords(tracksIds);
+	if (matchingCoords.empty())
+		return res;
+
+	auto releases = getReleases(session, matchingCoords);
+	uniqueAndSortedByOccurence(releases.begin(), releases.end(), std::back_inserter(res));
+
+	res.erase(std::remove_if(res.begin(), res.end(), [&](auto releaseId) { return releaseId == release.id(); }), res.end());
+
+	if (res.size() > maxCount)
+		res.resize(maxCount);
+
+	LMS_LOG(SIMILARITY, DEBUG) << "*** SIMILARITY RESULT *** :";
+	for (auto id : res)
+		LMS_LOG(SIMILARITY, DEBUG) << id;
+
+	return res;
+}
+
+std::vector<Database::IdType>
+SOMSearcher::getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount)
+{
+	std::vector<Database::IdType> res;
+
+	Wt::Dbo::Transaction transaction(session);
+
+	auto artist = Database::Artist::getById(session, artistId);
+	if (!artist)
+		return res;
+
+	auto tracks = artist->getTracks();
+
+	std::vector<Database::IdType> tracksIds;
+	for (auto track : tracks)
+		tracksIds.push_back(track.id());
+
+	auto matchingCoords = getMatchingCoords(tracksIds);
+	if (matchingCoords.empty())
+		return res;
+
+	auto artists = getArtists(session, matchingCoords);
+	uniqueAndSortedByOccurence(artists.begin(), artists.end(), std::back_inserter(res));
+
+	res.erase(std::remove_if(res.begin(), res.end(), [&](auto artistId) { return artistId == artist.id(); }), res.end());
+
+	if (res.size() > maxCount)
+		res.resize(maxCount);
+
+	LMS_LOG(SIMILARITY, DEBUG) << "*** SIMILARITY RESULT *** :";
+	for (auto id : res)
+		LMS_LOG(SIMILARITY, DEBUG) << id;
+
+	return res;
+}
+
+void
+SOMSearcher::dump(Wt::Dbo::Session& session, std::ostream& os) const
+{
+	os << "Number of tracks classified: " << _trackIdsCoords.size() << std::endl;
+	os << "Network size: " << _network.getWidth() << " * " << _network.getHeight() << std::endl;
+
+	Wt::Dbo::Transaction transaction(session);
+
+	for (std::size_t y = 0; y < _network.getHeight(); ++y)
+	{
+		for (std::size_t x = 0; x < _network.getWidth(); ++x)
+		{
+			const auto& trackIds = _tracksMap[{x, y}];
+
+			for (auto trackId : trackIds)
+			{
+				auto track = Database::Track::getById(session, trackId);
+				if (!track)
+					continue;
+
+				os << "{";
+				if (track->getArtist())
+					os << track->getArtist()->getName() << " ";
+				if (track->getRelease())
+					os << track->getRelease()->getName();
+				os << "} ";
+			}
+
+			os << "; ";
+		}
+		os << std::endl;
+	}
+}
+
+boost::optional<SOM::Coords>
+SOMSearcher::getBestMatchingCoords(const std::vector<Database::IdType>& tracksIds) const
+{
+	if (tracksIds.empty())
+		return boost::none;
+
+	std::map<SOM::Coords, std::size_t /*count*/> coordsCount;
+
+	for (auto trackId : tracksIds)
+	{
+		auto it = _trackIdsCoords.find(trackId);
+		if (it == _trackIdsCoords.end())
+			continue;
+
+		if (coordsCount.find(it->second) == coordsCount.end())
+			coordsCount[it->second] = 0;
+
+		coordsCount[it->second]++;
+	}
+
+	if (coordsCount.empty())
+		return boost::none;
+
+	auto bestCoords = std::max_element(std::begin(coordsCount), std::end(coordsCount),
+			[](const auto& a, const auto& b)
+			{
+				return a.second < b.second;
+			});
+
+	return bestCoords->first;
+}
+
+std::vector<SOM::Coords>
+SOMSearcher::getMatchingCoords(const std::vector<Database::IdType>& tracksIds) const
+{
+	std::vector<SOM::Coords> res;
+
+	if (tracksIds.empty())
+		return res;
+
+	for (auto trackId : tracksIds)
+	{
+		auto it = _trackIdsCoords.find(trackId);
+		if (it == _trackIdsCoords.end())
+			continue;
+
+		res.push_back(it->second);
+	}
+
+	return res;
+}
+
+std::vector<Database::IdType>
+SOMSearcher::getReleases(Wt::Dbo::Session& session, const std::vector<SOM::Coords>& coords) const
+{
+	std::vector<Database::IdType> res;
+	for (const auto& c : coords)
+	{
+		for (auto trackId : _tracksMap[c])
+		{
+			auto track = Database::Track::getById(session, trackId);
+			if (!track || !track->getRelease())
+				continue;
+
+			res.emplace_back(track->getRelease().id());
+		}
+	}
+
+	return res;
+}
+
+std::vector<Database::IdType>
+SOMSearcher::getArtists(Wt::Dbo::Session& session, const std::vector<SOM::Coords>& coords) const
+{
+	std::vector<Database::IdType> res;
+	for (const auto& c : coords)
+	{
+		for (auto trackId : _tracksMap[c])
+		{
+			auto track = Database::Track::getById(session, trackId);
+			if (!track || !track->getArtist())
+				continue;
+
+			res.emplace_back(track->getArtist().id());
+		}
+	}
+
+	return res;
+}
+
+} // ns Similarity

src/similarity/som/SimilaritySOMSearcher.hpp

@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <map>
+#include <boost/optional.hpp>
+
+#include "database/DatabaseHandler.hpp"
+#include "database/Types.hpp"
+#include "DataNormalizer.hpp"
+#include "Network.hpp"
+
+namespace Similarity {
+
+class SOMSearcher
+{
+	public:
+
+		struct ConstructionParams
+		{
+			SOM::Network network;
+			SOM::DataNormalizer normalizer;
+			SOM::Matrix<std::vector<Database::IdType>> tracksMap;
+			std::map<Database::IdType, SOM::Coords> trackIdsCoords;
+		};
+
+		SOMSearcher(ConstructionParams params);
+
+		std::vector<Database::IdType> getSimilarTracks(const std::vector<Database::IdType>& tracksId, std::size_t maxCount);
+		std::vector<Database::IdType> getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount);
+		std::vector<Database::IdType> getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount);
+
+		void dump(Wt::Dbo::Session& session, std::ostream& os) const;
+
+	private:
+		boost::optional<SOM::Coords> getBestMatchingCoords(const std::vector<Database::IdType>& tracksIds) const;
+		std::vector<SOM::Coords> getMatchingCoords(const std::vector<Database::IdType>& tracksIds) const;
+
+		std::vector<Database::IdType> getReleases(Wt::Dbo::Session& session, const std::vector<SOM::Coords>& coords) const;
+		std::vector<Database::IdType> getArtists(Wt::Dbo::Session& session, const std::vector<SOM::Coords>& coords) const;
+
+		SOM::Network _network;
+		SOM::DataNormalizer _normalizer;
+		SOM::Matrix<std::vector<Database::IdType>> _tracksMap;
+		std::map<Database::IdType, SOM::Coords> _trackIdsCoords;
+};
+
+} // ns Similarity

src/ui/LmsApplication.cpp

@@ -31,33 +31,33 @@
 #include <Wt/Auth/Identity.h>
 
 #include "config/config.h"
+#include "cover/CoverArtGrabber.hpp"
+#include "database/Artist.hpp"
+#include "database/Cluster.hpp"
+#include "database/Release.hpp"
+#include "explore/Explore.hpp"
+#include "main/Services.hpp"
 #include "utils/Logger.hpp"
 #include "utils/Utils.hpp"
 
-#include "explore/Explore.hpp"
-#include "MediaPlayer.hpp"
-#include "PlayHistoryView.hpp"
-#include "PlayQueueView.hpp"
-#include "SettingsView.hpp"
-
 #include "admin/InitWizardView.hpp"
 #include "admin/DatabaseSettingsView.hpp"
 #include "admin/UserView.hpp"
 #include "admin/UsersView.hpp"
-
 #include "resource/ImageResource.hpp"
 #include "resource/AudioResource.hpp"
+#include "MediaPlayer.hpp"
+#include "PlayHistoryView.hpp"
+#include "PlayQueueView.hpp"
+#include "SettingsView.hpp"
+
 
 namespace UserInterface {
 
 std::unique_ptr<Wt::WApplication>
-LmsApplication::create(const Wt::WEnvironment& env, Wt::Dbo::SqlConnectionPool& connectionPool, LmsApplicationGroupContainer& appGroups, Scanner::MediaScanner& scanner)
+LmsApplication::create(const Wt::WEnvironment& env, Wt::Dbo::SqlConnectionPool& connectionPool, LmsApplicationGroupContainer& appGroups)
 {
-	/*
+	return std::make_unique<LmsApplication>(env, connectionPool, appGroups);
-	 * You could read information from the environment to decide whether
-	 * the user has permission to start a new application
-	 */
-	return std::make_unique<LmsApplication>(env, connectionPool, appGroups, scanner);
 }
 
 LmsApplication*
@@ -66,19 +66,12 @@ LmsApplication::instance()
 	return reinterpret_cast<LmsApplication*>(Wt::WApplication::instance());
 }
 
-/*
+LmsApplication::LmsApplication(const Wt::WEnvironment& env,
- * The env argument contains information about the new session, and
+		Wt::Dbo::SqlConnectionPool& connectionPool,
- * the initial request. It must be passed to the Wt::WApplication
+		LmsApplicationGroupContainer& appGroups)
- * constructor so it is typically also an argument for your custom
- * application constructor.
-*/
-LmsApplication::LmsApplication(const Wt::WEnvironment& env, Wt::Dbo::SqlConnectionPool& connectionPool, LmsApplicationGroupContainer& appGroups, Scanner::MediaScanner& scanner)
 : Wt::WApplication(env),
   _db(connectionPool),
-  _appGroups(appGroups),
+  _appGroups(appGroups)
-  _scanner(scanner),
-  _imageResource(nullptr),
-  _audioResource(nullptr)
 {
 	auto  bootstrapTheme = std::make_unique<Wt::WBootstrapTheme>();
 	bootstrapTheme->setVersion(Wt::BootstrapVersion::v3);
@@ -96,6 +89,8 @@ LmsApplication::LmsApplication(const Wt::WEnvironment& env, Wt::Dbo::SqlConnecti
 	messageResourceBundle().use(appRoot() + "admin-users");
 	messageResourceBundle().use(appRoot() + "admin-initwizard");
 	messageResourceBundle().use(appRoot() + "artist");
+	messageResourceBundle().use(appRoot() + "artistinfo");
+	messageResourceBundle().use(appRoot() + "artistlink");
 	messageResourceBundle().use(appRoot() + "artists");
 	messageResourceBundle().use(appRoot() + "artistsinfo");
 	messageResourceBundle().use(appRoot() + "explore");
@@ -105,6 +100,8 @@ LmsApplication::LmsApplication(const Wt::WEnvironment& env, Wt::Dbo::SqlConnecti
 	messageResourceBundle().use(appRoot() + "playqueue");
 	messageResourceBundle().use(appRoot() + "playhistory");
 	messageResourceBundle().use(appRoot() + "release");
+	messageResourceBundle().use(appRoot() + "releaseinfo");
+	messageResourceBundle().use(appRoot() + "releaselink");
 	messageResourceBundle().use(appRoot() + "releases");
 	messageResourceBundle().use(appRoot() + "releasesinfo");
 	messageResourceBundle().use(appRoot() + "settings");
@@ -112,6 +109,9 @@ LmsApplication::LmsApplication(const Wt::WEnvironment& env, Wt::Dbo::SqlConnecti
 	messageResourceBundle().use(appRoot() + "tracks");
 	messageResourceBundle().use(appRoot() + "tracksinfo");
 
+	// hack since Server does not expose the docRoot
+	getServices().coverArtGrabber->setDefaultCover(Wt::WApplication::instance()->docRoot() + "/images/unknown-cover.jpg");
+
 	// Require js here to avoid async problems
 	requireJQuery("/js/jquery-1.10.2.min.js");
 	require("/js/mediaplayer.js");
@@ -462,7 +462,7 @@ LmsApplication::createHome()
 
 	// Events from MediaScanner
 	std::string sessionId = LmsApp->sessionId();
-	_scanner.scanComplete().connect([=] (Scanner::MediaScanner::Stats stats)
+	getServices().mediaScanner->scanComplete().connect([=] (Scanner::MediaScanner::Stats stats)
 	{
 		// Runs from media scanner context
 		Wt::WServer::instance()->post(sessionId, [=]

src/ui/LmsApplication.hpp

@@ -20,25 +20,29 @@
 #ifndef LMS_APPLICATION_HPP
 #define LMS_APPLICATION_HPP
 
+#include <boost/optional.hpp>
+
 #include <Wt/WApplication.h>
 #include <Wt/Dbo/SqlConnectionPool.h>
 
 #include "database/DatabaseHandler.hpp"
 #include "scanner/MediaScanner.hpp"
 
-#include "database/Artist.hpp"
-#include "database/Cluster.hpp"
-#include "database/Release.hpp"
-
 #include "LmsApplicationGroup.hpp"
 #include "Auth.hpp"
 
+namespace Database {
+	class Artist;
+	class Cluster;
+	class Release;
+}
+
 namespace UserInterface {
 
 class AudioResource;
 class ImageResource;
 
-// Events that can be listen to anywhere in the application
+// Events that can be listen from anywhere in the application
 struct Events
 {
 	// Events relative to group
@@ -67,10 +71,10 @@ enum class MsgType
 class LmsApplication : public Wt::WApplication
 {
 	public:
-		LmsApplication(const Wt::WEnvironment& env, Wt::Dbo::SqlConnectionPool& connectionPool, LmsApplicationGroupContainer& appGroups, Scanner::MediaScanner& scanner);
+		LmsApplication(const Wt::WEnvironment& env, Wt::Dbo::SqlConnectionPool& connectionPool, LmsApplicationGroupContainer& appGroups);
 
 		static std::unique_ptr<Wt::WApplication> create(const Wt::WEnvironment& env,
-				Wt::Dbo::SqlConnectionPool& connectionPool, LmsApplicationGroupContainer& appGroups, Scanner::MediaScanner& scanner);
+				Wt::Dbo::SqlConnectionPool& connectionPool, LmsApplicationGroupContainer& appGroups);
 		static LmsApplication* instance();
 
 		// Session application data
@@ -83,8 +87,6 @@ class LmsApplication : public Wt::WApplication
 		Database::User::pointer getUser() { return _db.getCurrentUser(); }
 		Wt::WString getUserIdentity() { return _userIdentity; }
 
-		Scanner::MediaScanner& getMediaScanner() { return _scanner; }
-
 		Events& getEvents() { return _events; }
 
 		// Utils
@@ -94,11 +96,11 @@ class LmsApplication : public Wt::WApplication
 		void post(std::function<void()> func);
 		void notifyMsg(MsgType type, const Wt::WString& message, std::chrono::milliseconds duration = std::chrono::milliseconds(4000));
 
-		static Wt::WLink createArtistLink(Database::Artist::pointer artist);
+		static Wt::WLink createArtistLink(Wt::Dbo::ptr<Database::Artist> artist);
-		static std::unique_ptr<Wt::WAnchor> createArtistAnchor(Database::Artist::pointer artist, bool addText = true);
+		static std::unique_ptr<Wt::WAnchor> createArtistAnchor(Wt::Dbo::ptr<Database::Artist> artist, bool addText = true);
-		static Wt::WLink createReleaseLink(Database::Release::pointer release);
+		static Wt::WLink createReleaseLink(Wt::Dbo::ptr<Database::Release> release);
-		static std::unique_ptr<Wt::WAnchor> createReleaseAnchor(Database::Release::pointer release, bool addText = true);
+		static std::unique_ptr<Wt::WAnchor> createReleaseAnchor(Wt::Dbo::ptr<Database::Release> release, bool addText = true);
-		static std::unique_ptr<Wt::WTemplate> createCluster(Database::Cluster::pointer cluster, bool canDelete = false);
+		static std::unique_ptr<Wt::WTemplate> createCluster(Wt::Dbo::ptr<Database::Cluster> cluster, bool canDelete = false);
 
 		// Signal emitted just before the session ends (user may already be logged out)
 		Wt::Signal<>&	preQuit() { return _preQuit; }
@@ -119,8 +121,7 @@ class LmsApplication : public Wt::WApplication
 		LmsApplicationGroupContainer&   _appGroups;
 		Events			_events;
 		Wt::WString		_userIdentity;
-		Auth*			_auth;
+		Auth*			_auth = nullptr;
-		Scanner::MediaScanner&	_scanner;
 		std::shared_ptr<ImageResource>	_imageResource;
 		std::shared_ptr<AudioResource>	_audioResource;
 		bool			_isAdmin = false;

src/ui/MediaPlayer.cpp

@@ -22,9 +22,10 @@
 #include "av/AvInfo.hpp"
 #include "utils/Logger.hpp"
 
+#include "database/Artist.hpp"
+#include "database/Release.hpp"
 #include "database/Track.hpp"
 
-
 #include "resource/ImageResource.hpp"
 #include "resource/AudioResource.hpp"
 

src/ui/PlayQueueView.cpp

@@ -19,15 +19,12 @@
 
 #include "PlayQueueView.hpp"
 
-#include <random>
-
-#include <Wt/WAnchor.h>
 #include <Wt/WText.h>
 
-#include "utils/Logger.hpp"
-
 #include "database/TrackList.hpp"
-
+#include "main/Services.hpp"
+#include "similarity/SimilaritySearcher.hpp"
+#include "utils/Logger.hpp"
 #include "LmsApplication.hpp"
 
 namespace UserInterface {
@@ -385,45 +382,26 @@ PlayQueue::addSome()
 void
 PlayQueue::addRadioTrack()
 {
-	auto now = std::chrono::system_clock::now();
-	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
-
 	auto tracklist = getTrackList();
 
-	std::set<Database::IdType> trackIds;
+	std::vector<Database::IdType> trackIds = getTrackList()->getTrackIds();
-	{
+	if (trackIds.empty())
-		auto ids = tracklist->getTrackIds();
-		trackIds = std::set<Database::IdType>(ids.begin(), ids.end());
-	}
-
-	// Get all the tracks of the tracklist, get the cluster that is mostly used
-	// and reuse it to get the next track
-	auto clusters = tracklist->getClusters();
-	if (clusters.empty())
 		return;
 
-	for (auto cluster : clusters)
+	auto res = getServices().similaritySearcher->getSimilarTracks(trackIds, 1);
+	for (auto trackId : res)
 	{
-		std::set<Database::IdType> clusterTrackIds = cluster->getTrackIds();
+		auto trackToAdd = Database::Track::getById(LmsApp->getDboSession(), trackId);
-
-		std::set<Database::IdType> candidateTrackIds;
-		std::set_difference(clusterTrackIds.begin(), clusterTrackIds.end(),
-				trackIds.begin(), trackIds.end(),
-				std::inserter(candidateTrackIds, candidateTrackIds.end()));
-
-		if (candidateTrackIds.empty())
-			continue;
-
-		std::uniform_int_distribution<int> dist(0, candidateTrackIds.size() - 1);
-
-		auto trackToAdd = Database::Track::getById(LmsApp->getDboSession(), *std::next(candidateTrackIds.begin(), dist(randGenerator)));
 		enqueueTrack(trackToAdd);
-
-		return;
 	}
 
-	LMS_LOG(UI, INFO) << "No more track to be added!";
 }
 
+void addRadioTrackFromSimilarity(std::shared_ptr<Similarity::Searcher> similaritySearcher)
+{
+
+}
+
+
 } // namespace UserInterface
 

src/ui/PlayQueueView.hpp

@@ -31,6 +31,11 @@
 #include "database/TrackList.hpp"
 #include "database/Track.hpp"
 
+namespace Similarity
+{
+	class Finder;
+}
+
 namespace UserInterface {
 
 class PlayQueue : public Wt::WTemplate
@@ -69,6 +74,9 @@ class PlayQueue : public Wt::WTemplate
 		void load(std::size_t pos, bool play);
 		void stop();
 
+		void addRadioTrackFromSimilarity(std::shared_ptr<Similarity::Finder> similarityFinder);
+		void addRadioTrackFromClusters();
+
 		bool _repeatAll = false;
 		bool _radioMode = false;
 		boost::optional<Database::IdType> _tracklistId;

src/ui/admin/DatabaseSettingsView.cpp

@@ -19,19 +19,20 @@
 
 #include "DatabaseSettingsView.hpp"
 
-#include <Wt/WString.h>
-#include <Wt/WPushButton.h>
 #include <Wt/WComboBox.h>
+#include <Wt/WFormModel.h>
 #include <Wt/WLineEdit.h>
+#include <Wt/WPushButton.h>
+#include <Wt/WString.h>
+#include <Wt/WStringListModel.h>
 #include <Wt/WTemplateFormView.h>
 
-#include <Wt/WFormModel.h>
+#include "database/Cluster.hpp"
-#include <Wt/WStringListModel.h>
+#include "main/Services.hpp"
-
-#include "common/Validators.hpp"
 #include "utils/Logger.hpp"
 #include "utils/Utils.hpp"
 
+#include "common/Validators.hpp"
 #include "LmsApplication.hpp"
 
 namespace UserInterface {
@@ -287,7 +288,7 @@ DatabaseSettingsView::refreshView()
 		{
 			model->saveData();
 
-			LmsApp->getMediaScanner().reschedule();
+			getServices().mediaScanner->reschedule();
 			LmsApp->notifyMsg(MsgType::Success, Wt::WString::tr("Lms.Admin.Database.settings-saved"));
 		}
 
@@ -304,7 +305,7 @@ DatabaseSettingsView::refreshView()
 
 	immScanBtn->clicked().connect([=] ()
 	{
-		LmsApp->getMediaScanner().scheduleImmediateScan();
+		getServices().mediaScanner->scheduleImmediateScan();
 		LmsApp->notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-launched"));
 	});
 

src/ui/explore/ArtistInfoView.cpp

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ArtistInfoView.hpp"
+
+#include "database/Artist.hpp"
+#include "main/Services.hpp"
+#include "similarity/SimilaritySearcher.hpp"
+#include "utils/Utils.hpp"
+
+#include "ArtistLink.hpp"
+#include "LmsApplication.hpp"
+
+using namespace Database;
+
+namespace UserInterface {
+
+ArtistInfo::ArtistInfo()
+: Wt::WTemplate(Wt::WString::tr("Lms.Explore.ArtistInfo.template"))
+{
+	addFunction("tr", &Wt::WTemplate::Functions::tr);
+
+	_similarArtistsContainer = bindNew<Wt::WContainerWidget>("similar-artists");
+
+	wApp->internalPathChanged().connect(std::bind([=]
+	{
+		refresh();
+	}));
+
+	LmsApp->getEvents().dbScanned.connect([=]
+	{
+		refresh();
+	});
+
+	refresh();
+}
+
+void
+ArtistInfo::refresh()
+{
+	_similarArtistsContainer->clear();
+
+	if (!wApp->internalPathMatches("/artist/"))
+		return;
+
+	auto artistId = readAs<Database::IdType>(wApp->internalPathNextPart("/artist/"));
+	if (!artistId)
+		return;
+
+	auto artistsIds = getServices().similaritySearcher->getSimilarArtists(LmsApp->getDboSession(), *artistId, 5);
+
+	Wt::Dbo::Transaction transaction(LmsApp->getDboSession());
+
+	std::vector<Database::Artist::pointer> artists;
+	for (auto artistId : artistsIds)
+	{
+		auto artist = Database::Artist::getById(LmsApp->getDboSession(), artistId);
+
+		if (artist)
+			artists.push_back(artist);
+	}
+
+	for (auto artist : artists)
+		_similarArtistsContainer->addNew<ArtistLink>(artist);
+}
+
+} // namespace UserInterface
+

src/ui/explore/ArtistInfoView.hpp

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/WContainerWidget.h>
+#include <Wt/WTemplate.h>
+
+namespace UserInterface {
+
+class ArtistInfo : public Wt::WTemplate
+{
+	public:
+		ArtistInfo();
+
+	private:
+		void refresh();
+
+		Wt::WContainerWidget* _similarArtistsContainer;
+};
+
+} // namespace UserInterface
+

src/ui/explore/ArtistLink.cpp

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ArtistLink.hpp"
+
+#include <Wt/WAnchor.h>
+
+#include "database/Artist.hpp"
+#include "LmsApplication.hpp"
+
+namespace UserInterface {
+
+ArtistLink::ArtistLink(Database::Artist::pointer artist)
+: Wt::WTemplate(Wt::WString::tr("Lms.Explore.ArtistLink.template"))
+{
+	bindWidget("name", LmsApplication::createArtistAnchor(artist));
+}
+
+} // namespace UserInterface

src/ui/explore/ArtistLink.hpp

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/WTemplate.h>
+
+namespace Database{
+	class Artist;
+}
+
+namespace UserInterface {
+
+class ArtistLink : public Wt::WTemplate
+{
+	public:
+		ArtistLink(Wt::Dbo::ptr<Database::Artist> artist);
+
+};
+
+}

src/ui/explore/ArtistView.cpp

@@ -25,6 +25,7 @@
 #include <Wt/WText.h>
 
 #include "database/Artist.hpp"
+#include "database/Release.hpp"
 
 #include "utils/Logger.hpp"
 #include "utils/Utils.hpp"

src/ui/explore/ArtistsInfoView.cpp

@@ -19,34 +19,16 @@
 
 #include "ArtistsInfoView.hpp"
 
-#include <Wt/WAnchor.h>
 #include <Wt/WLocalDateTime.h>
 
 #include "database/Artist.hpp"
 #include "database/TrackList.hpp"
-
 #include "utils/Utils.hpp"
-
+#include "ArtistLink.hpp"
 #include "LmsApplication.hpp"
 
 using namespace Database;
 
-namespace {
-
-using namespace UserInterface;
-
-void addEntries(Wt::WContainerWidget *container, const std::vector<Artist::pointer>& artists)
-{
-	for (auto artist : artists)
-	{
-		Wt::WTemplate* entry = container->addNew<Wt::WTemplate>(Wt::WString::tr("Lms.Explore.ArtistsInfo.template.entry"));
-
-		entry->bindWidget("name", LmsApplication::createArtistAnchor(artist));
-	}
-}
-
-}
-
 namespace UserInterface {
 
 ArtistsInfo::ArtistsInfo()
@@ -80,7 +62,8 @@ ArtistsInfo::refreshRecentlyAdded()
 	auto artists = Artist::getLastAdded(LmsApp->getDboSession(), after, 5);
 
 	_recentlyAddedContainer->clear();
-	addEntries(_recentlyAddedContainer, artists);
+	for (auto artist : artists)
+		_recentlyAddedContainer->addNew<ArtistLink>(artist);
 }
 
 void
@@ -90,7 +73,8 @@ ArtistsInfo::refreshMostPlayed()
 	auto artists = LmsApp->getUser()->getPlayedTrackList()->getTopArtists(5);
 
 	_mostPlayedContainer->clear();
-	addEntries(_mostPlayedContainer, artists);
+	for (auto artist : artists)
+		_mostPlayedContainer->addNew<ArtistLink>(artist);
 }
 
 } // namespace UserInterface

src/ui/explore/Explore.cpp

@@ -23,14 +23,18 @@
 #include <Wt/WTemplate.h>
 #include <Wt/WText.h>
 
+#include "database/Artist.hpp"
+#include "database/Release.hpp"
 #include "utils/Logger.hpp"
 
 #include "LmsApplication.hpp"
 
+#include "ArtistInfoView.hpp"
 #include "ArtistsInfoView.hpp"
 #include "ArtistsView.hpp"
 #include "ArtistView.hpp"
 #include "Filters.hpp"
+#include "ReleaseInfoView.hpp"
 #include "ReleasesInfoView.hpp"
 #include "ReleasesView.hpp"
 #include "ReleaseView.hpp"
@@ -79,7 +83,9 @@ handleInfoPathChange(Wt::WStackedWidget* stack)
 {
 	enum Idx
 	{
-		IdxArtists = 0,
+		IdxArtist = 0,
+		IdxArtists,
+		IdxRelease,
 		IdxReleases,
 		IdxTracks,
 	};
@@ -87,9 +93,9 @@ handleInfoPathChange(Wt::WStackedWidget* stack)
 	static const std::map<std::string, int> indexes =
 	{
 		{ "/artists",		IdxArtists },
-		{ "/artist",		IdxArtists },
+		{ "/artist",		IdxArtist },
 		{ "/releases",		IdxReleases },
-		{ "/release",		IdxReleases },
+		{ "/release",		IdxRelease },
 		{ "/tracks",		IdxTracks },
 	};
 
@@ -151,9 +157,15 @@ Explore::Explore()
 	// Info
 	Wt::WStackedWidget* infoStack = bindNew<Wt::WStackedWidget>("info");
 
+	auto artistInfo = std::make_unique<ArtistInfo>();
+	infoStack->addWidget(std::move(artistInfo));
+
 	auto artistsInfo = std::make_unique<ArtistsInfo>();
 	infoStack->addWidget(std::move(artistsInfo));
 
+	auto releaseInfo = std::make_unique<ReleaseInfo>();
+	infoStack->addWidget(std::move(releaseInfo));
+
 	auto releasesInfo = std::make_unique<ReleasesInfo>();
 	infoStack->addWidget(std::move(releasesInfo));
 

src/ui/explore/Filters.cpp

@@ -24,6 +24,8 @@
 #include <Wt/WPushButton.h>
 #include <Wt/WTemplate.h>
 
+#include "database/Cluster.hpp"
+
 #include "LmsApplication.hpp"
 
 namespace UserInterface {

src/ui/explore/ReleaseInfoView.cpp

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ReleaseInfoView.hpp"
+
+#include "database/Release.hpp"
+#include "main/Services.hpp"
+#include "similarity/SimilaritySearcher.hpp"
+#include "utils/Utils.hpp"
+
+#include "ReleaseLink.hpp"
+#include "LmsApplication.hpp"
+
+using namespace Database;
+
+namespace UserInterface {
+
+ReleaseInfo::ReleaseInfo()
+: Wt::WTemplate(Wt::WString::tr("Lms.Explore.ReleaseInfo.template"))
+{
+	addFunction("tr", &Wt::WTemplate::Functions::tr);
+
+	_similarReleasesContainer = bindNew<Wt::WContainerWidget>("similar-releases");
+
+	wApp->internalPathChanged().connect(std::bind([=]
+	{
+		refresh();
+	}));
+
+	LmsApp->getEvents().dbScanned.connect([=]
+	{
+		refresh();
+	});
+
+	refresh();
+}
+
+void
+ReleaseInfo::refresh()
+{
+	_similarReleasesContainer->clear();
+
+	if (!wApp->internalPathMatches("/release/"))
+		return;
+
+	auto releaseId = readAs<Database::IdType>(wApp->internalPathNextPart("/release/"));
+	if (!releaseId)
+		return;
+
+	auto releasesIds = getServices().similaritySearcher->getSimilarReleases(LmsApp->getDboSession(), *releaseId, 5);
+
+	Wt::Dbo::Transaction transaction(LmsApp->getDboSession());
+
+	std::vector<Database::Release::pointer> releases;
+	for (auto releaseId : releasesIds)
+	{
+		auto release = Database::Release::getById(LmsApp->getDboSession(), releaseId);
+
+		if (release)
+			releases.push_back(release);
+	}
+
+	for (auto release : releases)
+		_similarReleasesContainer->addNew<ReleaseLink>(release);
+}
+
+} // namespace UserInterface
+

src/ui/explore/ReleaseInfoView.hpp

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/WContainerWidget.h>
+#include <Wt/WTemplate.h>
+
+namespace UserInterface {
+
+class ReleaseInfo : public Wt::WTemplate
+{
+	public:
+		ReleaseInfo();
+
+	private:
+		void refresh();
+
+		Wt::WContainerWidget* _similarReleasesContainer;
+};
+
+} // namespace UserInterface
+

src/ui/explore/ReleaseLink.cpp

@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ReleaseLink.hpp"
+
+#include <Wt/WAnchor.h>
+#include <Wt/WImage.h>
+
+#include "database/Release.hpp"
+#include "resource/ImageResource.hpp"
+#include "utils/Utils.hpp"
+
+#include "LmsApplication.hpp"
+
+using namespace Database;
+
+namespace UserInterface {
+
+ReleaseLink::ReleaseLink(Database::Release::pointer release)
+: Wt::WTemplate(Wt::WString::tr("Lms.Explore.ReleaseLink.template"))
+{
+	bindWidget("release-name", LmsApplication::createReleaseAnchor(release));
+
+	Wt::WAnchor* anchor = bindWidget("cover", LmsApplication::createReleaseAnchor(release, false));
+	auto cover = std::make_unique<Wt::WImage>();
+	cover->setImageLink(LmsApp->getImageResource()->getReleaseUrl(release.id(), 48));
+	cover->setWidth(48);
+	anchor->setImage(std::move(cover));
+
+	auto artists = release->getArtists();
+	if (artists.size() > 1)
+	{
+		setCondition("if-has-artist", true);
+		bindString("artist-name", Wt::WString::tr("Lms.Explore.various-artists"));
+	}
+	else if (artists.size() == 1)
+	{
+		setCondition("if-has-artist", true);
+		bindWidget("artist-name", LmsApplication::createArtistAnchor(artists.front()));
+	}
+}
+
+} // namespace UserInterface
+

src/ui/explore/ReleaseLink.hpp

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/WTemplate.h>
+
+namespace Database {
+	class Release;
+}
+
+namespace UserInterface {
+
+class ReleaseLink : public Wt::WTemplate
+{
+	public:
+		ReleaseLink(Wt::Dbo::ptr<Database::Release>);
+};
+
+} // namespace UserInterface
+

src/ui/explore/ReleasesInfoView.cpp

@@ -19,54 +19,16 @@
 
 #include "ReleasesInfoView.hpp"
 
-#include <Wt/WAnchor.h>
-#include <Wt/WImage.h>
 #include <Wt/WLocalDateTime.h>
-#include <Wt/WTemplate.h>
 
 #include "database/Release.hpp"
 #include "database/TrackList.hpp"
-
 #include "resource/ImageResource.hpp"
-
+#include "ReleaseLink.hpp"
 #include "LmsApplication.hpp"
 
 using namespace Database;
 
-namespace {
-
-using namespace UserInterface;
-
-void addEntries(Wt::WContainerWidget* container, const std::vector<Release::pointer>& releases)
-{
-	for (auto release : releases)
-	{
-		Wt::WTemplate* entry = container->addNew<Wt::WTemplate>(Wt::WString::tr("Lms.Explore.ReleasesInfo.template.entry"));
-
-		entry->bindWidget("release-name", LmsApplication::createReleaseAnchor(release));
-
-		Wt::WAnchor* anchor = entry->bindWidget("cover", LmsApplication::createReleaseAnchor(release, false));
-		auto cover = std::make_unique<Wt::WImage>();
-		cover->setImageLink(LmsApp->getImageResource()->getReleaseUrl(release.id(), 48));
-		cover->setWidth(48);
-		anchor->setImage(std::move(cover));
-
-		auto artists = release->getArtists();
-		if (artists.size() > 1)
-		{
-			entry->setCondition("if-has-artist", true);
-			entry->bindString("artist-name", Wt::WString::tr("Lms.Explore.various-artists"));
-		}
-		else if (artists.size() == 1)
-		{
-			entry->setCondition("if-has-artist", true);
-			entry->bindWidget("artist-name", LmsApplication::createArtistAnchor(artists.front()));
-		}
-	}
-}
-
-} // namespace
-
 namespace UserInterface {
 
 ReleasesInfo::ReleasesInfo()
@@ -101,7 +63,8 @@ ReleasesInfo::refreshRecentlyAdded()
 	auto releases = Release::getLastAdded(LmsApp->getDboSession(), after, 5);
 
 	_recentlyAddedContainer->clear();
-	addEntries(_recentlyAddedContainer, releases);
+	for (auto release : releases)
+		_recentlyAddedContainer->addNew<ReleaseLink>(release);
 }
 
 void
@@ -112,7 +75,8 @@ ReleasesInfo::refreshMostPlayed()
 	auto releases = LmsApp->getUser()->getPlayedTrackList()->getTopReleases(5);
 
 	_mostPlayedContainer->clear();
-	addEntries(_mostPlayedContainer, releases);
+	for (auto release : releases)
+		_mostPlayedContainer->addNew<ReleaseLink>(release);
 }
 
 } // namespace UserInterface

src/ui/resource/ImageResource.cpp

@@ -22,16 +22,15 @@
 #include <Wt/WApplication.h>
 #include <Wt/Http/Response.h>
 
+#include "cover/CoverArtGrabber.hpp"
+#include "database/Track.hpp"
+#include "main/Services.hpp"
 #include "utils/Exception.hpp"
 #include "utils/Logger.hpp"
 #include "utils/Utils.hpp"
 
-#include "database/Track.hpp"
-
 #include "LmsApplication.hpp"
 
-#include "cover/CoverArtGrabber.hpp"
-
 namespace UserInterface {
 
 static const std::string unknownCoverPath = "/images/unknown-cover.jpg";
@@ -81,7 +80,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons
 		// transactions are not thread safe
 		{
 			Wt::WApplication::UpdateLock lock(LmsApp);
-			cover = CoverArt::Grabber::instance().getFromTrack(LmsApp->getDboSession(), *trackId, Image::Format::JPEG, *size);
+			cover = getServices().coverArtGrabber->getFromTrack(LmsApp->getDboSession(), *trackId, Image::Format::JPEG, *size);
 		}
 	}
 	else if (releaseIdStr)
@@ -93,7 +92,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons
 		// transactions are not thread safe
 		{
 			Wt::WApplication::UpdateLock lock(LmsApp);
-			cover = CoverArt::Grabber::instance().getFromRelease(LmsApp->getDboSession(), *releaseId, Image::Format::JPEG, *size);
+			cover = getServices().coverArtGrabber->getFromRelease(LmsApp->getDboSession(), *releaseId, Image::Format::JPEG, *size);
 		}
 	}
 	else

src/utils/Logger.cpp

@@ -32,6 +32,7 @@ std::string getModuleName(Module mod)
 		case Module::METADATA:		return "METADATA";
 		case Module::REMOTE:		return "REMOTE";
 		case Module::SERVICE:		return "SERVICE";
+		case Module::SIMILARITY:	return "SIMILARITY";
 		case Module::TRANSCODE:		return "TRANSCODE";
 		case Module::UI:		return "UI";
 	}

src/utils/Logger.hpp

@@ -44,6 +44,7 @@ enum class Module
 	METADATA,
 	REMOTE,
 	SERVICE,
+	SIMILARITY,
 	TRANSCODE,
 	UI,
 };

src/utils/Utils.hpp

@@ -21,6 +21,7 @@
 
 #include <chrono>
 #include <list>
+#include <map>
 #include <string>
 #include <sstream>
 #include <vector>
@@ -63,3 +64,34 @@ boost::optional<T> readAs(const std::string& str)
 std::string
 replaceInString(std::string str, const std::string& from, const std::string& to);
 
+// warning: not efficient
+template<class In, class Out, class U = typename std::iterator_traits<In>::value_type>
+void uniqueAndSortedByOccurence(In first, In last, Out out)
+{
+	std::map<U, std::size_t> occurencesMap;
+
+	for (In it = first; it != last; ++it)
+	{
+		if (occurencesMap.find(*it) == occurencesMap.end())
+			occurencesMap[*it] = 0;
+
+		occurencesMap[*it]++;
+	}
+
+	struct Item
+	{
+		U elem;
+		std::size_t count;
+	};
+
+	std::vector<Item> occurencesVector;
+	for (const auto& occurence : occurencesMap)
+		occurencesVector.emplace_back(Item{occurence.first, occurence.second});
+
+	std::sort(occurencesVector.begin(), occurencesVector.end(), [](const auto& a, const auto& b) { return a.count > b.count;});
+
+	for (const auto& occurence : occurencesVector)
+		*out++ = occurence.elem;
+}
+
+

tools/Makefile.am

@@ -1,2 +1,2 @@
-SUBDIRS = metadata clusterer
+SUBDIRS = feature-extractor metadata
 

tools/clusterer/LmsClusterer.cpp

@@ -1,209 +0,0 @@
-#include <stdlib.h>
-#include <stdexcept>
-#include <iostream>
-#include <string>
-
-#include <boost/filesystem.hpp>
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/json_parser.hpp>
-
-#include <curl/curl.h>
-
-#include "clusterer/SOM.hpp"
-#include "clusterer/DataNormalizer.hpp"
-#include "clusterer/Clusterer.hpp"
-#include "database/DatabaseHandler.hpp"
-#include "database/Track.hpp"
-#include "database/Artist.hpp"
-#include "database/Release.hpp"
-#include "utils/Config.hpp"
-
-static std::vector<std::string> features =
-{
-	"lowlevel.average_loudness",
-	"lowlevel.barkbands_flatness_db.mean",
-	"lowlevel.dissonance.mean",
-	"lowlevel.dynamic_complexity",
-	"lowlevel.hfc.mean",				// GOOD
-	"lowlevel.melbands_crest.mean",
-	"lowlevel.melbands_kurtosis.mean",
-	"lowlevel.melbands_skewness.mean",
-	"lowlevel.melbands_spread.mean",
-	"lowlevel.pitch_salience.mean",
-	"lowlevel.pitch_salience.var",
-	"lowlevel.silence_rate_30dB.mean",
-	"lowlevel.silence_rate_60dB.mean",
-	"lowlevel.spectral_centroid.mean",
-	"lowlevel.spectral_complexity.mean",
-	"lowlevel.spectral_decrease.mean",
-	"lowlevel.spectral_energy.mean",
-	"lowlevel.spectral_energyband_high.mean",
-	"lowlevel.spectral_energyband_low.mean",
-	"lowlevel.spectral_energyband_middle_high.mean",
-	"lowlevel.spectral_energyband_middle_low.mean",
-	"lowlevel.spectral_entropy.mean",
-	"lowlevel.spectral_flux.mean",
-	"lowlevel.spectral_kurtosis.mean",
-	"lowlevel.spectral_rms.mean",
-	"lowlevel.spectral_skewness.mean",
-	"lowlevel.spectral_spread.mean",
-	"lowlevel.spectral_strongpeak.mean",
-	"lowlevel.zerocrossingrate.mean",
-	"rhythm.beats_loudness.mean",		// BAD
-	"rhythm.bpm",
-	"tonal.chords_changes_rate",		// OK
-//	"tonal.chords_number_rate",		// BAD
-	"tonal.chords_strength.mean",		// OK
-	"tonal.hpcp_entropy.mean",		// GOOD
-};
-
-static size_t writeToFile(void *buffer, size_t size, size_t nmemb, void* ctx)
-{
-	std::ofstream& ofs = *reinterpret_cast<std::ofstream*>(ctx);
-
-	ofs.write(reinterpret_cast<char*>(buffer), size * nmemb);
-
-	return size * nmemb;
-}
-
-static void acousticBrainzGetLowLevel(const std::string& mbid, boost::filesystem::path output)
-{
-	std::string url = "http://acousticbrainz.org/api/v1/" + mbid + "/low-level";
-
-	std::cout << "GET " << url << std::endl;
-
-	CURL *curl;
-	CURLcode res;
-
-	curl = curl_easy_init();
-	if (!curl)
-	{
-		return;
-	}
-
-	std::ofstream ofs(output.string().c_str());
-	if (!ofs)
-	{
-		curl_easy_cleanup(curl);
-		std::cerr << "Cannot open " << output.string() << " for writing purpose" << std::endl;
-		return;
-	}
-
-	curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeToFile);
-	curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ofs);
-
-	res = curl_easy_perform(curl);
-
-	if (res != CURLE_OK)
-	{
-		std::cerr << "perform failed: " << curl_easy_strerror(res) << std::endl;
-	}
-
-	curl_easy_cleanup(curl);
-}
-
-static boost::filesystem::path getLowLevelFeaturePath(const std::string& mbid)
-{
-	return boost::filesystem::path(Config::instance().getPath("working-dir") / "features" / mbid);
-}
-
-std::vector<double> getFeatures(const std::string& mbid)
-{
-
-	std::vector<double> res;
-
-	try
-	{
-		boost::property_tree::ptree root;
-
-		boost::property_tree::read_json(getLowLevelFeaturePath(mbid).string(), root);
-
-		for (const auto& feature : features)
-		{
-			res.push_back(root.get<double>(feature));
-		}
-	}
-	catch (std::exception& e)
-	{
-		std::cerr << "Caught exception during processing " << mbid << std::endl;
-	}
-
-	return res;
-}
-
-int main(int argc, char *argv[])
-{
-	try
-	{
-		boost::filesystem::path configFilePath = "/etc/lms.conf";
-
-		if (argc >= 2)
-			configFilePath = std::string(argv[1], 0, 256);
-
-		Config::instance().setFile(configFilePath);
-
-		Database::Handler::configureAuth();
-		auto connectionPool = Database::Handler::createConnectionPool(Config::instance().getPath("working-dir") / "lms.db");
-		Database::Handler db(*connectionPool);
-
-		Wt::Dbo::Transaction transaction(db.getSession());
-
-		auto tracks = Database::Track::getAll(db.getSession());
-
-		std::vector<std::pair<std::vector<double>, Database::IdType>> entries;
-
-		std::cout << "Constructing input vectors..." << std::endl;
-
-		for (auto track : tracks)
-		{
-			if (track->getMBID().empty())
-				continue;
-
-			auto path = getLowLevelFeaturePath(track->getMBID());
-			if (!boost::filesystem::exists(path))
-				acousticBrainzGetLowLevel(track->getMBID(), path);
-
-			if (!boost::filesystem::exists(path))
-				continue;
-
-			std::pair<std::vector<double>, Database::IdType> entry;
-
-			entry.first = getFeatures(track->getMBID());
-			entry.second = track.id();
-
-			if (entry.first.size() == features.size())
-				entries.push_back(std::move(entry));
-		}
-
-		std::cout << "Constructing input vectors... DONE" << std::endl;
-
-		std::cout << "Clutering..." << std::endl;
-		Clusterer<Database::IdType> clusterer(entries, features.size(), 500);
-
-		std::cout << "Clusterer :" << std::endl;
-		clusterer.dump(std::cout);
-		std::cout << std::endl;
-
-		for (const auto& cluster : clusterer.getAllClusters())
-		{
-			std::cout << "******************" << std::endl;
-			for (const auto& value : cluster)
-			{
-				auto track = Database::Track::getById(db.getSession(), value);
-				auto artist = track->getArtist();
-				auto release = track->getRelease();
-
-				std::cout << "\t" << value << " - " << (artist ? artist->getName() : "") << " - " << (release ? release->getName() : "" ) << " - " << track->getName() << std::endl;
-			}
-			std::cout << std::endl;
-		}
-	}
-	catch( std::exception& e)
-	{
-		std::cerr << "Caught exception: " << e.what() << std::endl;
-	}
-
-	return EXIT_SUCCESS;
-}
-

tools/feature-extractor/LmsFeatureExtractor.cpp

@@ -0,0 +1,98 @@
+#include <stdlib.h>
+#include <stdexcept>
+#include <iostream>
+#include <string>
+
+#include <curl/curl.h>
+
+#include "database/DatabaseHandler.hpp"
+#include "database/Track.hpp"
+#include "utils/Config.hpp"
+
+static size_t writeToOstream(char *ptr, size_t size, size_t nmemb, void *userdata)
+{
+	std::ofstream& ofs = *reinterpret_cast<std::ofstream*>(userdata);
+	ofs.write(ptr, size*nmemb);
+	return size*nmemb;
+}
+
+static void acousticBrainzGetLowLevel(const std::string& mbid, boost::filesystem::path output)
+{
+	std::string url = "http://acousticbrainz.org/api/v1/" + mbid + "/low-level";
+
+	std::cout << "GET " << url << std::endl;
+
+	CURL *curl;
+	CURLcode res;
+
+	curl = curl_easy_init();
+	if (!curl)
+	{
+		return;
+	}
+
+	std::ofstream ofs(output.string().c_str());
+	if (!ofs)
+	{
+		curl_easy_cleanup(curl);
+		std::cerr << "Cannot open " << output.string() << " for writing purpose" << std::endl;
+		return;
+	}
+
+	curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeToOstream);
+	curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ofs);
+
+	res = curl_easy_perform(curl);
+
+	if (res != CURLE_OK)
+	{
+		std::cerr << "perform failed: " << curl_easy_strerror(res) << std::endl;
+	}
+
+	curl_easy_cleanup(curl);
+}
+
+static boost::filesystem::path getLowLevelFeaturePath(const std::string& mbid)
+{
+	return boost::filesystem::path(Config::instance().getPath("working-dir") / "features" / mbid);
+}
+
+int main(int argc, char *argv[])
+{
+	try
+	{
+		boost::filesystem::path configFilePath = "/etc/lms.conf";
+
+		if (argc >= 2)
+			configFilePath = std::string(argv[1], 0, 256);
+
+		Config::instance().setFile(configFilePath);
+
+		Database::Handler::configureAuth();
+		auto connectionPool = Database::Handler::createConnectionPool(Config::instance().getPath("working-dir") / "lms.db");
+		Database::Handler db(*connectionPool);
+
+		Wt::Dbo::Transaction transaction(db.getSession());
+
+		auto tracks = Database::Track::getAll(db.getSession());
+
+		for (auto track : tracks)
+		{
+			if (track->getMBID().empty())
+				continue;
+
+			auto path = getLowLevelFeaturePath(track->getMBID());
+			if (!boost::filesystem::exists(path))
+				acousticBrainzGetLowLevel(track->getMBID(), path);
+		}
+
+	}
+	catch( std::exception& e)
+	{
+		std::cerr << "Caught exception: " << e.what() << std::endl;
+	}
+
+	return EXIT_SUCCESS;
+}
+

tools/feature-extractor/Makefile.am

@@ -1,12 +1,11 @@
-bin_PROGRAMS = lms-clusterer
+bin_PROGRAMS = lms-feature-extractor
 
-lms_clusterer_SOURCES = \
+lms_feature_extractor_SOURCES = \
-	$(srcdir)/LmsClusterer.cpp			\
+	$(srcdir)/LmsFeatureExtractor.cpp		\
-	$(top_srcdir)/src/clusterer/DataNormalizer.cpp	\
-	$(top_srcdir)/src/clusterer/SOM.cpp		\
 	$(top_srcdir)/src/database/Artist.cpp		\
 	$(top_srcdir)/src/database/Cluster.cpp		\
 	$(top_srcdir)/src/database/DatabaseHandler.cpp	\
+	$(top_srcdir)/src/database/TrackFeature.cpp	\
 	$(top_srcdir)/src/database/TrackList.cpp	\
 	$(top_srcdir)/src/database/Release.cpp		\
 	$(top_srcdir)/src/database/ScanSettings.cpp	\
@@ -17,5 +16,5 @@ lms_clusterer_SOURCES = \
 	$(top_srcdir)/src/utils/Logger.cpp 		\
 	$(top_srcdir)/src/utils/Utils.cpp
 
-lms_clusterer_CXXFLAGS=-std=c++14 -Wall -I$(top_srcdir)/src -D_REENTRANT
+lms_feature_extractor_CXXFLAGS=-std=c++14 -Wall -I$(top_srcdir)/src -D_REENTRANT