WIP, first working genetic algorithm to train the neural network

configure.ac

@@ -15,7 +15,7 @@ fi
 AC_SUBST(MAGICKXX_CFLAGS)
 AC_SUBST(MAGICKXX_LIBS)
 
-AC_CHECK_HEADERS([Wt/WApplication.h pstreams/pstream.h],
+AC_CHECK_HEADERS([Wt/WApplication.h pstreams/pstream.h boost/asio.hpp],
                  [],
                  [AC_MSG_ERROR([Header not found or unusable !])])
 

src/Makefile.am

@@ -40,6 +40,8 @@ lms_SOURCES = \
 	$(srcdir)/database/ScanSettings.hpp			\
 	$(srcdir)/database/Session.cpp				\
 	$(srcdir)/database/Session.hpp				\
+	$(srcdir)/database/SessionPool.cpp				\
+	$(srcdir)/database/SessionPool.hpp				\
 	$(srcdir)/database/SimilaritySettings.cpp		\
 	$(srcdir)/database/SimilaritySettings.hpp		\
 	$(srcdir)/database/SqlQuery.cpp				\

src/api/subsonic/SubsonicResource.cpp

@@ -129,42 +129,6 @@ struct RequestContext
 	std::string userName;
 };
 
-using SessionMap = std::map<Db*, Session>;
-static std::map<std::thread::id, SessionMap> dbSessions;
-
-static
-Session&
-getOrCreateDbSession(Db& db)
-{
-	static std::mutex mutex;
-
-	SessionMap* sessionMap {};
-
-	{
-		std::unique_lock<std::mutex> lock {mutex};
-		sessionMap = &dbSessions[std::this_thread::get_id()];
-	}
-
-	auto it {sessionMap->find(&db)};
-	if (it != std::end(*sessionMap))
-		return it->second;
-
-	auto res { sessionMap->try_emplace(&db, db)};
-	assert(res.second);
-
-	LMS_LOG(API_SUBSONIC, DEBUG) << "Created db session";
-
-	return res.first->second;
-}
-
-static
-void
-clearDbSessions()
-{
-	dbSessions.clear();
-}
-
-
 static
 std::string
 makeNameFilesystemCompatible(const std::string& name)
@@ -275,16 +239,10 @@ struct MediaRetrievalResult
 };
 
 SubsonicResource::SubsonicResource(Db& db)
-: _db {db}
+: _sessionPool {db}
 {
 }
 
-SubsonicResource::~SubsonicResource()
-{
-	LMS_LOG(API_SUBSONIC, DEBUG) << "Cleaning db sessions...";
-	clearDbSessions();
-}
-
 static
 std::string parameterMapToDebugString(const Wt::Http::ParameterMap& parameterMap)
 {
@@ -1907,9 +1865,9 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp
 		// Mandatory parameters
 		const ClientInfo clientInfo {getClientInfo(parameters)};
 
-		Session& dbSession {getOrCreateDbSession(_db)};
+		SessionPool::ScopedSession dbSession {_sessionPool};
 
-		switch (ServiceProvider<Auth::PasswordService>::get()->checkUserPassword(dbSession,
+		switch (ServiceProvider<Auth::PasswordService>::get()->checkUserPassword(dbSession.get(),
 					boost::asio::ip::address::from_string(request.clientAddress()),
 					clientInfo.user, clientInfo.password))
 		{
@@ -1921,16 +1879,16 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp
 				throw LoginThrottledGenericError {};
 		}
 
-		RequestContext requestContext {.parameters = parameters, .dbSession = dbSession, .userName = clientInfo.user};
+		RequestContext requestContext {.parameters = parameters, .dbSession = dbSession.get(), .userName = clientInfo.user};
 
 		auto itEntryPoint {requestEntryPoints.find(requestPath)};
 		if (itEntryPoint != requestEntryPoints.end())
 		{
 			if (itEntryPoint->second.mustBeAdmin)
 			{
-				auto transaction {dbSession.createSharedTransaction()};
+				auto transaction {dbSession.get().createSharedTransaction()};
 
-				User::pointer user {User::getByLoginName(dbSession, clientInfo.user)};
+				User::pointer user {User::getByLoginName(dbSession.get(), clientInfo.user)};
 				if (!user || !user->isAdmin())
 					throw UserNotAuthorizedError {};
 			}

src/api/subsonic/SubsonicResource.hpp

@@ -21,6 +21,8 @@
 #include <Wt/WResource.h>
 #include <Wt/Http/Response.h>
 
+#include "database/SessionPool.hpp"
+
 namespace Database
 {
 	class Db;
@@ -33,14 +35,13 @@ class SubsonicResource final : public Wt::WResource
 {
 	public:
 		SubsonicResource(Database::Db& db);
-		~SubsonicResource();
 
 		static std::string getPath() { return "/rest/"; }
 	private:
 
 		void handleRequest(const Wt::Http::Request &request, Wt::Http::Response &response) override;
 
-		Database::Db& _db;
+		Database::SessionPool _sessionPool;
 };
 
 } // namespace

src/database/Session.cpp

@@ -19,6 +19,10 @@
 
 #include "Session.hpp"
 
+#include <map>
+#include <mutex>
+#include <thread>
+
 #include "utils/Exception.hpp"
 #include "utils/Logger.hpp"
 

src/database/Session.hpp

@@ -19,9 +19,11 @@
 
 #pragma once
 
-#include <shared_mutex>
 #include <mutex>
+#include <map>
 #include <memory>
+#include <shared_mutex>
+#include <vector>
 
 #include <Wt/Dbo/Dbo.h>
 #include <Wt/Dbo/SqlConnectionPool.h>

src/database/SessionPool.cpp

@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SessionPool.hpp"
+
+#include "utils/Exception.hpp"
+#include "utils/Logger.hpp"
+
+#include "Session.hpp"
+
+namespace Database {
+
+SessionPool::SessionPool(Db& database, std::size_t maxSessionCount)
+: _db {database},
+_maxSessionCount {maxSessionCount}
+{
+}
+
+Session&
+SessionPool::acquireSession()
+{
+	std::scoped_lock lock {_mutex};
+
+	if (_freeSessions.empty())
+	{
+		if (_acquiredSessions.size() == _maxSessionCount)
+			throw LmsException {"Too many database sessions!"};
+
+		_freeSessions.emplace_back(std::make_unique<Session>(_db));
+	}
+
+	std::unique_ptr<Session> session {std::move(_freeSessions.back())};
+	_freeSessions.pop_back();
+	_acquiredSessions.push_back(std::move(session));
+
+	return *_acquiredSessions.back().get();
+}
+
+void
+SessionPool::releaseSession(Session& sessionToRelease)
+{
+	std::scoped_lock lock {_mutex};
+
+	auto it {std::find_if(std::begin(_acquiredSessions), std::end(_acquiredSessions), [&](const std::unique_ptr<Session>& session) { return session.get() == &sessionToRelease; })};
+	if (it == std::end(_acquiredSessions))
+		throw LmsException {"Unknown released Session!"};
+
+	std::unique_ptr<Session> session {std::move(*it)};
+	_acquiredSessions.erase(it);
+	_freeSessions.push_back(std::move(session));
+}
+
+} // namespace Database

src/database/SessionPool.hpp

@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2013 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <memory>
+#include <mutex>
+#include <vector>
+
+#include "Session.hpp"
+
+namespace Database {
+
+class SessionPool
+{
+	public:
+		class ScopedSession
+		{
+			public:
+				ScopedSession(SessionPool& pool) : _pool {pool}, _session {_pool.acquireSession()} {}
+				~ScopedSession() { _pool.releaseSession(_session); }
+
+				ScopedSession(const ScopedSession&) = delete;
+				ScopedSession(ScopedSession&&) = delete;
+				ScopedSession& operator=(const ScopedSession&) = delete;
+				ScopedSession& operator=(ScopedSession&&) = delete;
+
+				Session& get() { return _session; }
+
+			private:
+				SessionPool& _pool;
+				Session& _session;
+		};
+
+		SessionPool(Db& database, std::size_t maxSessionCount = 30);
+
+		SessionPool(const SessionPool&) = delete;
+		SessionPool(SessionPool&&) = delete;
+		SessionPool& operator=(const SessionPool&) = delete;
+		SessionPool& operator=(SessionPool&&) = delete;
+
+	private:
+		friend class ScopedSession;
+		Session& acquireSession();
+		void releaseSession(Session& session);
+
+		std::mutex	_mutex;
+		Db&		_db;
+		std::size_t	_maxSessionCount;
+		std::vector<std::unique_ptr<Session>>	_freeSessions;
+		std::vector<std::unique_ptr<Session>>	_acquiredSessions;
+};
+
+} // namespace Database
+
+

src/database/Track.cpp

@@ -171,6 +171,20 @@ Track::getClusters(void) const
 	return clusters;
 }
 
+std::vector<IdType>
+Track::getClusterIds(void) const
+{
+	assert(self());
+	assert(IdIsValid(self()->id()));
+	assert(session());
+
+	Wt::Dbo::collection<IdType> res = session()->query<IdType>
+		("SELECT DISTINCT c.id FROM cluster c INNER JOIN track_cluster t_c ON t_c.cluster_id = c.id INNER JOIN track t ON t.id = t_c.track_id")
+		.where("t.id = ?").bind(self()->id());
+
+	return std::vector<IdType>(res.begin(), res.end());
+}
+
 bool
 Track::hasTrackFeatures() const
 {
@@ -377,6 +391,20 @@ Track::getArtists(TrackArtistLink::Type type) const
 	return std::vector<Wt::Dbo::ptr<Artist>>(artists.begin(), artists.end());
 }
 
+std::vector<IdType>
+Track::getArtistIds(TrackArtistLink::Type type) const
+{
+	assert(self());
+	assert(IdIsValid(self()->id()));
+	assert(session());
+
+	Wt::Dbo::collection<IdType> artists {session()->query<IdType>("SELECT a.id from artist a INNER JOIN track_artist_link t_a_l ON a.id = t_a_l.artist_id INNER JOIN track t ON t.id = t_a_l.track_id")
+		.where("t.id = ?").bind(self()->id())
+		.where("t_a_l.type = ?").bind(type)};
+
+	return std::vector<IdType>(artists.begin(), artists.end());
+}
+
 std::vector<Wt::Dbo::ptr<TrackArtistLink>>
 Track::getArtistLinks() const
 {

src/database/Track.hpp

@@ -115,9 +115,11 @@ class Track : public Wt::Dbo::Dbo<Track>
 		std::optional<std::string>		getCopyright() const;
 		std::optional<std::string>		getCopyrightURL() const;
 		std::vector<Wt::Dbo::ptr<Artist>>	getArtists(TrackArtistLink::Type type = TrackArtistLink::Type::Artist) const;
+		std::vector<IdType>			getArtistIds(TrackArtistLink::Type type = TrackArtistLink::Type::Artist) const;
 		std::vector<Wt::Dbo::ptr<TrackArtistLink>>	getArtistLinks() const;
 		Wt::Dbo::ptr<Release>			getRelease() const		{ return _release; }
 		std::vector<Wt::Dbo::ptr<Cluster>>	getClusters() const;
+		std::vector<IdType>			getClusterIds() const;
 		bool					hasTrackFeatures() const;
 		Wt::Dbo::ptr<TrackFeatures>		getTrackFeatures() const;
 

src/database/TrackFeatures.cpp

@@ -80,7 +80,7 @@ TrackFeatures::getFeatureValuesMap(const std::unordered_set<FeatureName>& featur
 	}
 	catch (boost::property_tree::ptree_error& error)
 	{
-		LMS_LOG(SIMILARITY, ERROR) << "Track " << _track.id() << ": ptree exception: " << error.what();
+		LMS_LOG(DB, ERROR) << "Track " << _track.id() << ": ptree exception: " << error.what();
 		return {};
 	}
 }

src/similarity/features/SimilarityFeaturesDefs.cpp

@@ -19,20 +19,291 @@
 
 #include "SimilarityFeaturesDefs.hpp"
 
-#include <unordered_map>
+#include <algorithm>
+#include <iterator>
+
 #include "utils/Exception.hpp"
 
 namespace Similarity {
 
 static const std::unordered_map<FeatureName, FeatureDef> featureDefinitions
 {
+	{ "lowlevel.average_loudness",			{1}},
+	{ "lowlevel.barkbands.dmean",			{27}},
+	{ "lowlevel.barkbands.dmean2",			{27}},
+	{ "lowlevel.barkbands.dvar",			{27}},
+	{ "lowlevel.barkbands.dvar2",			{27}},
+	{ "lowlevel.barkbands.max",			{27}},
+	{ "lowlevel.barkbands.mean",			{27}},
+	{ "lowlevel.barkbands.median",			{27}},
+	{ "lowlevel.barkbands.min",			{27}},
+	{ "lowlevel.barkbands.var",			{27}},
+	{ "lowlevel.barkbands_crest.dmean",		{1}},
+	{ "lowlevel.barkbands_crest.dmean2",		{1}},
+	{ "lowlevel.barkbands_crest.dvar",		{1}},
+	{ "lowlevel.barkbands_crest.dvar2",		{1}},
+	{ "lowlevel.barkbands_crest.max",		{1}},
+	{ "lowlevel.barkbands_crest.mean",		{1}},
+	{ "lowlevel.barkbands_crest.median",		{1}},
+	{ "lowlevel.barkbands_crest.min",		{1}},
+	{ "lowlevel.barkbands_crest.var",		{1}},
+	{ "lowlevel.barkbands_flatness_db.dmean",	{1}},
+	{ "lowlevel.barkbands_flatness_db.dmean2",	{1}},
+	{ "lowlevel.barkbands_flatness_db.dvar",	{1}},
+	{ "lowlevel.barkbands_flatness_db.dvar2",	{1}},
+	{ "lowlevel.barkbands_flatness_db.max",		{1}},
+	{ "lowlevel.barkbands_flatness_db.mean",	{1}},
+	{ "lowlevel.barkbands_flatness_db.median",	{1}},
+	{ "lowlevel.barkbands_flatness_db.min",		{1}},
+	{ "lowlevel.barkbands_flatness_db.var",		{1}},
+	{ "lowlevel.barkbands_kurtosis.dmean",		{1}},
+	{ "lowlevel.barkbands_kurtosis.dmean2",		{1}},
+	{ "lowlevel.barkbands_kurtosis.dvar",		{1}},
+	{ "lowlevel.barkbands_kurtosis.dvar2",		{1}},
+	{ "lowlevel.barkbands_kurtosis.max",		{1}},
+	{ "lowlevel.barkbands_kurtosis.mean",		{1}},
+	{ "lowlevel.barkbands_kurtosis.median",		{1}},
+	{ "lowlevel.barkbands_kurtosis.min",		{1}},
+	{ "lowlevel.barkbands_kurtosis.var",		{1}},
+	{ "lowlevel.barkbands_skewness.dmean",		{1}},
+	{ "lowlevel.barkbands_skewness.dmean2",		{1}},
+	{ "lowlevel.barkbands_skewness.dvar",		{1}},
+	{ "lowlevel.barkbands_skewness.dvar2",		{1}},
+	{ "lowlevel.barkbands_skewness.max",		{1}},
+	{ "lowlevel.barkbands_skewness.mean",		{1}},
+	{ "lowlevel.barkbands_skewness.median",		{1}},
+	{ "lowlevel.barkbands_skewness.min",		{1}},
+	{ "lowlevel.barkbands_skewness.var",		{1}},
+	{ "lowlevel.barkbands_spread.dmean",		{1}},
+	{ "lowlevel.barkbands_spread.dmean2",		{1}},
+	{ "lowlevel.barkbands_spread.dvar",		{1}},
+	{ "lowlevel.barkbands_spread.dvar2",		{1}},
+	{ "lowlevel.barkbands_spread.max",		{1}},
+	{ "lowlevel.barkbands_spread.mean",		{1}},
+	{ "lowlevel.barkbands_spread.median",		{1}},
+	{ "lowlevel.barkbands_spread.min",		{1}},
+	{ "lowlevel.barkbands_spread.var",		{1}},
+	{ "lowlevel.dissonance.dmean",			{1}},
+	{ "lowlevel.dissonance.dmean2",			{1}},
+	{ "lowlevel.dissonance.dvar",			{1}},
+	{ "lowlevel.dissonance.dvar2",			{1}},
+	{ "lowlevel.dissonance.max",			{1}},
+	{ "lowlevel.dissonance.mean",			{1}},
+	{ "lowlevel.dissonance.median",			{1}},
+	{ "lowlevel.dissonance.min",			{1}},
+	{ "lowlevel.dissonance.var",			{1}},
+	{ "lowlevel.dynamic_complexity",		{1}},
+	{ "lowlevel.spectral_contrast_coeffs.dmean",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.dmean2",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.dvar",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.dvar2",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.max",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.mean",	{6}},
 	{ "lowlevel.spectral_contrast_coeffs.median",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.min",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.var",	{6}},
+	{ "lowlevel.erbbands.dmean",			{40}},
+	{ "lowlevel.erbbands.dmean2",			{40}},
+	{ "lowlevel.erbbands.dvar",			{40}},
+	{ "lowlevel.erbbands.dvar2",			{40}},
+	{ "lowlevel.erbbands.max",			{40}},
+	{ "lowlevel.erbbands.mean",			{40}},
 	{ "lowlevel.erbbands.median",			{40}},
+	{ "lowlevel.erbbands.min",			{40}},
+	{ "lowlevel.erbbands.var",			{40}},
+	{ "lowlevel.gfcc.mean",				{13}},
+	{ "lowlevel.hfc.dmean",				{1}},
+	{ "lowlevel.hfc.dmean2",			{1}},
+	{ "lowlevel.hfc.dvar",				{1}},
+	{ "lowlevel.hfc.dvar2",				{1}},
+	{ "lowlevel.hfc.max",				{1}},
+	{ "lowlevel.hfc.mean",				{1}},
+	{ "lowlevel.hfc.median",			{1}},
+	{ "lowlevel.hfc.min",				{1}},
+	{ "lowlevel.hfc.var",				{1}},
 	{ "tonal.hpcp.median",				{36}},
 	{ "lowlevel.melbands.median",			{40}},
-	{ "lowlevel.barkbands.median",			{27}},
 	{ "lowlevel.mfcc.mean",				{13}},
-	{ "lowlevel.gfcc.mean",				{13}},
+	{ "lowlevel.pitch_salience.dmean",		{1}},
+	{ "lowlevel.pitch_salience.dmean2",		{1}},
+	{ "lowlevel.pitch_salience.dvar",		{1}},
+	{ "lowlevel.pitch_salience.dvar2",		{1}},
+	{ "lowlevel.pitch_salience.max",		{1}},
+	{ "lowlevel.pitch_salience.mean",		{1}},
+	{ "lowlevel.pitch_salience.median",		{1}},
+	{ "lowlevel.pitch_salience.min",		{1}},
+	{ "lowlevel.pitch_salience.var",		{1}},
+	{ "lowlevel.spectral_centroid.dmean",		{1}},
+	{ "lowlevel.spectral_centroid.dmean2",		{1}},
+	{ "lowlevel.spectral_centroid.dvar",		{1}},
+	{ "lowlevel.spectral_centroid.dvar2",		{1}},
+	{ "lowlevel.spectral_centroid.max",		{1}},
+	{ "lowlevel.spectral_centroid.mean",		{1}},
+	{ "lowlevel.spectral_centroid.median",		{1}},
+	{ "lowlevel.spectral_centroid.min",		{1}},
+	{ "lowlevel.spectral_centroid.var",		{1}},
+	{ "lowlevel.spectral_complexity.dmean",		{1}},
+	{ "lowlevel.spectral_complexity.dmean2",	{1}},
+	{ "lowlevel.spectral_complexity.dvar",		{1}},
+	{ "lowlevel.spectral_complexity.dvar2",		{1}},
+	{ "lowlevel.spectral_complexity.max",		{1}},
+	{ "lowlevel.spectral_complexity.mean",		{1}},
+	{ "lowlevel.spectral_complexity.median",	{1}},
+	{ "lowlevel.spectral_complexity.min",		{1}},
+	{ "lowlevel.spectral_complexity.var",		{1}},
+	{ "lowlevel.spectral_contrast_coeffs.dmean",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.dmean2",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.dvar",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.dvar2",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.max",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.mean",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.median",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.min",	{6}},
+	{ "lowlevel.spectral_contrast_coeffs.var",	{6}},
+	{ "lowlevel.spectral_contrast_valleys.dmean",	{6}},
+	{ "lowlevel.spectral_contrast_valleys.dmean2",	{6}},
+	{ "lowlevel.spectral_contrast_valleys.dvar",	{6}},
+	{ "lowlevel.spectral_contrast_valleys.dvar2",	{6}},
+	{ "lowlevel.spectral_contrast_valleys.max",	{6}},
+	{ "lowlevel.spectral_contrast_valleys.mean",	{6}},
+	{ "lowlevel.spectral_contrast_valleys.median",	{6}},
+	{ "lowlevel.spectral_contrast_valleys.min",	{6}},
+	{ "lowlevel.spectral_contrast_valleys.var",	{6}},
+	{ "lowlevel.spectral_decrease.dmean",		{1}},
+	{ "lowlevel.spectral_decrease.dmean2",		{1}},
+	{ "lowlevel.spectral_decrease.dvar",		{1}},
+	{ "lowlevel.spectral_decrease.dvar2",		{1}},
+	{ "lowlevel.spectral_decrease.max",		{1}},
+	{ "lowlevel.spectral_decrease.mean",		{1}},
+	{ "lowlevel.spectral_decrease.median",		{1}},
+	{ "lowlevel.spectral_decrease.min",		{1}},
+	{ "lowlevel.spectral_decrease.var",		{1}},
+	{ "lowlevel.spectral_energy.dmean",		{1}},
+	{ "lowlevel.spectral_energy.dmean2",		{1}},
+	{ "lowlevel.spectral_energy.dvar",		{1}},
+	{ "lowlevel.spectral_energy.dvar2",		{1}},
+	{ "lowlevel.spectral_energy.max",		{1}},
+	{ "lowlevel.spectral_energy.mean",		{1}},
+	{ "lowlevel.spectral_energy.median",		{1}},
+	{ "lowlevel.spectral_energy.min",		{1}},
+	{ "lowlevel.spectral_energy.var",		{1}},
+	{ "lowlevel.spectral_energyband_high.dmean",		{1}},
+	{ "lowlevel.spectral_energyband_high.dmean2",		{1}},
+	{ "lowlevel.spectral_energyband_high.dvar",		{1}},
+	{ "lowlevel.spectral_energyband_high.dvar2",		{1}},
+	{ "lowlevel.spectral_energyband_high.max",		{1}},
+	{ "lowlevel.spectral_energyband_high.mean",		{1}},
+	{ "lowlevel.spectral_energyband_high.median",		{1}},
+	{ "lowlevel.spectral_energyband_high.min",		{1}},
+	{ "lowlevel.spectral_energyband_high.var",		{1}},
+	{ "lowlevel.spectral_energyband_low.dmean",		{1}},
+	{ "lowlevel.spectral_energyband_low.dmean2",		{1}},
+	{ "lowlevel.spectral_energyband_low.dvar",		{1}},
+	{ "lowlevel.spectral_energyband_low.dvar2",		{1}},
+	{ "lowlevel.spectral_energyband_low.max",		{1}},
+	{ "lowlevel.spectral_energyband_low.mean",		{1}},
+	{ "lowlevel.spectral_energyband_low.median",		{1}},
+	{ "lowlevel.spectral_energyband_low.min",		{1}},
+	{ "lowlevel.spectral_energyband_low.var",		{1}},
+	{ "lowlevel.spectral_energyband_middle_high.dmean",		{1}},
+	{ "lowlevel.spectral_energyband_middle_high.dmean2",		{1}},
+	{ "lowlevel.spectral_energyband_middle_high.dvar",		{1}},
+	{ "lowlevel.spectral_energyband_middle_high.dvar2",		{1}},
+	{ "lowlevel.spectral_energyband_middle_high.max",		{1}},
+	{ "lowlevel.spectral_energyband_middle_high.mean",		{1}},
+	{ "lowlevel.spectral_energyband_middle_high.median",		{1}},
+	{ "lowlevel.spectral_energyband_middle_high.min",		{1}},
+	{ "lowlevel.spectral_energyband_middle_high.var",		{1}},
+	{ "lowlevel.spectral_energyband_middle_low.dmean",		{1}},
+	{ "lowlevel.spectral_energyband_middle_low.dmean2",		{1}},
+	{ "lowlevel.spectral_energyband_middle_low.dvar",		{1}},
+	{ "lowlevel.spectral_energyband_middle_low.dvar2",		{1}},
+	{ "lowlevel.spectral_energyband_middle_low.max",		{1}},
+	{ "lowlevel.spectral_energyband_middle_low.mean",		{1}},
+	{ "lowlevel.spectral_energyband_middle_low.median",		{1}},
+	{ "lowlevel.spectral_energyband_middle_low.min",		{1}},
+	{ "lowlevel.spectral_energyband_middle_low.var",		{1}},
+	{ "lowlevel.spectral_entropy.dmean",		{1}},
+	{ "lowlevel.spectral_entropy.dmean2",		{1}},
+	{ "lowlevel.spectral_entropy.dvar",		{1}},
+	{ "lowlevel.spectral_entropy.dvar2",		{1}},
+	{ "lowlevel.spectral_entropy.max",		{1}},
+	{ "lowlevel.spectral_entropy.mean",		{1}},
+	{ "lowlevel.spectral_entropy.median",		{1}},
+	{ "lowlevel.spectral_entropy.min",		{1}},
+	{ "lowlevel.spectral_entropy.var",		{1}},
+	{ "lowlevel.spectral_flux.dmean",		{1}},
+	{ "lowlevel.spectral_flux.dmean2",		{1}},
+	{ "lowlevel.spectral_flux.dvar",		{1}},
+	{ "lowlevel.spectral_flux.dvar2",		{1}},
+	{ "lowlevel.spectral_flux.max",			{1}},
+	{ "lowlevel.spectral_flux.mean",		{1}},
+	{ "lowlevel.spectral_flux.median",		{1}},
+	{ "lowlevel.spectral_flux.min",			{1}},
+	{ "lowlevel.spectral_flux.var",			{1}},
+	{ "lowlevel.spectral_kurtosis.dmean",		{1}},
+	{ "lowlevel.spectral_kurtosis.dmean2",		{1}},
+	{ "lowlevel.spectral_kurtosis.dvar",		{1}},
+	{ "lowlevel.spectral_kurtosis.dvar2",		{1}},
+	{ "lowlevel.spectral_kurtosis.max",		{1}},
+	{ "lowlevel.spectral_kurtosis.mean",		{1}},
+	{ "lowlevel.spectral_kurtosis.median",		{1}},
+	{ "lowlevel.spectral_kurtosis.min",		{1}},
+	{ "lowlevel.spectral_kurtosis.var",		{1}},
+	{ "lowlevel.spectral_rms.dmean",		{1}},
+	{ "lowlevel.spectral_rms.dmean2",		{1}},
+	{ "lowlevel.spectral_rms.dvar",			{1}},
+	{ "lowlevel.spectral_rms.dvar2",		{1}},
+	{ "lowlevel.spectral_rms.max",			{1}},
+	{ "lowlevel.spectral_rms.mean",			{1}},
+	{ "lowlevel.spectral_rms.median",		{1}},
+	{ "lowlevel.spectral_rms.min",			{1}},
+	{ "lowlevel.spectral_rms.var",			{1}},
+	{ "lowlevel.spectral_rolloff.dmean",		{1}},
+	{ "lowlevel.spectral_rolloff.dmean2",		{1}},
+	{ "lowlevel.spectral_rolloff.dvar",		{1}},
+	{ "lowlevel.spectral_rolloff.dvar2",		{1}},
+	{ "lowlevel.spectral_rolloff.max",		{1}},
+	{ "lowlevel.spectral_rolloff.mean",		{1}},
+	{ "lowlevel.spectral_rolloff.median",		{1}},
+	{ "lowlevel.spectral_rolloff.min",		{1}},
+	{ "lowlevel.spectral_rolloff.var",		{1}},
+	{ "lowlevel.spectral_skewness.dmean",		{1}},
+	{ "lowlevel.spectral_skewness.dmean2",		{1}},
+	{ "lowlevel.spectral_skewness.dvar",		{1}},
+	{ "lowlevel.spectral_skewness.dvar2",		{1}},
+	{ "lowlevel.spectral_skewness.max",		{1}},
+	{ "lowlevel.spectral_skewness.mean",		{1}},
+	{ "lowlevel.spectral_skewness.median",		{1}},
+	{ "lowlevel.spectral_skewness.min",		{1}},
+	{ "lowlevel.spectral_skewness.var",		{1}},
+	{ "lowlevel.spectral_spread.dmean",		{1}},
+	{ "lowlevel.spectral_spread.dmean2",		{1}},
+	{ "lowlevel.spectral_spread.dvar",		{1}},
+	{ "lowlevel.spectral_spread.dvar2",		{1}},
+	{ "lowlevel.spectral_spread.max",		{1}},
+	{ "lowlevel.spectral_spread.mean",		{1}},
+	{ "lowlevel.spectral_spread.median",		{1}},
+	{ "lowlevel.spectral_spread.min",		{1}},
+	{ "lowlevel.spectral_spread.var",		{1}},
+	{ "lowlevel.spectral_strongpeak.dmean",		{1}},
+	{ "lowlevel.spectral_strongpeak.dmean2",	{1}},
+	{ "lowlevel.spectral_strongpeak.dvar",		{1}},
+	{ "lowlevel.spectral_strongpeak.dvar2",		{1}},
+	{ "lowlevel.spectral_strongpeak.max",		{1}},
+	{ "lowlevel.spectral_strongpeak.mean",		{1}},
+	{ "lowlevel.spectral_strongpeak.median",	{1}},
+	{ "lowlevel.spectral_strongpeak.min",		{1}},
+	{ "lowlevel.spectral_strongpeak.var",		{1}},
+	{ "lowlevel.zerocrossingrate.dmean",		{1}},
+	{ "lowlevel.zerocrossingrate.dmean2",		{1}},
+	{ "lowlevel.zerocrossingrate.dvar",		{1}},
+	{ "lowlevel.zerocrossingrate.dvar2",		{1}},
+	{ "lowlevel.zerocrossingrate.max",		{1}},
+	{ "lowlevel.zerocrossingrate.mean",		{1}},
+	{ "lowlevel.zerocrossingrate.median",		{1}},
+	{ "lowlevel.zerocrossingrate.min",		{1}},
+	{ "lowlevel.zerocrossingrate.var",		{1}},
 };
 
 FeatureDef
@@ -45,5 +316,16 @@ getFeatureDef(const FeatureName& featureName)
 	return it->second;
 }
 
+FeatureNames
+getFeatureNames()
+{
+	FeatureNames res;
+
+	std::transform(std::cbegin(featureDefinitions), std::cend(featureDefinitions),
+			std::inserter(res, std::begin(res)), [](auto itFeature) { return itFeature.first; });
+
+	return res;
+}
+
 } // namespace Similarity
 

src/similarity/features/SimilarityFeaturesDefs.hpp

@@ -21,11 +21,13 @@
 
 #include <string>
 #include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 namespace Similarity {
 
 using FeatureName = std::string;
+using FeatureNames = std::unordered_set<FeatureName>;
 using FeatureValue = double;
 using FeatureValues = std::vector<FeatureValue>;
 using FeatureValuesMap = std::unordered_map<FeatureName, FeatureValues>;
@@ -36,6 +38,7 @@ struct FeatureDef
 };
 
 FeatureDef getFeatureDef(const FeatureName& featureName);
+FeatureNames getFeatureNames();
 
 struct FeatureSettings
 {

src/similarity/features/SimilarityFeaturesScannerAddon.cpp

@@ -151,9 +151,10 @@ FeaturesScannerAddon::updateSearcher()
 		return;
 	}
 
-	const auto features {getFeatureSettings(_dbSession)};
+	Similarity::FeaturesSearcher::TrainSettings trainSettings;
+	trainSettings.featureSettingsMap = getFeatureSettings(_dbSession);
 
-	auto searcher {std::make_shared<Similarity::FeaturesSearcher>(_dbSession, features, [&]() { return _stopRequested; })};
+	auto searcher {std::make_shared<Similarity::FeaturesSearcher>(_dbSession, trainSettings, [&]() { return _stopRequested; })};
 	if (searcher->isValid())
 	{
 		std::atomic_store(&_searcher, searcher);

src/similarity/features/SimilarityFeaturesSearcher.cpp

@@ -99,13 +99,13 @@ getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t
 }
 
 FeaturesSearcher::FeaturesSearcher(Database::Session& session,
-		const FeatureSettingsMap& featureSettingsMap,
+		const TrainSettings& trainSettings,
 		StopRequestedFunction stopRequested)
 {
 	LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher...";
 
 	std::unordered_set<FeatureName> featureNames;
-	std::transform(std::cbegin(featureSettingsMap), std::cend(featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)),
+	std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)),
 		[](const auto& itFeatureSetting) { return itFeatureSetting.first; });
 
 	const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0},
@@ -161,7 +161,7 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session,
 
 	SOM::Network network {size, size, nbDimensions};
 
-	SOM::InputVector weights {getInputVectorWeights(featureSettingsMap, nbDimensions)};
+	SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)};
 	network.setDataWeights(weights);
 
 	auto progressIndicator{[](const auto& iter)
@@ -170,7 +170,7 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session,
 		}};
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Training network...";
-	network.train(samples, 10, progressIndicator, stopRequested);
+	network.train(samples, trainSettings.nbIterations, progressIndicator, stopRequested);
 	LMS_LOG(SIMILARITY, DEBUG) << "Training network DONE";
 
 	if (stopRequested && stopRequested())

src/similarity/features/SimilarityFeaturesSearcher.hpp

@@ -48,7 +48,12 @@ class FeaturesSearcher
 		FeaturesSearcher(Database::Session& session, FeaturesCache cache, StopRequestedFunction stopRequested);
 
 		// Use training (may be very slow)
-		FeaturesSearcher(Database::Session& session, const FeatureSettingsMap& featuresSettingsMap, StopRequestedFunction stopRequested = {});
+		struct TrainSettings
+		{
+			std::size_t nbIterations {10};
+			FeatureSettingsMap featureSettingsMap;
+		};
+		FeaturesSearcher(Database::Session& session, const TrainSettings& trainSettings, StopRequestedFunction stopRequested = {});
 
 		bool isValid() const;
 

src/similarity/features/som/Network.cpp

@@ -279,7 +279,7 @@ Network::updateRefVectors(const Position& closestRefVectorPosition, const InputV
 			InputVector delta {input - refVector};
 			delta *= (learningFactor * _neighbourhoodFunc(norm, iteration));
 
-			refVector += delta; // * (learningFactor * _neighbourhoodFunc(norm, iteration));
+			refVector += delta;
 		}
 	}
 }

src/utils/Utils.cpp

@@ -166,6 +166,20 @@ stringFromHex(const std::string& str)
 	}
 
 	return res;
-
+}
+
+RandGenerator& getRandGenerator()
+{
+	static thread_local std::random_device rd;
+	static thread_local std::mt19937 randGenerator(rd());
+
+	return randGenerator;
+}
+
+int
+getRandom(int min, int max)
+{
+	std::uniform_int_distribution<> dist {min, max};
+	return dist (getRandGenerator());
 }
 

src/utils/Utils.hpp

@@ -110,24 +110,27 @@ constexpr T clamp(T v, T lo, T hi, Compare comp = {})
 	return comp(v, lo) ? lo : comp(hi, v) ? hi : v;
 }
 
+using RandGenerator = std::mt19937;
+RandGenerator& getRandGenerator();
+
+int
+getRandom(int min, int max);
+
 template <typename Container>
 void
 shuffleContainer(Container& container)
 {
-	auto now {std::chrono::system_clock::now()};
-	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
-	std::shuffle(std::begin(container), std::end(container), randGenerator);
+	std::shuffle(std::begin(container), std::end(container), getRandGenerator());
 }
 
 template <typename Container>
-typename Container::iterator
-pickRandom(Container& container)
+typename Container::const_iterator
+pickRandom(const Container& container)
 {
-	auto now {std::chrono::system_clock::now()};
-	std::mt19937 randGenerator (std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
-	std::uniform_int_distribution<> dist {0, static_cast<int>(container.size())};
+	if (container.empty())
+		return std::end(container);
 
-	return std::next(std::begin(container), dist(randGenerator ));
+	return std::next(std::begin(container), getRandom(0, static_cast<int>(container.size() - 1)));
 }
 
 

test/database/DatabaseTest.cpp

@@ -437,6 +437,8 @@ testSingleTrackSingleCluster(Session& session)
 		auto transaction {session.createSharedTransaction()};
 		auto clusters {Cluster::getAllOrphans(session)};
 		CHECK(clusters.size() == 2);
+		CHECK(track->getClusters().empty());
+		CHECK(track->getClusterIds().empty());
 	}
 
 	{
@@ -464,6 +466,18 @@ testSingleTrackSingleCluster(Session& session)
 		tracks = Track::getByClusters(session, {cluster2.getId()});
 		CHECK(tracks.empty());
 	}
+
+	{
+		auto transaction {session.createSharedTransaction()};
+
+		auto clusters {track->getClusters()};
+		CHECK(clusters.size() == 1);
+		CHECK(clusters.front().id() == cluster1.getId());
+
+		auto clusterIds {track->getClusterIds()};
+		CHECK(clusterIds.size() == 1);
+		CHECK(clusterIds.front() == cluster1.getId());
+	}
 }
 
 static
@@ -640,6 +654,12 @@ testSingleTrackSingleArtistMultiClusters(Session& session)
 		CHECK(Artist::getAllOrphans(session).empty());
 	}
 
+	{
+		auto transaction {session.createSharedTransaction()};
+		CHECK(track->getClusters().size() == 1);
+		CHECK(track->getClusterIds().size() == 1);
+	}
+
 	{
 		auto transaction {session.createSharedTransaction()};
 

tools/metadata/LmsMetadata.cpp

@@ -1,3 +1,22 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 #include <chrono>
 #include <optional>
 #include <stdexcept>

tools/similarity-parameters/GeneticAlgorithm.hpp

@@ -0,0 +1,137 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <numeric>
+
+#include "utils/Utils.hpp"
+#include "ParallelFor.hpp"
+
+template<typename Individual>
+class GeneticAlgorithm
+{
+	public:
+		using Score = float;
+
+		using BreedFunction = std::function<Individual(const Individual&, const Individual&)>;
+		using MutateFunction = std::function<void(Individual&)>;
+		using ScoreFunction = std::function<Score(const Individual&)>;
+
+		struct Params
+		{
+			std::size_t		nbWorkers {1};
+			std::size_t		nbGenerations;
+			float			mutationProbability {0.05};
+			BreedFunction	breedFunction;
+			MutateFunction		mutateFunction;
+			ScoreFunction		scoreFunction;
+		};
+
+		GeneticAlgorithm(const Params& params);
+
+		// Returns the individual that has the maximum score after processing the requested generations
+		Individual simulate(const std::vector<Individual>& initialPopulation);
+
+	private:
+
+		struct ScoredIndividual
+		{
+			Individual individual;
+			std::optional<Score> score {};
+		};
+
+		void scoreAndSortPopulation(std::vector<ScoredIndividual>& population);
+
+		Params _params;
+};
+
+template<typename Individual>
+GeneticAlgorithm<Individual>::GeneticAlgorithm(const Params& params)
+: _params {params}
+{
+}
+
+template<typename Individual>
+Individual
+GeneticAlgorithm<Individual>::simulate(const std::vector<Individual>& initialPopulation)
+{
+	if (initialPopulation.size() < 10)
+		throw std::runtime_error("Initial population must has at least 10 elements");
+
+	std::vector<ScoredIndividual> scoredPopulation;
+	scoredPopulation.reserve(initialPopulation.size());
+
+	std::transform(std::cbegin(initialPopulation), std::cend(initialPopulation), std::back_inserter(scoredPopulation ),
+			[](const Individual& individual) { return ScoredIndividual {individual};});
+
+	scoreAndSortPopulation(scoredPopulation);
+
+	for (std::size_t currentGeneration {}; currentGeneration  < _params.nbGenerations; ++currentGeneration)
+	{
+		std::cout << "Processing generation " << currentGeneration << "..." << std::endl;
+		// parent selection (elitist selection)
+		scoredPopulation.resize(scoredPopulation.size() / 2);
+
+		// breed the remaining individuals
+		std::vector<ScoredIndividual> children;
+		children.reserve(initialPopulation.size() - scoredPopulation.size());
+
+		while (children.size() + scoredPopulation.size() < initialPopulation.size())
+		{
+			// Select two random parents
+			const auto itParent1 {pickRandom(scoredPopulation)};
+			const auto itParent2 {pickRandom(scoredPopulation)};
+
+			if (itParent1 == itParent2)
+				continue;
+
+			ScoredIndividual child {_params.breedFunction(itParent1->individual, itParent2->individual)};
+			
+			if (getRandom(0, 100) <= _params.mutationProbability * 100)
+				_params.mutateFunction(child.individual);
+
+			children.emplace_back(std::move(child ));
+		}
+
+		scoredPopulation.insert(std::end(scoredPopulation), std::make_move_iterator(std::begin(children)), std::make_move_iterator(std::end(children)));
+		assert(scoredPopulation.size() == initialPopulation.size());
+
+		scoreAndSortPopulation(scoredPopulation);
+
+		std::cout << "Current best score = " << *scoredPopulation.front().score << std::endl;
+	}
+
+	std::cout << "Best score = " << *scoredPopulation.front().score << std::endl;
+	return scoredPopulation.front().individual;
+}
+
+
+template<typename Individual>
+void
+GeneticAlgorithm<Individual>::scoreAndSortPopulation(std::vector<ScoredIndividual>& scoredPopulation)
+{
+	parallel_foreach(_params.nbWorkers, std::begin(scoredPopulation), std::end(scoredPopulation),
+			[&](ScoredIndividual& scoredIndividual)
+			{
+				if (!scoredIndividual.score)
+					scoredIndividual.score = _params.scoreFunction(scoredIndividual.individual);
+			});
+
+	std::sort(std::begin(scoredPopulation), std::end(scoredPopulation), [](const ScoredIndividual& a, const ScoredIndividual& b) { return a.score > b.score; });
+}
+

tools/similarity-parameters/LmsSimilarityParameters.cpp

@@ -1,47 +1,354 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
 #include <iostream>
 #include <filesystem>
 #include <string>
 
+#include "database/Artist.hpp"
+#include "database/Cluster.hpp"
 #include "database/Db.hpp"
-#include "database/Session.hpp"
+#include "database/Release.hpp"
+#include "database/SessionPool.hpp"
+#include "database/Track.hpp"
+#include "similarity/features/SimilarityFeaturesSearcher.hpp"
 #include "utils/Config.hpp"
 #include "utils/Service.hpp"
 #include "utils/StreamLogger.hpp"
 
+#include "GeneticAlgorithm.hpp"
+
+using namespace Similarity;
+using SimilarityScore = GeneticAlgorithm<FeatureSettingsMap>::Score;
+
+// An individual is just a FeatureSettingsMap
+// The goal is to get the FeatureSettingsMap that maximize the score
+const FeatureSettingsMap featuresSettings
+{
+	{ "lowlevel.average_loudness",			{1}},
+	{ "lowlevel.barkbands.mean",			{1}},
+	{ "lowlevel.barkbands.median",			{1}},
+	{ "lowlevel.barkbands.var",			{1}},
+	{ "lowlevel.barkbands_crest.mean",		{1}},
+	{ "lowlevel.barkbands_crest.median",		{1}},
+	{ "lowlevel.barkbands_crest.var",		{1}},
+	{ "lowlevel.barkbands_flatness_db.mean",	{1}},
+	{ "lowlevel.barkbands_flatness_db.median",	{1}},
+	{ "lowlevel.barkbands_flatness_db.var",		{1}},
+	{ "lowlevel.barkbands_kurtosis.mean",		{1}},
+	{ "lowlevel.barkbands_kurtosis.median",		{1}},
+	{ "lowlevel.barkbands_kurtosis.var",		{1}},
+	{ "lowlevel.barkbands_skewness.mean",		{1}},
+	{ "lowlevel.barkbands_skewness.median",		{1}},
+	{ "lowlevel.barkbands_skewness.var",		{1}},
+	{ "lowlevel.barkbands_spread.mean",		{1}},
+	{ "lowlevel.barkbands_spread.median",		{1}},
+	{ "lowlevel.barkbands_spread.var",		{1}},
+	{ "lowlevel.dissonance.mean",			{1}},
+	{ "lowlevel.dissonance.median",			{1}},
+	{ "lowlevel.dissonance.var",			{1}},
+	{ "lowlevel.dynamic_complexity",		{1}},
+	{ "lowlevel.spectral_contrast_coeffs.mean",	{1}},
+	{ "lowlevel.spectral_contrast_coeffs.median",	{1}},
+	{ "lowlevel.spectral_contrast_coeffs.var",	{1}},
+	{ "lowlevel.erbbands.mean",			{1}},
+	{ "lowlevel.erbbands.median",			{1}},
+	{ "lowlevel.erbbands.var",			{1}},
+	{ "lowlevel.gfcc.mean",				{1}},
+	{ "lowlevel.hfc.mean",				{1}},
+	{ "lowlevel.hfc.median",			{1}},
+	{ "lowlevel.hfc.var",				{1}},
+	{ "tonal.hpcp.median",				{1}},
+	{ "lowlevel.melbands.median",			{1}},
+	{ "lowlevel.mfcc.mean",				{1}},
+	{ "lowlevel.pitch_salience.mean",		{1}},
+	{ "lowlevel.pitch_salience.median",		{1}},
+	{ "lowlevel.pitch_salience.var",		{1}},
+	{ "lowlevel.spectral_centroid.mean",		{1}},
+	{ "lowlevel.spectral_centroid.median",		{1}},
+	{ "lowlevel.spectral_centroid.var",		{1}},
+	{ "lowlevel.spectral_complexity.mean",		{1}},
+	{ "lowlevel.spectral_complexity.median",	{1}},
+	{ "lowlevel.spectral_complexity.var",		{1}},
+	{ "lowlevel.spectral_contrast_coeffs.mean",	{1}},
+	{ "lowlevel.spectral_contrast_coeffs.median",	{1}},
+	{ "lowlevel.spectral_contrast_coeffs.var",	{1}},
+	{ "lowlevel.spectral_contrast_valleys.mean",	{1}},
+	{ "lowlevel.spectral_contrast_valleys.median",	{1}},
+	{ "lowlevel.spectral_contrast_valleys.var",	{1}},
+	{ "lowlevel.spectral_decrease.mean",		{1}},
+	{ "lowlevel.spectral_decrease.median",		{1}},
+	{ "lowlevel.spectral_decrease.var",		{1}},
+	{ "lowlevel.spectral_energy.mean",		{1}},
+	{ "lowlevel.spectral_energy.median",		{1}},
+	{ "lowlevel.spectral_energy.var",		{1}},
+	{ "lowlevel.spectral_energyband_high.mean",	{1}},
+	{ "lowlevel.spectral_energyband_high.median",	{1}},
+	{ "lowlevel.spectral_energyband_high.var",	{1}},
+	{ "lowlevel.spectral_energyband_low.mean",	{1}},
+	{ "lowlevel.spectral_energyband_low.median",	{1}},
+	{ "lowlevel.spectral_energyband_low.var",	{1}},
+	{ "lowlevel.spectral_energyband_middle_high.mean",	{1}},
+	{ "lowlevel.spectral_energyband_middle_high.median",	{1}},
+	{ "lowlevel.spectral_energyband_middle_high.var",	{1}},
+	{ "lowlevel.spectral_energyband_middle_low.mean",	{1}},
+	{ "lowlevel.spectral_energyband_middle_low.median",	{1}},
+	{ "lowlevel.spectral_energyband_middle_low.var",	{1}},
+	{ "lowlevel.spectral_entropy.mean",		{1}},
+	{ "lowlevel.spectral_entropy.median",		{1}},
+	{ "lowlevel.spectral_entropy.var",		{1}},
+	{ "lowlevel.spectral_flux.mean",		{1}},
+	{ "lowlevel.spectral_flux.median",		{1}},
+	{ "lowlevel.spectral_flux.var",			{1}},
+	{ "lowlevel.spectral_kurtosis.mean",		{1}},
+	{ "lowlevel.spectral_kurtosis.median",		{1}},
+	{ "lowlevel.spectral_kurtosis.var",		{1}},
+	{ "lowlevel.spectral_rms.mean",			{1}},
+	{ "lowlevel.spectral_rms.median",		{1}},
+	{ "lowlevel.spectral_rms.var",			{1}},
+	{ "lowlevel.spectral_rolloff.mean",		{1}},
+	{ "lowlevel.spectral_rolloff.median",		{1}},
+	{ "lowlevel.spectral_rolloff.var",		{1}},
+	{ "lowlevel.spectral_skewness.mean",		{1}},
+	{ "lowlevel.spectral_skewness.median",		{1}},
+	{ "lowlevel.spectral_skewness.var",		{1}},
+	{ "lowlevel.spectral_spread.mean",		{1}},
+	{ "lowlevel.spectral_spread.median",		{1}},
+	{ "lowlevel.spectral_spread.var",		{1}},
+	{ "lowlevel.zerocrossingrate.mean",		{1}},
+	{ "lowlevel.zerocrossingrate.median",		{1}},
+	{ "lowlevel.zerocrossingrate.var",		{1}},
+};
+
+
+static
+void
+printFeatureSettingsMap(const FeatureSettingsMap& featureSettings)
+{
+	std::cout << "FeatureSettingsMap: (" << featureSettings.size() << " features)" << std::endl;
+	for (const auto& [name, settings] : featureSettings)
+		std::cout << "\t" << name << std::endl;
+}
+
+static
+std::string
+trackToString(Database::Session& session, Database::IdType trackId)
+{
+	std::string res;
+	auto transaction {session.createSharedTransaction()};
+	Database::Track::pointer track {Database::Track::getById(session, trackId)};
+
+	res += track->getName();
+	if (track->getRelease())
+		res += " [" + track->getRelease()->getName() + "]";
+	for (auto artist : track->getArtists())
+		res += " - " + artist->getName();
+	for (auto cluster : track->getClusters())
+		res += " {" + cluster->getType()->getName() + "-"+ cluster->getName() + "}";
+
+	return res;
+}
+
+static
+SimilarityScore
+computeTrackScore(Database::Session& session, Database::IdType track1Id, Database::IdType track2Id)
+{
+	SimilarityScore score {};
+
+	auto transaction {session.createSharedTransaction()};
+
+	auto track1 {Database::Track::getById(session, track1Id)};
+	auto track2 {Database::Track::getById(session, track2Id)};
+
+	if (track1->getRelease() == track2->getRelease())
+		score += 1;
+
+	// Artists in common
+	{
+		auto track1ArtistIds {track1->getArtistIds()};
+		auto track2ArtistIds {track2->getArtistIds()};
+
+		std::vector<Database::IdType> commonArtistIds;
+		std::set_intersection(std::cbegin(track1ArtistIds), std::cend(track1ArtistIds),
+				std::cbegin(track2ArtistIds), std::cend(track2ArtistIds),
+				std::back_inserter(commonArtistIds));
+
+		score += commonArtistIds.size();
+	}
+
+	// Clusters in common
+	{
+		auto track1ClusterIds {track1->getClusterIds()};
+		auto track2ClusterIds {track2->getClusterIds()};
+
+		std::vector<Database::IdType> commonClusterIds;
+		std::set_intersection(std::cbegin(track1ClusterIds), std::cend(track1ClusterIds),
+				std::cbegin(track2ClusterIds), std::cend(track2ClusterIds),
+				std::back_inserter(commonClusterIds));
+
+		score += commonClusterIds.size();
+	}
+
+	return score;
+}
+
+static
+SimilarityScore
+computeSimilarityScore(Database::Session& session, const FeatureSettingsMap& featuresSettings)
+{
+	std::cout << "Compute score of: ";
+	printFeatureSettingsMap(featuresSettings);
+	std::cout << std::endl;
+
+	FeaturesSearcher::TrainSettings trainSettings;
+	trainSettings.nbIterations = 10;
+	trainSettings.featureSettingsMap = featuresSettings;
+	FeaturesSearcher searcher {session, trainSettings};
+
+	const std::vector<Database::IdType> trackIds = std::invoke([&]()
+			{
+				auto transaction {session.createSharedTransaction()};
+				return Database::Track::getAllIds(session);
+			});
+
+	SimilarityScore score {};
+	for (Database::IdType trackId : trackIds)
+	{
+//		std::cout << "Processing track '" << trackToString(session, trackId) << "'" << std::endl;
+		SimilarityScore factor {1};		
+		for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, 3))
+		{
+			SimilarityScore trackScore {computeTrackScore(session, trackId, similarTrackId)};
+//			std::cout << "\tScore = " << trackScore << " (*" << factor << ") with track '" << trackToString(session, similarTrackId) << "'" << std::endl;
+			trackScore *= factor;
+			score += trackScore;
+
+			factor -= (SimilarityScore {1}/3);
+		}
+	}
+
+	std::cout << "Total score = " << score << std::endl;
+
+	return score;
+}
+
+static
+FeatureSettingsMap 
+breedFeatureSettingsMap(const FeatureSettingsMap& a, const FeatureSettingsMap& b)
+{
+	FeatureSettingsMap res;
+
+	res.insert(std::cbegin(a), std::cend(a));
+	res.insert(std::cbegin(b), std::cend(b));
+
+	// just kill random elements until size is good
+	while (res.size() > a.size())
+	{
+		const auto itFeature {pickRandom(res)};
+		res.erase(itFeature);
+	}
+
+	return res;
+}
+
+static
+void
+mutateFeatureSettingsMap(FeatureSettingsMap& a)
+{
+	const std::size_t size {a.size()};
+	// Replace one of the feature with another one, random
+	a.erase(pickRandom(a));
+
+	while (a.size() != size)
+	{
+		const auto itFeatureSetting {pickRandom(featuresSettings)};
+		a.emplace(itFeatureSetting->first, itFeatureSetting->second);
+	}
+}
 
 int main(int argc, char *argv[])
 {
 	try
 	{
+
 		// log to stdout
 		ServiceProvider<Logger>::create<StreamLogger>(std::cout);
 
-		std::filesystem::path configFilePath {"/etc/lms.conf"};
-		if (argc >= 2)
-			configFilePath = std::string(argv[1], 0, 256);
+		if (argc != 3)
+		{
+			std::cerr << "usage: <lms_conf_file> <nb_workers>" << std::endl;
+			return EXIT_FAILURE;
+		}
+
+		const std::filesystem::path configFilePath {std::string(argv[1], 0, 256)};
+		const std::size_t nbWorkers = atoi(argv[2]);
 
 		ServiceProvider<Config>::create(configFilePath);
 
 		Database::Db db {ServiceProvider<Config>::get()->getPath("working-dir") / "lms.db"};
-		Database::Session session {db};
+		Database::SessionPool sessionPool {db, nbWorkers};
 
-/*		const FeatureSettings
+		// Create some random settings (i.e random population)
+		std::vector<FeatureSettingsMap> initialPopulation;
+
+		constexpr std::size_t populationSize {100};
+		constexpr std::size_t nbFeatures {5};
+
+		for (std::size_t i {}; i < populationSize; ++i)
 		{
-			{ "lowlevel.average_loudness",			1 },
-			{ "lowlevel.dynamic_complexity",		1 },
-			{ "lowlevel.spectral_contrast_coeffs.median",	6 },
-			{ "lowlevel.erbbands.median",			40 },
-			{ "tonal.hpcp.median",				36 },
-			{ "lowlevel.melbands.median",			40 },
-			{ "lowlevel.barkbands.median",			27 },
-			{ "lowlevel.mfcc.mean",				13 },
-			{ "lowlevel.gfcc.mean",				13 },
-		};
+			FeatureSettingsMap settings;
 
-		const TrackFeaturesMap trackFeaturesMap {getAllTrackFeatures(*session)};
+			while (settings.size() < nbFeatures)
+			{
+				const auto itFeatureSetting {pickRandom(featuresSettings)};
+				settings.emplace(itFeatureSetting->first, itFeatureSetting->second);
+			}
 
-		std::cout << "Found " << trackFeaturesMap.size() << " tracks with features!" << std::endl;*/
+			initialPopulation.emplace_back(std::move(settings));
+		}
+
+		GeneticAlgorithm<FeatureSettingsMap>::Params params;
+		params.nbWorkers = nbWorkers;
+		params.nbGenerations = 300;
+		params.mutationProbability = 0.2;
+		params.breedFunction = breedFeatureSettingsMap;
+		params.mutateFunction = mutateFeatureSettingsMap;
+		params.scoreFunction =
+			[&](const FeatureSettingsMap& settings)
+			{
+				Database::SessionPool::ScopedSession scopedSession {sessionPool};
+				return computeSimilarityScore(scopedSession.get(), settings);
+			};
+
+		GeneticAlgorithm<FeatureSettingsMap> geneticAlgorithm {params};
+
+		std::cout << "Parameters:\n"
+			<< "\tnb generations = " << params.nbGenerations << "\n"
+			<< "\tpopulationSize = " << populationSize << "\n"
+			<< "\tnbFeatures = " << nbFeatures << "\n"
+			<< "\tmutationProbability = " << params.mutationProbability << "\n"
+			<< std::endl;
+			
+		std::cout << "Starting simulation..." << std::endl;
+		const FeatureSettingsMap selectedSettings {geneticAlgorithm.simulate(initialPopulation)};
+		std::cout << "Simulation complete! Best result:" << std::endl;
+		printFeatureSettingsMap(selectedSettings);
 	}
 	catch (std::exception& e)
 	{

tools/similarity-parameters/Makefile.am

@@ -10,12 +10,16 @@ lms_similarity_parameters_SOURCES = \
 	$(top_srcdir)/src/database/Release.cpp		\
 	$(top_srcdir)/src/database/ScanSettings.cpp	\
 	$(top_srcdir)/src/database/Session.cpp		\
+	$(top_srcdir)/src/database/SessionPool.cpp	\
 	$(top_srcdir)/src/database/SimilaritySettings.cpp	\
 	$(top_srcdir)/src/database/SqlQuery.cpp		\
 	$(top_srcdir)/src/database/Track.cpp		\
 	$(top_srcdir)/src/database/User.cpp		\
 	$(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp		\
 	$(top_srcdir)/src/similarity/features/som/Network.cpp			\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp	\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp	\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp	\
 	$(top_srcdir)/src/utils/Config.cpp 		\
 	$(top_srcdir)/src/utils/Logger.cpp 		\
 	$(top_srcdir)/src/utils/StreamLogger.cpp 	\

tools/similarity-parameters/ParallelFor.hpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <functional>
+#include <thread>
+#include <boost/asio/io_context.hpp>
+
+template <typename It, typename Func>
+void parallel_foreach(std::size_t nbWorkers, It begin, It end, Func&& func)
+{
+	if (nbWorkers == 0)
+		throw std::runtime_error("Invalid worker count");
+
+	boost::asio::io_context ioContext;
+
+	for (It it {begin}; it != end; ++it)
+	{
+		auto refValue {std::ref<typename It::value_type>(*it)};
+		ioContext.post([refValue, &func]() { std::cout << "EXEC FROM WORKER" << std::endl; func(refValue); std::cout << "END EXEC FROM WORKER" << std::endl; });
+	}
+
+	std::vector<std::thread> threads;
+	for (std::size_t i {}; i < nbWorkers - 1; ++i)
+		threads.emplace_back([&]() { ioContext.run(); });
+
+	ioContext.run();
+
+	for (std::thread& t : threads)
+		t.join();
+}
+

tools/similarity/LmsSimilarity.cpp

@@ -1,3 +1,22 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 #include <filesystem>
 #include <iostream>
 #include <stdexcept>
@@ -25,8 +44,8 @@ int main(int argc, char *argv[])
 
 		const FeatureSettingsMap featuresSettings
 		{
-//			{ "lowlevel.average_loudness",			1 },
-//			{ "lowlevel.dynamic_complexity",		1 },
+/*			{ "lowlevel.average_loudness",			1 },
+			{ "lowlevel.dynamic_complexity",		1 },
 			{ "lowlevel.spectral_contrast_coeffs.median",	{1} },
 			{ "lowlevel.erbbands.median",			{1} },
 			{ "tonal.hpcp.median",				{1} },
@@ -34,6 +53,41 @@ int main(int argc, char *argv[])
 			{ "lowlevel.barkbands.median",			{1} },
 			{ "lowlevel.mfcc.mean",				{1} },
 			{ "lowlevel.gfcc.mean",				{1} },
+*/
+			{ "lowlevel.spectral_kurtosis.median",	{1}},
+			{ "lowlevel.spectral_kurtosis.mean",	{1}},
+			{ "lowlevel.spectral_complexity.var",	{1}},
+			{ "lowlevel.barkbands.median",	{1}},
+			{ "lowlevel.barkbands_kurtosis.mean",	{1}},
+				/*
+				   {"lowlevel.spectral_centroid.dvar2",		{1} },
+				   {"lowlevel.barkbands.median",		{1} },
+				   {        "lowlevel.barkbands.dvar",		{1} },
+				{        "lowlevel.spectral_complexity.min",		{1} },
+				{        "lowlevel.pitch_salience.dmean2",		{1} },
+				{        "lowlevel.spectral_contrast_valleys.dmean",		{1} },
+				{        "lowlevel.pitch_salience.max",		{1} },
+				{        "lowlevel.barkbands.mean",		{1} },
+				{        "lowlevel.spectral_complexity.mean",		{1} },
+				{        "lowlevel.dissonance.dvar",		{1} },
+				*/
+/*
+			{ "lowlevel.spectral_energy.dvar",		{1} },
+			{ "lowlevel.barkbands.min",			{1} },
+			{ "lowlevel.spectral_centroid.median",		{1} },
+			{"lowlevel.barkbands_kurtosis.median",		{1} },
+			{"lowlevel.spectral_energy.median",		{1} },
+			{"lowlevel.barkbands.max",		{1} },
+			{"lowlevel.barkbands_spread.var",		{1} },
+			{"lowlevel.spectral_decrease.var",		{1} },
+			{"lowlevel.spectral_contrast_valleys.dmean",		{1} },
+			{"lowlevel.barkbands_crest.mean",		{1} },
+			{"lowlevel.spectral_entropy.var",		{1} },
+			{"lowlevel.barkbands_crest.max",		{1} },
+			{"lowlevel.hfc.dvar",		{1} },
+			{"lowlevel.barkbands_skewness.dvar2",		{1} },
+			{"lowlevel.spectral_centroid.max",		{1} },
+			*/
 		};
 
 		std::filesystem::path configFilePath {"/etc/lms.conf"};
@@ -47,7 +101,10 @@ int main(int argc, char *argv[])
 
 		std::cout << "Classifying tracks..." << std::endl;
 		// may be long...
-		FeaturesSearcher searcher {session, featuresSettings};
+		struct FeaturesSearcher::TrainSettings trainSettings;
+		trainSettings.nbIterations = 10;
+		trainSettings.featureSettingsMap = featuresSettings;
+		FeaturesSearcher searcher {session, trainSettings};
 		std::cout << "Classifying tracks DONE" << std::endl;
 
 		const std::vector<Database::IdType> trackIds = std::invoke([&]()