WIP

src/tools/Makefile.am

@@ -0,0 +1,4 @@
+if BUILD_TOOLS
+SUBDIRS = similarity similarity-parameters metadata
+endif
+

src/tools/metadata/LmsMetadata.cpp

@@ -0,0 +1,177 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <chrono>
+#include <optional>
+#include <stdexcept>
+#include <stdlib.h>
+#include <iostream>
+
+#include <Wt/WDate.h>
+
+#include "av/AvInfo.hpp"
+#include "metadata/AvFormat.hpp"
+#include "metadata/TagLibParser.hpp"
+#include "utils/StreamLogger.hpp"
+
+std::ostream& operator<<(std::ostream& os, const MetaData::Artist& artist)
+{
+	os << artist.name;
+
+	if (artist.musicBrainzArtistID)
+		os << " (" << artist.musicBrainzArtistID->getAsString() << ")";
+
+	return os;
+}
+
+std::ostream& operator<<(std::ostream& os, const MetaData::Album& album)
+{
+	os << album.name;
+
+	if (album.musicBrainzAlbumID)
+		os << " (" << album.musicBrainzAlbumID->getAsString() << ")";
+
+	return os;
+}
+
+
+
+void parse(MetaData::Parser& parser, const std::filesystem::path& file)
+{
+	using namespace MetaData;
+
+	parser.setClusterTypeNames( {"MOOD", "GENRE"} );
+
+	const auto start {std::chrono::steady_clock::now()};
+	std::optional<Track> track {parser.parse(file, true)};
+	if (!track)
+	{
+		std::cerr << "Parsing failed" << std::endl;
+		return;
+	}
+	const auto end {std::chrono::steady_clock::now()};
+
+	std::cout << "Parsing time: " << std::chrono::duration_cast<std::chrono::microseconds>(end - start).count() / 1000. << "ms" << std::endl;
+
+	std::cout << "Track metadata:" << std::endl;
+
+	for (const Artist& artist : track->artists)
+		std::cout << "Artist: " << artist << std::endl;
+
+	for (const Artist& artist: track->albumArtists)
+		std::cout << "Album artist: " << artist << std::endl;
+
+	if (track->album)
+		std::cout << "Album: " << *track->album << std::endl;
+
+	std::cout << "Title: " << track->title << std::endl;
+
+	if (track->musicBrainzTrackID)
+		std::cout << "MB TrackID = " << track->musicBrainzTrackID->getAsString() << std::endl;
+
+	if (track->musicBrainzRecordID)
+		std::cout << "MB RecordID = " << track->musicBrainzRecordID->getAsString() << std::endl;
+
+	for (const auto& cluster : track->clusters)
+	{
+		std::cout << "Cluster: " << cluster.first << std::endl;
+		for (const auto& name : cluster.second)
+		{
+			std::cout << "\t" << name << std::endl;
+		}
+	}
+
+	std::cout << "Duration: " << track->duration.count() / 1000 << "s" << std::endl;
+
+	if (track->trackNumber)
+		std::cout << "Track: " << *track->trackNumber << std::endl;
+
+	if (track->totalTrack)
+		std::cout << "TotalTrack: " << *track->totalTrack << std::endl;
+
+	if (track->discNumber)
+		std::cout << "Disc: " << *track->discNumber << std::endl;
+
+	if (track->totalDisc)
+		std::cout << "TotalDisc: " << *track->totalDisc << std::endl;
+
+	if (track->year)
+		std::cout << "Year: " << *track->year << std::endl;
+
+	if (track->originalYear)
+		std::cout << "Original year: " << *track->originalYear << std::endl;
+
+	std::cout << "HasCover = " << std::boolalpha << track->hasCover << std::endl;
+
+	for (const auto& audioStream : track->audioStreams)
+		std::cout << "Audio stream: " << audioStream.bitRate << " bps" << std::endl;
+
+	if (track->acoustID)
+		std::cout << "AcoustID: " << track->acoustID->getAsString() << std::endl;
+
+	if (!track->copyright.empty())
+		std::cout << "Copyright: " << track->copyright << std::endl;
+
+	if (!track->copyrightURL.empty())
+		std::cout << "CopyrightURL: " << track->copyrightURL << std::endl;
+
+	std::cout << std::endl;
+}
+
+int main(int argc, char *argv[])
+{
+	if (argc == 1)
+	{
+		std::cerr << "Usage: <file> [<file> ...]" << std::endl;
+		return EXIT_FAILURE;
+	}
+
+	try
+	{
+		// log to stdout
+		ServiceProvider<Logger>::create<StreamLogger>(std::cout);
+
+		for (std::size_t  i {}; i < static_cast<std::size_t>(argc - 1); ++i)
+		{
+			std::filesystem::path file {argv[i + 1]};
+
+			std::cout << "Parsing file '" << file << "'" << std::endl;
+
+			{
+				std::cout << "Using av:" << std::endl;
+				MetaData::AvFormat parser;
+				parse(parser, file);
+			}
+
+			{
+				std::cout << "Using TagLib:" << std::endl;
+				MetaData::TagLibParser parser;
+				parse(parser, file);
+			}
+		}
+
+	}
+	catch (std::exception& e)
+	{
+		std::cerr << "Caught exception: " << e.what() << std::endl;
+		return EXIT_FAILURE;
+	}
+
+	return EXIT_SUCCESS;
+}

src/tools/metadata/Makefile.am

@@ -0,0 +1,14 @@
+bin_PROGRAMS = lms-metadata
+
+lms_metadata_SOURCES = \
+	$(srcdir)/LmsMetadata.cpp			\
+	$(top_srcdir)/src/av/AvInfo.cpp 		\
+	$(top_srcdir)/src/metadata/AvFormat.cpp		\
+	$(top_srcdir)/src/metadata/TagLibParser.cpp	\
+	$(top_srcdir)/src/utils/Logger.cpp 		\
+	$(top_srcdir)/src/utils/StreamLogger.cpp 	\
+	$(top_srcdir)/src/utils/String.cpp 		\
+	$(top_srcdir)/src/utils/UUID.cpp
+
+lms_metadata_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT
+

src/tools/similarity-parameters/GeneticAlgorithm.hpp

@@ -0,0 +1,174 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <numeric>
+
+#include "utils/Random.hpp"
+
+#include "ParallelFor.hpp"
+
+template<typename Individual>
+class GeneticAlgorithm
+{
+	public:
+		using Score = float;
+
+		using BreedFunction = std::function<Individual(const Individual&, const Individual&)>;
+		using MutateFunction = std::function<void(Individual&)>;
+		using ScoreFunction = std::function<Score(const Individual&)>;
+
+		struct Params
+		{
+			std::size_t		nbWorkers {1};
+			std::size_t		nbGenerations;
+			float			crossoverRatio {0.5};
+			float			mutationProbability {0.05};
+			BreedFunction	breedFunction;
+			MutateFunction		mutateFunction;
+			ScoreFunction		scoreFunction;
+		};
+
+		GeneticAlgorithm(const Params& params);
+
+		// Returns the individual that has the maximum score after processing the requested generations
+		Individual simulate(const std::vector<Individual>& initialPopulation);
+
+	private:
+
+		struct ScoredIndividual
+		{
+			Individual individual;
+			std::optional<Score> score {};
+		};
+
+		void scoreAndSortPopulation(std::vector<ScoredIndividual>& population);
+		Score getTotalScore(const std::vector<ScoredIndividual>& population) const;
+		typename std::vector<ScoredIndividual>::const_iterator pickRandomRouletteWheel(const std::vector<ScoredIndividual>& population, Score totalScore);
+
+		Params _params;
+};
+
+template<typename Individual>
+GeneticAlgorithm<Individual>::GeneticAlgorithm(const Params& params)
+: _params {params}
+{
+}
+
+
+template<typename Individual>
+Individual
+GeneticAlgorithm<Individual>::simulate(const std::vector<Individual>& initialPopulation)
+{
+	const std::size_t childrenCountPerGeneration {static_cast<std::size_t>(initialPopulation.size() * _params.crossoverRatio)};
+	if (initialPopulation.size() < 10)
+		throw std::runtime_error("Initial population must has at least 10 elements");
+
+	std::vector<ScoredIndividual> scoredPopulation;
+	scoredPopulation.reserve(initialPopulation.size());
+
+	std::transform(std::cbegin(initialPopulation), std::cend(initialPopulation), std::back_inserter(scoredPopulation ),
+			[](const Individual& individual) { return ScoredIndividual {individual};});
+
+	scoreAndSortPopulation(scoredPopulation);
+
+	for (std::size_t currentGeneration {}; currentGeneration  < _params.nbGenerations; ++currentGeneration)
+	{
+		assert(scoredPopulation.size() == initialPopulation.size());
+		std::cout << "Processing generation " << currentGeneration << "..." << std::endl;
+		std::cout << "Need to create " << childrenCountPerGeneration << " new children" << std::endl;
+
+		// breed
+		const Score populationTotalScore {getTotalScore(scoredPopulation)};
+		std::vector<ScoredIndividual> children;
+		children.reserve(childrenCountPerGeneration);
+
+		while (children.size() < childrenCountPerGeneration)
+		{
+			// Select two random parents using their score as weight
+			const auto itParent1 {pickRandomRouletteWheel(scoredPopulation, populationTotalScore)};
+			const auto itParent2 {pickRandomRouletteWheel(scoredPopulation, populationTotalScore)};
+
+			if (itParent1 == itParent2)
+				continue;
+
+			ScoredIndividual child {_params.breedFunction(itParent1->individual, itParent2->individual)};
+			
+			if (Random::getRealRandom(float {}, float {1}) <= _params.mutationProbability)
+				_params.mutateFunction(child.individual);
+
+			children.emplace_back(std::move(child));
+		}
+
+		// Elitist selection
+		scoredPopulation.resize(initialPopulation.size() - childrenCountPerGeneration);
+
+		scoredPopulation.insert(std::end(scoredPopulation), std::make_move_iterator(std::begin(children)), std::make_move_iterator(std::end(children)));
+		assert(scoredPopulation.size() == initialPopulation.size());
+
+		scoreAndSortPopulation(scoredPopulation);
+
+		std::cout << "Mean score = " << getTotalScore(scoredPopulation) / scoredPopulation.size() << std::endl;
+		std::cout << "Current best score = " << *scoredPopulation.front().score << std::endl;
+	}
+
+	std::cout << "Best score = " << *scoredPopulation.front().score << std::endl;
+	return scoredPopulation.front().individual;
+}
+
+
+template<typename Individual>
+void
+GeneticAlgorithm<Individual>::scoreAndSortPopulation(std::vector<ScoredIndividual>& scoredPopulation)
+{
+	parallel_foreach(_params.nbWorkers, std::begin(scoredPopulation), std::end(scoredPopulation),
+			[&](ScoredIndividual& scoredIndividual)
+			{
+				if (!scoredIndividual.score)
+					scoredIndividual.score = _params.scoreFunction(scoredIndividual.individual);
+			});
+
+	std::sort(std::begin(scoredPopulation), std::end(scoredPopulation), [](const ScoredIndividual& a, const ScoredIndividual& b) { return a.score > b.score; });
+}
+
+template<typename Individual>
+typename GeneticAlgorithm<Individual>::Score
+GeneticAlgorithm<Individual>::getTotalScore(const std::vector<ScoredIndividual>& scoredPopulation) const
+{
+	return std::accumulate(std::cbegin(scoredPopulation), std::cend(scoredPopulation), Score {}, [](Score score, const ScoredIndividual& individual) { return score + *individual.score; });
+}
+
+template<typename Individual>
+typename std::vector<typename GeneticAlgorithm<Individual>::ScoredIndividual>::const_iterator
+GeneticAlgorithm<Individual>::pickRandomRouletteWheel(const std::vector<ScoredIndividual>& population, Score totalScore)
+{
+	const Score randomScore {Random::getRealRandom(Score {}, totalScore)};
+
+	Score curScore{};
+	for (auto itScoredIndividual {std::cbegin(population)}; itScoredIndividual != std::cend(population); ++itScoredIndividual )
+	{
+		if (curScore + *itScoredIndividual->score > randomScore)
+			return itScoredIndividual;
+
+		curScore += *itScoredIndividual->score;
+	}
+
+	throw std::runtime_error("bad random or empty population");
+}
+
+

src/tools/similarity-parameters/LmsSimilarityParameters.cpp

@@ -0,0 +1,489 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <iostream>
+#include <filesystem>
+#include <string>
+
+#include "database/Artist.hpp"
+#include "database/Cluster.hpp"
+#include "database/Db.hpp"
+#include "database/Release.hpp"
+#include "database/SessionPool.hpp"
+#include "database/Track.hpp"
+#include "database/TrackFeatures.hpp"
+#include "similarity/features/SimilarityFeaturesSearcher.hpp"
+#include "utils/Config.hpp"
+#include "utils/Service.hpp"
+#include "utils/StreamLogger.hpp"
+
+#include "GeneticAlgorithm.hpp"
+
+using namespace Similarity;
+using SimilarityScore = GeneticAlgorithm<FeatureSettingsMap>::Score;
+
+// An individual is just a FeatureSettingsMap
+// The goal is to get the FeatureSettingsMap that maximize the score
+const FeatureSettingsMap featuresSettings
+{
+	{ "lowlevel.average_loudness",			{1}},
+	{ "lowlevel.barkbands.mean",			{1}},
+	{ "lowlevel.barkbands.median",			{1}},
+	{ "lowlevel.barkbands.var",			{1}},
+	{ "lowlevel.barkbands_crest.mean",		{1}},
+	{ "lowlevel.barkbands_crest.median",		{1}},
+	{ "lowlevel.barkbands_crest.var",		{1}},
+	{ "lowlevel.barkbands_flatness_db.mean",	{1}},
+	{ "lowlevel.barkbands_flatness_db.median",	{1}},
+	{ "lowlevel.barkbands_flatness_db.var",		{1}},
+	{ "lowlevel.barkbands_kurtosis.mean",		{1}},
+	{ "lowlevel.barkbands_kurtosis.median",		{1}},
+	{ "lowlevel.barkbands_kurtosis.var",		{1}},
+	{ "lowlevel.barkbands_skewness.mean",		{1}},
+	{ "lowlevel.barkbands_skewness.median",		{1}},
+	{ "lowlevel.barkbands_skewness.var",		{1}},
+	{ "lowlevel.barkbands_spread.mean",		{1}},
+	{ "lowlevel.barkbands_spread.median",		{1}},
+	{ "lowlevel.barkbands_spread.var",		{1}},
+	{ "lowlevel.dissonance.mean",			{1}},
+	{ "lowlevel.dissonance.median",			{1}},
+	{ "lowlevel.dissonance.var",			{1}},
+	{ "lowlevel.dynamic_complexity",		{1}},
+	{ "lowlevel.spectral_contrast_coeffs.mean",	{1}},
+	{ "lowlevel.spectral_contrast_coeffs.median",	{1}},
+	{ "lowlevel.spectral_contrast_coeffs.var",	{1}},
+	{ "lowlevel.erbbands.mean",			{1}},
+	{ "lowlevel.erbbands.median",			{1}},
+	{ "lowlevel.erbbands.var",			{1}},
+	{ "lowlevel.gfcc.mean",				{1}},
+	{ "lowlevel.hfc.mean",				{1}},
+	{ "lowlevel.hfc.median",			{1}},
+	{ "lowlevel.hfc.var",				{1}},
+	{ "tonal.hpcp.median",				{1}},
+	{ "lowlevel.melbands.mean",			{1}},
+	{ "lowlevel.melbands.median",			{1}},
+	{ "lowlevel.melbands.var",			{1}},
+	{ "lowlevel.melbands_crest.mean",		{1}},
+	{ "lowlevel.melbands_crest.median",		{1}},
+	{ "lowlevel.melbands_crest.var",		{1}},
+	{ "lowlevel.melbands_flatness_db.mean",		{1}},
+	{ "lowlevel.melbands_flatness_db.median",	{1}},
+	{ "lowlevel.melbands_flatness_db.var",		{1}},
+	{ "lowlevel.melbands_kurtosis.mean",		{1}},
+	{ "lowlevel.melbands_kurtosis.median",		{1}},
+	{ "lowlevel.melbands_kurtosis.var",		{1}},
+	{ "lowlevel.melbands_skewness.mean",		{1}},
+	{ "lowlevel.melbands_skewness.median",		{1}},
+	{ "lowlevel.melbands_skewness.var",		{1}},
+	{ "lowlevel.melbands_spread.mean",		{1}},
+	{ "lowlevel.melbands_spread.median",		{1}},
+	{ "lowlevel.melbands_spread.var",		{1}},
+	{ "lowlevel.mfcc.mean",				{1}},
+	{ "lowlevel.pitch_salience.mean",		{1}},
+	{ "lowlevel.pitch_salience.median",		{1}},
+	{ "lowlevel.pitch_salience.var",		{1}},
+	{ "lowlevel.silence_rate_30dB.mean",		{1}},
+	{ "lowlevel.silence_rate_30dB.median",		{1}},
+	{ "lowlevel.silence_rate_30dB.var",		{1}},
+	{ "lowlevel.silence_rate_60dB.mean",		{1}},
+	{ "lowlevel.silence_rate_60dB.median",		{1}},
+	{ "lowlevel.silence_rate_60dB.var",		{1}},
+	{ "lowlevel.spectral_centroid.mean",		{1}},
+	{ "lowlevel.spectral_centroid.median",		{1}},
+	{ "lowlevel.spectral_centroid.var",		{1}},
+	{ "lowlevel.spectral_complexity.mean",		{1}},
+	{ "lowlevel.spectral_complexity.median",	{1}},
+	{ "lowlevel.spectral_complexity.var",		{1}},
+	{ "lowlevel.spectral_contrast_coeffs.mean",	{1}},
+	{ "lowlevel.spectral_contrast_coeffs.median",	{1}},
+	{ "lowlevel.spectral_contrast_coeffs.var",	{1}},
+	{ "lowlevel.spectral_contrast_valleys.mean",	{1}},
+	{ "lowlevel.spectral_contrast_valleys.median",	{1}},
+	{ "lowlevel.spectral_contrast_valleys.var",	{1}},
+	{ "lowlevel.spectral_decrease.mean",		{1}},
+	{ "lowlevel.spectral_decrease.median",		{1}},
+	{ "lowlevel.spectral_decrease.var",		{1}},
+	{ "lowlevel.spectral_energy.mean",		{1}},
+	{ "lowlevel.spectral_energy.median",		{1}},
+	{ "lowlevel.spectral_energy.var",		{1}},
+	{ "lowlevel.spectral_energyband_high.mean",	{1}},
+	{ "lowlevel.spectral_energyband_high.median",	{1}},
+	{ "lowlevel.spectral_energyband_high.var",	{1}},
+	{ "lowlevel.spectral_energyband_low.mean",	{1}},
+	{ "lowlevel.spectral_energyband_low.median",	{1}},
+	{ "lowlevel.spectral_energyband_low.var",	{1}},
+	{ "lowlevel.spectral_energyband_middle_high.mean",	{1}},
+	{ "lowlevel.spectral_energyband_middle_high.median",	{1}},
+	{ "lowlevel.spectral_energyband_middle_high.var",	{1}},
+	{ "lowlevel.spectral_energyband_middle_low.mean",	{1}},
+	{ "lowlevel.spectral_energyband_middle_low.median",	{1}},
+	{ "lowlevel.spectral_energyband_middle_low.var",	{1}},
+	{ "lowlevel.spectral_entropy.mean",		{1}},
+	{ "lowlevel.spectral_entropy.median",		{1}},
+	{ "lowlevel.spectral_entropy.var",		{1}},
+	{ "lowlevel.spectral_flux.mean",		{1}},
+	{ "lowlevel.spectral_flux.median",		{1}},
+	{ "lowlevel.spectral_flux.var",			{1}},
+	{ "lowlevel.spectral_kurtosis.mean",		{1}},
+	{ "lowlevel.spectral_kurtosis.median",		{1}},
+	{ "lowlevel.spectral_kurtosis.var",		{1}},
+	{ "lowlevel.spectral_rms.mean",			{1}},
+	{ "lowlevel.spectral_rms.median",		{1}},
+	{ "lowlevel.spectral_rms.var",			{1}},
+	{ "lowlevel.spectral_rolloff.mean",		{1}},
+	{ "lowlevel.spectral_rolloff.median",		{1}},
+	{ "lowlevel.spectral_rolloff.var",		{1}},
+	{ "lowlevel.spectral_skewness.mean",		{1}},
+	{ "lowlevel.spectral_skewness.median",		{1}},
+	{ "lowlevel.spectral_skewness.var",		{1}},
+	{ "lowlevel.spectral_spread.mean",		{1}},
+	{ "lowlevel.spectral_spread.median",		{1}},
+	{ "lowlevel.spectral_spread.var",		{1}},
+	{ "lowlevel.zerocrossingrate.mean",		{1}},
+	{ "lowlevel.zerocrossingrate.median",		{1}},
+	{ "lowlevel.zerocrossingrate.var",		{1}},
+};
+
+static
+std::unordered_map<Database::IdType, FeatureValuesMap>
+constructFeaturesCache(Database::Session& session, const FeatureSettingsMap& featureSettings)
+{
+	std::unordered_map<Database::IdType, FeatureValuesMap> cache;
+
+	std::unordered_set<FeatureName> names;
+	std::transform(std::cbegin(featureSettings), std::cend(featureSettings), std::inserter(names, std::begin(names)),
+			[](const auto& itFeature) { return itFeature.first; });
+
+	auto transaction {session.createSharedTransaction()};
+
+	for (auto trackId : Database::Track::getAllIdsWithFeatures(session))
+	{
+		const Database::Track::pointer track {Database::Track::getById(session, trackId)};
+		const Database::TrackFeatures::pointer trackFeatures {track->getTrackFeatures()};
+
+		cache[trackId] = trackFeatures->getFeatureValuesMap(names);
+	}
+
+	return cache;
+}
+
+static
+std::optional<FeatureValuesMap>
+getFeaturesFromCache(const std::unordered_map<Database::IdType, FeatureValuesMap>& cache, Database::IdType trackId, const FeatureNames& names)
+{
+	std::optional<FeatureValuesMap> res;
+
+	auto it {cache.find(trackId)};
+	if (it == std::cend(cache))
+		return res;
+
+	res = FeatureValuesMap{};
+
+	const FeatureValuesMap& trackFeatures {it->second};
+	for (const FeatureName& name : names)
+	{
+		auto itFeatures {trackFeatures.find(name)};
+		if (itFeatures == std::cend(trackFeatures))
+		{
+			res.reset();
+			break;
+		}
+
+		res->emplace(name, itFeatures ->second);
+	}
+
+	return res;
+}
+
+static
+void
+printFeatureSettingsMap(const FeatureSettingsMap& featureSettings)
+{
+	std::cout << "FeatureSettingsMap: (" << featureSettings.size() << " features)" << std::endl;
+	for (const auto& [name, settings] : featureSettings)
+		std::cout << "\t" << name << std::endl;
+}
+
+static
+std::string
+trackToString(Database::Session& session, Database::IdType trackId)
+{
+	std::string res;
+	auto transaction {session.createSharedTransaction()};
+	Database::Track::pointer track {Database::Track::getById(session, trackId)};
+
+	res += track->getName();
+	if (track->getRelease())
+		res += " [" + track->getRelease()->getName() + "]";
+	for (auto artist : track->getArtists())
+		res += " - " + artist->getName();
+	for (auto cluster : track->getClusters())
+		res += " {" + cluster->getType()->getName() + "-"+ cluster->getName() + "}";
+
+	return res;
+}
+
+static
+SimilarityScore
+computeTrackScore(Database::Session& session, Database::IdType track1Id, Database::IdType track2Id)
+{
+	SimilarityScore score {};
+
+	auto transaction {session.createSharedTransaction()};
+
+	auto track1 {Database::Track::getById(session, track1Id)};
+	auto track2 {Database::Track::getById(session, track2Id)};
+
+	if (track1->getRelease() == track2->getRelease())
+		score += 1;
+
+	// Artists in common
+	{
+		auto track1ArtistIds {track1->getArtistIds()};
+		auto track2ArtistIds {track2->getArtistIds()};
+
+		std::vector<Database::IdType> commonArtistIds;
+		std::set_intersection(std::cbegin(track1ArtistIds), std::cend(track1ArtistIds),
+				std::cbegin(track2ArtistIds), std::cend(track2ArtistIds),
+				std::back_inserter(commonArtistIds));
+
+		score += commonArtistIds.size();
+	}
+
+	// Clusters in common
+	{
+		auto track1ClusterIds {track1->getClusterIds()};
+		auto track2ClusterIds {track2->getClusterIds()};
+
+		std::vector<Database::IdType> commonClusterIds;
+		std::set_intersection(std::cbegin(track1ClusterIds), std::cend(track1ClusterIds),
+				std::cbegin(track2ClusterIds), std::cend(track2ClusterIds),
+				std::back_inserter(commonClusterIds));
+
+		score += commonClusterIds.size();
+	}
+
+	return score;
+}
+
+static
+SimilarityScore
+computeSimilarityScore(Database::Session& session, FeaturesSearcher::TrainSettings trainSettings)
+{
+	std::cout << "Compute score of: ";
+	printFeatureSettingsMap(trainSettings.featureSettingsMap);
+	std::cout << std::endl;
+
+	FeaturesSearcher searcher {session, trainSettings};
+
+	const std::vector<Database::IdType> trackIds = std::invoke([&]()
+			{
+				auto transaction {session.createSharedTransaction()};
+				return Database::Track::getAllIdsWithFeatures(session);
+			});
+
+	SimilarityScore score {};
+	for (Database::IdType trackId : trackIds)
+	{
+		constexpr std::size_t nbSimilarTracks {3};
+//		std::cout << "Processing track '" << trackToString(session, trackId) << "'" << std::endl;
+		SimilarityScore factor {1};		
+		for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, nbSimilarTracks))
+		{
+			SimilarityScore trackScore {computeTrackScore(session, trackId, similarTrackId)};
+//			std::cout << "\tScore = " << trackScore << " (*" << factor << ") with track '" << trackToString(session, similarTrackId) << "'" << std::endl;
+			trackScore *= factor;
+			score += trackScore;
+
+			factor -= (SimilarityScore {1}/nbSimilarTracks );
+		}
+	}
+
+	std::cout << "Total score = " << score << std::endl;
+
+	return score;
+}
+
+static
+void
+printBadlyClassifiedTracks(Database::Session& session, FeaturesSearcher::TrainSettings trainSettings)
+{
+	
+	FeaturesSearcher searcher {session, trainSettings};
+
+	const std::vector<Database::IdType> trackIds = std::invoke([&]()
+			{
+				auto transaction {session.createSharedTransaction()};
+				return Database::Track::getAllIdsWithFeatures(session);
+			});
+
+	for (Database::IdType trackId : trackIds)
+	{
+		constexpr std::size_t nbSimilarTracks {3};
+		for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, nbSimilarTracks))
+		{
+			SimilarityScore trackScore {computeTrackScore(session, trackId, similarTrackId)};
+			if (trackScore == 0)
+				std::cout << "Badly classified tracks: '" << trackToString(session, trackId) << "'\n\twith track '" << trackToString(session, similarTrackId) << "'" <<std::endl;
+		}
+	}
+}
+
+
+static
+FeatureSettingsMap 
+breedFeatureSettingsMap(const FeatureSettingsMap& a, const FeatureSettingsMap& b)
+{
+	FeatureSettingsMap res;
+
+	res.insert(std::cbegin(a), std::cend(a));
+	res.insert(std::cbegin(b), std::cend(b));
+
+	// just kill random elements until size is good
+	while (res.size() > a.size())
+	{
+		const auto itFeature {Random::pickRandom(res)};
+		res.erase(itFeature);
+	}
+
+	return res;
+}
+
+static
+void
+mutateFeatureSettingsMap(FeatureSettingsMap& a)
+{
+	const std::size_t size {a.size()};
+	// Replace one of the feature with another one, random
+	a.erase(Random::pickRandom(a));
+
+	while (a.size() != size)
+	{
+		const auto itFeatureSetting {Random::pickRandom(featuresSettings)};
+		a.emplace(itFeatureSetting->first, itFeatureSetting->second);
+	}
+}
+
+int main(int argc, char *argv[])
+{
+	try
+	{
+
+		// log to stdout
+//		ServiceProvider<Logger>::create<StreamLogger>(std::cout);
+
+		if (argc != 3)
+		{
+			std::cerr << "usage: <lms_conf_file> <nb_workers>" << std::endl;
+			return EXIT_FAILURE;
+		}
+
+		const std::filesystem::path configFilePath {std::string(argv[1], 0, 256)};
+		const std::size_t nbWorkers = atoi(argv[2]);
+
+		ServiceProvider<Config>::create(configFilePath);
+
+		Database::Db db {ServiceProvider<Config>::get()->getPath("working-dir") / "lms.db"};
+		Database::SessionPool sessionPool {db, nbWorkers};
+
+		std::cout << "Caching all features..." << std::endl;
+		// Cache all the features of all the music in order to speed up the multiple trainings
+		const auto cachedFeatures { constructFeaturesCache(Database::SessionPool::ScopedSession {sessionPool}.get(), featuresSettings) };
+		std::cout << "Caching all features DONE" << std::endl;
+
+		FeaturesSearcher::setFeaturesFetchFunc(
+				[&](Database::IdType trackId, const FeatureNames& featureNames)
+				{
+					return getFeaturesFromCache(cachedFeatures, trackId, featureNames);
+				});
+
+		// Create some random settings (i.e random population)
+		std::vector<FeatureSettingsMap> initialPopulation;
+
+		constexpr std::size_t populationSize {200};
+		constexpr std::size_t nbFeatures {5};
+
+		for (std::size_t i {}; i < populationSize; ++i)
+		{
+			FeatureSettingsMap settings;
+
+			while (settings.size() < nbFeatures)
+			{
+				const auto itFeatureSetting {Random::pickRandom(featuresSettings)};
+				settings.emplace(itFeatureSetting->first, itFeatureSetting->second);
+			}
+
+			initialPopulation.emplace_back(std::move(settings));
+		}
+
+		FeaturesSearcher::TrainSettings trainSettings;
+		trainSettings.iterationCount = 8;
+		trainSettings.sampleCountPerNeuron = 1.5;
+
+		GeneticAlgorithm<FeatureSettingsMap>::Params params;
+		params.nbWorkers = nbWorkers;
+		params.nbGenerations = 1;
+		params.crossoverRatio = 0.78;
+		params.mutationProbability = 0.2;
+		params.breedFunction = breedFeatureSettingsMap;
+		params.mutateFunction = mutateFeatureSettingsMap;
+		params.scoreFunction =
+			[&](const FeatureSettingsMap& featureSettings)
+			{
+				FeaturesSearcher::TrainSettings settings {trainSettings};
+				settings.featureSettingsMap = featureSettings;
+
+				Database::SessionPool::ScopedSession scopedSession {sessionPool};
+				return computeSimilarityScore(scopedSession.get(), settings);
+			};
+
+		GeneticAlgorithm<FeatureSettingsMap> geneticAlgorithm {params};
+
+		std::cout << "Parameters:\n"
+			<< "\tnb total settings = "<< featuresSettings.size() << "\n"
+			<< "\tnb generations = " << params.nbGenerations << "\n"
+			<< "\tpopulationSize = " << populationSize << "\n"
+			<< "\tnbFeatures = " << nbFeatures << "\n"
+			<< "\tcrossoverRatio = " << params.crossoverRatio << "\n"
+			<< "\tmutationProbability = " << params.mutationProbability << "\n"
+			<< std::endl;
+			
+		std::cout << "Starting simulation..." << std::endl;
+		const FeatureSettingsMap selectedSettings {geneticAlgorithm.simulate(initialPopulation)};
+		std::cout << "Simulation complete! Best result:" << std::endl;
+		printFeatureSettingsMap(selectedSettings);
+
+		// print all badly classified tracks
+		{
+			FeaturesSearcher::TrainSettings settings {trainSettings};
+			settings.featureSettingsMap = selectedSettings;
+
+			Database::SessionPool::ScopedSession scopedSession {sessionPool};
+			printBadlyClassifiedTracks(scopedSession.get(), settings);
+		}
+	}
+	catch (std::exception& e)
+	{
+		std::cerr << "Caught exception: " << e.what() << std::endl;
+	}
+
+	return EXIT_SUCCESS;
+}
+
+

src/tools/similarity-parameters/Makefile.am

@@ -0,0 +1,29 @@
+noinst_PROGRAMS = lms-similarity-parameters
+
+lms_similarity_parameters_SOURCES = \
+	$(srcdir)/LmsSimilarityParameters.cpp			\
+	$(top_srcdir)/src/database/Artist.cpp		\
+	$(top_srcdir)/src/database/Cluster.cpp		\
+	$(top_srcdir)/src/database/Db.cpp		\
+	$(top_srcdir)/src/database/TrackFeatures.cpp	\
+	$(top_srcdir)/src/database/TrackList.cpp	\
+	$(top_srcdir)/src/database/Release.cpp		\
+	$(top_srcdir)/src/database/ScanSettings.cpp	\
+	$(top_srcdir)/src/database/Session.cpp		\
+	$(top_srcdir)/src/database/SessionPool.cpp	\
+	$(top_srcdir)/src/database/SqlQuery.cpp		\
+	$(top_srcdir)/src/database/Track.cpp		\
+	$(top_srcdir)/src/database/User.cpp		\
+	$(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp		\
+	$(top_srcdir)/src/similarity/features/som/Network.cpp			\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp	\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp	\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp	\
+	$(top_srcdir)/src/utils/Config.cpp 		\
+	$(top_srcdir)/src/utils/Logger.cpp 		\
+	$(top_srcdir)/src/utils/Random.cpp 		\
+	$(top_srcdir)/src/utils/StreamLogger.cpp 	\
+	$(top_srcdir)/src/utils/String.cpp
+
+lms_similarity_parameters_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT
+

src/tools/similarity-parameters/ParallelFor.hpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <functional>
+#include <thread>
+#include <boost/asio/io_context.hpp>
+
+template <typename It, typename Func>
+void parallel_foreach(std::size_t nbWorkers, It begin, It end, Func&& func)
+{
+	if (nbWorkers == 0)
+		throw std::runtime_error("Invalid worker count");
+
+	boost::asio::io_context ioContext;
+
+	for (It it {begin}; it != end; ++it)
+	{
+		auto refValue {std::ref<typename It::value_type>(*it)};
+		ioContext.post([refValue, &func]() { std::cout << "EXEC FROM WORKER" << std::endl; func(refValue); std::cout << "END EXEC FROM WORKER" << std::endl; });
+	}
+
+	std::vector<std::thread> threads;
+	for (std::size_t i {}; i < nbWorkers - 1; ++i)
+		threads.emplace_back([&]() { ioContext.run(); });
+
+	ioContext.run();
+
+	for (std::thread& t : threads)
+		t.join();
+}
+

src/tools/similarity/LmsSimilarity.cpp

@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <filesystem>
+#include <iostream>
+#include <stdexcept>
+#include <stdlib.h>
+
+#include "database/Artist.hpp"
+#include "database/Cluster.hpp"
+#include "database/Db.hpp"
+#include "database/Release.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+#include "utils/Config.hpp"
+#include "utils/Service.hpp"
+#include "utils/StreamLogger.hpp"
+#include "similarity/features/SimilarityFeaturesSearcher.hpp"
+
+int main(int argc, char *argv[])
+{
+	try
+	{
+		using namespace Similarity;
+
+		// log to stdout
+		ServiceProvider<Logger>::create<StreamLogger>(std::cout);
+
+		std::filesystem::path configFilePath {"/etc/lms.conf"};
+		if (argc >= 2)
+			configFilePath = std::string(argv[1], 0, 256);
+
+		ServiceProvider<Config>::create(configFilePath);
+
+		Database::Db db {ServiceProvider<Config>::get()->getPath("working-dir") / "lms.db"};
+		Database::Session session {db};
+
+		std::cout << "Classifying tracks..." << std::endl;
+		// may be long...
+		struct FeaturesSearcher::TrainSettings trainSettings;
+		trainSettings.featureSettingsMap = FeaturesSearcher::getDefaultTrainFeatureSettings();
+		FeaturesSearcher searcher {session, trainSettings};
+		std::cout << "Classifying tracks DONE" << std::endl;
+
+		const std::vector<Database::IdType> trackIds = std::invoke([&]()
+					{
+						auto transaction {session.createSharedTransaction()};
+						return Database::Track::getAllIdsWithFeatures(session);
+					});
+
+		std::cout << "*** Tracks (" << trackIds.size() << ") ***" << std::endl;
+		for (Database::IdType trackId : trackIds)
+		{
+			auto trackToString = [&](Database::IdType trackId)
+			{
+				std::string res;
+				auto transaction {session.createSharedTransaction()};
+				Database::Track::pointer track {Database::Track::getById(session, trackId)};
+
+				res += track->getName();
+				if (track->getRelease())
+					res += " [" + track->getRelease()->getName() + "]";
+				for (auto artist : track->getArtists())
+					res += " - " + artist->getName();
+				for (auto cluster : track->getClusters())
+					res += " {" + cluster->getType()->getName() + "-"+ cluster->getName() + "}";
+
+				return res;
+			};
+
+			std::cout << "Processing track '" << trackToString(trackId) << std::endl;
+			for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, 3))
+				std::cout << "\t- Similar track '" << trackToString(similarTrackId) << std::endl;
+		}
+
+		const std::vector<Database::IdType> releaseIds = std::invoke([&]()
+					{
+						auto transaction {session.createSharedTransaction()};
+						return Database::Release::getAllIds(session);
+					});
+
+		std::cout << "*** Releases ***" << std::endl;
+		for (Database::IdType releaseId : releaseIds)
+		{
+			auto releaseToString = [&](Database::IdType releaseId)
+			{
+				auto transaction {session.createSharedTransaction()};
+
+				Database::Release::pointer release {Database::Release::getById(session, releaseId)};
+				return release->getName();
+			};
+
+			std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl;
+			for (Database::IdType similarReleaseId : searcher.getSimilarReleases({releaseId}, 3))
+				std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl;
+		}
+
+		const std::vector<Database::IdType> artistIds = std::invoke([&]()
+					{
+						auto transaction {session.createSharedTransaction()};
+						return Database::Artist::getAllIds(session);
+					});
+
+		std::cout << "*** Artists ***" << std::endl;
+		for (Database::IdType artistId : artistIds)
+		{
+			auto artistToString = [&](Database::IdType artistId)
+			{
+				auto transaction {session.createSharedTransaction()};
+
+				Database::Artist::pointer artist {Database::Artist::getById(session, artistId)};
+				return artist->getName();
+			};
+
+			std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl;
+			for (Database::IdType similarArtistId : searcher.getSimilarArtists({artistId}, 3))
+				std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl;
+		}
+
+	}
+	catch( std::exception& e)
+	{
+		std::cerr << "Caught exception: " << e.what() << std::endl;
+		return EXIT_FAILURE;
+	}
+
+	return EXIT_SUCCESS;
+}
+

src/tools/similarity/Makefile.am

@@ -0,0 +1,27 @@
+noinst_PROGRAMS = lms-similarity
+
+lms_similarity_SOURCES = \
+	$(srcdir)/LmsSimilarity.cpp			\
+	$(top_srcdir)/src/database/Artist.cpp		\
+	$(top_srcdir)/src/database/Cluster.cpp		\
+	$(top_srcdir)/src/database/Db.cpp		\
+	$(top_srcdir)/src/database/TrackFeatures.cpp	\
+	$(top_srcdir)/src/database/TrackList.cpp	\
+	$(top_srcdir)/src/database/Release.cpp		\
+	$(top_srcdir)/src/database/ScanSettings.cpp	\
+	$(top_srcdir)/src/database/Session.cpp		\
+	$(top_srcdir)/src/database/SqlQuery.cpp		\
+	$(top_srcdir)/src/database/Track.cpp		\
+	$(top_srcdir)/src/database/User.cpp		\
+	$(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp		\
+	$(top_srcdir)/src/similarity/features/som/Network.cpp			\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp	\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp	\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp	\
+	$(top_srcdir)/src/utils/Config.cpp 		\
+	$(top_srcdir)/src/utils/Logger.cpp 		\
+	$(top_srcdir)/src/utils/StreamLogger.cpp 	\
+	$(top_srcdir)/src/utils/String.cpp
+
+lms_similarity_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT
+