Added a cache for track features, made the crossover ratio configurable, now using a fitness proportional selection

src/similarity/features/SimilarityFeaturesSearcher.cpp

@@ -36,43 +36,52 @@
 namespace Similarity {
 
 static
-std::optional<SOM::InputVector>
-getInputVectorFromTrack(Database::Session& session, Database::IdType trackId, const std::unordered_set<FeatureName>& featureNames, std::size_t nbDimensions)
+std::optional<FeatureValuesMap>
+getTrackFeatureValues(FeaturesSearcher::FeaturesFetchFunc func, Database::IdType trackId, const std::unordered_set<FeatureName>& featureNames)
 {
-	FeatureValuesMap featureValuesMap;
+	return func(trackId, featureNames);
+}
 
+static
+std::optional<FeatureValuesMap>
+getTrackFeatureValuesFromDb(Database::Session& session, Database::IdType trackId, const std::unordered_set<FeatureName>& featureNames)
+{
+	auto func = [&](Database::IdType trackId, const std::unordered_set<FeatureName>& featureNames)
 	{
+		std::optional<FeatureValuesMap> res;
+
 		auto transaction {session.createSharedTransaction()};
 
 		Database::Track::pointer track {Database::Track::getById(session, trackId)};
 		if (!track)
-			return std::nullopt;
+			return res;
 
-		featureValuesMap = track->getTrackFeatures()->getFeatureValuesMap(featureNames);
-		if (featureValuesMap.empty())
-			return std::nullopt;
-	}
+		res = track->getTrackFeatures()->getFeatureValuesMap(featureNames);
+		if (res->empty())
+			res.reset();
 
+		return res;
+	};
+
+	return getTrackFeatureValues(func, trackId, featureNames);
+}
+
+static
+std::optional<SOM::InputVector>
+convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions)
+{
 	std::size_t i {};
 	std::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
-	for (const auto& featureName : featureNames)
+	for (const auto& [featureName, values] : featureValuesMap)
 	{
-		const auto it {featureValuesMap.find(featureName)};
-		if (it == std::cend(featureValuesMap))
+		if (values.size() != getFeatureDef(featureName).nbDimensions)
 		{
-			LMS_LOG(SIMILARITY, WARNING) << "Cannot find feature '" << featureName << "' for track id'" << trackId << "'";
+			LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size();
 			res.reset();
 			break;
 		}
 
-		if (it->second.size() != getFeatureDef(featureName).nbDimensions)
-		{
-			LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << it->second.size() << ", trackId = " << trackId;
-			res.reset();
-			break;
-		}
-
-		for (double val : it->second)
+		for (double val : values)
 			(*res)[i++] = val;
 	}
 
@@ -134,7 +143,17 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session,
 		if (stopRequested && stopRequested())
 			return;
 
-		std::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(session, trackId, featureNames, nbDimensions)};
+		std::optional<FeatureValuesMap> featureValuesMap;
+
+		if (_featuresFetchFunc)
+			featureValuesMap = getTrackFeatureValues(_featuresFetchFunc, trackId, featureNames);
+		else
+			featureValuesMap = getTrackFeatureValuesFromDb(session, trackId, featureNames);
+
+		if (!featureValuesMap)
+			continue;
+
+		std::optional<SOM::InputVector> inputVector {convertFeatureValuesMapToInputVector(*featureValuesMap, nbDimensions)};
 		if (!inputVector)
 			continue;
 

src/similarity/features/SimilarityFeaturesSearcher.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include <map>
+#include <optional>
 #include <set>
 #include <string>
 
@@ -70,6 +71,11 @@ class FeaturesSearcher
 
 		FeaturesCache toCache() const;
 
+		using FeaturesFetchFunc = std::function<std::optional<std::unordered_map<std::string, std::vector<double>>>(Database::IdType /*trackId*/, const std::unordered_set<std::string>& /*features*/)>;
+		// Default is to retrieve the features from the database (may be slow).
+		// Use this only if you want to train different searchers with the same data
+		static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; }
+
 	private:
 
 		using ObjectPositions = std::map<Database::IdType, std::set<SOM::Position>>;
@@ -96,6 +102,7 @@ class FeaturesSearcher
 		SOM::Matrix<std::set<Database::IdType>> 	_tracksMap;
 		ObjectPositions					_trackPositions;
 
+		static inline FeaturesFetchFunc _featuresFetchFunc;
 };
 
 } // ns Similarity

src/utils/Utils.cpp

@@ -176,10 +176,3 @@ RandGenerator& getRandGenerator()
 	return randGenerator;
 }
 
-int
-getRandom(int min, int max)
-{
-	std::uniform_int_distribution<> dist {min, max};
-	return dist (getRandGenerator());
-}
-

src/utils/Utils.hpp

@@ -113,8 +113,21 @@ constexpr T clamp(T v, T lo, T hi, Compare comp = {})
 using RandGenerator = std::mt19937;
 RandGenerator& getRandGenerator();
 
-int
-getRandom(int min, int max);
+template <typename T>
+T
+getRandom(T min, T max)
+{
+	std::uniform_int_distribution<> dist {min, max};
+	return dist (getRandGenerator());
+}
+
+template <typename T>
+T
+getRealRandom(T min, T max)
+{
+	std::uniform_real_distribution<> dist {min, max};
+	return dist (getRandGenerator());
+}
 
 template <typename Container>
 void

tools/similarity-parameters/GeneticAlgorithm.hpp

@@ -36,6 +36,7 @@ class GeneticAlgorithm
 		{
 			std::size_t		nbWorkers {1};
 			std::size_t		nbGenerations;
+			float			crossoverRatio {0.5};
 			float			mutationProbability {0.05};
 			BreedFunction	breedFunction;
 			MutateFunction		mutateFunction;
@@ -56,6 +57,8 @@ class GeneticAlgorithm
 		};
 
 		void scoreAndSortPopulation(std::vector<ScoredIndividual>& population);
+		Score getTotalScore(const std::vector<ScoredIndividual>& population) const;
+		typename std::vector<ScoredIndividual>::const_iterator pickRandomRouletteWheel(const std::vector<ScoredIndividual>& population);
 
 		Params _params;
 };
@@ -66,10 +69,12 @@ GeneticAlgorithm<Individual>::GeneticAlgorithm(const Params& params)
 {
 }
 
+
 template<typename Individual>
 Individual
 GeneticAlgorithm<Individual>::simulate(const std::vector<Individual>& initialPopulation)
 {
+	const std::size_t childrenCountPerGeneration {static_cast<std::size_t>(initialPopulation.size() * _params.crossoverRatio)};
 	if (initialPopulation.size() < 10)
 		throw std::runtime_error("Initial population must has at least 10 elements");
 
@@ -83,36 +88,42 @@ GeneticAlgorithm<Individual>::simulate(const std::vector<Individual>& initialPop
 
 	for (std::size_t currentGeneration {}; currentGeneration  < _params.nbGenerations; ++currentGeneration)
 	{
+		assert(scoredPopulation.size() == initialPopulation.size());
 		std::cout << "Processing generation " << currentGeneration << "..." << std::endl;
-		// parent selection (elitist selection)
-		scoredPopulation.resize(scoredPopulation.size() / 2);
 
-		// breed the remaining individuals
+		// breed
 		std::vector<ScoredIndividual> children;
-		children.reserve(initialPopulation.size() - scoredPopulation.size());
+		children.reserve(childrenCountPerGeneration);
 
-		while (children.size() + scoredPopulation.size() < initialPopulation.size())
+		while (children.size() < childrenCountPerGeneration)
 		{
-			// Select two random parents
-			const auto itParent1 {pickRandom(scoredPopulation)};
-			const auto itParent2 {pickRandom(scoredPopulation)};
+			// Select two random parents using their score as weight
+			const auto itParent1 {pickRandomRouletteWheel(scoredPopulation)};
+			const auto itParent2 {pickRandomRouletteWheel(scoredPopulation)};
 
 			if (itParent1 == itParent2)
 				continue;
 
+			std::cout << "Parent1 = " << std::distance(std::cbegin(scoredPopulation), itParent1) << std::endl;
+			std::cout << "Parent2 = " << std::distance(std::cbegin(scoredPopulation), itParent2) << std::endl;
+
 			ScoredIndividual child {_params.breedFunction(itParent1->individual, itParent2->individual)};
 			
-			if (getRandom(0, 100) <= _params.mutationProbability * 100)
+			if (getRealRandom(float {}, float {1}) <= _params.mutationProbability)
 				_params.mutateFunction(child.individual);
 
-			children.emplace_back(std::move(child ));
+			children.emplace_back(std::move(child));
 		}
 
+		// Elitist selection
+		scoredPopulation.resize(initialPopulation.size() - childrenCountPerGeneration);
+
 		scoredPopulation.insert(std::end(scoredPopulation), std::make_move_iterator(std::begin(children)), std::make_move_iterator(std::end(children)));
 		assert(scoredPopulation.size() == initialPopulation.size());
 
 		scoreAndSortPopulation(scoredPopulation);
 
+		std::cout << "Mean score = " << getTotalScore(scoredPopulation) / scoredPopulation.size() << std::endl;
 		std::cout << "Current best score = " << *scoredPopulation.front().score << std::endl;
 	}
 
@@ -135,3 +146,31 @@ GeneticAlgorithm<Individual>::scoreAndSortPopulation(std::vector<ScoredIndividua
 	std::sort(std::begin(scoredPopulation), std::end(scoredPopulation), [](const ScoredIndividual& a, const ScoredIndividual& b) { return a.score > b.score; });
 }
 
+template<typename Individual>
+typename GeneticAlgorithm<Individual>::Score
+GeneticAlgorithm<Individual>::getTotalScore(const std::vector<ScoredIndividual>& scoredPopulation) const
+{
+	return std::accumulate(std::cbegin(scoredPopulation), std::cend(scoredPopulation), Score {}, [](Score score, const ScoredIndividual& individual) { return score + *individual.score; });
+}
+
+template<typename Individual>
+typename std::vector<typename GeneticAlgorithm<Individual>::ScoredIndividual>::const_iterator
+GeneticAlgorithm<Individual>::pickRandomRouletteWheel(const std::vector<ScoredIndividual>& population)
+{
+	const Score randomScore {getRealRandom(Score {}, getTotalScore(population))};
+
+	std::cout << "Random = " << randomScore << ", total = " << getTotalScore(population) << std::endl;
+
+	Score curScore{};
+	for (auto itScoredIndividual {std::cbegin(population)}; itScoredIndividual != std::cend(population); ++itScoredIndividual )
+	{
+		if (curScore + *itScoredIndividual->score > randomScore)
+			return itScoredIndividual;
+
+		curScore += *itScoredIndividual->score;
+	}
+
+	throw std::runtime_error("bad random or empty population");
+}
+
+

tools/similarity-parameters/LmsSimilarityParameters.cpp

@@ -27,6 +27,7 @@
 #include "database/Release.hpp"
 #include "database/SessionPool.hpp"
 #include "database/Track.hpp"
+#include "database/TrackFeatures.hpp"
 #include "similarity/features/SimilarityFeaturesSearcher.hpp"
 #include "utils/Config.hpp"
 #include "utils/Service.hpp"
@@ -136,6 +137,56 @@ const FeatureSettingsMap featuresSettings
 	{ "lowlevel.zerocrossingrate.var",		{1}},
 };
 
+static
+std::unordered_map<Database::IdType, FeatureValuesMap>
+constructFeaturesCache(Database::Session& session, const FeatureSettingsMap& featureSettings)
+{
+	std::unordered_map<Database::IdType, FeatureValuesMap> cache;
+
+	std::unordered_set<FeatureName> names;
+	std::transform(std::cbegin(featureSettings), std::cend(featureSettings), std::inserter(names, std::begin(names)),
+			[](const auto& itFeature) { return itFeature.first; });
+
+	auto transaction {session.createSharedTransaction()};
+
+	for (auto trackId : Database::Track::getAllIdsWithFeatures(session))
+	{
+		const Database::Track::pointer track {Database::Track::getById(session, trackId)};
+		const Database::TrackFeatures::pointer trackFeatures {track->getTrackFeatures()};
+
+		cache[trackId] = trackFeatures->getFeatureValuesMap(names);
+	}
+
+	return cache;
+}
+
+static
+std::optional<FeatureValuesMap>
+getFeaturesFromCache(const std::unordered_map<Database::IdType, FeatureValuesMap>& cache, Database::IdType trackId, const FeatureNames& names)
+{
+	std::optional<FeatureValuesMap> res;
+
+	auto it {cache.find(trackId)};
+	if (it == std::cend(cache))
+		return res;
+
+	res = FeatureValuesMap{};
+
+	const FeatureValuesMap& trackFeatures {it->second};
+	for (const FeatureName& name : names)
+	{
+		auto itFeatures {trackFeatures.find(name)};
+		if (itFeatures == std::cend(trackFeatures))
+		{
+			res.reset();
+			break;
+		}
+
+		res->emplace(name, itFeatures ->second);
+	}
+
+	return res;
+}
 
 static
 void
@@ -228,12 +279,12 @@ computeSimilarityScore(Database::Session& session, FeaturesSearcher::TrainSettin
 	for (Database::IdType trackId : trackIds)
 	{
 		constexpr std::size_t nbSimilarTracks {3};
-		std::cout << "Processing track '" << trackToString(session, trackId) << "'" << std::endl;
+//		std::cout << "Processing track '" << trackToString(session, trackId) << "'" << std::endl;
 		SimilarityScore factor {1};		
 		for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, nbSimilarTracks))
 		{
 			SimilarityScore trackScore {computeTrackScore(session, trackId, similarTrackId)};
-			std::cout << "\tScore = " << trackScore << " (*" << factor << ") with track '" << trackToString(session, similarTrackId) << "'" << std::endl;
+//			std::cout << "\tScore = " << trackScore << " (*" << factor << ") with track '" << trackToString(session, similarTrackId) << "'" << std::endl;
 			trackScore *= factor;
 			score += trackScore;
 
@@ -286,7 +337,7 @@ int main(int argc, char *argv[])
 	{
 
 		// log to stdout
-		ServiceProvider<Logger>::create<StreamLogger>(std::cout);
+//		ServiceProvider<Logger>::create<StreamLogger>(std::cout);
 
 		if (argc != 3)
 		{
@@ -302,10 +353,21 @@ int main(int argc, char *argv[])
 		Database::Db db {ServiceProvider<Config>::get()->getPath("working-dir") / "lms.db"};
 		Database::SessionPool sessionPool {db, nbWorkers};
 
+		std::cout << "Caching all features..." << std::endl;
+		// Cache all the features of all the music in order to speed up the multiple trainings
+		const auto cachedFeatures { constructFeaturesCache(Database::SessionPool::ScopedSession {sessionPool}.get(), featuresSettings) };
+		std::cout << "Caching all features DONE" << std::endl;
+
+		FeaturesSearcher::setFeaturesFetchFunc(
+				[&](Database::IdType trackId, const FeatureNames& featureNames)
+				{
+					return getFeaturesFromCache(cachedFeatures, trackId, featureNames);
+				});
+
 		// Create some random settings (i.e random population)
 		std::vector<FeatureSettingsMap> initialPopulation;
 
-		constexpr std::size_t populationSize {100};
+		constexpr std::size_t populationSize {10};
 		constexpr std::size_t nbFeatures {5};
 
 		for (std::size_t i {}; i < populationSize; ++i)
@@ -324,7 +386,8 @@ int main(int argc, char *argv[])
 
 		GeneticAlgorithm<FeatureSettingsMap>::Params params;
 		params.nbWorkers = nbWorkers;
-		params.nbGenerations = 300;
+		params.nbGenerations = 5;
+		params.crossoverRatio = 0.78;
 		params.mutationProbability = 0.2;
 		params.breedFunction = breedFeatureSettingsMap;
 		params.mutateFunction = mutateFeatureSettingsMap;
@@ -332,7 +395,7 @@ int main(int argc, char *argv[])
 			[&](const FeatureSettingsMap& settings)
 			{
 				FeaturesSearcher::TrainSettings trainSettings;
-				trainSettings.iterationCount = 10;
+				trainSettings.iterationCount = 8;
 				trainSettings.sampleCountPerNeuron = 1.5;
 				trainSettings.featureSettingsMap = settings;
 
@@ -346,6 +409,7 @@ int main(int argc, char *argv[])
 			<< "\tnb generations = " << params.nbGenerations << "\n"
 			<< "\tpopulationSize = " << populationSize << "\n"
 			<< "\tnbFeatures = " << nbFeatures << "\n"
+			<< "\tcrossoverRatio = " << params.crossoverRatio << "\n"
 			<< "\tmutationProbability = " << params.mutationProbability << "\n"
 			<< std::endl;