WIP, reworking features

configure.ac

@@ -80,6 +80,7 @@ AC_CONFIG_FILES([Makefile
 		 test/Makefile
 		 tools/Makefile
 		 tools/similarity/Makefile
+		 tools/similarity-parameters/Makefile
 		 tools/metadata/Makefile])
 
 AC_ARG_ENABLE([tools],

src/Makefile.am

@@ -69,6 +69,8 @@ lms_SOURCES = \
 	$(srcdir)/similarity/features/AcousticBrainzUtils.hpp	\
 	$(srcdir)/similarity/features/SimilarityFeaturesCache.cpp	\
 	$(srcdir)/similarity/features/SimilarityFeaturesCache.hpp	\
+	$(srcdir)/similarity/features/SimilarityFeaturesDefs.cpp	\
+	$(srcdir)/similarity/features/SimilarityFeaturesDefs.hpp	\
 	$(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.cpp	\
 	$(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.hpp	\
 	$(srcdir)/similarity/features/SimilarityFeaturesSearcher.cpp	\

src/database/SimilaritySettings.cpp

@@ -27,38 +27,30 @@
 
 namespace Database {
 
-struct TrackFeatureInfo
+static const std::map<std::string, double> defaultFeatures =
 {
-	std::string	name;
+	{ "lowlevel.spectral_contrast_coeffs.median",	1. },
-	std::size_t	nbDimensions;
+	{ "lowlevel.erbbands.median",			1. },
-	double		weight;
+	{ "tonal.hpcp.median",				1. },
+	{ "lowlevel.melbands.median",			1. },
+	{ "lowlevel.barkbands.median",			1. },
+	{ "lowlevel.mfcc.mean",				1. },
+	{ "lowlevel.gfcc.mean",				1. },
 };
 
-static const std::vector<TrackFeatureInfo> defaultFeatures =
+SimilaritySettingsFeature::SimilaritySettingsFeature(Wt::Dbo::ptr<SimilaritySettings> settings, const std::string& name, double weight)
-{
+: _name {name},
-	{ "lowlevel.spectral_contrast_coeffs.median",	6,	1. },
+_weight {weight},
-	{ "lowlevel.erbbands.median",			40,	1. },
+_settings {settings}
-	{ "tonal.hpcp.median",				36,	1. },
-	{ "lowlevel.melbands.median",			40,	1. },
-	{ "lowlevel.barkbands.median",			27,	1. },
-	{ "lowlevel.mfcc.mean",				13,	1. },
-	{ "lowlevel.gfcc.mean",				13,	1. },
-};
-
-SimilaritySettingsFeature::SimilaritySettingsFeature(Wt::Dbo::ptr<SimilaritySettings> settings, const std::string& name, std::size_t nbDimensions, double weight)
-: _name(name),
-_nbDimensions(nbDimensions),
-_weight(weight),
-_settings(settings)
 {
 }
 
 SimilaritySettingsFeature::pointer
-SimilaritySettingsFeature::create(Session& session, Wt::Dbo::ptr<SimilaritySettings> settings, const std::string& name, std::size_t nbDimensions, double weight)
+SimilaritySettingsFeature::create(Session& session, Wt::Dbo::ptr<SimilaritySettings> settings, const std::string& name, double weight)
 {
 	session.checkUniqueLocked();
 
-	SimilaritySettingsFeature::pointer res {session.getDboSession().add(std::make_unique<SimilaritySettingsFeature>(settings, name, nbDimensions, weight))};
+	SimilaritySettingsFeature::pointer res {session.getDboSession().add(std::make_unique<SimilaritySettingsFeature>(settings, name, weight))};
 	session.getDboSession().flush();
 
 	return res;
@@ -74,8 +66,8 @@ SimilaritySettings::init(Session& session)
 		return;
 
 	settings = session.getDboSession().add(std::make_unique<SimilaritySettings>());
-	for (const auto& feature : defaultFeatures)
+	for (const auto& [feature, weight] : defaultFeatures)
-		SimilaritySettingsFeature::create(session, settings, feature.name, feature.nbDimensions, feature.weight);
+		SimilaritySettingsFeature::create(session, settings, feature, weight);
 }
 
 

src/database/SimilaritySettings.hpp

@@ -32,19 +32,17 @@ class SimilaritySettingsFeature :  public Wt::Dbo::Dbo<SimilaritySettingsFeature
 		using pointer = Wt::Dbo::ptr<SimilaritySettingsFeature>;
 
 		SimilaritySettingsFeature() = default;
-		SimilaritySettingsFeature(Wt::Dbo::ptr<SimilaritySettings> settings, const std::string& name, std::size_t nbDimensions, double weight);
+		SimilaritySettingsFeature(Wt::Dbo::ptr<SimilaritySettings> settings, const std::string& name, double weight);
 
-		static pointer create(Session& session, Wt::Dbo::ptr<SimilaritySettings> settings, const std::string& name, std::size_t nbDimensions, double weight = 1);
+		static pointer create(Session& session, Wt::Dbo::ptr<SimilaritySettings> settings, const std::string& name, double weight = 1);
 
 		const std::string&	getName() const { return _name; } ;
-		std::size_t 		getNbDimensions() const { return static_cast<std::size_t>(_nbDimensions); }
 		double			getWeight() const { return _weight; }
 
 		template<class Action>
 		void persist(Action& a)
 		{
 			Wt::Dbo::field(a, _name,		"name");
-			Wt::Dbo::field(a, _nbDimensions,	"dimension_count");
 			Wt::Dbo::field(a, _weight,		"weight");
 
 			Wt::Dbo::belongsTo(a, _settings, "similarity_settings", Wt::Dbo::OnDeleteCascade);
@@ -52,8 +50,7 @@ class SimilaritySettingsFeature :  public Wt::Dbo::Dbo<SimilaritySettingsFeature
 
 	private:
 		std::string	_name;
-		int 		_nbDimensions;
+		double		_weight {1};
-		double		_weight;
 
 		Wt::Dbo::ptr<SimilaritySettings> _settings;
 };
@@ -75,8 +72,8 @@ class SimilaritySettings : public Wt::Dbo::Dbo<SimilaritySettings>
 		static pointer get(Session& session);
 
 		// Accessors Read
-		std::size_t		getVersion() const { return _settingsVersion; }
+		std::size_t		getVersion() const	{ return _settingsVersion; }
-		EngineType		getEngineType() const { return _engineType; }
+		EngineType		getEngineType() const	{ return _engineType; }
 		std::vector<Wt::Dbo::ptr<SimilaritySettingsFeature>> getFeatures() const;
 
 		// Setters

src/database/TrackFeatures.cpp

@@ -41,53 +41,47 @@ TrackFeatures::create(Session& session, Wt::Dbo::ptr<Track> track, const std::st
 	return session.getDboSession().add(std::make_unique<TrackFeatures>(track, jsonEncodedFeatures));
 }
 
-std::vector<double>
+FeatureValues
-TrackFeatures::getFeatures(const std::string& featureNode) const
+TrackFeatures::getFeatureValues(const FeatureName& featureNode) const
 {
-	std::vector<double> res;
+	FeatureValuesMap featuresValuesMap {getFeatureValuesMap({featureNode})};
-
+	return std::move(featuresValuesMap[featureNode]);
-	std::map<std::string, std::vector<double>> features = { {featureNode, {}} };
-	if (!getFeatures( features ))
-		return res;
-
-	res = std::move(features[featureNode]);
-
-	return res;
 }
 
-bool
+FeatureValuesMap
-TrackFeatures::getFeatures(std::map<std::string /*name*/, std::vector<double> /*values*/>& features) const
+TrackFeatures::getFeatureValuesMap(const std::unordered_set<FeatureName>& featureNames) const
 {
 	try
 	{
+		std::istringstream iss {_data};
 		boost::property_tree::ptree root;
 
-		std::istringstream iss(_data);
 		boost::property_tree::read_json(iss, root);
 
-		for (auto& featureNode : features)
+		FeatureValuesMap res;
+		for (const FeatureName& featureName : featureNames)
 		{
-			auto node = root.get_child(featureNode.first);
+			FeatureValues& featureValues {res[featureName]};
+
+			auto node {root.get_child(featureName)};
 
 			bool hasChildren = false;
 			for (const auto& child : node.get_child(""))
 			{
 				hasChildren = true;
-				featureNode.second.push_back(child.second.get_value<double>());
+				featureValues.push_back(child.second.get_value<double>());
 			}
 
 			if (!hasChildren)
-			{
+				featureValues.push_back(node.get_value<double>());
-				featureNode.second.push_back(node.get_value<double>());
-			}
 		}
 
-		return true;
+		return res;
 	}
 	catch (boost::property_tree::ptree_error& error)
 	{
 		LMS_LOG(SIMILARITY, ERROR) << "Track " << _track.id() << ": ptree exception: " << error.what();
-		return false;
+		return {};
 	}
 }
 

src/database/TrackFeatures.hpp

@@ -20,6 +20,9 @@
 #pragma once
 
 #include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
 
 #include <Wt/Dbo/Dbo.h>
 
@@ -30,6 +33,10 @@ namespace Database {
 class Session;
 class Track;
 
+using FeatureName = std::string;
+using FeatureValues  = std::vector<double>;
+using FeatureValuesMap = std::unordered_map<FeatureName, FeatureValues>;
+
 class TrackFeatures : public Wt::Dbo::Dbo<TrackFeatures>
 {
 	public:
@@ -42,8 +49,8 @@ class TrackFeatures : public Wt::Dbo::Dbo<TrackFeatures>
 		// Create utility
 		static pointer create(Session& session, Wt::Dbo::ptr<Track> track, const std::string& jsonEncodedFeatures);
 
-		std::vector<double> getFeatures(const std::string& featureNode) const;
+		FeatureValues		getFeatureValues(const FeatureName& feature) const;
-		bool getFeatures(std::map<std::string /*featureNode*/, std::vector<double> /*values*/>& featureNodes) const;
+		FeatureValuesMap	getFeatureValuesMap(const std::unordered_set<FeatureName>& featureNames) const;
 
 		template<class Action>
 		void persist(Action& a)

src/similarity/features/SimilarityFeaturesDefs.cpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SimilarityFeaturesDefs.hpp"
+
+#include <unordered_map>
+#include "utils/Exception.hpp"
+
+namespace Similarity {
+
+static const std::unordered_map<FeatureName, FeatureDef> featureDefinitions
+{
+	{ "lowlevel.spectral_contrast_coeffs.median",	{6}},
+	{ "lowlevel.erbbands.median",			{40}},
+	{ "tonal.hpcp.median",				{36}},
+	{ "lowlevel.melbands.median",			{40}},
+	{ "lowlevel.barkbands.median",			{27}},
+	{ "lowlevel.mfcc.mean",				{13}},
+	{ "lowlevel.gfcc.mean",				{13}},
+};
+
+FeatureDef
+getFeatureDef(const FeatureName& featureName)
+{
+	auto it {featureDefinitions.find(featureName)};
+	if (it == std::cend(featureDefinitions))
+		throw LmsException {"Unhandled requested feature '" + featureName + "'"};
+
+	return it->second;
+}
+
+} // namespace Similarity
+

src/similarity/features/SimilarityFeaturesDefs.hpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace Similarity {
+
+using FeatureName = std::string;
+using FeatureValue = double;
+using FeatureValues = std::vector<FeatureValue>;
+using FeatureValuesMap = std::unordered_map<FeatureName, FeatureValues>;
+
+struct FeatureDef
+{
+	std::size_t nbDimensions {};
+};
+
+FeatureDef getFeatureDef(const FeatureName& featureName);
+
+struct FeatureSettings
+{
+	double weight {};
+};
+using FeatureSettingsMap = std::unordered_map<FeatureName, FeatureSettings>;
+
+} // namespace Similarity

src/similarity/features/SimilarityFeaturesScannerAddon.cpp

@@ -29,7 +29,27 @@
 
 namespace Similarity {
 
-namespace {
+static
+FeatureSettingsMap
+getFeatureSettings(Database::Session& session)
+{
+	FeatureSettingsMap res;
+
+	auto transaction {session.createSharedTransaction()};
+
+	for (const auto& feature : Database::SimilaritySettings::get(session)->getFeatures())
+		res[feature->getName()] = {feature->getWeight()};
+
+	return res;
+}
+
+static
+bool
+hasAtLeastOneTrackWithFeatures(Database::Session& session)
+{
+	auto transaction {session.createSharedTransaction()};
+	return !Database::Track::getAllIdsWithFeatures(session).empty();
+}
 
 struct TrackInfo
 {
@@ -37,6 +57,7 @@ struct TrackInfo
 	std::string mbid;
 };
 
+static
 std::vector<TrackInfo>
 getTracksWithMBIDAndMissingFeatures(Database::Session& dbSession)
 {
@@ -51,8 +72,6 @@ getTracksWithMBIDAndMissingFeatures(Database::Session& dbSession)
 	return res;
 }
 
-} // namespace
-
 FeaturesScannerAddon::FeaturesScannerAddon(std::unique_ptr<Database::Session> dbSession)
 : _dbSession {std::move(dbSession)}
 {
@@ -125,20 +144,16 @@ FeaturesScannerAddon::updateSearcher()
 {
 	LMS_LOG(SIMILARITY, INFO) << "Updating searcher...";
 
-	std::vector<Database::IdType> trackIds;
+	if (hasAtLeastOneTrackWithFeatures(*_dbSession))
-	{
-		auto transaction {_dbSession->createSharedTransaction()};
-		trackIds = Database::Track::getAllIdsWithFeatures(*_dbSession);
-	}
-
-	if (trackIds.empty())
 	{
 		LMS_LOG(DBUPDATER, INFO) << "No track suitable for features similarity clustering";
 		std::atomic_store(&_searcher, std::shared_ptr<FeaturesSearcher>{});
 		return;
 	}
 
-	auto searcher {std::make_shared<Similarity::FeaturesSearcher>(*_dbSession, [&]() { return _stopRequested; })};
+	const auto features {getFeatureSettings(*_dbSession)};
+
+	auto searcher {std::make_shared<Similarity::FeaturesSearcher>(*_dbSession, features, [&]() { return _stopRequested; })};
 	if (searcher->isValid())
 	{
 		std::atomic_store(&_searcher, searcher);

src/similarity/features/SimilarityFeaturesSearcher.cpp

@@ -20,6 +20,7 @@
 #include "SimilarityFeaturesSearcher.hpp"
 
 #include <random>
+#include <unordered_map>
 
 #include "database/Artist.hpp"
 #include "database/SimilaritySettings.hpp"
@@ -34,71 +35,44 @@
 
 namespace Similarity {
 
-struct FeatureInfo
-{
-	std::size_t	nbDimensions;
-	double		weight;
-};
-
-using FeatureInfoMap = std::map<std::string, FeatureInfo>;
-
-static
-FeatureInfoMap
-getFeatureInfoMap(Database::Session& session)
-{
-	auto transaction {session.createSharedTransaction()};
-
-	auto settings {Database::SimilaritySettings::get(session)};
-
-	std::map<std::string, FeatureInfo> featuresInfo;
-	for (auto feature : settings->getFeatures())
-	{
-		LMS_LOG(SIMILARITY, DEBUG) << "Feature '" << feature->getName() << "', nbDimns = " << feature->getNbDimensions() << ", weight = " << feature->getWeight() ;
-		featuresInfo[feature->getName()] = { feature->getNbDimensions(), feature->getWeight() };
-	}
-
-	return featuresInfo;
-}
-
-static
-std::size_t
-getFeatureInfoMapNbDimensions(const FeatureInfoMap& featureInfoMap)
-{
-	return std::accumulate(featureInfoMap.begin(), featureInfoMap.end(), 0, [](std::size_t sum, auto it) { return sum + it.second.nbDimensions; });
-}
-
 static
 std::optional<SOM::InputVector>
-getInputVectorFromTrack(Database::Session& session, Database::IdType trackId, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
+getInputVectorFromTrack(Database::Session& session, Database::IdType trackId, const std::unordered_set<FeatureName>& featureNames, std::size_t nbDimensions)
 {
-	std::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
+	FeatureValuesMap featureValuesMap;
 
-	std::map<std::string, std::vector<double>> features;
+	{
-	for (auto itFeatureInfo : featuresInfo)
+		auto transaction {session.createSharedTransaction()};
-		features[itFeatureInfo.first] = {};
 
-	auto transaction {session.createSharedTransaction()};
+		Database::Track::pointer track {Database::Track::getById(session, trackId)};
+		if (!track)
+			return std::nullopt;
 
-	Database::Track::pointer track {Database::Track::getById(session, trackId)};
+		featureValuesMap = track->getTrackFeatures()->getFeatureValuesMap(featureNames);
-	if (!track)
+		if (featureValuesMap.empty())
-		return res;
+			return std::nullopt;
-
+	}
-	if (!track->getTrackFeatures()->getFeatures(features))
-		return res;
 
 	std::size_t i {};
-	for (const auto& feature : features)
+	std::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
+	for (const auto& featureName : featureNames)
 	{
-		// Check dimensions for each feature
+		const auto it {featureValuesMap.find(featureName)};
-		auto it {featuresInfo.find(feature.first)};
+		if (it == std::cend(featureValuesMap))
-		if (it == featuresInfo.end() || it->second.nbDimensions != feature.second.size())
 		{
-			LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << feature.first << "'. Expected " << it->second.nbDimensions << ", got " << feature.second.size();
+			LMS_LOG(SIMILARITY, WARNING) << "Cannot find feature '" << featureName << "' for track id'" << trackId << "'";
 			res.reset();
 			break;
 		}
 
-		for (double val : feature.second)
+		if (it->second.size() != getFeatureDef(featureName).nbDimensions)
+		{
+			LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << it->second.size() << ", trackId = " << trackId;
+			res.reset();
+			break;
+		}
+
+		for (double val : it->second)
 			(*res)[i++] = val;
 	}
 
@@ -107,25 +81,35 @@ getInputVectorFromTrack(Database::Session& session, Database::IdType trackId, co
 
 static
 SOM::InputVector
-getInputVectorWeights(const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
+getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions)
 {
 	SOM::InputVector weights {nbDimensions};
 	std::size_t index {};
-	for (const auto& featureInfo : featuresInfo)
+	for (const auto& [featureName, featureSettings] : featureSettingsMap)
 	{
-		for (std::size_t i {}; i < featureInfo.second.nbDimensions; ++i)
+		const std::size_t featureNbDimensions {getFeatureDef(featureName).nbDimensions};
-			weights[index++] = (1. / featureInfo.second.nbDimensions * featureInfo.second.weight);
+
+		for (std::size_t i {}; i < featureNbDimensions; ++i)
+			weights[index++] = (1. / featureNbDimensions * featureSettings.weight);
 	}
 
+	assert(index == nbDimensions);
+
 	return weights;
 }
 
-FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function<bool()> stopRequested)
+FeaturesSearcher::FeaturesSearcher(Database::Session& session,
+		const FeatureSettingsMap& featureSettingsMap,
+		StopRequestedFunction stopRequested)
 {
 	LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher...";
 
-	const FeatureInfoMap featuresInfo {getFeatureInfoMap(session)};
+	std::unordered_set<FeatureName> featureNames;
-	const std::size_t nbDimensions {getFeatureInfoMapNbDimensions(featuresInfo)};
+	std::transform(std::cbegin(featureSettingsMap), std::cend(featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)),
+		[](const auto& itFeatureSetting) { return itFeatureSetting.first; });
+
+	const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0},
+			[](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; })};
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Features dimension = " << nbDimensions;
 
@@ -147,10 +131,10 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function<boo
 	LMS_LOG(SIMILARITY, DEBUG) << "Extracting features...";
 	for (Database::IdType trackId : trackIds)
 	{
-		if (stopRequested())
+		if (stopRequested && stopRequested())
 			return;
 
-		std::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(session, trackId, featuresInfo, nbDimensions)};
+		std::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(session, trackId, featureNames, nbDimensions)};
 		if (!inputVector)
 			continue;
 
@@ -177,7 +161,7 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function<boo
 
 	SOM::Network network {size, size, nbDimensions};
 
-	SOM::InputVector weights {getInputVectorWeights(featuresInfo, nbDimensions)};
+	SOM::InputVector weights {getInputVectorWeights(featureSettingsMap, nbDimensions)};
 	network.setDataWeights(weights);
 
 	auto progressIndicator{[](const auto& iter)
@@ -189,14 +173,14 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function<boo
 	network.train(samples, 10, progressIndicator, stopRequested);
 	LMS_LOG(SIMILARITY, DEBUG) << "Training network DONE";
 
-	if (stopRequested())
+	if (stopRequested && stopRequested())
 		return;
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks...";
 	std::map<Database::IdType, std::set<SOM::Position>> trackPositions;
 	for (std::size_t i {}; i < samples.size(); ++i)
 	{
-		if (stopRequested())
+		if (stopRequested && stopRequested())
 			return;
 
 		const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
@@ -211,7 +195,7 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function<boo
 	LMS_LOG(SIMILARITY, INFO) << "Successfully constructed features searcher";
 }
 
-FeaturesSearcher::FeaturesSearcher(Database::Session& session, FeaturesCache cache, std::function<bool()> stopRequested)
+FeaturesSearcher::FeaturesSearcher(Database::Session& session, FeaturesCache cache, StopRequestedFunction stopRequested)
 {
 	LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher from cache...";
 
@@ -341,7 +325,7 @@ FeaturesSearcher::init(Database::Session& session,
 
 	for (auto itTrackCoord : tracksPosition)
 	{
-		if (stopRequested())
+		if (stopRequested && stopRequested())
 			return;
 
 		auto transaction {session.createSharedTransaction()};

src/similarity/features/SimilarityFeaturesSearcher.hpp

@@ -21,11 +21,13 @@
 
 #include <map>
 #include <set>
+#include <string>
 
 #include "database/Types.hpp"
 #include "som/DataNormalizer.hpp"
 #include "som/Network.hpp"
 #include "SimilarityFeaturesCache.hpp"
+#include "SimilarityFeaturesDefs.hpp"
 
 namespace Database
 {
@@ -34,15 +36,19 @@ namespace Database
 
 namespace Similarity {
 
+using FeatureWeight = double;
+
 class FeaturesSearcher
 {
 	public:
 
+		using StopRequestedFunction = std::function<bool()>; // return true if stop requested
+
 		// Use cache
-		FeaturesSearcher(Database::Session& session, FeaturesCache cache, std::function<bool()> stopRequested);
+		FeaturesSearcher(Database::Session& session, FeaturesCache cache, StopRequestedFunction stopRequested);
 
 		// Use training (may be very slow)
-		FeaturesSearcher(Database::Session& session, std::function<bool()> stopRequested);
+		FeaturesSearcher(Database::Session& session, const FeatureSettingsMap& featuresSettingsMap, StopRequestedFunction stopRequested = {});
 
 		bool isValid() const;
 
@@ -65,7 +71,7 @@ class FeaturesSearcher
 		void init(Database::Session& session,
 				SOM::Network network,
 				ObjectPositions tracksPosition,
-				std::function<bool()> stopRequested);
+				StopRequestedFunction stopRequested);
 
 		std::vector<Database::IdType> getSimilarObjects(const std::set<Database::IdType>& ids,
 				const SOM::Matrix<std::set<Database::IdType>>& objectsMap,

src/utils/Logger.cpp

@@ -19,7 +19,7 @@
 
 #include "Logger.hpp"
 
-std::string getModuleName(Module mod)
+const char* getModuleName(Module mod)
 {
 	switch (mod)
 	{
@@ -41,7 +41,7 @@ std::string getModuleName(Module mod)
 	return "";
 }
 
-std::string getSeverityName(Severity sev)
+const char* getSeverityName(Severity sev)
 {
 	switch (sev)
 	{

src/utils/Logger.hpp

@@ -51,8 +51,14 @@ enum class Module
 	UI,
 };
 
-std::string getModuleName(Module mod);
+const char* getModuleName(Module mod);
-std::string getSeverityName(Severity sev);
+const char* getSeverityName(Severity sev);
+
+
+TODO class logger
+TODO class log entry
+
+TODO configure logger to redirect to either Wt's logger or to a ostream
 
 #define LMS_LOG(module, level)	Wt::log(getSeverityName(Severity::level)) << Wt::WLogger::sep << "[" << getModuleName(Module::module) << "]" <<  Wt::WLogger::sep
 

tools/Makefile.am

@@ -1,4 +1,4 @@
 if BUILD_TOOLS
-SUBDIRS = similarity metadata
+SUBDIRS = similarity similarity-parameters metadata
 endif
 

tools/similarity-parameters/LmsSimilarityParameters.cpp

@@ -0,0 +1,49 @@
+
+#include <iostream>
+#include <filesystem>
+#include <string>
+
+#include "database/Db.hpp"
+#include "utils/Config.hpp"
+#include "utils/Service.hpp"
+
+
+int main(int argc, char *argv[])
+{
+	try
+	{
+		std::filesystem::path configFilePath {"/etc/lms.conf"};
+		if (argc >= 2)
+			configFilePath = std::string(argv[1], 0, 256);
+
+		ServiceProvider<Config>::create(configFilePath);
+
+		Database::Db db {getService<Config>()->getPath("working-dir") / "lms.db"};
+		auto session {db.createSession()};
+
+/*		const FeatureSettings
+		{
+			{ "lowlevel.average_loudness",			1 },
+			{ "lowlevel.dynamic_complexity",		1 },
+			{ "lowlevel.spectral_contrast_coeffs.median",	6 },
+			{ "lowlevel.erbbands.median",			40 },
+			{ "tonal.hpcp.median",				36 },
+			{ "lowlevel.melbands.median",			40 },
+			{ "lowlevel.barkbands.median",			27 },
+			{ "lowlevel.mfcc.mean",				13 },
+			{ "lowlevel.gfcc.mean",				13 },
+		};
+
+		const TrackFeaturesMap trackFeaturesMap {getAllTrackFeatures(*session)};
+
+		std::cout << "Found " << trackFeaturesMap.size() << " tracks with features!" << std::endl;*/
+	}
+	catch (std::exception& e)
+	{
+		std::cerr << "Caught exception: " << e.what() << std::endl;
+	}
+
+	return EXIT_SUCCESS;
+}
+
+

tools/similarity-parameters/Makefile.am

@@ -0,0 +1,24 @@
+noinst_PROGRAMS = lms-similarity-parameters
+
+lms_similarity_parameters_SOURCES = \
+	$(srcdir)/LmsSimilarityParameters.cpp			\
+	$(top_srcdir)/src/database/Artist.cpp		\
+	$(top_srcdir)/src/database/Cluster.cpp		\
+	$(top_srcdir)/src/database/Db.cpp		\
+	$(top_srcdir)/src/database/TrackFeatures.cpp	\
+	$(top_srcdir)/src/database/TrackList.cpp	\
+	$(top_srcdir)/src/database/Release.cpp		\
+	$(top_srcdir)/src/database/ScanSettings.cpp	\
+	$(top_srcdir)/src/database/Session.cpp		\
+	$(top_srcdir)/src/database/SimilaritySettings.cpp	\
+	$(top_srcdir)/src/database/SqlQuery.cpp		\
+	$(top_srcdir)/src/database/Track.cpp		\
+	$(top_srcdir)/src/database/User.cpp		\
+	$(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp		\
+	$(top_srcdir)/src/similarity/features/som/Network.cpp			\
+	$(top_srcdir)/src/utils/Config.cpp 		\
+	$(top_srcdir)/src/utils/Logger.cpp 		\
+	$(top_srcdir)/src/utils/Utils.cpp
+
+lms_similarity_parameters_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT
+

tools/similarity/LmsSimilarity.cpp

@@ -1,88 +1,32 @@
-#include <chrono>
 #include <filesystem>
 #include <iostream>
 #include <stdexcept>
 #include <stdlib.h>
-#include <string>
 
 #include "database/Db.hpp"
 #include "database/Session.hpp"
-#include "database/Track.hpp"
-#include "database/Artist.hpp"
-#include "database/Cluster.hpp"
-#include "database/Release.hpp"
-#include "database/TrackFeatures.hpp"
 #include "utils/Config.hpp"
 #include "utils/Service.hpp"
-#include "similarity/features/som/DataNormalizer.hpp"
+#include "similarity/features/SimilarityFeaturesSearcher.hpp"
-#include "similarity/features/som/Network.hpp"
-
-static
-std::ostream& operator<<(std::ostream& os, const Database::Track::pointer& track)
-{
-	os << "[";
-	for (auto artist : track->getArtists())
-		os << artist->getName() << " - ";
-	if (track->getRelease())
-		os << track->getRelease()->getName() << " - ";
-	os << track->getName() << "]";
-
-	return os;
-}
-
-static
-bool
-getTrackFeatures(Database::Session&, const Database::Track::pointer& track, const std::map<std::string, std::size_t>& featuresSettings, SOM::InputVector& res)
-{
-	std::map<std::string, std::vector<double>> features;
-	for (const auto& featureSettings : featuresSettings)
-		features[featureSettings.first] = {};
-
-	if (!track->getTrackFeatures()->getFeatures(features))
-	{
-		std::cout << "Skipping track '" << track->getMBID() << "': missing item" << std::endl;
-		return false;
-	};
-
-	std::size_t index {};
-	for (const auto& feature : features)
-	{
-		auto it = featuresSettings.find(feature.first);
-		if (it == featuresSettings.end() || (feature.second.size() != it->second))
-			return false;
-
-		for (double value : feature.second)
-			res[index++] = value;
-	}
-
-	return true;
-}
-
 
 int main(int argc, char *argv[])
 {
 	try
 	{
-		const std::size_t width = 5;
+		using namespace Similarity;
-		const std::size_t height = 5;
-		const std::size_t nbIterations = 10;
-		std::size_t nbTracks = 5000;
 
-		const std::map<std::string, std::size_t> featuresSettings =
+		const FeatureSettingsMap featuresSettings
 		{
 //			{ "lowlevel.average_loudness",			1 },
 //			{ "lowlevel.dynamic_complexity",		1 },
-			{ "lowlevel.spectral_contrast_coeffs.median",	6 },
+			{ "lowlevel.spectral_contrast_coeffs.median",	{1} },
-			{ "lowlevel.erbbands.median",			40 },
+			{ "lowlevel.erbbands.median",			{1} },
-			{ "tonal.hpcp.median",				36 },
+			{ "tonal.hpcp.median",				{1} },
-			{ "lowlevel.melbands.median",			40 },
+			{ "lowlevel.melbands.median",			{1} },
-			{ "lowlevel.barkbands.median",			27 },
+			{ "lowlevel.barkbands.median",			{1} },
-			{ "lowlevel.mfcc.mean",				13 },
+			{ "lowlevel.mfcc.mean",				{1} },
-			{ "lowlevel.gfcc.mean",				13 },
+			{ "lowlevel.gfcc.mean",				{1} },
 		};
-		std::size_t nbDims = 0;
-		for (const auto& featureSettings : featuresSettings)
-			nbDims += featureSettings.second;
 
 		std::filesystem::path configFilePath {"/etc/lms.conf"};
 		if (argc >= 2)
@@ -94,147 +38,17 @@ int main(int argc, char *argv[])
 		auto session {db.createSession()};
 
 		std::cout << "Getting all features..." << std::endl;
-		auto transaction {session->createUniqueTransaction()};
-
-		std::vector<Database::IdType> trackIds {Database::Track::getAllIdsWithFeatures(*session, nbTracks)};
-
-		nbTracks = trackIds.size();
-		std::cout << "Getting features DONE (" << nbTracks << " tracks)" << std::endl;
-
-		std::cout << "Reading features..." << std::endl;
-		std::vector<SOM::InputVector> tracksFeatures;
-
-		for (Database::IdType trackId : trackIds)
-		{
-			Database::Track::pointer track {Database::Track::getById(*session, trackId)};
-			if (!track)
-				continue;
-
-			SOM::InputVector features {nbDims};
-			if (!getTrackFeatures(*session, track, featuresSettings, features))
-				continue;
-
-			tracksFeatures.emplace_back(std::move(features));
-		}
-		std::cout << "Reading features DONE" << std::endl;
-
-		SOM::Network network {width, height, nbDims};
-		SOM::DataNormalizer normalizer {nbDims};
-
-		SOM::InputVector weights {nbDims};
-		{
-			std::size_t index {};
-			for (const auto& featureSettings : featuresSettings)
-			{
-				for (std::size_t i {}; i < featureSettings.second; ++i)
-					weights[index++] = SOM::InputVector::value_type{1. / featureSettings.second};
-			}
-		}
-
-		network.setDataWeights(weights);
-
-		std::cout << "Weights: " << weights << std::endl;
-
-		std::cout << "Normalizing..." << std::endl;
-		normalizer.computeNormalizationFactors(tracksFeatures);
-
-		std::cout << "Dumping normalizer: " << std::endl;
-		normalizer.dump(std::cout);
-		std::cout << "Dumping normalizer DONE" << std::endl;
-
-		for (SOM::InputVector& features : tracksFeatures)
-			normalizer.normalizeData(features);
-		std::cout << "Normalizing DONE" << std::endl;
-
-		auto progress {[](const SOM::Network::CurrentIteration& iteration)
-		{
-			std::cout << "Iteration " << iteration.idIteration + 1 << " of " << iteration.iterationCount << std::endl;;
-		}};
-
-		std::cout << "Training..." << std::endl;
-		network.train(tracksFeatures, nbIterations, progress);
-		std::cout << "Training DONE" << std::endl;
-
-		auto meanDistance = network.computeRefVectorsDistanceMean();
-		std::cout << "MEAN distance = " << meanDistance << std::endl;
-		auto medianDistance = network.computeRefVectorsDistanceMedian();
-		std::cout << "MEDIAN distance = " << medianDistance << std::endl;
 
 		std::cout << "Classifying tracks..." << std::endl;
-
+		// may be long...
-		SOM::Matrix< std::vector<Database::Track::pointer> > tracksMap(width, height);
+		FeaturesSearcher searcher {*session, featuresSettings};
-		for (Database::IdType trackId : trackIds)
-		{
-			Database::Track::pointer track {Database::Track::getById(*session, trackId)};
-			if (!track)
-				continue;
-
-			SOM::InputVector features {nbDims};
-			if (!getTrackFeatures(*session, track, featuresSettings, features))
-				continue;
-
-			normalizer.normalizeData(features);
-
-			SOM::Position position = network.getClosestRefVectorPosition(features);
-			tracksMap[position].push_back(track);
-		}
 
 		std::cout << "Classifying tracks DONE" << std::endl;
-
-		// Dump tracks
-
-		for (SOM::Coordinate y = 0; y < tracksMap.getHeight(); ++y)
-		{
-			for (SOM::Coordinate x = 0; x < tracksMap.getWidth(); ++x)
-			{
-				std::cout << "{" << x << ", " << y << "}" << std::endl;
-				const auto& tracks = tracksMap[{x, y}];
-
-				for (const auto& track : tracks)
-				{
-					std::cout << " - " << track << std::endl;
-				}
-			}
-		}
-
-		// For each track, get the nearest tracks
-		for (Database::IdType trackId : trackIds)
-		{
-			Database::Track::pointer track {Database::Track::getById(*session, trackId)};
-			if (!track)
-				continue;
-
-			SOM::InputVector features {nbDims};
-			if (!getTrackFeatures(*session, track, featuresSettings, features))
-				continue;
-
-			normalizer.normalizeData(features);
-
-			SOM::Position refVectorPosition {network.getClosestRefVectorPosition(features)};
-
-			std::cout << "Getting nearest songs for track " << track << " in {" << refVectorPosition.x << ", " << refVectorPosition.y << "}:" << std::endl;
-			for (auto similarTrack : tracksMap[refVectorPosition])
-				std::cout << " - " << similarTrack << std::endl;
-
-			std::set<SOM::Position> neighbourPosition {refVectorPosition};
-			for (std::size_t i {}; i < 3; ++i)
-			{
-				auto position = network.getClosestRefVectorPosition(neighbourPosition, medianDistance);
-				if (!position)
-					break;
-
-				std::cout << " - in {" << position->x << ", " << position->y << "}, dist = " << network.getRefVectorsDistance(*position, refVectorPosition) << std::endl;
-				for (const auto& similarTrack : tracksMap[*position])
-					std::cout << "    - " << similarTrack << std::endl;
-
-				neighbourPosition.insert(*position);
-			}
-
-		}
 	}
 	catch( std::exception& e)
 	{
 		std::cerr << "Caught exception: " << e.what() << std::endl;
+		return EXIT_FAILURE;
 	}
 
 	return EXIT_SUCCESS;

tools/similarity/Makefile.am

@@ -16,6 +16,9 @@ lms_similarity_SOURCES = \
 	$(top_srcdir)/src/database/User.cpp		\
 	$(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp		\
 	$(top_srcdir)/src/similarity/features/som/Network.cpp			\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp	\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp	\
+	$(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp	\
 	$(top_srcdir)/src/utils/Config.cpp 		\
 	$(top_srcdir)/src/utils/Logger.cpp 		\
 	$(top_srcdir)/src/utils/Utils.cpp