Refactored the inputvector part of the som implementation

configure.ac

@@ -85,7 +85,7 @@ AC_CONFIG_FILES([Makefile
 		 src/Makefile
 		 test/Makefile
 		 tools/Makefile
-		 tools/feature-extractor/Makefile
+		 tools/similarity/Makefile
 		 tools/metadata/Makefile])
 
 AC_OUTPUT

src/Makefile.am

@@ -24,6 +24,7 @@ lms_SOURCES = \
 	$(srcdir)/similarity/SimilaritySearcher.cpp		\
 	$(srcdir)/similarity/cluster/SimilarityClusterSearcher.cpp \
 	$(srcdir)/similarity/features/AcousticBrainzUtils.cpp	\
+	$(srcdir)/similarity/features/SimilarityFeaturesCache.cpp	\
 	$(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.cpp	\
 	$(srcdir)/similarity/features/SimilarityFeaturesSearcher.cpp	\
 	$(srcdir)/similarity/features/som/DataNormalizer.cpp	\

src/database/Track.cpp

@@ -119,11 +119,15 @@ Track::getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session)
 }
 
 std::vector<Track::pointer>
-Track::getAllWithFeatures(Wt::Dbo::Session& session)
+Track::getAllWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit)
 {
+	int size {limit ? static_cast<int>(*limit) : -1};
+
 	Wt::Dbo::collection<pointer> res = session.query<pointer>
 		("SELECT t FROM track t")
-		.where("EXISTS (SELECT * from track_features t_f WHERE t_f.track_id = t.id)");
+		.where("EXISTS (SELECT * from track_features t_f WHERE t_f.track_id = t.id)")
+		.limit(size);
+
 	return std::vector<pointer>(res.begin(), res.end());
 }
 

src/database/Track.hpp

@@ -70,7 +70,7 @@ class Track : public Wt::Dbo::Dbo<Track>
 		static std::vector<pointer> getChecksumDuplicates(Wt::Dbo::Session& session);
 		static std::vector<pointer> getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1);
 		static std::vector<pointer> getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction
-		static std::vector<pointer> getAllWithFeatures(Wt::Dbo::Session& session); // nested transaction
+		static std::vector<pointer> getAllWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit = {}); // nested transaction
 
 		// Create utility
 		static pointer	create(Wt::Dbo::Session& session, const boost::filesystem::path& p);

src/similarity/features/SimilarityFeaturesCache.cpp

@@ -0,0 +1,254 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SimilarityFeaturesCache.hpp"
+
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/xml_parser.hpp>
+
+#include "utils/Config.hpp"
+#include "utils/Logger.hpp"
+#include "utils/Utils.hpp"
+
+namespace Similarity {
+
+
+static
+boost::filesystem::path getCacheDirectory()
+{
+	return Config::instance().getPath("working-dir") / "cache" / "features";
+}
+
+static boost::filesystem::path getCacheNetworkFilePath()
+{
+	return getCacheDirectory() / "network";
+};
+
+static boost::filesystem::path getCacheTrackPositionsFilePath()
+{
+	return getCacheDirectory() / "track_positions";
+}
+
+static
+bool
+networkToCacheFile(const SOM::Network& network, boost::filesystem::path path)
+{
+	try
+	{
+		boost::property_tree::ptree root;
+
+		root.put("width", network.getWidth());
+		root.put("height", network.getHeight());
+		root.put("dim_count", network.getInputDimCount());
+
+		for (SOM::InputVector::value_type weight : network.getDataWeights())
+			root.add("weights.weight", weight);
+
+		for (SOM::Coordinate x = 0; x < network.getWidth(); ++x)
+		{
+			for (SOM::Coordinate y = 0; y < network.getWidth(); ++y)
+			{
+				const auto& refVector = network.getRefVector({x, y});
+
+				boost::property_tree::ptree node;
+				for (auto value : refVector)
+					node.add("values.value", value);
+
+				node.put("coord_x", x);
+				node.put("coord_y", y);
+
+				root.add_child("ref_vectors.ref_vector", node);
+			}
+		}
+
+		boost::property_tree::write_xml(path.string(), root);
+
+		LMS_LOG(SIMILARITY, DEBUG) << "Created network cache";
+		return true;
+	}
+	catch (boost::property_tree::ptree_error& error)
+	{
+		LMS_LOG(SIMILARITY, ERROR) << "Cannot create network cache: " << error.what();
+		return false;
+	}
+}
+
+static
+boost::optional<SOM::Network>
+createNetworkFromCacheFile(boost::filesystem::path path)
+{
+	try
+	{
+		boost::property_tree::ptree root;
+
+		boost::property_tree::read_xml(path.string(), root);
+
+		SOM::Coordinate width {root.get<SOM::Coordinate>("width")};
+		SOM::Coordinate height {root.get<SOM::Coordinate>("height")};
+		std::size_t dimCount {root.get<std::size_t>("dim_count")};
+
+		SOM::Network res {width, height, dimCount};
+
+		{
+			SOM::InputVector weights {dimCount};
+			std::size_t i {};
+			for (const auto& val : root.get_child("weights"))
+				weights[i++] = val.second.get_value<double>();
+
+			res.setDataWeights(weights);
+		}
+
+		for (const auto& node : root.get_child("ref_vectors"))
+		{
+			SOM::Coordinate x {node.second.get<SOM::Coordinate>("coord_x")};
+			SOM::Coordinate y {node.second.get<SOM::Coordinate>("coord_y")};
+
+			SOM::InputVector refVector {dimCount};
+			std::size_t i {};
+			for (const auto& val : node.second.get_child("values"))
+				refVector[i++] = val.second.get_value<SOM::InputVector::value_type>();
+
+			res.setRefVector({x, y}, refVector);
+		}
+
+		LMS_LOG(SIMILARITY, DEBUG) << "Successfully read network from cache";
+
+		return res;
+	}
+	catch (boost::property_tree::ptree_error& error)
+	{
+		LMS_LOG(SIMILARITY, ERROR) << "Cannot read network cache: " << error.what();
+		return boost::none;
+	}
+}
+
+static
+bool
+objectPositionToCacheFile(const std::map<Database::IdType, std::set<SOM::Position>>& objectsPosition, boost::filesystem::path path)
+{
+	try
+	{
+		boost::property_tree::ptree root;
+
+		for (const auto& objectPosition : objectsPosition)
+		{
+			boost::property_tree::ptree node;
+
+			node.put("id", objectPosition.first);
+
+			for (const auto& position : objectPosition.second)
+			{
+				boost::property_tree::ptree positionNode;
+				positionNode.put("x", position.x);
+				positionNode.put("y", position.y);
+
+				node.add_child("position.position", positionNode);
+			}
+
+			root.add_child("objects.object", node);
+		}
+
+		boost::property_tree::write_xml(path.string(), root);
+		return true;
+	}
+	catch (boost::property_tree::ptree_error& error)
+	{
+		LMS_LOG(SIMILARITY, ERROR) << "Cannot cache object position: " << error.what();
+		return false;
+	}
+}
+
+static
+boost::optional<std::map<Database::IdType, std::set<SOM::Position>>>
+createObjectPositionsFromCacheFile(boost::filesystem::path path)
+{
+	try
+	{
+		boost::property_tree::ptree root;
+
+		boost::property_tree::read_xml(path.string(), root);
+
+		std::map<Database::IdType, std::set<SOM::Position>> res;
+
+		for (const auto& object : root.get_child("objects"))
+		{
+			auto id = object.second.get<Database::IdType>("id");
+			for (const auto& position : object.second.get_child("position"))
+			{
+				auto x = position.second.get<SOM::Coordinate>("x");
+				auto y = position.second.get<SOM::Coordinate>("y");
+
+				res[id].insert({x, y});
+			}
+		}
+
+		LMS_LOG(SIMILARITY, DEBUG) << "Successfully read object position from cache";
+
+		return res;
+	}
+	catch (boost::property_tree::ptree_error& error)
+	{
+		LMS_LOG(SIMILARITY, ERROR) << "Cannot create object position from cache file: " << error.what();
+		return boost::none;
+	}
+}
+
+void
+FeaturesCache::invalidate()
+{
+	boost::filesystem::remove(getCacheNetworkFilePath());
+	boost::filesystem::remove(getCacheTrackPositionsFilePath());
+}
+
+boost::optional<FeaturesCache>
+FeaturesCache::read()
+{
+	boost::optional<FeaturesCache> res;
+
+	auto network{createNetworkFromCacheFile(getCacheNetworkFilePath())};
+	if (!network)
+		return res;
+
+	auto trackPositions{createObjectPositionsFromCacheFile(getCacheTrackPositionsFilePath())};
+	if (!trackPositions)
+		return res;
+
+	return FeaturesCache{std::move(*network), std::move(*trackPositions)};
+}
+
+void
+FeaturesCache::write()
+{
+	boost::filesystem::create_directories(Config::instance().getPath("working-dir") / "cache" / "features");
+
+	if (!networkToCacheFile(_network, getCacheNetworkFilePath())
+		|| !objectPositionToCacheFile(_trackPositions, getCacheTrackPositionsFilePath()))
+	{
+		invalidate();
+	}
+}
+
+FeaturesCache::FeaturesCache(SOM::Network network, ObjectPositions trackPositions)
+: _network{std::move(network)},
+_trackPositions{std::move(trackPositions)}
+{
+}
+
+
+} // namespace Similarity

src/similarity/features/SimilarityFeaturesCache.hpp

@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <map>
+#include <set>
+
+#include "database/Types.hpp"
+#include "som/Network.hpp"
+
+namespace Similarity {
+
+class FeaturesCache
+{
+	public:
+
+		static void invalidate();
+
+		static boost::optional<FeaturesCache> read();
+		void write();
+
+	private:
+		using ObjectPositions = std::map<Database::IdType, std::set<SOM::Position>>;
+
+		FeaturesCache(SOM::Network network, ObjectPositions trackPositions);
+
+		friend class FeaturesSearcher;
+
+		SOM::Network		_network;
+		ObjectPositions		_trackPositions;
+};
+
+} // namespace Similarity

src/similarity/features/SimilarityFeaturesScannerAddon.cpp

@@ -19,15 +19,13 @@
 
 #include "SimilarityFeaturesScannerAddon.hpp"
 
-
+#include "AcousticBrainzUtils.hpp"
 #include "database/Track.hpp"
 #include "database/TrackFeatures.hpp"
+#include "similarity/features/SimilarityFeaturesCache.hpp"
 #include "utils/Config.hpp"
 #include "utils/Logger.hpp"
 
-#include "AcousticBrainzUtils.hpp"
-
-
 namespace Similarity {
 
 namespace {
@@ -45,8 +43,8 @@ getTracksWithMBIDAndMissingFeatures(Wt::Dbo::Session& session)
 
 	Wt::Dbo::Transaction transaction(session);
 
-	auto tracks = Database::Track::getAllWithMBIDAndMissingFeatures(session);
+	auto tracks {Database::Track::getAllWithMBIDAndMissingFeatures(session)};
-	for (auto track : tracks)
+	for (const Database::Track::pointer& track : tracks)
 		res.push_back({track.id(), track->getMBID()});
 
 	return res;
@@ -57,11 +55,13 @@ getTracksWithMBIDAndMissingFeatures(Wt::Dbo::Session& session)
 FeaturesScannerAddon::FeaturesScannerAddon(Wt::Dbo::SqlConnectionPool& connectionPool)
 : _db(connectionPool)
 {
-	boost::filesystem::create_directories(Config::instance().getPath("working-dir") / "cache" / "features");
+	boost::optional<Similarity::FeaturesCache> cache {Similarity::FeaturesCache::read()};
-
+	if (cache)
-	auto searcher = std::make_shared<Similarity::FeaturesSearcher>();
+	{
-	if (searcher->initFromCache(_db.getSession()))
+		auto searcher {std::make_shared<Similarity::FeaturesSearcher>(_db.getSession(), *cache)};
-		std::atomic_store(&_searcher, searcher);
+		if (searcher->isValid())
+			std::atomic_store(&_searcher, searcher);
+	}
 }
 
 std::shared_ptr<Similarity::FeaturesSearcher>
@@ -92,13 +92,13 @@ void
 FeaturesScannerAddon::preScanComplete()
 {
 	LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features...";
-	auto tracksInfo = getTracksWithMBIDAndMissingFeatures(_db.getSession());
+	std::vector<TrackInfo> tracksInfo {getTracksWithMBIDAndMissingFeatures(_db.getSession())};
 	LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features DONE (found " << tracksInfo.size() << ")";
 
-	for (const auto& trackInfo : tracksInfo)
+	for (const TrackInfo& trackInfo : tracksInfo)
 		fetchFeatures(trackInfo.id, trackInfo.mbid);
 
-	FeaturesSearcher::invalidateCache();
+	Similarity::FeaturesCache::invalidate();
 	updateSearcher();
 }
 
@@ -112,15 +112,24 @@ FeaturesScannerAddon::updateSearcher()
 	if (tracks.empty())
 	{
 		LMS_LOG(DBUPDATER, INFO) << "No track suitable for features similarity clustering";
-		std::atomic_store(&_searcher, std::shared_ptr<FeaturesSearcher>());
+		std::atomic_store(&_searcher, std::shared_ptr<FeaturesSearcher>{});
 		return;
 	}
 
-	auto searcher {std::make_shared<Similarity::FeaturesSearcher>()};
+	auto searcher {std::make_shared<Similarity::FeaturesSearcher>(_db.getSession(), _stopRequested)};
-	if (searcher->init(_db.getSession(), _stopRequested))
+	if (searcher->isValid())
+	{
 		std::atomic_store(&_searcher, searcher);
+		FeaturesCache cache{searcher->toCache()};
+		cache.write();
+
+		searcher->dump(_db.getSession(), std::cout);
+
+		LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated";
+	}
+	else
+		std::atomic_store(&_searcher, std::shared_ptr<FeaturesSearcher>{});
 
-	LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated";
 }
 
 bool
@@ -129,7 +138,7 @@ FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const std::string&
 	std::map<std::string, double> features;
 
 	LMS_LOG(DBUPDATER, DEBUG) << "Fetching low level features for track '" << MBID << "'";
-	std::string data = AcousticBrainz::extractLowLevelFeatures(MBID);
+	std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)};
 
 	if (data.empty())
 	{
@@ -139,9 +148,9 @@ FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const std::string&
 
 	// TODO check if the expected features are here
 
-	Wt::Dbo::Transaction transaction(_db.getSession());
+	Wt::Dbo::Transaction transaction{_db.getSession()};
 
-	Wt::Dbo::ptr<Database::Track> track = Database::Track::getById(_db.getSession(), trackId);
+	Wt::Dbo::ptr<Database::Track> track {Database::Track::getById(_db.getSession(), trackId)};
 	if (!track)
 		return false;
 

src/similarity/features/SimilarityFeaturesSearcher.cpp

@@ -20,8 +20,6 @@
 #include "SimilarityFeaturesSearcher.hpp"
 
 #include <random>
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/xml_parser.hpp>
 
 #include "database/Artist.hpp"
 #include "database/SimilaritySettings.hpp"
@@ -29,237 +27,82 @@
 #include "database/Track.hpp"
 #include "database/TrackFeatures.hpp"
 #include "som/DataNormalizer.hpp"
-#include "utils/Config.hpp"
 #include "utils/Logger.hpp"
 #include "utils/Utils.hpp"
 
 
 namespace Similarity {
 
-static
+struct FeatureInfo
-boost::filesystem::path getCacheDirectory()
 {
-	return Config::instance().getPath("working-dir") / "cache" / "features";
+	std::size_t	nbDimensions;
-}
+	double		weight;
-
-static boost::filesystem::path getCacheNetworkFilePath()
-{
-	return getCacheDirectory() / "network";
 };
 
-static boost::filesystem::path getCacheTrackPositionsFilePath()
+using FeatureInfoMap = std::map<std::string, FeatureInfo>;
+
+static
+FeatureInfoMap
+getFeatureInfoMap(Wt::Dbo::Session& session)
 {
-	return getCacheDirectory() / "track_positions";
+	Wt::Dbo::Transaction transaction {session};
+
+	auto settings {Database::SimilaritySettings::get(session)};
+
+	std::map<std::string, FeatureInfo> featuresInfo;
+	for (auto feature : settings->getFeatures())
+	{
+		LMS_LOG(SIMILARITY, DEBUG) << "Feature '" << feature->getName() << "', nbDimns = " << feature->getNbDimensions() << ", weight = " << feature->getWeight() ;
+		featuresInfo[feature->getName()] = { feature->getNbDimensions(), feature->getWeight() };
+	}
+
+	return featuresInfo;
 }
 
 static
-bool
+std::size_t
-networkToCacheFile(const SOM::Network& network, boost::filesystem::path path)
+getFeatureInfoMapNbDimensions(const FeatureInfoMap& featureInfoMap)
 {
-	try
+	return std::accumulate(featureInfoMap.begin(), featureInfoMap.end(), 0, [](std::size_t sum, auto it) { return sum + it.second.nbDimensions; });
-	{
-		boost::property_tree::ptree root;
-
-		root.put("width", network.getWidth());
-		root.put("height", network.getHeight());
-		root.put("dim_count", network.getInputDimCount());
-
-		for (auto weight : network.getDataWeights())
-			root.add("weights.weight", weight);
-
-		for (SOM::Coordinate x = 0; x < network.getWidth(); ++x)
-		{
-			for (SOM::Coordinate y = 0; y < network.getWidth(); ++y)
-			{
-				const auto& refVector = network.getRefVector({x, y});
-
-				boost::property_tree::ptree node;
-				for (auto value : refVector)
-					node.add("values.value", value);
-
-				node.put("coord_x", x);
-				node.put("coord_y", y);
-
-				root.add_child("ref_vectors.ref_vector", node);
-			}
-		}
-
-		boost::property_tree::write_xml(path.string(), root);
-
-		LMS_LOG(SIMILARITY, DEBUG) << "Created network cache";
-		return true;
-	}
-	catch (boost::property_tree::ptree_error& error)
-	{
-		LMS_LOG(SIMILARITY, ERROR) << "Cannot create network cache: " << error.what();
-		return false;
-	}
 }
 
-static
+FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested)
-boost::optional<SOM::Network>
-createNetworkFromCacheFile(boost::filesystem::path path)
-{
-	try
-	{
-		boost::property_tree::ptree root;
-
-		boost::property_tree::read_xml(path.string(), root);
-
-		auto width = root.get<double>("width");
-		auto height = root.get<double>("height");
-		auto dimCount = root.get<std::size_t>("dim_count");
-
-		SOM::Network res(width, height, dimCount);
-
-		SOM::InputVector weights;
-		for (const auto& val : root.get_child("weights"))
-			weights.push_back(val.second.get_value<double>());
-
-		res.setDataWeights(weights);
-
-		for (const auto& node : root.get_child("ref_vectors"))
-		{
-			auto x = node.second.get<SOM::Coordinate>("coord_x");
-			auto y = node.second.get<SOM::Coordinate>("coord_y");
-
-			std::vector<double> values;
-			for (const auto& val : node.second.get_child("values"))
-				values.push_back(val.second.get_value<double>());
-
-			res.setRefVector({x, y}, values);
-		}
-
-		LMS_LOG(SIMILARITY, DEBUG) << "Successfully read network from cache";
-
-		return res;
-	}
-	catch (boost::property_tree::ptree_error& error)
-	{
-		LMS_LOG(SIMILARITY, ERROR) << "Cannot read network cache: " << error.what();
-		return boost::none;
-	}
-}
-
-static
-bool
-objectPositionToCacheFile(const std::map<Database::IdType, std::set<SOM::Position>>& objectsPosition, boost::filesystem::path path)
-{
-	try
-	{
-		boost::property_tree::ptree root;
-
-		for (const auto& objectPosition : objectsPosition)
-		{
-			boost::property_tree::ptree node;
-
-			node.put("id", objectPosition.first);
-
-			for (const auto& position : objectPosition.second)
-			{
-				boost::property_tree::ptree positionNode;
-				positionNode.put("x", position.x);
-				positionNode.put("y", position.y);
-
-				node.add_child("position.position", positionNode);
-			}
-
-			root.add_child("objects.object", node);
-		}
-
-		boost::property_tree::write_xml(path.string(), root);
-		return true;
-	}
-	catch (boost::property_tree::ptree_error& error)
-	{
-		LMS_LOG(SIMILARITY, ERROR) << "Cannot cache object position: " << error.what();
-		return false;
-	}
-}
-
-static
-boost::optional<std::map<Database::IdType, std::set<SOM::Position>>>
-createObjectPositionsFromCacheFile(boost::filesystem::path path)
-{
-	try
-	{
-		boost::property_tree::ptree root;
-
-		boost::property_tree::read_xml(path.string(), root);
-
-		std::map<Database::IdType, std::set<SOM::Position>> res;
-
-		for (const auto& object : root.get_child("objects"))
-		{
-			auto id = object.second.get<Database::IdType>("id");
-			for (const auto& position : object.second.get_child("position"))
-			{
-				auto x = position.second.get<SOM::Coordinate>("x");
-				auto y = position.second.get<SOM::Coordinate>("y");
-
-				res[id].insert({x, y});
-			}
-		}
-
-		LMS_LOG(SIMILARITY, DEBUG) << "Successfully read object position from cache";
-
-		return res;
-	}
-	catch (boost::property_tree::ptree_error& error)
-	{
-		LMS_LOG(SIMILARITY, ERROR) << "Cannot create object position from cache file: " << error.what();
-		return boost::none;
-	}
-}
-
-bool
-FeaturesSearcher::init(Wt::Dbo::Session& session, bool& stopRequested)
 {
 	Wt::Dbo::Transaction transaction(session);
 
-	auto settings = Database::SimilaritySettings::get(session);
+	FeatureInfoMap featuresInfo {getFeatureInfoMap(session)};
+	std::size_t nbDimensions {getFeatureInfoMapNbDimensions(featuresInfo)};
 
-	struct FeatureInfo
+	LMS_LOG(SIMILARITY, DEBUG) << "Features dimension = " << nbDimensions;
-	{
-		std::size_t	nbDimensions;
-		double		weight;
-	};
-
-	std::map<std::string, FeatureInfo> featuresInfo;
-	std::size_t nbDimensions = 0;
-	for (auto feature : settings->getFeatures())
-	{
-		featuresInfo[feature->getName()] = { feature->getNbDimensions(), feature->getWeight() };
-		nbDimensions += feature->getNbDimensions();
-	}
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features...";
-	auto tracks = Database::Track::getAllWithFeatures(session);
+	auto tracks {Database::Track::getAllWithFeatures(session)};
 	LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features DONE";
 
 	std::vector<SOM::InputVector> samples;
 	std::vector<Database::IdType> tracksIds;
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Extracting features...";
-	for (auto track : tracks)
+	for (const Database::Track::pointer& track : tracks)
 	{
 		if (stopRequested)
-			return false;
+			return;
 
-		SOM::InputVector sample;
+		SOM::InputVector sample {nbDimensions};
 
 		std::map<std::string, std::vector<double>> features;
-		for (const auto& featureInfo : featuresInfo)
+		for (auto itFeatureInfo : featuresInfo)
-			features[featureInfo.first] = {};
+			features[itFeatureInfo.first] = {};
 
 		if (!track->getTrackFeatures()->getFeatures(features))
 			continue;
 
-		// Check dimensions for each feature
+		bool ok {true};
-		bool ok = true;
+		std::size_t i {};
 		for (const auto& feature : features)
 		{
-			auto it = featuresInfo.find(feature.first);
+			// Check dimensions for each feature
+			auto it {featuresInfo.find(feature.first)};
 			if (it == featuresInfo.end() || it->second.nbDimensions != feature.second.size())
 			{
 				LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << feature.first << "'. Expected " << it->second.nbDimensions << ", got " << feature.second.size();
@@ -267,7 +110,8 @@ FeaturesSearcher::init(Wt::Dbo::Session& session, bool& stopRequested)
 				break;
 			}
 
-			sample.insert( sample.end(), feature.second.begin(), feature.second.end() );
+			for (double val : feature.second)
+				sample[i++] = val;
 		}
 
 		if (!ok)
@@ -283,7 +127,7 @@ FeaturesSearcher::init(Wt::Dbo::Session& session, bool& stopRequested)
 	if (tracksIds.empty())
 	{
 		LMS_LOG(SIMILARITY, INFO) << "Nothing to classify!";
-		return false;
+		return;
 	}
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Normalizing data...";
@@ -293,17 +137,21 @@ FeaturesSearcher::init(Wt::Dbo::Session& session, bool& stopRequested)
 	for (auto& sample : samples)
 		dataNormalizer.normalizeData(sample);
 
-	std::size_t size = std::sqrt(samples.size()/2);
+	SOM::InputVector weights {nbDimensions};
-	LMS_LOG(SIMILARITY, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network";
-
-	std::vector<double> weights;
-	for (const auto& featureInfo : featuresInfo)
 	{
-		for (std::size_t i = 0; i < featureInfo.second.nbDimensions; ++i)
+		std::size_t index {};
-			weights.push_back(1. / featureInfo.second.nbDimensions * featureInfo.second.weight);
+		for (const auto& featureInfo : featuresInfo)
+		{
+			for (std::size_t i {}; i < featureInfo.second.nbDimensions; ++i)
+				weights[index++] = (1. / featureInfo.second.nbDimensions * featureInfo.second.weight);
+		}
 	}
 
-	SOM::Network network(size, size, nbDimensions);
+	SOM::Coordinate size {static_cast<SOM::Coordinate>(std::sqrt(samples.size() / 4))};
+	LMS_LOG(SIMILARITY, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network";
+
+	SOM::Network network {size, size, nbDimensions};
+	std::cout << "Weights = '" << weights << "'";
 	network.setDataWeights(weights);
 
 	auto progressIndicator{[](const auto& iter)
@@ -314,116 +162,100 @@ FeaturesSearcher::init(Wt::Dbo::Session& session, bool& stopRequested)
 	auto stopper{[&]() { return stopRequested; }};
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Training network...";
-	network.train(samples, 1, progressIndicator, stopper);
+	network.train(samples, 10, progressIndicator, stopper);
 	LMS_LOG(SIMILARITY, DEBUG) << "Training network DONE";
 
 	if (stopRequested)
-		return false;
+		return;
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks...";
-	std::map<Database::IdType, std::set<SOM::Position>> trackPosition;
+	std::map<Database::IdType, std::set<SOM::Position>> trackPositions;
-	for (std::size_t i = 0; i < samples.size(); ++i)
+	for (std::size_t i {}; i < samples.size(); ++i)
 	{
 		if (stopRequested)
-			return false;
+			return;
 
-		Wt::Dbo::Transaction transaction(session);
+		Wt::Dbo::Transaction transaction {session};
 
 		const auto& sample = samples[i];
 		auto trackId = tracksIds[i];
 		auto position = network.getClosestRefVectorPosition(sample);
 
-		trackPosition[trackId].insert(position);
+		trackPositions[trackId].insert(position);
 	}
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE";
 
-	init(session, std::move(network), std::move(trackPosition));
+	init(session, std::move(network), std::move(trackPositions));
+}
 
-	saveToCache();
+FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache)
+{
+	init(session, std::move(cache._network), std::move(cache._trackPositions));
 
-	return true;
+	LMS_LOG(SIMILARITY, DEBUG) << "Init from cache DONE";
 }
 
 bool
-FeaturesSearcher::initFromCache(Wt::Dbo::Session& session)
+FeaturesSearcher::isValid() const
 {
-	auto network{createNetworkFromCacheFile(getCacheNetworkFilePath())};
+	return _network.get() != nullptr;
-	if (!network)
-	{
-		clearCache();
-		return false;
-	}
-
-	auto trackPositions{createObjectPositionsFromCacheFile(getCacheTrackPositionsFilePath())};
-	if (!trackPositions)
-	{
-		clearCache();
-		return false;
-	}
-
-	init(session, std::move(*network), std::move(*trackPositions));
-
-	LMS_LOG(SIMILARITY, DEBUG) << "Init from cache OK";
-
-	return true;
-}
-
-void
-FeaturesSearcher::invalidateCache()
-{
-	boost::filesystem::remove(getCacheNetworkFilePath());
-	boost::filesystem::remove(getCacheTrackPositionsFilePath());
 }
 
 std::vector<Database::IdType>
 FeaturesSearcher::getSimilarTracks(const std::set<Database::IdType>& tracksIds, std::size_t maxCount) const
 {
-	return getSimilarObjects(tracksIds, _tracksMap, _trackPosition, maxCount);
+	return getSimilarObjects(tracksIds, _tracksMap, _trackPositions, maxCount);
 }
 
 std::vector<Database::IdType>
 FeaturesSearcher::getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const
 {
-	return getSimilarObjects({releaseId}, _releasesMap, _releasePosition, maxCount);
+	return getSimilarObjects({releaseId}, _releasesMap, _releasePositions, maxCount);
 }
 
 std::vector<Database::IdType>
 FeaturesSearcher::getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const
 {
-	return getSimilarObjects({artistId}, _artistsMap, _artistPosition, maxCount);
+	return getSimilarObjects({artistId}, _artistsMap, _artistPositions, maxCount);
 }
 
 void
 FeaturesSearcher::dump(Wt::Dbo::Session& session, std::ostream& os) const
 {
-	os << "Number of tracks classified: " << _trackPosition.size() << std::endl;
+	if (!isValid())
-	os << "Network size: " << _network.getWidth() << " * " << _network.getHeight() << std::endl;
+	{
+		os << "Invalid searcher" << std::endl;
+		return;
+	}
+
+	os << "Number of tracks classified: " << _trackPositions.size() << std::endl;
+	os << "Network size: " << _network->getWidth() << " * " << _network->getHeight() << std::endl;
 	os << "Ref vectors median distance = " << _networkRefVectorsDistanceMedian << std::endl;
 
 	Wt::Dbo::Transaction transaction(session);
 
-	for (SOM::Coordinate y = 0; y < _network.getHeight(); ++y)
+	for (SOM::Coordinate y {}; y < _network->getHeight(); ++y)
 	{
-		for (SOM::Coordinate x = 0; x < _network.getWidth(); ++x)
+		for (SOM::Coordinate x {}; x < _network->getWidth(); ++x)
 		{
-			const auto& trackIds = _tracksMap[{x, y}];
+			const auto& trackIds {_tracksMap[{x, y}]};
 
 			os << "{" << x << ", " << y << "}";
 
 			if (y > 0)
-				os << " - {" << x << ", " << y - 1 << "}: " << _network.getRefVectorsDistance({x, y}, {x, y - 1});
+				os << " - {" << x << ", " << y - 1 << "}: " << _network->getRefVectorsDistance({x, y}, {x, y - 1});
 			if (x > 0)
-				os << " - {" << x - 1 << ", " << y << "}: " << _network.getRefVectorsDistance({x, y}, {x - 1, y});
+				os << " - {" << x - 1 << ", " << y << "}: " << _network->getRefVectorsDistance({x, y}, {x - 1, y});
-			if (y !=  _network.getHeight() - 1)
+			if (y !=  _network->getHeight() - 1)
-				os << " - {" << x << ", " << y + 1 << "}: " << _network.getRefVectorsDistance({x, y}, {x, y + 1});
+				os << " - {" << x << ", " << y + 1 << "}: " << _network->getRefVectorsDistance({x, y}, {x, y + 1});
-			if (x !=  _network.getWidth() - 1)
+			if (x !=  _network->getWidth() - 1)
-				os << " - {" << x + 1 << ", " << y << "}: " << _network.getRefVectorsDistance({x, y}, {x + 1, y});
+				os << " - {" << x + 1 << ", " << y << "}: " << _network->getRefVectorsDistance({x, y}, {x + 1, y});
 			os << std::endl;
 
-			for (auto trackId : trackIds)
+			for (Database::IdType trackId : trackIds)
 			{
-				auto track = Database::Track::getById(session, trackId);
+				auto track {Database::Track::getById(session, trackId)};
 				if (!track)
 					continue;
 
@@ -440,48 +272,52 @@ FeaturesSearcher::dump(Wt::Dbo::Session& session, std::ostream& os) const
 	}
 }
 
+FeaturesCache
+FeaturesSearcher::toCache() const
+{
+	return FeaturesCache{*_network, _trackPositions};
+}
 
 void
 FeaturesSearcher::init(Wt::Dbo::Session& session,
 			SOM::Network network,
 			std::map<Database::IdType, std::set<SOM::Position>> tracksPosition)
 {
-	_network = std::move(network);
+	_network = std::make_unique<SOM::Network>(std::move(network));
-
+	_networkRefVectorsDistanceMedian = _network->computeRefVectorsDistanceMedian();
-	_networkRefVectorsDistanceMedian = _network.computeRefVectorsDistanceMedian();
 	LMS_LOG(SIMILARITY, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian;
 
-	auto width = _network.getWidth();
+	SOM::Coordinate width {_network->getWidth()};
-	auto height = _network.getHeight();
+	SOM::Coordinate height {_network->getHeight()};
 
 	_artistsMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
 	_releasesMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
 	_tracksMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
 
-	Wt::Dbo::Transaction transaction(session);
+	Wt::Dbo::Transaction transaction {session};
 
 	for (auto itTrackCoord : tracksPosition)
 	{
-		auto trackId = itTrackCoord.first;
+		Database::IdType trackId {itTrackCoord.first};
-		const auto& positionSet = itTrackCoord.second;
+		const std::set<SOM::Position>& positionSet {itTrackCoord.second};
 
-		auto track = Database::Track::getById(session, trackId);
+		auto track {Database::Track::getById(session, trackId)};
 		if (!track)
 			continue;
 
-		for (const auto& position : positionSet)
+		for (const SOM::Position& position : positionSet)
 		{
 			_tracksMap[position].insert(trackId);
-			_trackPosition[trackId].insert(position);
+			_trackPositions[trackId].insert(position);
 
 			if (track->getRelease())
 			{
-				_releasePosition[track->getRelease().id()].insert(position);
+				_releasePositions[track->getRelease().id()].insert(position);
 				_releasesMap[position].insert(track->getRelease().id());
 			}
 			if (track->getArtist())
 			{
-				_artistPosition[track->getArtist().id()].insert(position);
+				_artistPositions[track->getArtist().id()].insert(position);
 				_artistsMap[position].insert(track->getArtist().id());
 			}
 		}
@@ -491,25 +327,6 @@ FeaturesSearcher::init(Wt::Dbo::Session& session,
 
 }
 
-void
-FeaturesSearcher::saveToCache() const
-{
-	if (!networkToCacheFile(_network, getCacheNetworkFilePath())
-		|| !objectPositionToCacheFile(_trackPosition, getCacheTrackPositionsFilePath()))
-	{
-		LMS_LOG(SIMILARITY, ERROR) << "Failed cache data";
-		clearCache();
-	}
-}
-
-void
-FeaturesSearcher::clearCache() const
-{
-	for (boost::filesystem::directory_iterator itEnd, it(getCacheDirectory()); it != itEnd; ++it)
-		boost::filesystem::remove_all(it->path());
-}
-
-
 static
 std::set<SOM::Position>
 getMatchingRefVectorsPosition(const std::set<Database::IdType>& ids, const std::map<Database::IdType, std::set<SOM::Position>>& objectPosition)
@@ -555,16 +372,19 @@ FeaturesSearcher::getSimilarObjects(const std::set<Database::IdType>& ids,
 {
 	std::vector<Database::IdType> res;
 
-	auto now = std::chrono::system_clock::now();
+	if (!isValid())
-	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
+		return res;
 
-	std::set<SOM::Position> searchedRefVectorsPosition = getMatchingRefVectorsPosition(ids, objectPosition);
+	auto now {std::chrono::system_clock::now()};
+	std::mt19937 randGenerator{static_cast<std::mt19937::result_type>(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count())};
+
+	std::set<SOM::Position> searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPosition)};
 	if (searchedRefVectorsPosition.empty())
 		return res;
 
 	while (1)
 	{
-		std::set<Database::IdType> closestObjectIds = getObjectsIds(searchedRefVectorsPosition, objectsMap);
+		std::set<Database::IdType> closestObjectIds {getObjectsIds(searchedRefVectorsPosition, objectsMap)};
 
 		// Remove objects that are already in input or already reported
 		for (auto id : ids)
@@ -574,8 +394,7 @@ FeaturesSearcher::getSimilarObjects(const std::set<Database::IdType>& ids,
 			closestObjectIds.erase(id);
 
 		{
-			std::vector<Database::IdType> objectIdsToAdd(closestObjectIds.begin(), closestObjectIds.end());
+			std::vector<Database::IdType> objectIdsToAdd {closestObjectIds.begin(), closestObjectIds.end()};
-
 			std::shuffle(objectIdsToAdd.begin(), objectIdsToAdd.end(), randGenerator);
 			std::copy(objectIdsToAdd.begin(), objectIdsToAdd.end(), std::back_inserter(res));
 		}
@@ -587,7 +406,7 @@ FeaturesSearcher::getSimilarObjects(const std::set<Database::IdType>& ids,
 			break;
 
 		// If there is not enough objects, try again with closest neighbour until there is too much distance
-		auto closestRefVectorPosition = _network.getClosestRefVectorPosition(searchedRefVectorsPosition, _networkRefVectorsDistanceMedian * 0.75);
+		boost::optional<SOM::Position> closestRefVectorPosition {_network->getClosestRefVectorPosition(searchedRefVectorsPosition, _networkRefVectorsDistanceMedian * 0.75)};
 		if (!closestRefVectorPosition)
 			break;
 

src/similarity/features/SimilarityFeaturesSearcher.hpp

@@ -26,17 +26,22 @@
 #include "database/Types.hpp"
 #include "som/DataNormalizer.hpp"
 #include "som/Network.hpp"
+#include "SimilarityFeaturesCache.hpp"
 
 namespace Similarity {
 
+
 class FeaturesSearcher
 {
 	public:
 
-		bool init(Wt::Dbo::Session& session, bool& stopRequested);
+		// Use cache
-		bool initFromCache(Wt::Dbo::Session& session);
+		FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache);
 
-		static void invalidateCache();
+		// Use training (may be very slow)
+		FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested);
+
+		bool isValid() const;
 
 		std::vector<Database::IdType> getSimilarTracks(const std::set<Database::IdType>& tracksId, std::size_t maxCount) const;
 		std::vector<Database::IdType> getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const;
@@ -44,31 +49,32 @@ class FeaturesSearcher
 
 		void dump(Wt::Dbo::Session& session, std::ostream& os) const;
 
+		FeaturesCache toCache() const;
+
 	private:
 
+		using ObjectPositions = std::map<Database::IdType, std::set<SOM::Position>>;
+
 		void init(Wt::Dbo::Session& session,
 				SOM::Network network,
-				std::map<Database::IdType, std::set<SOM::Position>> tracksPosition);
+				ObjectPositions tracksPosition);
-
-		void saveToCache() const;
-		void clearCache() const;
 
 		std::vector<Database::IdType> getSimilarObjects(const std::set<Database::IdType>& ids,
 				const SOM::Matrix<std::set<Database::IdType>>& objectsMap,
-				const std::map<Database::IdType, std::set<SOM::Position>>& objectPosition,
+				const ObjectPositions& objectPosition,
 				std::size_t maxCount) const;
 
-		SOM::Network	_network;
+		std::unique_ptr<SOM::Network>	_network;
-		double		_networkRefVectorsDistanceMedian = 0;
+		double				_networkRefVectorsDistanceMedian {};
 
-		SOM::Matrix<std::set<Database::IdType>> 		_artistsMap;
+		SOM::Matrix<std::set<Database::IdType>> 	_artistsMap;
-		std::map<Database::IdType, std::set<SOM::Position>>	_artistPosition;
+		ObjectPositions					_artistPositions;
 
-		SOM::Matrix<std::set<Database::IdType>> 		_releasesMap;
+		SOM::Matrix<std::set<Database::IdType>> 	_releasesMap;
-		std::map<Database::IdType, std::set<SOM::Position>>	_releasePosition;
+		ObjectPositions					_releasePositions;
 
-		SOM::Matrix<std::set<Database::IdType>> 		_tracksMap;
+		SOM::Matrix<std::set<Database::IdType>> 	_tracksMap;
-		std::map<Database::IdType, std::set<SOM::Position>>	_trackPosition;
+		ObjectPositions					_trackPositions;
 
 };
 

src/similarity/features/som/DataNormalizer.cpp

@@ -31,14 +31,14 @@ static
 T
 variance(const std::vector<T>& vec)
 {
-	std::size_t size = vec.size();
+	std::size_t size {vec.size()};
 
 	if (size == 1)
-		return T{0.};
+		return T {};
 
-	T mean = std::accumulate(vec.begin(), vec.end(), T{0.}) / size;
+	const T mean {std::accumulate(vec.begin(), vec.end(), T{}) / size};
 
-	return std::accumulate(vec.begin(), vec.end(), T{0.},
+	return std::accumulate(vec.begin(), vec.end(), T {},
 		[mean, size] (T accumulator, const T& val)
 		{
 			return accumulator + ((val - mean) * (val - mean) / (size - 1));
@@ -46,7 +46,7 @@ variance(const std::vector<T>& vec)
 }
 
 DataNormalizer::DataNormalizer(std::size_t inputDimCount)
-: _inputDimCount(inputDimCount)
+: _inputDimCount{inputDimCount}
 {
 }
 
@@ -66,13 +66,13 @@ void
 DataNormalizer::computeNormalizationFactors(const std::vector<InputVector>& inputVectors)
 {
 	if (inputVectors.empty())
-		throw SOMException("Empty input vectors");
+		throw Exception("Empty input vectors");
 
 	// For each dimension of the input, compute the min/max
 	_minmax.clear();
 	_minmax.resize(_inputDimCount);
 
-	for (std::size_t dimId = 0; dimId < _inputDimCount; ++dimId)
+	for (std::size_t dimId {}; dimId < _inputDimCount; ++dimId)
 	{
 		std::vector<InputVector::value_type> values;
 
@@ -82,7 +82,7 @@ DataNormalizer::computeNormalizationFactors(const std::vector<InputVector>& inpu
 			values.push_back(inputVector[dimId]);
 		}
 
-		auto result = std::minmax_element(values.begin(), values.end());
+		auto result {std::minmax_element(values.begin(), values.end())};
 		_minmax[dimId] = {*result.first, *result.second};
 	}
 }
@@ -104,7 +104,7 @@ DataNormalizer::normalizeData(InputVector& a) const
 {
 	checkSameDimensions(a, _inputDimCount);
 
-	for (std::size_t dimId = 0; dimId < _inputDimCount; ++dimId)
+	for (std::size_t dimId {}; dimId < _inputDimCount; ++dimId)
 	{
 		a[dimId] = normalizeValue(a[dimId], dimId);
 	}
@@ -113,7 +113,7 @@ DataNormalizer::normalizeData(InputVector& a) const
 void
 DataNormalizer::dump(std::ostream& os) const
 {
-	for (std::size_t i = 0; i < _inputDimCount; ++i)
+	for (std::size_t i {}; i < _inputDimCount; ++i)
 		os << "(" << _minmax[i].min << ", " << _minmax[i].max << ")";
 }
 

src/similarity/features/som/DataNormalizer.hpp

@@ -53,7 +53,7 @@ class DataNormalizer
 	private:
 		InputVector::value_type normalizeValue(InputVector::value_type value, std::size_t dimensionId) const;
 
-		std::size_t _inputDimCount;
+		const std::size_t _inputDimCount;
 
 		std::vector<MinMax> _minmax; // Indexed min/max used to normalize data
 };

src/similarity/features/som/InputVector.hpp

@@ -0,0 +1,194 @@
+
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <vector>
+#include <cmath>
+
+namespace SOM
+{
+
+class Exception : public LmsException
+{
+	public:
+		Exception(const std::string& msg) : LmsException(msg) {}
+};
+
+class InputVector
+{
+	public:
+		using value_type = double;
+		using Norm = double;
+		using Distance = double;
+
+		InputVector(std::size_t nbDimensions, value_type defaultValue = value_type {}) : _values(nbDimensions, defaultValue) {}
+
+		bool hasSameDimension(const InputVector& other) const
+		{
+			return _values.size() == other._values.size();
+		}
+
+		std::size_t getNbDimensions() const
+		{
+			return _values.size();
+		}
+
+		value_type& operator[](std::size_t index)
+		{
+			if (index >= getNbDimensions())
+				throw Exception("Bad range");
+
+			return _values[index];
+		}
+
+		value_type operator[](std::size_t index) const
+		{
+			if (index >= getNbDimensions())
+				throw Exception("Bad range");
+
+			return _values[index];
+		}
+
+		InputVector& operator+=(const InputVector& other)
+		{
+			if (!hasSameDimension(other.getNbDimensions()))
+				throw Exception {"Not the same dimension count"};
+
+			for (std::size_t i {}; i < _values.size(); ++i)
+			{
+				_values[i] += other[i];
+			}
+
+			return *this;
+		}
+
+		InputVector& operator-=(const InputVector& other)
+		{
+			if (!hasSameDimension(other.getNbDimensions()))
+				throw Exception {"Not the same dimension count"};
+
+			for (std::size_t i {}; i < _values.size(); ++i)
+			{
+				_values[i] -= other[i];
+			}
+
+			return *this;
+		}
+
+		InputVector& operator*=(value_type factor)
+		{
+			for (std::size_t i {}; i < _values.size(); ++i)
+			{
+				_values[i] *= factor;
+			}
+
+			return *this;
+		}
+
+		Norm computeNorm() const
+		{
+			Norm res {};
+			for (value_type val : _values)
+				res += val * val;
+			return std::sqrt(res);
+		}
+
+		Distance computeEuclidianSquareDistance(const InputVector& other, const InputVector& weights) const
+		{
+			if (!hasSameDimension(other.getNbDimensions())
+				|| !hasSameDimension(weights.getNbDimensions()))
+			{
+				throw Exception {"Not the same dimension count"};
+			}
+
+			Distance res {};
+
+			for (std::size_t i {}; i < getNbDimensions(); ++i)
+			{
+				const InputVector::value_type diff {_values[i] - other._values[i]};
+				res += diff * diff * weights._values[i];
+			}
+
+			return res;
+		}
+
+		std::vector<value_type>::iterator begin()
+		{
+			return _values.begin();
+		}
+
+		std::vector<value_type>::const_iterator begin() const
+		{
+			return _values.cbegin();
+		}
+
+		std::vector<value_type>::const_iterator cbegin() const
+		{
+			return _values.cbegin();
+		}
+
+		std::vector<value_type>::iterator end()
+		{
+			return _values.end();
+		}
+
+		std::vector<value_type>::const_iterator end() const
+		{
+			return _values.cend();
+		}
+
+		std::vector<value_type>::const_iterator cend() const
+		{
+			return _values.cend();
+		}
+
+	private:
+
+		friend class InputVector operator-(const InputVector& a, const InputVector& b)
+		{
+			if (!a.hasSameDimension(b.getNbDimensions()))
+				throw Exception {"Not the same dimension count"};
+
+			InputVector res {a.getNbDimensions()};
+
+			for (std::size_t i {}; i < res._values.size(); ++i)
+				res._values[i] = a._values[i] - b._values[i];
+
+			return res;
+		}
+
+		friend std::ostream&
+		operator<<(std::ostream& os, const InputVector& a)
+		{
+			os << "[";
+			for (value_type val : a._values)
+			{
+				os << val << " ";
+			}
+			os << "]";
+
+			return os;
+		}
+
+		std::vector<value_type> _values;
+};
+
+}

src/similarity/features/som/Matrix.hpp

@@ -28,6 +28,7 @@ namespace SOM
 {
 
 using Coordinate = unsigned;
+using Norm = InputVector::value_type;
 
 struct Position
 {
@@ -56,18 +57,18 @@ class Matrix
 		Matrix() = default;
 
 		Matrix(Coordinate width, Coordinate height)
-		: _width(width),
+		: _width{width},
-		_height(height)
+		_height{height}
 		{
 			_values.resize(_width*_height);
 		}
 
-		Matrix(std::size_t width, std::size_t height, std::vector<T> values)
+		template<typename... CtArgs>
-		: _width(width),
+		Matrix(Coordinate width, Coordinate height, CtArgs... args)
-		_height(height),
+		: _width{width},
-		_values(std::move(values))
+		_height{height}
 		{
-			assert(_values.size() == _width * _height);
+			 _values.resize(_width*_height, T{args...});
 		}
 
 		void clear()
@@ -101,17 +102,17 @@ class Matrix
 		{
 			assert(!_values.empty());
 
-			auto it = std::min_element(_values.begin(), _values.end(), func);
+			auto it {std::min_element(_values.begin(), _values.end(), std::move(func))};
-			auto index = static_cast<Coordinate>(std::distance(_values.begin(), it));
+			auto index {static_cast<Coordinate>(std::distance(_values.begin(), it))};
 
 			return {index % _height, index / _height};
 		}
 
 	private:
 
-		Coordinate	_width = 0;
+		Coordinate		_width {};
-		Coordinate	_height = 0;
+		Coordinate		_height {};
-		std::vector<T>	_values;
+		std::vector<T>		_values;
 };
 
 } // ns SOM

src/similarity/features/som/Network.cpp

@@ -33,151 +33,69 @@ namespace SOM
 void
 checkSameDimensions(const InputVector& a, const InputVector& b)
 {
-	if (a.size() != b.size())
+	if (!a.hasSameDimension(b))
-		throw SOMException("Bad data dimension count");
+		throw Exception("Bad data dimension count");
 }
 
 void
 checkSameDimensions(const InputVector& a, std::size_t inputDimCount)
 {
-	if (a.size() != inputDimCount)
+	if (a.getNbDimensions() != inputDimCount)
-		throw SOMException("Bad data dimension count");
+		throw Exception("Bad data dimension count");
 }
 
-static FeatureType
+static LearningFactor
 defaultLearningFactor(Network::CurrentIteration iteration)
 {
-	constexpr FeatureType initialValue = 1;
+	static const LearningFactor initialValue{1};
 
-	return initialValue * exp(-((iteration.idIteration + 1) / static_cast<FeatureType>(iteration.iterationCount)));
+	return initialValue * exp(-((iteration.idIteration + 1) / static_cast<LearningFactor>(iteration.iterationCount)));
 }
 
-static FeatureType
+static InputVector::Distance
 euclidianSquareDistance(const InputVector& a, const InputVector& b, const InputVector& weights)
 {
-	checkSameDimensions(a, b);
+	return a.computeEuclidianSquareDistance(b, weights);
-	checkSameDimensions(a, weights);
-
-	FeatureType res = 0;
-
-	for (std::size_t i = 0; i < a.size(); ++i)
-	{
-		res += (a[i] - b[i]) * (a[i] - b[i]) * weights[i];
-	}
-
-	return res;
 }
 
 static
-FeatureType
+InputVector::value_type
 sigmaFunc(Network::CurrentIteration iteration)
 {
-	constexpr FeatureType sigma0 = 1;
+	constexpr InputVector::value_type sigma0 {1};
 
-	return sigma0 * std::exp(- ((iteration.idIteration + 1) / static_cast<FeatureType>(iteration.iterationCount)));
+	return sigma0 * std::exp(- ((iteration.idIteration + 1) / static_cast<InputVector::value_type>(iteration.iterationCount)));
 }
 
 static
-FeatureType
+InputVector::value_type
-defaultNeighbourhoodFunc(FeatureType norm, Network::CurrentIteration iteration)
+defaultNeighbourhoodFunc(Norm norm, const Network::CurrentIteration& iteration)
 {
-	auto sigma = sigmaFunc(iteration);
+	InputVector::value_type sigma {sigmaFunc(iteration)};
 
 	return exp(-norm / (2 * sigma * sigma));
 }
 
-
-std::ostream&
-operator<<(std::ostream& os, const InputVector& a)
-{
-	os << "[";
-	for (const auto& val : a)
-	{
-		os << val << " ";
-	}
-	os << "]";
-
-	return os;
-}
-
-
-static
-FeatureType
-norm(const InputVector& a)
-{
-	FeatureType res = 0;
-
-	for (auto val : a)
-		res += val * val;
-
-	return std::sqrt(res);
-}
-
-static
-InputVector
-operator+(const InputVector& a, const InputVector& b)
-{
-	checkSameDimensions(a, b);
-
-	InputVector res;
-	res.reserve(a.size());
-
-	for (std::size_t dimId = 0; dimId < a.size(); ++dimId)
-		res.push_back(a[dimId] + b[dimId]);
-
-	return res;
-}
-
-static
-InputVector
-operator-(const InputVector& a, const InputVector& b)
-{
-	checkSameDimensions(a, b);
-
-	InputVector res;
-	res.reserve(a.size());
-
-	for (std::size_t dimId = 0; dimId < a.size(); ++dimId)
-		res.push_back(a[dimId] - b[dimId]);
-
-	return res;
-}
-
-static
-InputVector
-operator*(const InputVector& a, FeatureType factor)
-{
-	InputVector res;
-	res.reserve(a.size());
-
-	for (std::size_t dimId = 0; dimId < a.size(); ++dimId)
-		res.push_back(a[dimId] * factor);
-
-	return res;
-}
-
 Network::Network(Coordinate width, Coordinate height, std::size_t inputDimCount)
 :
 _inputDimCount(inputDimCount),
-_weights(inputDimCount, static_cast<FeatureType>(1)),
+_weights(inputDimCount, static_cast<InputVector::value_type>(1)),
-_refVectors(width, height),
+_refVectors(width, height, _inputDimCount),
 _distanceFunc(euclidianSquareDistance),
 _learningFactorFunc(defaultLearningFactor),
 _neighbourhoodFunc(defaultNeighbourhoodFunc)
 {
-	auto now = std::chrono::system_clock::now();
+	auto now {std::chrono::system_clock::now()};
-	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
+	std::mt19937 randGenerator {static_cast<std::mt19937::result_type>(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count())};
 
 	// init each vector with a random normalized value
-	std::uniform_real_distribution<FeatureType> dist(0, 1);
+	std::uniform_real_distribution<InputVector::value_type> dist{0, 1};
 
-	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y {}; y < _refVectors.getHeight(); ++y)
 	{
-		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x {}; x < _refVectors.getWidth(); ++x)
 		{
-			auto& refVector = _refVectors.get({x,y});
+			for (InputVector::value_type& val : _refVectors.get({x,y}))
-			refVector.resize(_inputDimCount);
-			for (auto& val : refVector)
 				val = dist(randGenerator);
 		}
 	}
@@ -199,25 +117,25 @@ Network::setRefVector(const Position& position, const InputVector& data)
 	_refVectors[position] = data;
 }
 
-double
+InputVector::Distance
 Network::getRefVectorsDistance(const Position& position1, const Position& position2) const
 {
 	return _distanceFunc(_refVectors.get(position1), _refVectors.get(position2), _weights);
 }
 
-double
+InputVector::Distance
 Network::computeRefVectorsDistanceMean() const
 {
-	std::vector<double> values;
+	std::vector<InputVector::Distance> values;
-	values.reserve(2*_refVectors.getHeight()*_refVectors.getWidth() - _refVectors.getWidth() - _refVectors.getHeight());
+	values.reserve(2 * _refVectors.getHeight()*_refVectors.getWidth() - _refVectors.getWidth() - _refVectors.getHeight());
-	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y {}; y < _refVectors.getHeight(); ++y)
 	{
-		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x {}; x < _refVectors.getWidth(); ++x)
 		{
 			if (x != _refVectors.getWidth() - 1)
-				values.push_back(getRefVectorsDistance( {x, y}, {x + 1, y}));
+				values.emplace_back(getRefVectorsDistance( {x, y}, {x + 1, y}));
 			if (y != _refVectors.getHeight() - 1)
-				values.push_back(getRefVectorsDistance( {x, y}, {x, y + 1}));
+				values.emplace_back(getRefVectorsDistance( {x, y}, {x, y + 1}));
 		}
 	}
 
@@ -227,20 +145,22 @@ Network::computeRefVectorsDistanceMean() const
 double
 Network::computeRefVectorsDistanceMedian() const
 {
-	std::vector<double> values;
+	std::vector<InputVector::Distance> values;
 	values.reserve(2*_refVectors.getHeight()*_refVectors.getWidth() - _refVectors.getWidth() - _refVectors.getHeight());
-	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y {}; y < _refVectors.getHeight(); ++y)
 	{
-		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x {}; x < _refVectors.getWidth(); ++x)
 		{
 			if (x != _refVectors.getWidth() - 1)
-				values.push_back(getRefVectorsDistance( {x, y}, {x + 1, y}));
+				values.emplace_back(getRefVectorsDistance( {x, y}, {x + 1, y}));
 			if (y != _refVectors.getHeight() - 1)
-				values.push_back(getRefVectorsDistance( {x, y}, {x, y + 1}));
+				values.emplace_back(getRefVectorsDistance( {x, y}, {x, y + 1}));
 		}
 	}
 
-	return values[values.size()/2 - 1];
+	std::sort(values.begin(), values.end());
+
+	return values[values.size() > 1 ? values.size()/2 - 1 : 0];
 }
 
 void
@@ -248,9 +168,9 @@ Network::dump(std::ostream& os) const
 {
 	os << "Width: " << _refVectors.getWidth() << ", Height: " << _refVectors.getHeight() << std::endl;;
 
-	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y {}; y < _refVectors.getHeight(); ++y)
 	{
-		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x {}; x < _refVectors.getWidth(); ++x)
 		{
 			os << _refVectors.get({x, y}) << " ";
 		}
@@ -270,21 +190,18 @@ Network::getClosestRefVectorPosition(const InputVector& data) const
 }
 
 boost::optional<Position>
-Network::getClosestRefVectorPosition(const InputVector& data, double maxDistance) const
+Network::getClosestRefVectorPosition(const InputVector& data, InputVector::Distance maxDistance) const
 {
-	Position position = _refVectors.getPositionMinElement([&](const auto& a, const auto& b)
+	boost::optional<Position> position {getClosestRefVectorPosition(data)};
-			{
-				return (_distanceFunc(a, data, _weights) < _distanceFunc(b, data, _weights));
-			});
 
-	if (_distanceFunc(data, _refVectors.get(position), _weights) > maxDistance)
+	if (_distanceFunc(data, _refVectors.get(*position), _weights) > maxDistance)
-		return boost::none;
+		position.reset();
 
 	return position;
 }
 
 boost::optional<Position>
-Network::getClosestRefVectorPosition(const std::set<Position>& refVectorsPosition, double maxDistance) const
+Network::getClosestRefVectorPosition(const std::set<Position>& refVectorsPosition, InputVector::Distance maxDistance) const
 {
 	std::set<Position> neighboursPosition;
 	for (const Position& refVectorPosition : refVectorsPosition)
@@ -322,49 +239,47 @@ Network::getClosestRefVectorPosition(const std::set<Position>& refVectorsPositio
 					return (this->getRefVectorsDistance(a, neighbourPosition) < this->getRefVectorsDistance(b, neighbourPosition));
 				});
 
-		double distance = getRefVectorsDistance(neighbourPosition, *min);
+		InputVector::Distance distance {getRefVectorsDistance(neighbourPosition, *min)};
 		if (distance > maxDistance)
 			continue;
 
-		neighboursInfo.push_back({neighbourPosition, distance});
+		neighboursInfo.emplace_back(NeighbourInfo {neighbourPosition, distance});
 	}
 
 	if (neighboursInfo.empty())
 		return boost::none;
 
-	auto min = std::min_element(neighboursInfo.begin(), neighboursInfo.end(),
+	auto min {std::min_element(neighboursInfo.begin(), neighboursInfo.end(),
 		[&](const auto& a, const auto& b)
 		{
 			return a.distance < b.distance;
-		});
+		})};
 
 
 	return min->position;
 }
 
-static FeatureType
+static Norm
-computePositionNorm(Position c1, Position c2)
+computePositionNorm(const Position& c1, const Position& c2)
 {
-	std::vector<FeatureType> a { static_cast<FeatureType>(c1.x), static_cast<FeatureType>(c1.y) };
+	return std::sqrt((c1.x - c2.x) * (c1.x - c2.x) + (c1.y - c2.y) * (c1.y - c2.y));
-	std::vector<FeatureType> b { static_cast<FeatureType>(c2.x), static_cast<FeatureType>(c2.y) };
-
-	return norm(a - b);
 }
 
-
 void
-Network::updateRefVectors(const Position& closestRefVectorPosition, const InputVector& input, FeatureType learningFactor, const CurrentIteration& iteration)
+Network::updateRefVectors(const Position& closestRefVectorPosition, const InputVector& input, LearningFactor learningFactor, const CurrentIteration& iteration)
 {
-	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y {}; y < _refVectors.getHeight(); ++y)
 	{
-		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x {}; x < _refVectors.getWidth(); ++x)
 		{
-			auto& refVector = _refVectors.get({x, y});
+			InputVector& refVector {_refVectors.get({x, y})};
 
-			auto delta = input - refVector;
+			const Norm norm {computePositionNorm({x, y}, closestRefVectorPosition)};
-			auto n = computePositionNorm({x, y}, closestRefVectorPosition);
 
-			refVector = refVector + delta * (learningFactor * _neighbourhoodFunc(n, iteration));
+			InputVector delta {input - refVector};
+			delta *= (learningFactor * _neighbourhoodFunc(norm, iteration));
+
+			refVector += delta; // * (learningFactor * _neighbourhoodFunc(norm, iteration));
 		}
 	}
 }
@@ -372,26 +287,26 @@ Network::updateRefVectors(const Position& closestRefVectorPosition, const InputV
 void
 Network::train(const std::vector<InputVector>& inputData, std::size_t nbIterations, ProgressCallback progressCallback, RequestStopCallback requestStopCallback)
 {
-	bool stopRequested{false};
+	bool stopRequested {false};
 	std::vector<const InputVector*> inputDataShuffled;
 	inputDataShuffled.reserve(inputData.size());
 
 	for (const auto& input : inputData)
 		inputDataShuffled.push_back(&input);
 
-	auto now = std::chrono::system_clock::now();
+	auto now {std::chrono::system_clock::now()};
-	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
+	std::mt19937 randGenerator{static_cast<std::mt19937::result_type>(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count())};
 
-	for (std::size_t i = 0; i < nbIterations; ++i)
+	for (std::size_t i {}; i < nbIterations; ++i)
 	{
-		CurrentIteration curIter{i, nbIterations};
+		CurrentIteration curIter {i, nbIterations};
 
 		if (progressCallback)
 			progressCallback(curIter);
 
 		std::shuffle(inputDataShuffled.begin(), inputDataShuffled.end(), randGenerator);
 
-		const auto learningFactor = _learningFactorFunc(curIter);
+		const LearningFactor learningFactor {_learningFactorFunc(curIter)};
 
 		for (const InputVector* input : inputDataShuffled)
 		{

src/similarity/features/som/Network.hpp

@@ -26,40 +26,29 @@
 
 #include <boost/optional.hpp>
 
-#include "Matrix.hpp"
-
 #include "utils/Exception.hpp"
+#include "InputVector.hpp"
+#include "Matrix.hpp"
 
 namespace SOM
 {
 
-using FeatureType = double;
+using LearningFactor = InputVector::value_type;
-using InputVector = std::vector<FeatureType>;
+
 void checkSameDimensions(const InputVector& a, const InputVector& b);
 void checkSameDimensions(const InputVector& a, std::size_t inputDimCount);
 std::ostream& operator<<(std::ostream& os, const InputVector& a);
 
-class SOMException : public LmsException
-{
-	public:
-		SOMException(const std::string& msg) : LmsException(msg) {}
-};
-
 
 class Network
 {
 	public:
 
-		Network() = default;
-
 		// Init a network with random values
 		Network(Coordinate width, Coordinate height, std::size_t inputDimCount);
 
-		// Init a network with serialized values
+		Coordinate getWidth() const { return _refVectors.getWidth(); }
-		Network(const std::string& data);
+		Coordinate getHeight() const { return _refVectors.getHeight(); }
-
-		std::size_t getWidth() const { return _refVectors.getWidth(); }
-		std::size_t getHeight() const { return _refVectors.getHeight(); }
 		std::size_t getInputDimCount() const { return _inputDimCount; }
 		const InputVector& getDataWeights() const { return _weights; }
 
@@ -81,14 +70,14 @@ class Network
 
 		const InputVector& getRefVector(const Position& position) const;
 		Position getClosestRefVectorPosition(const InputVector& data) const;
-		boost::optional<Position> getClosestRefVectorPosition(const InputVector& data, double maxDistance) const;
+		boost::optional<Position> getClosestRefVectorPosition(const InputVector& data, InputVector::Distance maxDistance) const;
 
-		boost::optional<Position> getClosestRefVectorPosition(const std::set<Position>& refVectorsPosition, double maxDistance) const;
+		boost::optional<Position> getClosestRefVectorPosition(const std::set<Position>& refVectorsPosition, InputVector::Distance maxDistance) const;
 
-		double getRefVectorsDistance(const Position& position1, const Position& position2) const;
+		InputVector::Distance getRefVectorsDistance(const Position& position1, const Position& position2) const;
 
-		double computeRefVectorsDistanceMean() const;
+		InputVector::Distance computeRefVectorsDistanceMean() const;
-		double computeRefVectorsDistanceMedian() const;
+		InputVector::Distance computeRefVectorsDistanceMedian() const;
 
 		void dump(std::ostream& os) const;
 
@@ -96,20 +85,21 @@ class Network
 		// i is the current iteration
 		// refVector(i+1) = refVector(i) + LearningFactor(i) * NeighbourhoodFunc(i) * (MatchingRefVector - refVector)
 
-		using DistanceFunc = std::function<FeatureType(const InputVector& /* a */, const InputVector& /* b */, const InputVector& /* weights */)>;
+		using DistanceFunc = std::function<InputVector::Distance(const InputVector& /* a */, const InputVector& /* b */, const InputVector& /* weights */)>;
 		void setDistanceFunc(DistanceFunc distanceFunc);
+		DistanceFunc getDistanceFunc() { return _distanceFunc; }
 
-		using LearningFactorFunc = std::function<FeatureType(const CurrentIteration&)>;
+		using LearningFactorFunc = std::function<LearningFactor(const CurrentIteration&)>;
 		void setLearningFactorFunc(LearningFactorFunc learningFactorFunc);
 
-		using NeighbourhoodFunc = std::function<FeatureType(FeatureType /* norm(Position - CoordMatchingRefVector) */, const CurrentIteration&)>;
+		using NeighbourhoodFunc = std::function<InputVector::value_type(Norm /* norm(Position - CoordMatchingRefVector) */, const CurrentIteration&)>;
 		void setNeighbourhoodFunc(NeighbourhoodFunc neighbourhoodFunc);
 
 	private:
 
-		void updateRefVectors(const Position& closestRefVectorPosition, const InputVector& input, FeatureType learningFactor, const CurrentIteration& iteration);
+		void updateRefVectors(const Position& closestRefVectorPosition, const InputVector& input, LearningFactor learningFactor, const CurrentIteration& iteration);
 
-		std::size_t _inputDimCount = 0;
+		std::size_t _inputDimCount {};
 		InputVector _weights;	// weight for each dimension
 		Matrix<InputVector> _refVectors;
 

test/Makefile.am

@@ -1,7 +1,14 @@
 
-TESTS =
+TESTS = som-test
 
-check_PROGRAMS =
+check_PROGRAMS = som-test
+
+som_test_SOURCES = \
+	$(srcdir)/som-test/SomTest.cpp					\
+	$(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp	\
+	$(top_srcdir)/src/similarity/features/som/Network.cpp
+
+som_test_CXXFLAGS=-std=c++14 -Wall -I${top_srcdir}/src/ -I${top_srcdir}/src/similarity/features/som/
 
 
 

tools/Makefile.am

@@ -1,2 +1,2 @@
-SUBDIRS = feature-extractor metadata
+SUBDIRS = similarity metadata
 

tools/similarity/LmsSimilarity.cpp

@@ -34,11 +34,9 @@ std::ostream& operator<<(std::ostream& os, const Database::Track::pointer& track
 }
 
 static
-std::vector<double>
+bool
-getTrackFeatures(Wt::Dbo::Session &session, Database::Track::pointer track, const std::map<std::string, std::size_t>& featuresSettings)
+getTrackFeatures(Wt::Dbo::Session &session, const Database::Track::pointer& track, const std::map<std::string, std::size_t>& featuresSettings, SOM::InputVector& res)
 {
-	std::vector<double> res;
-
 	std::map<std::string, std::vector<double>> features;
 	for (const auto& featureSettings : featuresSettings)
 		features[featureSettings.first] = {};
@@ -46,22 +44,21 @@ getTrackFeatures(Wt::Dbo::Session &session, Database::Track::pointer track, cons
 	if (!track->getTrackFeatures()->getFeatures(features))
 	{
 		std::cout << "Skipping track '" << track->getMBID() << "': missing item" << std::endl;
-		return res;
+		return false;
 	};
 
+	std::size_t index {};
 	for (const auto& feature : features)
 	{
 		auto it = featuresSettings.find(feature.first);
 		if (it == featuresSettings.end() || (feature.second.size() != it->second))
-		{
+			return false;
-			res.clear();
-			break;
-		}
 
-		res.insert( res.end(), feature.second.begin(), feature.second.end() );
+		for (double value : feature.second)
+			res[index++] = value;
 	}
 
-	return res;
+	return true;
 }
 
 
@@ -69,10 +66,10 @@ int main(int argc, char *argv[])
 {
 	try
 	{
-		const std::size_t width = 15;
+		const std::size_t width = 10;
-		const std::size_t height = 15;
+		const std::size_t height = 10;
-		const std::size_t nbIterations = 2;
+		const std::size_t nbIterations = 20;
-		const std::size_t nbTracks = 5000;
+		std::size_t nbTracks = 5000;
 
 		const std::map<std::string, std::size_t> featuresSettings =
 		{
@@ -91,7 +88,6 @@ int main(int argc, char *argv[])
 			nbDims += featureSettings.second;
 
 		boost::filesystem::path configFilePath = "/etc/lms.conf";
-
 		if (argc >= 2)
 			configFilePath = std::string(argv[1], 0, 256);
 
@@ -104,38 +100,32 @@ int main(int argc, char *argv[])
 		std::cout << "Getting all features..." << std::endl;
 		Wt::Dbo::Transaction transaction(db.getSession());
 
-		auto tracks = Database::Track::getAllWithFeatures(db.getSession());
+		auto tracks = Database::Track::getAllWithFeatures(db.getSession(), nbTracks);
 
-		std::cout << "Getting all features DONE" << std::endl;
+		nbTracks = tracks.size();
-
+		std::cout << "Getting features DONE (" << nbTracks << " tracks)" << std::endl;
-/*		auto now = std::chrono::system_clock::now();
-		std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
-		std::shuffle(tracks.begin(), tracks.end(), randGenerator);
-*/
-		tracks.resize(nbTracks);
 
 		std::cout << "Reading features..." << std::endl;
-		std::vector< std::vector<double> > tracksFeatures;
+		std::vector<SOM::InputVector> tracksFeatures;
 
-		for (auto track : tracks)
+		for (const auto& track : tracks)
 		{
-			auto features = getTrackFeatures(db.getSession(), track, featuresSettings);
+			SOM::InputVector features {nbDims};
-
+			if (!getTrackFeatures(db.getSession(), track, featuresSettings, features))
-			if (features.empty())
 				continue;
 
 			tracksFeatures.emplace_back(std::move(features));
 		}
 		std::cout << "Reading features DONE" << std::endl;
 
-		SOM::Network network(width, height, nbDims);
+		SOM::Network network {width, height, nbDims};
-		SOM::DataNormalizer normalizer(nbDims);
+		SOM::DataNormalizer normalizer {nbDims};
 
-		std::vector<double> weights;
+		SOM::InputVector weights {nbDims};
 		for (const auto& featureSettings : featuresSettings)
 		{
-			for (std::size_t i = 0; i < featureSettings.second; ++i)
+			for (std::size_t i {}; i < featureSettings.second; ++i)
-				weights.push_back(1. / featureSettings.second);
+				weights[i] = SOM::InputVector::value_type{1. / featureSettings.second};
 		}
 
 		network.setDataWeights(weights);
@@ -147,12 +137,17 @@ int main(int argc, char *argv[])
 		normalizer.dump(std::cout);
 		std::cout << "Dumping normalizer DONE" << std::endl;
 
-		for (auto& features : tracksFeatures)
+		for (SOM::InputVector& features : tracksFeatures)
 			normalizer.normalizeData(features);
 		std::cout << "Normalizing DONE" << std::endl;
 
+		auto progress {[](const SOM::Network::CurrentIteration& iteration)
+		{
+			std::cout << "Iteration " << iteration.idIteration + 1 << " of " << iteration.iterationCount << std::endl;;
+		}};
+
 		std::cout << "Training..." << std::endl;
-		network.train(tracksFeatures, nbIterations);
+		network.train(tracksFeatures, nbIterations, progress);
 		std::cout << "Training DONE" << std::endl;
 
 		auto meanDistance = network.computeRefVectorsDistanceMean();
@@ -160,20 +155,18 @@ int main(int argc, char *argv[])
 		auto medianDistance = network.computeRefVectorsDistanceMedian();
 		std::cout << "MEDIAN distance = " << medianDistance << std::endl;
 
-#if 0
 		std::cout << "Classifying tracks..." << std::endl;
 
 		SOM::Matrix< std::vector<Database::Track::pointer> > tracksMap(width, height);
 		for (auto track : tracks)
 		{
-			auto features = getTrackFeatures(db.getSession(), track, featuresSettings);
+			SOM::InputVector features {nbDims};
-
+			if (!getTrackFeatures(db.getSession(), track, featuresSettings, features))
-			if (features.empty())
 				continue;
 
 			normalizer.normalizeData(features);
 
-			auto position = network.getClosestRefVectorPosition(features);
+			SOM::Position position = network.getClosestRefVectorPosition(features);
 			tracksMap[position].push_back(track);
 		}
 
@@ -188,7 +181,7 @@ int main(int argc, char *argv[])
 				std::cout << "{" << x << ", " << y << "}" << std::endl;
 				const auto& tracks = tracksMap[{x, y}];
 
-				for (auto track : tracks)
+				for (const auto& track : tracks)
 				{
 					std::cout << " - " << track << std::endl;
 				}
@@ -196,39 +189,35 @@ int main(int argc, char *argv[])
 		}
 
 		// For each track, get the nearest tracks
-		for (auto track : tracks)
+		for (const auto& track : tracks)
 		{
-			auto features = getTrackFeatures(db.getSession(), track, featuresSettings);
+			SOM::InputVector features {nbDims};
-
+			if (!getTrackFeatures(db.getSession(), track, featuresSettings, features))
-			if (features.empty())
 				continue;
 
 			normalizer.normalizeData(features);
 
-			auto refVectorPosition = network.getClosestRefVectorPosition(features);
+			SOM::Position refVectorPosition {network.getClosestRefVectorPosition(features)};
 
 			std::cout << "Getting nearest songs for track " << track << " in {" << refVectorPosition.x << ", " << refVectorPosition.y << "}:" << std::endl;
 			for (auto similarTrack : tracksMap[refVectorPosition])
 				std::cout << " - " << similarTrack << std::endl;
 
-			std::set<SOM::Position> neighbourPosition = {refVectorPosition};
+			std::set<SOM::Position> neighbourPosition {refVectorPosition};
-			for (std::size_t i = 0; i < 5; ++i)
+			for (std::size_t i {}; i < 3; ++i)
 			{
 				auto position = network.getClosestRefVectorPosition(neighbourPosition, medianDistance);
 				if (!position)
 					break;
 
 				std::cout << " - in {" << position->x << ", " << position->y << "}, dist = " << network.getRefVectorsDistance(*position, refVectorPosition) << std::endl;
-				for (auto similarTrack : tracksMap[*position])
+				for (const auto& similarTrack : tracksMap[*position])
 					std::cout << "    - " << similarTrack << std::endl;
 
 				neighbourPosition.insert(*position);
 			}
 
 		}
-#endif
-
-		std::cout << "Classifying tracks DONE" << std::endl;
 	}
 	catch( std::exception& e)
 	{

tools/similarity/Makefile.am

@@ -1,7 +1,7 @@
-bin_PROGRAMS = lms-feature-extractor
+bin_PROGRAMS = lms-similarity
 
-lms_feature_extractor_SOURCES = \
+lms_similarity_SOURCES = \
-	$(srcdir)/LmsFeatureExtractor.cpp		\
+	$(srcdir)/LmsSimilarity.cpp			\
 	$(top_srcdir)/src/database/Artist.cpp		\
 	$(top_srcdir)/src/database/Cluster.cpp		\
 	$(top_srcdir)/src/database/DatabaseHandler.cpp	\
@@ -18,5 +18,5 @@ lms_feature_extractor_SOURCES = \
 	$(top_srcdir)/src/utils/Logger.cpp 		\
 	$(top_srcdir)/src/utils/Utils.cpp
 
-lms_feature_extractor_CXXFLAGS=-std=c++14 -Wall -I$(top_srcdir)/src -D_REENTRANT
+lms_similarity_CXXFLAGS=-std=c++14 -Wall -I$(top_srcdir)/src -D_REENTRANT