Breakable training, added cache

src/Makefile.am

@@ -23,11 +23,11 @@ lms_SOURCES = \
 	$(srcdir)/scanner/MediaScanner.cpp			\
 	$(srcdir)/similarity/SimilaritySearcher.cpp		\
 	$(srcdir)/similarity/cluster/SimilarityClusterSearcher.cpp \
+	$(srcdir)/similarity/features/AcousticBrainzUtils.cpp	\
 	$(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.cpp	\
 	$(srcdir)/similarity/features/SimilarityFeaturesSearcher.cpp	\
-	$(srcdir)/similarity/features/som/AcousticBrainzUtils.cpp	\
+	$(srcdir)/similarity/features/som/DataNormalizer.cpp	\
-	$(srcdir)/similarity/features/som/DataNormalizer.cpp		\
+	$(srcdir)/similarity/features/som/Network.cpp		\
-	$(srcdir)/similarity/features/som/Network.cpp			\
 	$(srcdir)/ui/Auth.cpp					\
 	$(srcdir)/ui/LmsApplication.cpp				\
 	$(srcdir)/ui/LmsApplicationGroup.cpp			\

src/av/AvTranscoder.hpp

@@ -48,9 +48,7 @@ struct TranscodeParameters
 	Encoding	encoding = Encoding::MP3;
 	std::size_t	bitrate = 128000;
 	boost::optional<std::size_t> stream = boost::none; // Id of the stream to be transcoded (auto detect by default)
-	boost::optional<std::chrono::seconds> offset = boost::none;;
+	boost::optional<std::chrono::seconds> offset = boost::none;
-
-	TranscodeParameters() = default;
 };
 
 class Transcoder

src/database/Artist.hpp

@@ -39,7 +39,7 @@ class Artist : public Wt::Dbo::Dbo<Artist>
 {
 	public:
 
-		typedef Wt::Dbo::ptr<Artist> pointer;
+		using pointer = Wt::Dbo::ptr<Artist>;
 
 		Artist() {}
 		Artist(const std::string& name, const std::string& MBID = "");

src/database/Release.hpp

@@ -38,7 +38,7 @@ class Release : public Wt::Dbo::Dbo<Release>
 {
 	public:
 
-		typedef Wt::Dbo::ptr<Release> pointer;
+		using pointer = Wt::Dbo::ptr<Release>;
 
 		Release() {}
 		Release(const std::string& name, const std::string& MBID = "");

src/database/SimilaritySettings.cpp

@@ -44,7 +44,6 @@ static std::vector<TrackFeatureInfo> defaultFeatures =
 	{ "lowlevel.gfcc.mean",				13,	1. },
 };
 
-
 SimilaritySettingsFeature::SimilaritySettingsFeature(Wt::Dbo::ptr<SimilaritySettings> settings, const std::string& name, std::size_t nbDimensions, double weight)
 : _name(name),
 _nbDimensions(nbDimensions),
@@ -80,6 +79,5 @@ SimilaritySettings::getFeatures() const
 	return std::vector<Wt::Dbo::ptr<SimilaritySettingsFeature>>(_features.begin(), _features.end());
 }
 
-
 } // namespace Database
 

src/database/SimilaritySettings.hpp

@@ -73,11 +73,12 @@ class SimilaritySettings : public Wt::Dbo::Dbo<SimilaritySettings>
 		// Utils
 		static pointer get(Wt::Dbo::Session& session);
 
-		// Accessors
+		// Accessors Read
-		std::size_t getVersion() const { return _settingsVersion; }
+		std::size_t		getVersion() const { return _settingsVersion; }
-		PreferredMethod getPreferredMethod() const { return _preferredMethod; }
+		PreferredMethod		getPreferredMethod() const { return _preferredMethod; }
 		std::vector<Wt::Dbo::ptr<SimilaritySettingsFeature>> getFeatures() const;
 
+
 		template<class Action>
 		void persist(Action& a)
 		{
@@ -91,6 +92,7 @@ class SimilaritySettings : public Wt::Dbo::Dbo<SimilaritySettings>
 
 		int			_settingsVersion = 0;
 		PreferredMethod		_preferredMethod = PreferredMethod::Auto;
+
 		Wt::Dbo::collection<Wt::Dbo::ptr<SimilaritySettingsFeature>> _features;
 };
 

src/main/main.cpp

@@ -96,9 +96,8 @@ int main(int argc, char* argv[])
 		Config::instance().setFile(configFilePath);
 
 		// Make sure the working directory exists
-		// TODO check with boost::system::error_code ec;
 		boost::filesystem::create_directories(Config::instance().getPath("working-dir"));
-		boost::filesystem::create_directories(Config::instance().getPath("working-dir") / "features");
+		boost::filesystem::create_directories(Config::instance().getPath("working-dir") / "cache");
 
 		// Construct WT configuration and get the argc/argv back
 		std::vector<std::string> wtServerArgs = generateWtConfig(argv[0]);

src/scanner/MediaScanner.cpp

@@ -225,6 +225,9 @@ MediaScanner::stop(void)
 {
 	_running = false;
 
+	for (auto& addon : _addons)
+		addon->requestStop();
+
 	_scheduleTimer.cancel();
 
 	_ioService.stop();

src/scanner/MediaScannerAddon.hpp

@@ -28,6 +28,7 @@ class MediaScannerAddon
 	public:
 
 		virtual void refreshSettings() = 0;
+		virtual void requestStop() = 0;
 
 		virtual void trackAdded(Database::IdType trackId) = 0;
 		virtual void trackToRemove(Database::IdType trackId) = 0;

src/similarity/features/SimilarityFeaturesScannerAddon.cpp

@@ -22,9 +22,11 @@
 
 #include "database/Track.hpp"
 #include "database/TrackFeatures.hpp"
-#include "som/AcousticBrainzUtils.hpp"
+#include "utils/Config.hpp"
 #include "utils/Logger.hpp"
 
+#include "AcousticBrainzUtils.hpp"
+
 
 namespace Similarity {
 
@@ -55,6 +57,11 @@ getTracksWithMBIDAndMissingFeatures(Wt::Dbo::Session& session)
 FeaturesScannerAddon::FeaturesScannerAddon(Wt::Dbo::SqlConnectionPool& connectionPool)
 : _db(connectionPool)
 {
+	boost::filesystem::create_directories(Config::instance().getPath("working-dir") / "cache" / "features");
+
+	auto searcher = std::make_shared<Similarity::FeaturesSearcher>();
+	if (searcher->initFromCache(_db.getSession()))
+		std::atomic_store(&_searcher, searcher);
 }
 
 std::shared_ptr<Similarity::FeaturesSearcher>
@@ -63,6 +70,12 @@ FeaturesScannerAddon::getSearcher()
 	return std::atomic_load(&_searcher);
 }
 
+void
+FeaturesScannerAddon::requestStop()
+{
+	_stopRequested = true;
+}
+
 void
 FeaturesScannerAddon::trackUpdated(Database::IdType trackId)
 {
@@ -85,14 +98,15 @@ FeaturesScannerAddon::preScanComplete()
 	for (const auto& trackInfo : tracksInfo)
 		fetchFeatures(trackInfo.id, trackInfo.mbid);
 
+	FeaturesSearcher::invalidateCache();
 	updateSearcher();
 }
 
 void
 FeaturesScannerAddon::updateSearcher()
 {
-	Wt::Dbo::Transaction transaction(_db.getSession());
+	Wt::Dbo::Transaction transaction {_db.getSession()};
-	auto tracks = Database::Track::getAllWithFeatures(_db.getSession());
+	auto tracks {Database::Track::getAllWithFeatures(_db.getSession())};
 	transaction.commit();
 
 	if (tracks.empty())
@@ -102,9 +116,10 @@ FeaturesScannerAddon::updateSearcher()
 		return;
 	}
 
-	auto searcher = std::make_shared<Similarity::FeaturesSearcher>(_db.getSession());
+	auto searcher {std::make_shared<Similarity::FeaturesSearcher>()};
+	if (searcher->init(_db.getSession(), _stopRequested))
+		std::atomic_store(&_searcher, searcher);
 
-	std::atomic_store(&_searcher, searcher);
 	LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated";
 }
 

src/similarity/features/SimilarityFeaturesScannerAddon.hpp

@@ -39,6 +39,7 @@ class FeaturesScannerAddon final : public Scanner::MediaScannerAddon
 	private:
 
 		void refreshSettings() override {}
+		void requestStop() override;
 		void trackAdded(Database::IdType trackId) override {}
 		void trackToRemove(Database::IdType trackId) override {}
 		void trackUpdated(Database::IdType trackId) override;
@@ -49,7 +50,8 @@ class FeaturesScannerAddon final : public Scanner::MediaScannerAddon
 		void updateSearcher();
 
 		Database::Handler			_db;
-		std::shared_ptr<FeaturesSearcher>		_searcher;
+		std::shared_ptr<FeaturesSearcher>	_searcher;
+		bool					_stopRequested{false};
 };
 
 FeaturesScannerAddon* setFeaturesScannerAddon(FeaturesScannerAddon addon);

src/similarity/features/SimilarityFeaturesSearcher.cpp

@@ -20,6 +20,8 @@
 #include "SimilarityFeaturesSearcher.hpp"
 
 #include <random>
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/xml_parser.hpp>
 
 #include "database/Artist.hpp"
 #include "database/SimilaritySettings.hpp"
@@ -27,14 +29,191 @@
 #include "database/Track.hpp"
 #include "database/TrackFeatures.hpp"
 #include "som/DataNormalizer.hpp"
+#include "utils/Config.hpp"
 #include "utils/Logger.hpp"
 #include "utils/Utils.hpp"
 
 
 namespace Similarity {
 
+static
+boost::filesystem::path getCacheDirectory()
+{
+	return Config::instance().getPath("working-dir") / "cache" / "features";
+}
 
-FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session)
+static boost::filesystem::path getCacheNetworkFilePath()
+{
+	return getCacheDirectory() / "network";
+};
+
+static boost::filesystem::path getCacheTrackPositionsFilePath()
+{
+	return getCacheDirectory() / "track_positions";
+}
+
+static
+bool
+networkToCacheFile(const SOM::Network& network, boost::filesystem::path path)
+{
+	try
+	{
+		boost::property_tree::ptree root;
+
+		root.put("width", network.getWidth());
+		root.put("height", network.getHeight());
+		root.put("dim_count", network.getInputDimCount());
+
+		for (auto weight : network.getDataWeights())
+			root.add("weights.weight", weight);
+
+		for (SOM::Coordinate x = 0; x < network.getWidth(); ++x)
+		{
+			for (SOM::Coordinate y = 0; y < network.getWidth(); ++y)
+			{
+				const auto& refVector = network.getRefVector({x, y});
+
+				boost::property_tree::ptree node;
+				for (auto value : refVector)
+					node.add("values.value", value);
+
+				node.put("coord_x", x);
+				node.put("coord_y", y);
+
+				root.add_child("ref_vectors.ref_vector", node);
+			}
+		}
+
+		boost::property_tree::write_xml(path.string(), root);
+
+		LMS_LOG(SIMILARITY, DEBUG) << "Created network cache";
+		return true;
+	}
+	catch (boost::property_tree::ptree_error& error)
+	{
+		LMS_LOG(SIMILARITY, ERROR) << "Cannot create network cache: " << error.what();
+		return false;
+	}
+}
+
+static
+boost::optional<SOM::Network>
+createNetworkFromCacheFile(boost::filesystem::path path)
+{
+	try
+	{
+		boost::property_tree::ptree root;
+
+		boost::property_tree::read_xml(path.string(), root);
+
+		auto width = root.get<double>("width");
+		auto height = root.get<double>("height");
+		auto dimCount = root.get<std::size_t>("dim_count");
+
+		SOM::Network res(width, height, dimCount);
+
+		SOM::InputVector weights;
+		for (const auto& val : root.get_child("weights"))
+			weights.push_back(val.second.get_value<double>());
+
+		res.setDataWeights(weights);
+
+		for (const auto& node : root.get_child("ref_vectors"))
+		{
+			auto x = node.second.get<SOM::Coordinate>("coord_x");
+			auto y = node.second.get<SOM::Coordinate>("coord_y");
+
+			std::vector<double> values;
+			for (const auto& val : node.second.get_child("values"))
+				values.push_back(val.second.get_value<double>());
+
+			res.setRefVector({x, y}, values);
+		}
+
+		LMS_LOG(SIMILARITY, DEBUG) << "Successfully read network from cache";
+
+		return res;
+	}
+	catch (boost::property_tree::ptree_error& error)
+	{
+		LMS_LOG(SIMILARITY, ERROR) << "Cannot read network cache: " << error.what();
+		return boost::none;
+	}
+}
+
+static
+bool
+objectPositionToCacheFile(const std::map<Database::IdType, std::set<SOM::Position>>& objectsPosition, boost::filesystem::path path)
+{
+	try
+	{
+		boost::property_tree::ptree root;
+
+		for (const auto& objectPosition : objectsPosition)
+		{
+			boost::property_tree::ptree node;
+
+			node.put("id", objectPosition.first);
+
+			for (const auto& position : objectPosition.second)
+			{
+				boost::property_tree::ptree positionNode;
+				positionNode.put("x", position.x);
+				positionNode.put("y", position.y);
+
+				node.add_child("position.position", positionNode);
+			}
+
+			root.add_child("objects.object", node);
+		}
+
+		boost::property_tree::write_xml(path.string(), root);
+		return true;
+	}
+	catch (boost::property_tree::ptree_error& error)
+	{
+		LMS_LOG(SIMILARITY, ERROR) << "Cannot cache object position: " << error.what();
+		return false;
+	}
+}
+
+static
+boost::optional<std::map<Database::IdType, std::set<SOM::Position>>>
+createObjectPositionsFromCacheFile(boost::filesystem::path path)
+{
+	try
+	{
+		boost::property_tree::ptree root;
+
+		boost::property_tree::read_xml(path.string(), root);
+
+		std::map<Database::IdType, std::set<SOM::Position>> res;
+
+		for (const auto& object : root.get_child("objects"))
+		{
+			auto id = object.second.get<Database::IdType>("id");
+			for (const auto& position : object.second.get_child("position"))
+			{
+				auto x = position.second.get<SOM::Coordinate>("x");
+				auto y = position.second.get<SOM::Coordinate>("y");
+
+				res[id].insert({x, y});
+			}
+		}
+
+		LMS_LOG(SIMILARITY, DEBUG) << "Successfully read object position from cache";
+
+		return res;
+	}
+	catch (boost::property_tree::ptree_error& error)
+	{
+		LMS_LOG(SIMILARITY, ERROR) << "Cannot create object position from cache file: " << error.what();
+		return boost::none;
+	}
+}
+
+bool
+FeaturesSearcher::init(Wt::Dbo::Session& session, bool& stopRequested)
 {
 	Wt::Dbo::Transaction transaction(session);
 
@@ -64,6 +243,9 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session)
 	LMS_LOG(SIMILARITY, DEBUG) << "Extracting features...";
 	for (auto track : tracks)
 	{
+		if (stopRequested)
+			return false;
+
 		SOM::InputVector sample;
 
 		std::map<std::string, std::vector<double>> features;
@@ -101,24 +283,19 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session)
 	if (tracksIds.empty())
 	{
 		LMS_LOG(SIMILARITY, INFO) << "Nothing to classify!";
-		return;
+		return false;
 	}
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Normalizing data...";
-	SOM::DataNormalizer normalizer(nbDimensions);
+	SOM::DataNormalizer dataNormalizer(nbDimensions);
 
-	normalizer.computeNormalizationFactors(samples);
+	dataNormalizer.computeNormalizationFactors(samples);
 	for (auto& sample : samples)
-		normalizer.normalizeData(sample);
+		dataNormalizer.normalizeData(sample);
 
 	std::size_t size = std::sqrt(samples.size()/2);
 	LMS_LOG(SIMILARITY, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network";
 
-	_network = std::make_unique<SOM::Network>(size, size, nbDimensions);
-	_artistsMap = SOM::Matrix<std::set<Database::IdType>>(size, size);
-	_releasesMap = SOM::Matrix<std::set<Database::IdType>>(size, size);
-	_tracksMap = SOM::Matrix<std::set<Database::IdType>>(size, size);
-
 	std::vector<double> weights;
 	for (const auto& featureInfo : featuresInfo)
 	{
@@ -126,113 +303,230 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session)
 			weights.push_back(1. / featureInfo.second.nbDimensions * featureInfo.second.weight);
 	}
 
+	SOM::Network network(size, size, nbDimensions);
+	network.setDataWeights(weights);
+
+	auto progressIndicator{[](const auto& iter)
+		{
+			LMS_LOG(SIMILARITY, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount;
+		}};
+
+	auto stopper{[&]() { return stopRequested; }};
+
 	LMS_LOG(SIMILARITY, DEBUG) << "Training network...";
-	_network->train(samples, 20);
+	network.train(samples, 1, progressIndicator, stopper);
 	LMS_LOG(SIMILARITY, DEBUG) << "Training network DONE";
 
-	LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks...";
+	if (stopRequested)
+		return false;
 
+	LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks...";
+	std::map<Database::IdType, std::set<SOM::Position>> trackPosition;
 	for (std::size_t i = 0; i < samples.size(); ++i)
 	{
+		if (stopRequested)
+			return false;
+
 		Wt::Dbo::Transaction transaction(session);
 
 		const auto& sample = samples[i];
 		auto trackId = tracksIds[i];
+		auto position = network.getClosestRefVectorPosition(sample);
 
-		auto coords = _network->getClosestRefVectorCoords(sample);
+		trackPosition[trackId].insert(position);
-
-		_trackCoords[trackId].insert(coords);
-		_tracksMap[coords].insert(trackId);
-
-		auto track = Database::Track::getById(session, trackId);
-		if (track->getRelease())
-		{
-			_releaseCoords[track->getRelease().id()].insert(coords);
-			_releasesMap[coords].insert(track->getRelease().id());
-		}
-		if (track->getArtist())
-		{
-			_artistCoords[track->getArtist().id()].insert(coords);
-			_artistsMap[coords].insert(track->getArtist().id());
-		}
 	}
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE";
 
+	init(session, std::move(network), std::move(trackPosition));
+
+	saveToCache();
+
+	return true;
 }
 
+bool
+FeaturesSearcher::initFromCache(Wt::Dbo::Session& session)
+{
+	auto network{createNetworkFromCacheFile(getCacheNetworkFilePath())};
+	if (!network)
+	{
+		clearCache();
+		return false;
+	}
+
+	auto trackPositions{createObjectPositionsFromCacheFile(getCacheTrackPositionsFilePath())};
+	if (!trackPositions)
+	{
+		clearCache();
+		return false;
+	}
+
+	init(session, std::move(*network), std::move(*trackPositions));
+
+	LMS_LOG(SIMILARITY, DEBUG) << "Init from cache OK";
+
+	return true;
+}
+
+void
+FeaturesSearcher::invalidateCache()
+{
+	boost::filesystem::remove(getCacheNetworkFilePath());
+	boost::filesystem::remove(getCacheTrackPositionsFilePath());
+}
 
 std::vector<Database::IdType>
 FeaturesSearcher::getSimilarTracks(const std::set<Database::IdType>& tracksIds, std::size_t maxCount) const
 {
-	return getSimilarObjects(tracksIds, _tracksMap, _trackCoords, maxCount);
+	return getSimilarObjects(tracksIds, _tracksMap, _trackPosition, maxCount);
 }
 
 std::vector<Database::IdType>
 FeaturesSearcher::getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const
 {
-	return getSimilarObjects({releaseId}, _releasesMap, _releaseCoords, maxCount);
+	return getSimilarObjects({releaseId}, _releasesMap, _releasePosition, maxCount);
 }
 
 std::vector<Database::IdType>
 FeaturesSearcher::getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const
 {
-	return getSimilarObjects({artistId}, _artistsMap, _artistCoords, maxCount);
+	return getSimilarObjects({artistId}, _artistsMap, _artistPosition, maxCount);
 }
-#if 0
+
 void
 FeaturesSearcher::dump(Wt::Dbo::Session& session, std::ostream& os) const
 {
-	os << "Number of tracks classified: " << _trackIdsCoords.size() << std::endl;
+	os << "Number of tracks classified: " << _trackPosition.size() << std::endl;
 	os << "Network size: " << _network.getWidth() << " * " << _network.getHeight() << std::endl;
+	os << "Ref vectors median distance = " << _networkRefVectorsDistanceMedian << std::endl;
 
 	Wt::Dbo::Transaction transaction(session);
 
-	for (std::size_t y = 0; y < _network.getHeight(); ++y)
+	for (SOM::Coordinate y = 0; y < _network.getHeight(); ++y)
 	{
-		for (std::size_t x = 0; x < _network.getWidth(); ++x)
+		for (SOM::Coordinate x = 0; x < _network.getWidth(); ++x)
 		{
 			const auto& trackIds = _tracksMap[{x, y}];
 
+			os << "{" << x << ", " << y << "}";
+
+			if (y > 0)
+				os << " - {" << x << ", " << y - 1 << "}: " << _network.getRefVectorsDistance({x, y}, {x, y - 1});
+			if (x > 0)
+				os << " - {" << x - 1 << ", " << y << "}: " << _network.getRefVectorsDistance({x, y}, {x - 1, y});
+			if (y !=  _network.getHeight() - 1)
+				os << " - {" << x << ", " << y + 1 << "}: " << _network.getRefVectorsDistance({x, y}, {x, y + 1});
+			if (x !=  _network.getWidth() - 1)
+				os << " - {" << x + 1 << ", " << y << "}: " << _network.getRefVectorsDistance({x, y}, {x + 1, y});
+			os << std::endl;
+
 			for (auto trackId : trackIds)
 			{
 				auto track = Database::Track::getById(session, trackId);
 				if (!track)
 					continue;
 
-				os << "{";
+				os << "\t - " << track->getName() << " - ";
 				if (track->getArtist())
-					os << track->getArtist()->getName() << " ";
+					os << track->getArtist()->getName() << " - ";
 				if (track->getRelease())
 					os << track->getRelease()->getName();
-				os << "} ";
+				os << std::endl;
 			}
 
-			os << "; ";
 		}
 		os << std::endl;
 	}
 }
-#endif
+
+
+void
+FeaturesSearcher::init(Wt::Dbo::Session& session,
+			SOM::Network network,
+			std::map<Database::IdType, std::set<SOM::Position>> tracksPosition)
+{
+	_network = std::move(network);
+
+	_networkRefVectorsDistanceMedian = _network.computeRefVectorsDistanceMedian();
+	LMS_LOG(SIMILARITY, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian;
+
+	auto width = _network.getWidth();
+	auto height = _network.getHeight();
+
+	_artistsMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
+	_releasesMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
+	_tracksMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
+
+	Wt::Dbo::Transaction transaction(session);
+
+	for (auto itTrackCoord : tracksPosition)
+	{
+		auto trackId = itTrackCoord.first;
+		const auto& positionSet = itTrackCoord.second;
+
+		auto track = Database::Track::getById(session, trackId);
+		if (!track)
+			continue;
+
+		for (const auto& position : positionSet)
+		{
+			_tracksMap[position].insert(trackId);
+			_trackPosition[trackId].insert(position);
+
+			if (track->getRelease())
+			{
+				_releasePosition[track->getRelease().id()].insert(position);
+				_releasesMap[position].insert(track->getRelease().id());
+			}
+			if (track->getArtist())
+			{
+				_artistPosition[track->getArtist().id()].insert(position);
+				_artistsMap[position].insert(track->getArtist().id());
+			}
+		}
+	}
+
+	LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE";
+
+}
+
+void
+FeaturesSearcher::saveToCache() const
+{
+	if (!networkToCacheFile(_network, getCacheNetworkFilePath())
+		|| !objectPositionToCacheFile(_trackPosition, getCacheTrackPositionsFilePath()))
+	{
+		LMS_LOG(SIMILARITY, ERROR) << "Failed cache data";
+		clearCache();
+	}
+}
+
+void
+FeaturesSearcher::clearCache() const
+{
+	for (boost::filesystem::directory_iterator itEnd, it(getCacheDirectory()); it != itEnd; ++it)
+		boost::filesystem::remove_all(it->path());
+}
 
 
 static
-std::set<SOM::Coords>
+std::set<SOM::Position>
-getMatchingRefVectorsCoords(const std::set<Database::IdType>& ids, const std::map<Database::IdType, std::set<SOM::Coords>>& objectCoords)
+getMatchingRefVectorsPosition(const std::set<Database::IdType>& ids, const std::map<Database::IdType, std::set<SOM::Position>>& objectPosition)
 {
-	std::set<SOM::Coords> res;
+	std::set<SOM::Position> res;
 
 	if (ids.empty())
 		return res;
 
 	for (auto id : ids)
 	{
-		auto it = objectCoords.find(id);
+		auto it = objectPosition.find(id);
-		if (it == objectCoords.end())
+		if (it == objectPosition.end())
 			continue;
 
-		for (const auto& coords : it->second)
+		for (const auto& position : it->second)
-			res.insert(coords);
+			res.insert(position);
 	}
 
 	return res;
@@ -240,13 +534,13 @@ getMatchingRefVectorsCoords(const std::set<Database::IdType>& ids, const std::ma
 
 static
 std::set<Database::IdType>
-getObjectsIds(const std::set<SOM::Coords>& coordsSet, const SOM::Matrix<std::set<Database::IdType>>& objectsMap )
+getObjectsIds(const std::set<SOM::Position>& positionSet, const SOM::Matrix<std::set<Database::IdType>>& objectsMap )
 {
 	std::set<Database::IdType> res;
 
-	for (const auto& coords : coordsSet)
+	for (const auto& position : positionSet)
 	{
-		for (auto id : objectsMap.get(coords))
+		for (auto id : objectsMap.get(position))
 			res.insert(id);
 	}
 
@@ -256,7 +550,7 @@ getObjectsIds(const std::set<SOM::Coords>& coordsSet, const SOM::Matrix<std::set
 std::vector<Database::IdType>
 FeaturesSearcher::getSimilarObjects(const std::set<Database::IdType>& ids,
 		const SOM::Matrix<std::set<Database::IdType>>& objectsMap,
-		const std::map<Database::IdType, std::set<SOM::Coords>>& objectCoords,
+		const std::map<Database::IdType, std::set<SOM::Position>>& objectPosition,
 		std::size_t maxCount) const
 {
 	std::vector<Database::IdType> res;
@@ -264,18 +558,21 @@ FeaturesSearcher::getSimilarObjects(const std::set<Database::IdType>& ids,
 	auto now = std::chrono::system_clock::now();
 	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
 
-	std::set<SOM::Coords> searchedRefVectorsCoords = getMatchingRefVectorsCoords(ids, objectCoords);
+	std::set<SOM::Position> searchedRefVectorsPosition = getMatchingRefVectorsPosition(ids, objectPosition);
-	if (searchedRefVectorsCoords.empty())
+	if (searchedRefVectorsPosition.empty())
 		return res;
 
 	while (1)
 	{
-		std::set<Database::IdType> closestObjectIds = getObjectsIds(searchedRefVectorsCoords, objectsMap);
+		std::set<Database::IdType> closestObjectIds = getObjectsIds(searchedRefVectorsPosition, objectsMap);
 
-		// Remove objects that are already in input
+		// Remove objects that are already in input or already reported
 		for (auto id : ids)
 			closestObjectIds.erase(id);
 
+		for (auto id : res)
+			closestObjectIds.erase(id);
+
 		{
 			std::vector<Database::IdType> objectIdsToAdd(closestObjectIds.begin(), closestObjectIds.end());
 
@@ -290,11 +587,11 @@ FeaturesSearcher::getSimilarObjects(const std::set<Database::IdType>& ids,
 			break;
 
 		// If there is not enough objects, try again with closest neighbour until there is too much distance
-		auto closestRefVectorCoords = _network->getClosestRefVectorCoords(searchedRefVectorsCoords, _networkRefVectorsDistanceMedian * 0.75);
+		auto closestRefVectorPosition = _network.getClosestRefVectorPosition(searchedRefVectorsPosition, _networkRefVectorsDistanceMedian * 0.75);
-		if (!closestRefVectorCoords)
+		if (!closestRefVectorPosition)
 			break;
 
-		searchedRefVectorsCoords.insert(*closestRefVectorCoords);
+		searchedRefVectorsPosition.insert(*closestRefVectorPosition);
 	}
 
 	return res;

src/similarity/features/SimilarityFeaturesSearcher.hpp

@@ -24,6 +24,7 @@
 
 #include "database/DatabaseHandler.hpp"
 #include "database/Types.hpp"
+#include "som/DataNormalizer.hpp"
 #include "som/Network.hpp"
 
 namespace Similarity {
@@ -32,7 +33,10 @@ class FeaturesSearcher
 {
 	public:
 
-		FeaturesSearcher(Wt::Dbo::Session& session);
+		bool init(Wt::Dbo::Session& session, bool& stopRequested);
+		bool initFromCache(Wt::Dbo::Session& session);
+
+		static void invalidateCache();
 
 		std::vector<Database::IdType> getSimilarTracks(const std::set<Database::IdType>& tracksId, std::size_t maxCount) const;
 		std::vector<Database::IdType> getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const;
@@ -42,22 +46,29 @@ class FeaturesSearcher
 
 	private:
 
+		void init(Wt::Dbo::Session& session,
+				SOM::Network network,
+				std::map<Database::IdType, std::set<SOM::Position>> tracksPosition);
+
+		void saveToCache() const;
+		void clearCache() const;
+
 		std::vector<Database::IdType> getSimilarObjects(const std::set<Database::IdType>& ids,
 				const SOM::Matrix<std::set<Database::IdType>>& objectsMap,
-				const std::map<Database::IdType, std::set<SOM::Coords>>& objectCoords,
+				const std::map<Database::IdType, std::set<SOM::Position>>& objectPosition,
 				std::size_t maxCount) const;
 
-		std::unique_ptr<SOM::Network>	_network;
+		SOM::Network	_network;
-		double				_networkRefVectorsDistanceMedian = 0;
+		double		_networkRefVectorsDistanceMedian = 0;
 
 		SOM::Matrix<std::set<Database::IdType>> 		_artistsMap;
-		std::map<Database::IdType, std::set<SOM::Coords>>	_artistCoords;
+		std::map<Database::IdType, std::set<SOM::Position>>	_artistPosition;
 
 		SOM::Matrix<std::set<Database::IdType>> 		_releasesMap;
-		std::map<Database::IdType, std::set<SOM::Coords>>	_releaseCoords;
+		std::map<Database::IdType, std::set<SOM::Position>>	_releasePosition;
 
 		SOM::Matrix<std::set<Database::IdType>> 		_tracksMap;
-		std::map<Database::IdType, std::set<SOM::Coords>>	_trackCoords;
+		std::map<Database::IdType, std::set<SOM::Position>>	_trackPosition;
 
 };
 

src/similarity/features/som/DataNormalizer.cpp

@@ -50,6 +50,18 @@ DataNormalizer::DataNormalizer(std::size_t inputDimCount)
 {
 }
 
+const DataNormalizer::MinMax&
+DataNormalizer::getValue(std::size_t index) const
+{
+	return _minmax[index];
+}
+
+void
+DataNormalizer::setValue(std::size_t index, const MinMax& minMax)
+{
+	_minmax[index] = minMax;
+}
+
 void
 DataNormalizer::computeNormalizationFactors(const std::vector<InputVector>& inputVectors)
 {

src/similarity/features/som/DataNormalizer.hpp

@@ -31,28 +31,31 @@ class DataNormalizer
 {
 	public:
 
+		struct MinMax
+		{
+			InputVector::value_type min;
+			InputVector::value_type max;
+		};
+
 		DataNormalizer(std::size_t inputDimCount);
 
+		std::size_t getInputDimCount() const { return _inputDimCount; }
+		const MinMax& getValue(std::size_t index) const;
+
+		void setValue(std::size_t index, const MinMax& minMax);
+
 		void computeNormalizationFactors(const std::vector<InputVector>& dataSamples);
 
 		void normalizeData(InputVector& data) const;
 
-		std::string serializeTo() const;
-
 		void dump(std::ostream& os) const;
 
 	private:
-		void serializeFrom(const std::string& data);
 		InputVector::value_type normalizeValue(InputVector::value_type value, std::size_t dimensionId) const;
 
 		std::size_t _inputDimCount;
 
-		struct minmax
+		std::vector<MinMax> _minmax; // Indexed min/max used to normalize data
-		{
-			InputVector::value_type min;
-			InputVector::value_type max;
-		};
-		std::vector<minmax> _minmax; // Indexed min/max used to normalize data
 };
 
 } // namespace SOM

src/similarity/features/som/Matrix.hpp

@@ -27,12 +27,14 @@
 namespace SOM
 {
 
-struct Coords
+using Coordinate = unsigned;
-{
-	std::size_t x;
-	std::size_t y;
 
-	bool operator<(const Coords& other) const
+struct Position
+{
+	Coordinate x;
+	Coordinate y;
+
+	bool operator<(const Position& other) const
 	{
 		if (x == other.x)
 			return y < other.y;
@@ -40,7 +42,7 @@ struct Coords
 			return x < other.x;
 	}
 
-	bool operator==(const Coords& other) const
+	bool operator==(const Position& other) const
 	{
 		return x == other.x && y == other.y;
 	}
@@ -53,7 +55,7 @@ class Matrix
 
 		Matrix() = default;
 
-		Matrix(std::size_t width, std::size_t height)
+		Matrix(Coordinate width, Coordinate height)
 		: _width(width),
 		_height(height)
 		{
@@ -74,42 +76,42 @@ class Matrix
 			_values.swap(values);
 		}
 
-		std::size_t getHeight() const { return _height; }
+		Coordinate getHeight() const { return _height; }
-		std::size_t getWidth() const { return _width; }
+		Coordinate getWidth() const { return _width; }
 
-		T& get(Coords coords)
+		T& get(const Position& position)
 		{
-			assert(coords.x < _width);
+			assert(position.x < _width);
-			assert(coords.y < _height);
+			assert(position.y < _height);
-			return _values[coords.x + _width*coords.y];
+			return _values[position.x + _width*position.y];
 		}
 
-		const T& get(Coords coords) const
+		const T& get(const Position& position) const
 		{
-			assert(coords.x < _width);
+			assert(position.x < _width);
-			assert(coords.y < _height);
+			assert(position.y < _height);
-			return _values[coords.x + _width*coords.y];
+			return _values[position.x + _width*position.y];
 		}
 
-		T& operator[](Coords coords) { return get(coords); }
+		T& operator[](const Position& position) { return get(position); }
-		const T& operator[](Coords coords) const { return get(coords); }
+		const T& operator[](const Position& position) const { return get(position); }
 
 		template <typename Func>
-		Coords getCoordsMinElement(Func func) const
+		Position getPositionMinElement(Func func) const
 		{
 			assert(!_values.empty());
 
 			auto it = std::min_element(_values.begin(), _values.end(), func);
-			auto index = std::distance(_values.begin(), it);
+			auto index = static_cast<Coordinate>(std::distance(_values.begin(), it));
 
 			return {index % _height, index / _height};
 		}
 
 	private:
 
-		std::size_t _width = 0;
+		Coordinate	_width = 0;
-		std::size_t _height = 0;
+		Coordinate	_height = 0;
-		std::vector<T> _values;
+		std::vector<T>	_values;
 };
 
 } // ns SOM

src/similarity/features/som/Network.cpp

@@ -44,21 +44,21 @@ checkSameDimensions(const InputVector& a, std::size_t inputDimCount)
 		throw SOMException("Bad data dimension count");
 }
 
-static InputVector::value_type
+static FeatureType
 defaultLearningFactor(Network::CurrentIteration iteration)
 {
-	constexpr InputVector::value_type initialValue = 1;
+	constexpr FeatureType initialValue = 1;
 
-	return initialValue * exp(-((iteration.idIteration + 1) / static_cast<InputVector::value_type>(iteration.iterationCount)));
+	return initialValue * exp(-((iteration.idIteration + 1) / static_cast<FeatureType>(iteration.iterationCount)));
 }
 
-static InputVector::value_type
+static FeatureType
 euclidianSquareDistance(const InputVector& a, const InputVector& b, const InputVector& weights)
 {
 	checkSameDimensions(a, b);
 	checkSameDimensions(a, weights);
 
-	InputVector::value_type res = 0;
+	FeatureType res = 0;
 
 	for (std::size_t i = 0; i < a.size(); ++i)
 	{
@@ -69,17 +69,17 @@ euclidianSquareDistance(const InputVector& a, const InputVector& b, const InputV
 }
 
 static
-InputVector::value_type
+FeatureType
 sigmaFunc(Network::CurrentIteration iteration)
 {
-	constexpr InputVector::value_type sigma0 = 1;
+	constexpr FeatureType sigma0 = 1;
 
-	return sigma0 * exp(- ((iteration.idIteration + 1) / static_cast<InputVector::value_type>(iteration.iterationCount)));
+	return sigma0 * std::exp(- ((iteration.idIteration + 1) / static_cast<FeatureType>(iteration.iterationCount)));
 }
 
 static
-InputVector::value_type
+FeatureType
-defaultNeighbourhoodFunc(InputVector::value_type norm, Network::CurrentIteration iteration)
+defaultNeighbourhoodFunc(FeatureType norm, Network::CurrentIteration iteration)
 {
 	auto sigma = sigmaFunc(iteration);
 
@@ -102,17 +102,15 @@ operator<<(std::ostream& os, const InputVector& a)
 
 
 static
-InputVector::value_type
+FeatureType
 norm(const InputVector& a)
 {
-	InputVector::value_type res = 0;
+	FeatureType res = 0;
 
-	for (const auto& val : a)
+	for (auto val : a)
-	{
 		res += val * val;
-	}
 
-	return sqrt(res);
+	return std::sqrt(res);
 }
 
 static
@@ -121,12 +119,11 @@ operator+(const InputVector& a, const InputVector& b)
 {
 	checkSameDimensions(a, b);
 
-	InputVector res(a.size(), 0);
+	InputVector res;
+	res.reserve(a.size());
 
 	for (std::size_t dimId = 0; dimId < a.size(); ++dimId)
-	{
+		res.push_back(a[dimId] + b[dimId]);
-		res[dimId] = a[dimId] + b[dimId];
-	}
 
 	return res;
 }
@@ -137,34 +134,32 @@ operator-(const InputVector& a, const InputVector& b)
 {
 	checkSameDimensions(a, b);
 
-	InputVector res(a.size(), 0);
+	InputVector res;
+	res.reserve(a.size());
 
 	for (std::size_t dimId = 0; dimId < a.size(); ++dimId)
-	{
+		res.push_back(a[dimId] - b[dimId]);
-		res[dimId] = a[dimId] - b[dimId];
-	}
 
 	return res;
 }
 
 static
 InputVector
-operator*(const InputVector& a, InputVector::value_type factor)
+operator*(const InputVector& a, FeatureType factor)
 {
-	InputVector res(a.size(), 0);
+	InputVector res;
+	res.reserve(a.size());
 
 	for (std::size_t dimId = 0; dimId < a.size(); ++dimId)
-	{
+		res.push_back(a[dimId] * factor);
-		res[dimId] = a[dimId] * factor;
-	}
 
 	return res;
 }
 
-Network::Network(std::size_t width, std::size_t height, std::size_t inputDimCount)
+Network::Network(Coordinate width, Coordinate height, std::size_t inputDimCount)
 :
 _inputDimCount(inputDimCount),
-_weights(inputDimCount, static_cast<InputVector::value_type>(1)),
+_weights(inputDimCount, static_cast<FeatureType>(1)),
 _refVectors(width, height),
 _distanceFunc(euclidianSquareDistance),
 _learningFactorFunc(defaultLearningFactor),
@@ -174,11 +169,11 @@ _neighbourhoodFunc(defaultNeighbourhoodFunc)
 	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
 
 	// init each vector with a random normalized value
-	std::uniform_real_distribution<InputVector::value_type> dist(0, 1);
+	std::uniform_real_distribution<FeatureType> dist(0, 1);
 
-	for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
 	{
-		for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
 		{
 			auto& refVector = _refVectors.get({x,y});
 			refVector.resize(_inputDimCount);
@@ -196,10 +191,18 @@ Network::setDataWeights(const InputVector& weights)
 	_weights = weights;
 }
 
-double
+void
-Network::getRefVectorsDistance(Coords coords1, Coords coords2) const
+Network::setRefVector(const Position& position, const InputVector& data)
 {
-	return _distanceFunc(_refVectors.get(coords1), _refVectors.get(coords2), _weights);
+	checkSameDimensions(data, _inputDimCount);
+
+	_refVectors[position] = data;
+}
+
+double
+Network::getRefVectorsDistance(const Position& position1, const Position& position2) const
+{
+	return _distanceFunc(_refVectors.get(position1), _refVectors.get(position2), _weights);
 }
 
 double
@@ -207,9 +210,9 @@ Network::computeRefVectorsDistanceMean() const
 {
 	std::vector<double> values;
 	values.reserve(2*_refVectors.getHeight()*_refVectors.getWidth() - _refVectors.getWidth() - _refVectors.getHeight());
-	for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
 	{
-		for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
 		{
 			if (x != _refVectors.getWidth() - 1)
 				values.push_back(getRefVectorsDistance( {x, y}, {x + 1, y}));
@@ -226,9 +229,9 @@ Network::computeRefVectorsDistanceMedian() const
 {
 	std::vector<double> values;
 	values.reserve(2*_refVectors.getHeight()*_refVectors.getWidth() - _refVectors.getWidth() - _refVectors.getHeight());
-	for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
 	{
-		for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
 		{
 			if (x != _refVectors.getWidth() - 1)
 				values.push_back(getRefVectorsDistance( {x, y}, {x + 1, y}));
@@ -245,9 +248,9 @@ Network::dump(std::ostream& os) const
 {
 	os << "Width: " << _refVectors.getWidth() << ", Height: " << _refVectors.getHeight() << std::endl;;
 
-	for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
 	{
-		for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
 		{
 			os << _refVectors.get({x, y}) << " ";
 		}
@@ -257,73 +260,73 @@ Network::dump(std::ostream& os) const
 	os << std::endl;
 }
 
-Coords
+Position
-Network::getClosestRefVectorCoords(const InputVector& data) const
+Network::getClosestRefVectorPosition(const InputVector& data) const
 {
-	return _refVectors.getCoordsMinElement([&](const auto& a, const auto& b)
+	return _refVectors.getPositionMinElement([&](const auto& a, const auto& b)
 			{
 				return (_distanceFunc(a, data, _weights) < _distanceFunc(b, data, _weights));
 			});
 }
 
-boost::optional<Coords>
+boost::optional<Position>
-Network::getClosestRefVectorCoords(const InputVector& data, double maxDistance) const
+Network::getClosestRefVectorPosition(const InputVector& data, double maxDistance) const
 {
-	Coords coords = _refVectors.getCoordsMinElement([&](const auto& a, const auto& b)
+	Position position = _refVectors.getPositionMinElement([&](const auto& a, const auto& b)
 			{
 				return (_distanceFunc(a, data, _weights) < _distanceFunc(b, data, _weights));
 			});
 
-	if (_distanceFunc(data, _refVectors.get(coords), _weights) > maxDistance)
+	if (_distanceFunc(data, _refVectors.get(position), _weights) > maxDistance)
 		return boost::none;
 
-	return coords;
+	return position;
 }
 
-boost::optional<Coords>
+boost::optional<Position>
-Network::getClosestRefVectorCoords(const std::set<Coords>& refVectorsCoords, double maxDistance) const
+Network::getClosestRefVectorPosition(const std::set<Position>& refVectorsPosition, double maxDistance) const
 {
-	std::set<Coords> neighboursCoords;
+	std::set<Position> neighboursPosition;
-	for (const Coords& refVectorCoords : refVectorsCoords)
+	for (const Position& refVectorPosition : refVectorsPosition)
 	{
-		if (refVectorCoords.y > 0)
+		if (refVectorPosition.y > 0)
-			neighboursCoords.insert({ refVectorCoords.x, refVectorCoords.y - 1 });
+			neighboursPosition.insert({ refVectorPosition.x, refVectorPosition.y - 1 });
-		if (refVectorCoords.y < _refVectors.getHeight() - 1)
+		if (refVectorPosition.y < _refVectors.getHeight() - 1)
-			neighboursCoords.insert({ refVectorCoords.x, refVectorCoords.y + 1 });
+			neighboursPosition.insert({ refVectorPosition.x, refVectorPosition.y + 1 });
-		if (refVectorCoords.x > 0)
+		if (refVectorPosition.x > 0)
-			neighboursCoords.insert({ refVectorCoords.x - 1, refVectorCoords.y });
+			neighboursPosition.insert({ refVectorPosition.x - 1, refVectorPosition.y });
-		if (refVectorCoords.x < _refVectors.getWidth() - 1)
+		if (refVectorPosition.x < _refVectors.getWidth() - 1)
-			neighboursCoords.insert({ refVectorCoords.x + 1, refVectorCoords.y });
+			neighboursPosition.insert({ refVectorPosition.x + 1, refVectorPosition.y });
 	}
 
-	// remove coords that are in the input coords
+	// remove position that are in the input position
-	for (const auto& refVectorCoords : refVectorsCoords)
+	for (const auto& refVectorPosition : refVectorsPosition)
-		neighboursCoords.erase(refVectorCoords);
+		neighboursPosition.erase(refVectorPosition);
 
-	if (neighboursCoords.empty())
+	if (neighboursPosition.empty())
 		return boost::none;
 
 	// Now compute the distance for each neighbour
 	struct NeighbourInfo
 	{
-		Coords coords;
+		Position position;
 		double distance;
 	};
 
 	std::vector<NeighbourInfo> neighboursInfo;
-	for (const Coords& neighbourCoords : neighboursCoords)
+	for (const Position& neighbourPosition : neighboursPosition)
 	{
-		auto min = std::min_element(refVectorsCoords.begin(), refVectorsCoords.end(),
+		auto min = std::min_element(refVectorsPosition.begin(), refVectorsPosition.end(),
-				[this, neighbourCoords](const auto& a, const auto& b)
+				[this, neighbourPosition](const auto& a, const auto& b)
 				{
-					return (this->getRefVectorsDistance(a, neighbourCoords) < this->getRefVectorsDistance(b, neighbourCoords));
+					return (this->getRefVectorsDistance(a, neighbourPosition) < this->getRefVectorsDistance(b, neighbourPosition));
 				});
 
-		double distance = getRefVectorsDistance(neighbourCoords, *min);
+		double distance = getRefVectorsDistance(neighbourPosition, *min);
 		if (distance > maxDistance)
 			continue;
 
-		neighboursInfo.push_back({neighbourCoords, distance});
+		neighboursInfo.push_back({neighbourPosition, distance});
 	}
 
 	if (neighboursInfo.empty())
@@ -336,65 +339,82 @@ Network::getClosestRefVectorCoords(const std::set<Coords>& refVectorsCoords, dou
 		});
 
 
-	return min->coords;
+	return min->position;
 }
 
-static InputVector::value_type
+static FeatureType
-computeCoordsNorm(Coords c1, Coords c2)
+computePositionNorm(Position c1, Position c2)
 {
-	std::vector<InputVector::value_type> a = { static_cast<InputVector::value_type>(c1.x), static_cast<InputVector::value_type>(c1.y) };
+	std::vector<FeatureType> a { static_cast<FeatureType>(c1.x), static_cast<FeatureType>(c1.y) };
-	std::vector<InputVector::value_type> b = { static_cast<InputVector::value_type>(c2.x), static_cast<InputVector::value_type>(c2.y) };
+	std::vector<FeatureType> b { static_cast<FeatureType>(c2.x), static_cast<FeatureType>(c2.y) };
 
 	return norm(a - b);
 }
 
 
 void
-Network::updateRefVectors(Coords closestRefVectorCoords, const InputVector& input, CurrentIteration iteration)
+Network::updateRefVectors(const Position& closestRefVectorPosition, const InputVector& input, FeatureType learningFactor, const CurrentIteration& iteration)
 {
-	for (std::size_t y = 0; y < _refVectors.getHeight(); ++y)
+	for (Coordinate y = 0; y < _refVectors.getHeight(); ++y)
 	{
-		for (std::size_t x = 0; x < _refVectors.getWidth(); ++x)
+		for (Coordinate x = 0; x < _refVectors.getWidth(); ++x)
 		{
 			auto& refVector = _refVectors.get({x, y});
 
 			auto delta = input - refVector;
-			auto n = computeCoordsNorm({x, y}, closestRefVectorCoords);
+			auto n = computePositionNorm({x, y}, closestRefVectorPosition);
 
-			auto oldRefVector = refVector;
+			refVector = refVector + delta * (learningFactor * _neighbourhoodFunc(n, iteration));
-			refVector = refVector + delta * (_learningFactorFunc(iteration) * _neighbourhoodFunc(n, iteration));
 		}
 	}
 }
 
 void
-Network::train(const std::vector<InputVector>& inputData, std::size_t nbIterations)
+Network::train(const std::vector<InputVector>& inputData, std::size_t nbIterations, ProgressCallback progressCallback, RequestStopCallback requestStopCallback)
 {
-
+	bool stopRequested{false};
 	std::vector<const InputVector*> inputDataShuffled;
 	inputDataShuffled.reserve(inputData.size());
 
 	for (const auto& input : inputData)
-	{
 		inputDataShuffled.push_back(&input);
-	}
 
 	auto now = std::chrono::system_clock::now();
 	std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
 
 	for (std::size_t i = 0; i < nbIterations; ++i)
 	{
+		CurrentIteration curIter{i, nbIterations};
+
+		if (progressCallback)
+			progressCallback(curIter);
+
 		std::shuffle(inputDataShuffled.begin(), inputDataShuffled.end(), randGenerator);
 
-		for (auto input : inputDataShuffled)
+		const auto learningFactor = _learningFactorFunc(curIter);
-		{
-			Coords closestRefVectorCoords = getClosestRefVectorCoords(*input);
 
-			updateRefVectors(closestRefVectorCoords, *input, {i, nbIterations});
+		for (const InputVector* input : inputDataShuffled)
+		{
+			if (requestStopCallback)
+			{
+				stopRequested = requestStopCallback();
+				break;
+			}
+
+			updateRefVectors(getClosestRefVectorPosition(*input), *input, learningFactor, curIter);
 		}
+
+		if (stopRequested)
+			break;
 	}
 }
 
+const InputVector&
+Network::getRefVector(const Position& position) const
+{
+	return _refVectors[position];
+}
+
 
 } // namespace SOM
 

src/similarity/features/som/Network.hpp

@@ -33,7 +33,8 @@
 namespace SOM
 {
 
-using InputVector = std::vector<double>;
+using FeatureType = double;
+using InputVector = std::vector<FeatureType>;
 void checkSameDimensions(const InputVector& a, const InputVector& b);
 void checkSameDimensions(const InputVector& a, std::size_t inputDimCount);
 std::ostream& operator<<(std::ostream& os, const InputVector& a);
@@ -49,27 +50,42 @@ class Network
 {
 	public:
 
+		Network() = default;
+
 		// Init a network with random values
-		Network(std::size_t width, std::size_t height, std::size_t inputDimCount);
+		Network(Coordinate width, Coordinate height, std::size_t inputDimCount);
 
 		// Init a network with serialized values
 		Network(const std::string& data);
 
 		std::size_t getWidth() const { return _refVectors.getWidth(); }
 		std::size_t getHeight() const { return _refVectors.getHeight(); }
-		std::size_t getInputDimCount() const {return _inputDimCount;}
+		std::size_t getInputDimCount() const { return _inputDimCount; }
+		const InputVector& getDataWeights() const { return _weights; }
+
 		// Set weight for each dimension (default is 1 for each weight)
 		void setDataWeights(const InputVector& weights);
 
+		// use this to manually construct a network without training
+		void setRefVector(const Position& position, const InputVector& data);
+
 		// <!> data must be normalized
-		void train(const std::vector<InputVector>& dataSamples, std::size_t nbIterations);
+		struct CurrentIteration
+		{
+			std::size_t idIteration;
+			std::size_t iterationCount;
+		};
+		using ProgressCallback = std::function<void(const CurrentIteration&)>;
+		using RequestStopCallback = std::function<bool()>;
+		void train(const std::vector<InputVector>& dataSamples, std::size_t nbIterations, ProgressCallback = ProgressCallback{}, RequestStopCallback = RequestStopCallback{});
 
-		Coords getClosestRefVectorCoords(const InputVector& data) const;
+		const InputVector& getRefVector(const Position& position) const;
-		boost::optional<Coords> getClosestRefVectorCoords(const InputVector& data, double maxDistance) const;
+		Position getClosestRefVectorPosition(const InputVector& data) const;
+		boost::optional<Position> getClosestRefVectorPosition(const InputVector& data, double maxDistance) const;
 
-		boost::optional<Coords> getClosestRefVectorCoords(const std::set<Coords>& refVectorsCoords, double maxDistance) const;
+		boost::optional<Position> getClosestRefVectorPosition(const std::set<Position>& refVectorsPosition, double maxDistance) const;
 
-		double getRefVectorsDistance(Coords coords1, Coords coords2) const;
+		double getRefVectorsDistance(const Position& position1, const Position& position2) const;
 
 		double computeRefVectorsDistanceMean() const;
 		double computeRefVectorsDistanceMedian() const;
@@ -80,26 +96,20 @@ class Network
 		// i is the current iteration
 		// refVector(i+1) = refVector(i) + LearningFactor(i) * NeighbourhoodFunc(i) * (MatchingRefVector - refVector)
 
-		using DistanceFunc = std::function<InputVector::value_type(const InputVector& /* a */, const InputVector& /* b */, const InputVector& /* weights */)>;
+		using DistanceFunc = std::function<FeatureType(const InputVector& /* a */, const InputVector& /* b */, const InputVector& /* weights */)>;
 		void setDistanceFunc(DistanceFunc distanceFunc);
 
-		struct CurrentIteration
+		using LearningFactorFunc = std::function<FeatureType(const CurrentIteration&)>;
-		{
-			std::size_t idIteration;
-			std::size_t iterationCount;
-		};
-
-		using LearningFactorFunc = std::function<InputVector::value_type(CurrentIteration)>;
 		void setLearningFactorFunc(LearningFactorFunc learningFactorFunc);
 
-		using NeighbourhoodFunc = std::function<InputVector::value_type(InputVector::value_type /* norm(Coords - CoordMatchingRefVector) */, CurrentIteration)>;
+		using NeighbourhoodFunc = std::function<FeatureType(FeatureType /* norm(Position - CoordMatchingRefVector) */, const CurrentIteration&)>;
 		void setNeighbourhoodFunc(NeighbourhoodFunc neighbourhoodFunc);
 
 	private:
 
-		void updateRefVectors(Coords closestRefVectorCoords, const InputVector& input, CurrentIteration iteration);
+		void updateRefVectors(const Position& closestRefVectorPosition, const InputVector& input, FeatureType learningFactor, const CurrentIteration& iteration);
 
-		std::size_t _inputDimCount;
+		std::size_t _inputDimCount = 0;
 		InputVector _weights;	// weight for each dimension
 		Matrix<InputVector> _refVectors;
 

src/utils/Path.cpp

@@ -37,7 +37,7 @@ boost::filesystem::path searchExecPath(std::string filename)
 		throw LmsException("Environment variable PATH not found");
 
 	std::string result;
-	typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
+	using tokenizer = boost::tokenizer<boost::char_separator<char>>;
 	boost::char_separator<char> sep(":");
 	tokenizer tok(path, sep);
 	for (tokenizer::iterator it = tok.begin(); it != tok.end(); ++it)
@@ -53,11 +53,10 @@ boost::filesystem::path searchExecPath(std::string filename)
 	return result;
 }
 
-typedef boost::crc_32_type crc_type;
-
 void computeCrc(const boost::filesystem::path& p, std::vector<unsigned char>& crc)
 {
-	crc_type	result;
+	using crc_type = boost::crc_32_type;
+	crc_type result;
 
 	std::ifstream  ifs( p.string().c_str(), std::ios_base::binary );
 
@@ -78,7 +77,6 @@ void computeCrc(const boost::filesystem::path& p, std::vector<unsigned char>& cr
 		throw LmsException("Failed to open file '" + p.string() + "'" );
 	}
 
-	// Copy back result into the vector
 
 	// Copy the result into a vector of unsigned char
 	const crc_type::value_type checksum = result.checksum();

tools/feature-extractor/LmsFeatureExtractor.cpp

@@ -5,9 +5,6 @@
 #include <chrono>
 #include <random>
 
-#include <curl/curl.h>
-
-
 #include "database/DatabaseHandler.hpp"
 #include "database/Track.hpp"
 #include "database/Artist.hpp"
@@ -17,7 +14,6 @@
 #include "utils/Config.hpp"
 #include "similarity/features/som/DataNormalizer.hpp"
 #include "similarity/features/som/Network.hpp"
-#include "similarity/features/som/AcousticBrainzUtils.hpp"
 
 static
 std::ostream& operator<<(std::ostream& os, const Database::Track::pointer& track)
@@ -73,14 +69,11 @@ int main(int argc, char *argv[])
 {
 	try
 	{
-		constexpr std::size_t width = 10;
+		const std::size_t width = 15;
-		constexpr std::size_t height = 10;
+		const std::size_t height = 15;
-//		constexpr std::size_t nbTracks = 80;
+		const std::size_t nbIterations = 2;
-		constexpr std::size_t nbIterations = 100;
+		const std::size_t nbTracks = 5000;
 
-//		std::vector<std::string> items = { "lowlevel.barkbands.median", "lowlevel.erbbands.median", "lowlevel.melbands.median"};
-//		constexpr std::size_t nbDims = 27 + 40 + 40;
-		//std::vector<std::string> items = { "tonal.hpcp.median"};
 		const std::map<std::string, std::size_t> featuresSettings =
 		{
 //			{ "lowlevel.average_loudness",			1 },
@@ -111,41 +104,20 @@ int main(int argc, char *argv[])
 		std::cout << "Getting all features..." << std::endl;
 		Wt::Dbo::Transaction transaction(db.getSession());
 
-		auto tracks = Database::Track::getAll(db.getSession());
+		auto tracks = Database::Track::getAllWithFeatures(db.getSession());
-
-		std::vector<Database::Track::pointer> trainingTracks;
-		for (auto track : tracks)
-		{
-			if (track->getMBID().empty())
-				continue;
-
-			if (!track->hasTrackFeatures())
-			{
-				std::string features = AcousticBrainz::extractLowLevelFeatures(track->getMBID());
-
-				if (features.empty())
-					continue;
-
-				Database::TrackFeatures::create(db.getSession(), track, features);
-			}
-
-			trainingTracks.push_back(track);
-		}
 
 		std::cout << "Getting all features DONE" << std::endl;
 
 /*		auto now = std::chrono::system_clock::now();
 		std::mt19937 randGenerator(std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count());
-		std::shuffle(trainingTracks.begin(), trainingTracks.end(), randGenerator);
+		std::shuffle(tracks.begin(), tracks.end(), randGenerator);
-
-		trainingTracks.resize(nbTracks);
 */
-		std::cout << "Getting all features DONE" << std::endl;
+		tracks.resize(nbTracks);
 
 		std::cout << "Reading features..." << std::endl;
 		std::vector< std::vector<double> > tracksFeatures;
 
-		for (auto track : trainingTracks)
+		for (auto track : tracks)
 		{
 			auto features = getTrackFeatures(db.getSession(), track, featuresSettings);
 
@@ -188,10 +160,11 @@ int main(int argc, char *argv[])
 		auto medianDistance = network.computeRefVectorsDistanceMedian();
 		std::cout << "MEDIAN distance = " << medianDistance << std::endl;
 
+#if 0
 		std::cout << "Classifying tracks..." << std::endl;
 
 		SOM::Matrix< std::vector<Database::Track::pointer> > tracksMap(width, height);
-		for (auto track : trainingTracks)
+		for (auto track : tracks)
 		{
 			auto features = getTrackFeatures(db.getSession(), track, featuresSettings);
 
@@ -200,17 +173,17 @@ int main(int argc, char *argv[])
 
 			normalizer.normalizeData(features);
 
-			auto coords = network.getClosestRefVectorCoords(features);
+			auto position = network.getClosestRefVectorPosition(features);
-			tracksMap[coords].push_back(track);
+			tracksMap[position].push_back(track);
 		}
 
 		std::cout << "Classifying tracks DONE" << std::endl;
 
 		// Dump tracks
 
-		for (std::size_t y = 0; y < tracksMap.getHeight(); ++y)
+		for (SOM::Coordinate y = 0; y < tracksMap.getHeight(); ++y)
 		{
-			for (std::size_t x = 0; x < tracksMap.getWidth(); ++x)
+			for (SOM::Coordinate x = 0; x < tracksMap.getWidth(); ++x)
 			{
 				std::cout << "{" << x << ", " << y << "}" << std::endl;
 				const auto& tracks = tracksMap[{x, y}];
@@ -223,7 +196,7 @@ int main(int argc, char *argv[])
 		}
 
 		// For each track, get the nearest tracks
-		for (auto track : trainingTracks)
+		for (auto track : tracks)
 		{
 			auto features = getTrackFeatures(db.getSession(), track, featuresSettings);
 
@@ -232,27 +205,28 @@ int main(int argc, char *argv[])
 
 			normalizer.normalizeData(features);
 
-			auto refVectorCoords = network.getClosestRefVectorCoords(features);
+			auto refVectorPosition = network.getClosestRefVectorPosition(features);
 
-			std::cout << "Getting nearest songs for track " << track << " in {" << refVectorCoords.x << ", " << refVectorCoords.y << "}:" << std::endl;
+			std::cout << "Getting nearest songs for track " << track << " in {" << refVectorPosition.x << ", " << refVectorPosition.y << "}:" << std::endl;
-			for (auto similarTrack : tracksMap[refVectorCoords])
+			for (auto similarTrack : tracksMap[refVectorPosition])
 				std::cout << " - " << similarTrack << std::endl;
 
-			std::set<SOM::Coords> neighbourCoords = {refVectorCoords};
+			std::set<SOM::Position> neighbourPosition = {refVectorPosition};
 			for (std::size_t i = 0; i < 5; ++i)
 			{
-				auto coords = network.getClosestRefVectorCoords(neighbourCoords, medianDistance);
+				auto position = network.getClosestRefVectorPosition(neighbourPosition, medianDistance);
-				if (!coords)
+				if (!position)
 					break;
 
-				std::cout << " - in {" << coords->x << ", " << coords->y << "}, dist = " << network.getRefVectorsDistance(*coords, refVectorCoords) << std::endl;
+				std::cout << " - in {" << position->x << ", " << position->y << "}, dist = " << network.getRefVectorsDistance(*position, refVectorPosition) << std::endl;
-				for (auto similarTrack : tracksMap[*coords])
+				for (auto similarTrack : tracksMap[*position])
 					std::cout << "    - " << similarTrack << std::endl;
 
-				neighbourCoords.insert(*coords);
+				neighbourPosition.insert(*position);
 			}
 
 		}
+#endif
 
 		std::cout << "Classifying tracks DONE" << std::endl;
 	}

tools/feature-extractor/Makefile.am

@@ -12,7 +12,6 @@ lms_feature_extractor_SOURCES = \
 	$(top_srcdir)/src/database/SqlQuery.cpp		\
 	$(top_srcdir)/src/database/Track.cpp		\
 	$(top_srcdir)/src/database/User.cpp		\
-	$(top_srcdir)/src/similarity/features/som/AcousticBrainzUtils.cpp	\
 	$(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp		\
 	$(top_srcdir)/src/similarity/features/som/Network.cpp			\
 	$(top_srcdir)/src/utils/Config.cpp 		\