Fixed bug on training + removed some debug info

src/database/TrackFeatures.cpp

@@ -84,8 +84,7 @@ TrackFeatures::getFeatures(std::map<std::string /*name*/, std::vector<double> /*
 	}
 	catch (boost::property_tree::ptree_error& error)
 	{
-		LMS_LOG(DB, ERROR) << "ptree exception: " << error.what();
+		LMS_LOG(SIMILARITY, ERROR) << "Track " << _track.id() << ": ptree exception: " << error.what();
-		std::cout << "ptree exception: " << error.what() << std::endl;
 		return false;
 	}
 }

src/similarity/features/SimilarityFeaturesScannerAddon.cpp

@@ -123,8 +123,6 @@ FeaturesScannerAddon::updateSearcher()
 		FeaturesCache cache{searcher->toCache()};
 		cache.write();
 
-		searcher->dump(_db.getSession(), std::cout);
-
 		LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated";
 	}
 	else
@@ -139,15 +137,12 @@ FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const std::string&
 
 	LMS_LOG(DBUPDATER, DEBUG) << "Fetching low level features for track '" << MBID << "'";
 	std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)};
-
 	if (data.empty())
 	{
 		LMS_LOG(DBUPDATER, ERROR) << "Cannot extract features using AcousticBrainz!";
 		return false;
 	}
 
-	// TODO check if the expected features are here
-
 	Wt::Dbo::Transaction transaction{_db.getSession()};
 
 	Wt::Dbo::ptr<Database::Track> track {Database::Track::getById(_db.getSession(), trackId)};

src/similarity/features/SimilarityFeaturesSearcher.cpp

@@ -66,9 +66,56 @@ getFeatureInfoMapNbDimensions(const FeatureInfoMap& featureInfoMap)
 	return std::accumulate(featureInfoMap.begin(), featureInfoMap.end(), 0, [](std::size_t sum, auto it) { return sum + it.second.nbDimensions; });
 }
 
+static
+boost::optional<SOM::InputVector>
+getInputVectorFromTrack(const Database::Track::pointer& track, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
+{
+	boost::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
+
+	std::map<std::string, std::vector<double>> features;
+	for (auto itFeatureInfo : featuresInfo)
+		features[itFeatureInfo.first] = {};
+
+	if (!track->getTrackFeatures()->getFeatures(features))
+		return res;
+
+	std::size_t i {};
+	for (const auto& feature : features)
+	{
+		// Check dimensions for each feature
+		auto it {featuresInfo.find(feature.first)};
+		if (it == featuresInfo.end() || it->second.nbDimensions != feature.second.size())
+		{
+			LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << feature.first << "'. Expected " << it->second.nbDimensions << ", got " << feature.second.size();
+			res.reset();
+			break;
+		}
+
+		for (double val : feature.second)
+			(*res)[i++] = val;
+	}
+
+	return res;
+}
+
+static
+SOM::InputVector
+getInputVectorWeights(const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
+{
+	SOM::InputVector weights {nbDimensions};
+	std::size_t index {};
+	for (const auto& featureInfo : featuresInfo)
+	{
+		for (std::size_t i {}; i < featureInfo.second.nbDimensions; ++i)
+			weights[index++] = (1. / featureInfo.second.nbDimensions * featureInfo.second.weight);
+	}
+
+	return weights;
+}
+
 FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested)
 {
-	Wt::Dbo::Transaction transaction(session);
+	Wt::Dbo::Transaction transaction{session};
 
 	FeatureInfoMap featuresInfo {getFeatureInfoMap(session)};
 	std::size_t nbDimensions {getFeatureInfoMapNbDimensions(featuresInfo)};
@@ -82,42 +129,18 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
 	std::vector<SOM::InputVector> samples;
 	std::vector<Database::IdType> tracksIds;
 
+	samples.reserve(tracks.size());
+	tracksIds.reserve(tracks.size());
+
 	LMS_LOG(SIMILARITY, DEBUG) << "Extracting features...";
 	for (const Database::Track::pointer& track : tracks)
 	{
 		if (stopRequested)
 			return;
 
-		SOM::InputVector sample {nbDimensions};
+		boost::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(track, featuresInfo, nbDimensions)};
 
-		std::map<std::string, std::vector<double>> features;
+		samples.emplace_back(std::move(*inputVector));
-		for (auto itFeatureInfo : featuresInfo)
-			features[itFeatureInfo.first] = {};
-
-		if (!track->getTrackFeatures()->getFeatures(features))
-			continue;
-
-		bool ok {true};
-		std::size_t i {};
-		for (const auto& feature : features)
-		{
-			// Check dimensions for each feature
-			auto it {featuresInfo.find(feature.first)};
-			if (it == featuresInfo.end() || it->second.nbDimensions != feature.second.size())
-			{
-				LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << feature.first << "'. Expected " << it->second.nbDimensions << ", got " << feature.second.size();
-				ok = false;
-				break;
-			}
-
-			for (double val : feature.second)
-				sample[i++] = val;
-		}
-
-		if (!ok)
-			continue;
-
-		samples.emplace_back(std::move(sample));
 		tracksIds.emplace_back(track.id());
 	}
 	LMS_LOG(SIMILARITY, DEBUG) << "Extracting features DONE";
@@ -131,27 +154,18 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
 	}
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Normalizing data...";
-	SOM::DataNormalizer dataNormalizer(nbDimensions);
+	SOM::DataNormalizer dataNormalizer {nbDimensions};
 
 	dataNormalizer.computeNormalizationFactors(samples);
 	for (auto& sample : samples)
 		dataNormalizer.normalizeData(sample);
 
-	SOM::InputVector weights {nbDimensions};
-	{
-		std::size_t index {};
-		for (const auto& featureInfo : featuresInfo)
-		{
-			for (std::size_t i {}; i < featureInfo.second.nbDimensions; ++i)
-				weights[index++] = (1. / featureInfo.second.nbDimensions * featureInfo.second.weight);
-		}
-	}
-
 	SOM::Coordinate size {static_cast<SOM::Coordinate>(std::sqrt(samples.size() / 4))};
 	LMS_LOG(SIMILARITY, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network";
 
 	SOM::Network network {size, size, nbDimensions};
-	std::cout << "Weights = '" << weights << "'";
+
+	SOM::InputVector weights {getInputVectorWeights(featuresInfo, nbDimensions)};
 	network.setDataWeights(weights);
 
 	auto progressIndicator{[](const auto& iter)
@@ -175,13 +189,9 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
 		if (stopRequested)
 			return;
 
-		Wt::Dbo::Transaction transaction {session};
+		const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
 
-		const auto& sample = samples[i];
+		trackPositions[tracksIds[i]].insert(position);
-		auto trackId = tracksIds[i];
-		auto position = network.getClosestRefVectorPosition(sample);
-
-		trackPositions[trackId].insert(position);
 	}
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE";
@@ -290,9 +300,9 @@ FeaturesSearcher::init(Wt::Dbo::Session& session,
 	SOM::Coordinate width {_network->getWidth()};
 	SOM::Coordinate height {_network->getHeight()};
 
-	_artistsMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
+	_artistsMap = SOM::Matrix<std::set<Database::IdType>>{width, height};
-	_releasesMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
+	_releasesMap = SOM::Matrix<std::set<Database::IdType>>{width, height};
-	_tracksMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
+	_tracksMap = SOM::Matrix<std::set<Database::IdType>>{width, height};
 
 	Wt::Dbo::Transaction transaction {session};
 
@@ -301,7 +311,7 @@ FeaturesSearcher::init(Wt::Dbo::Session& session,
 		Database::IdType trackId {itTrackCoord.first};
 		const std::set<SOM::Position>& positionSet {itTrackCoord.second};
 
-		auto track {Database::Track::getById(session, trackId)};
+		Database::Track::pointer track {Database::Track::getById(session, trackId)};
 		if (!track)
 			continue;
 

src/similarity/features/som/Network.cpp

@@ -289,8 +289,8 @@ Network::train(const std::vector<InputVector>& inputData, std::size_t nbIteratio
 {
 	bool stopRequested {false};
 	std::vector<const InputVector*> inputDataShuffled;
-	inputDataShuffled.reserve(inputData.size());
 
+	inputDataShuffled.reserve(inputData.size());
 	for (const auto& input : inputData)
 		inputDataShuffled.push_back(&input);
 
@@ -311,16 +311,16 @@ Network::train(const std::vector<InputVector>& inputData, std::size_t nbIteratio
 		for (const InputVector* input : inputDataShuffled)
 		{
 			if (requestStopCallback)
-			{
 				stopRequested = requestStopCallback();
-				break;
+
-			}
+			if (stopRequested)
+				return;
 
 			updateRefVectors(getClosestRefVectorPosition(*input), *input, learningFactor, curIter);
 		}
 
 		if (stopRequested)
-			break;
+			return;
 	}
 }
 

tools/similarity/LmsSimilarity.cpp

@@ -66,9 +66,9 @@ int main(int argc, char *argv[])
 {
 	try
 	{
-		const std::size_t width = 10;
+		const std::size_t width = 5;
-		const std::size_t height = 10;
+		const std::size_t height = 5;
-		const std::size_t nbIterations = 20;
+		const std::size_t nbIterations = 10;
 		std::size_t nbTracks = 5000;
 
 		const std::map<std::string, std::size_t> featuresSettings =
@@ -122,14 +122,19 @@ int main(int argc, char *argv[])
 		SOM::DataNormalizer normalizer {nbDims};
 
 		SOM::InputVector weights {nbDims};
-		for (const auto& featureSettings : featuresSettings)
 		{
-			for (std::size_t i {}; i < featureSettings.second; ++i)
+			std::size_t index {};
-				weights[i] = SOM::InputVector::value_type{1. / featureSettings.second};
+			for (const auto& featureSettings : featuresSettings)
+			{
+				for (std::size_t i {}; i < featureSettings.second; ++i)
+					weights[index++] = SOM::InputVector::value_type{1. / featureSettings.second};
+			}
 		}
 
 		network.setDataWeights(weights);
 
+		std::cout << "Weights: " << weights << std::endl;
+
 		std::cout << "Normalizing..." << std::endl;
 		normalizer.computeNormalizationFactors(tracksFeatures);