Merge branch 'develop' into multi-artists

src/database/Track.cpp

@@ -118,17 +118,17 @@ Track::getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session)
 	return std::vector<pointer>(res.begin(), res.end());
 }
 
-std::vector<Track::pointer>
+std::vector<IdType>
-Track::getAllWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit)
+Track::getAllIdsWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit)
 {
 	int size {limit ? static_cast<int>(*limit) : -1};
 
-	Wt::Dbo::collection<pointer> res = session.query<pointer>
+	Wt::Dbo::collection<IdType> res = session.query<IdType>
-		("SELECT t FROM track t")
+		("SELECT t.id FROM track t")
 		.where("EXISTS (SELECT * from track_features t_f WHERE t_f.track_id = t.id)")
 		.limit(size);
 
-	return std::vector<pointer>(res.begin(), res.end());
+	return std::vector<IdType>(res.begin(), res.end());
 }
 
 std::vector<Cluster::pointer>

src/database/Track.hpp

@@ -64,13 +64,13 @@ class Track : public Wt::Dbo::Dbo<Track>
 							bool& moreExpected);
 
 		static Wt::Dbo::collection< pointer > getAll(Wt::Dbo::Session& session);
-		static std::vector<IdType> getAllIds(Wt::Dbo::Session& session); // nested transaction
+		static std::vector<IdType>	getAllIds(Wt::Dbo::Session& session); // nested transaction
 		static std::vector<boost::filesystem::path> getAllPaths(Wt::Dbo::Session& session); // nested transaction
-		static std::vector<pointer> getMBIDDuplicates(Wt::Dbo::Session& session);
+		static std::vector<pointer>	getMBIDDuplicates(Wt::Dbo::Session& session);
-		static std::vector<pointer> getChecksumDuplicates(Wt::Dbo::Session& session);
+		static std::vector<pointer>	getChecksumDuplicates(Wt::Dbo::Session& session);
-		static std::vector<pointer> getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1);
+		static std::vector<pointer>	getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1);
-		static std::vector<pointer> getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction
+		static std::vector<pointer>	getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction
-		static std::vector<pointer> getAllWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit = {}); // nested transaction
+		static std::vector<IdType>	getAllIdsWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit = {}); // nested transaction
 
 		// Create utility
 		static pointer	create(Wt::Dbo::Session& session, const boost::filesystem::path& p);

src/similarity/features/AcousticBrainzUtils.cpp

@@ -81,6 +81,12 @@ getJsonData(const std::string& mbid)
 std::string
 extractLowLevelFeatures(const std::string& mbid)
 {
+	if (boost::filesystem::exists("/storage/emeric/lms-dev/features/" + mbid))
+	{
+		std::ifstream ifs{std::string{"/storage/emeric/lms-dev/features/" + mbid}.c_str()};
+		return std::string {std::istreambuf_iterator<char>{ifs}, std::istreambuf_iterator<char>{}};
+	}
+
 	return getJsonData(mbid);
 }
 

src/similarity/features/SimilarityFeaturesScannerAddon.cpp

@@ -106,10 +106,10 @@ void
 FeaturesScannerAddon::updateSearcher()
 {
 	Wt::Dbo::Transaction transaction {_db.getSession()};
-	auto tracks {Database::Track::getAllWithFeatures(_db.getSession())};
+	std::vector<Database::IdType> trackIds {Database::Track::getAllIdsWithFeatures(_db.getSession())};
 	transaction.commit();
 
-	if (tracks.empty())
+	if (trackIds.empty())
 	{
 		LMS_LOG(DBUPDATER, INFO) << "No track suitable for features similarity clustering";
 		std::atomic_store(&_searcher, std::shared_ptr<FeaturesSearcher>{});

src/similarity/features/SimilarityFeaturesSearcher.cpp

@@ -68,7 +68,7 @@ getFeatureInfoMapNbDimensions(const FeatureInfoMap& featureInfoMap)
 
 static
 boost::optional<SOM::InputVector>
-getInputVectorFromTrack(const Database::Track::pointer& track, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
+getInputVectorFromTrack(Wt::Dbo::Session& session, Database::IdType trackId, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
 {
 	boost::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
 
@@ -76,6 +76,12 @@ getInputVectorFromTrack(const Database::Track::pointer& track, const FeatureInfo
 	for (auto itFeatureInfo : featuresInfo)
 		features[itFeatureInfo.first] = {};
 
+	Wt::Dbo::Transaction transaction {session};
+
+	Database::Track::pointer track {Database::Track::getById(session, trackId)};
+	if (!track)
+		return res;
+
 	if (!track->getTrackFeatures()->getFeatures(features))
 		return res;
 
@@ -115,7 +121,7 @@ getInputVectorWeights(const FeatureInfoMap& featuresInfo, std::size_t nbDimensio
 
 FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested)
 {
-	Wt::Dbo::Transaction transaction{session};
+	Wt::Dbo::Transaction transaction {session};
 
 	FeatureInfoMap featuresInfo {getFeatureInfoMap(session)};
 	std::size_t nbDimensions {getFeatureInfoMapNbDimensions(featuresInfo)};
@@ -123,31 +129,33 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
 	LMS_LOG(SIMILARITY, DEBUG) << "Features dimension = " << nbDimensions;
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features...";
-	auto tracks {Database::Track::getAllWithFeatures(session)};
+	std::vector<Database::IdType> trackIds {Database::Track::getAllIdsWithFeatures(session)};
 	LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features DONE";
 
-	std::vector<SOM::InputVector> samples;
+	transaction.commit();
-	std::vector<Database::IdType> tracksIds;
 
-	samples.reserve(tracks.size());
+	std::vector<SOM::InputVector> samples;
-	tracksIds.reserve(tracks.size());
+	std::vector<Database::IdType> samplesTrackIds;
+
+	samples.reserve(trackIds.size());
+	samplesTrackIds.reserve(trackIds.size());
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Extracting features...";
-	for (const Database::Track::pointer& track : tracks)
+	for (Database::IdType trackId : trackIds)
 	{
 		if (stopRequested)
 			return;
 
-		boost::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(track, featuresInfo, nbDimensions)};
+		boost::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(session, trackId, featuresInfo, nbDimensions)};
+		if (!inputVector)
+			continue;
 
 		samples.emplace_back(std::move(*inputVector));
-		tracksIds.emplace_back(track.id());
+		samplesTrackIds.emplace_back(trackId);
 	}
 	LMS_LOG(SIMILARITY, DEBUG) << "Extracting features DONE";
 
-	transaction.commit();
+	if (samples.empty())
-
-	if (tracksIds.empty())
 	{
 		LMS_LOG(SIMILARITY, INFO) << "Nothing to classify!";
 		return;
@@ -191,12 +199,13 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
 
 		const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
 
-		trackPositions[tracksIds[i]].insert(position);
+		trackPositions[samplesTrackIds[i]].insert(position);
 	}
 
 	LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE";
 
 	init(session, std::move(network), std::move(trackPositions));
+
 }
 
 FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache)
@@ -269,12 +278,12 @@ FeaturesSearcher::dump(Wt::Dbo::Session& session, std::ostream& os) const
 				if (!track)
 					continue;
 
-				os << "\t - " << track->getName() << " - ";
+				os << "\t";
 				if (track->getArtist())
 					os << track->getArtist()->getName() << " - ";
 				if (track->getRelease())
-					os << track->getRelease()->getName();
+					os << track->getRelease()->getName() << " - ";
-				os << std::endl;
+				os << track->getName() << std::endl;
 			}
 
 		}

tools/similarity/LmsSimilarity.cpp

@@ -100,16 +100,20 @@ int main(int argc, char *argv[])
 		std::cout << "Getting all features..." << std::endl;
 		Wt::Dbo::Transaction transaction(db.getSession());
 
-		auto tracks = Database::Track::getAllWithFeatures(db.getSession(), nbTracks);
+		std::vector<Database::IdType> trackIds {Database::Track::getAllIdsWithFeatures(db.getSession(), nbTracks)};
 
-		nbTracks = tracks.size();
+		nbTracks = trackIds.size();
 		std::cout << "Getting features DONE (" << nbTracks << " tracks)" << std::endl;
 
 		std::cout << "Reading features..." << std::endl;
 		std::vector<SOM::InputVector> tracksFeatures;
 
-		for (const auto& track : tracks)
+		for (Database::IdType trackId : trackIds)
 		{
+			Database::Track::pointer track {Database::Track::getById(db.getSession(), trackId)};
+			if (!track)
+				continue;
+
 			SOM::InputVector features {nbDims};
 			if (!getTrackFeatures(db.getSession(), track, featuresSettings, features))
 				continue;
@@ -163,8 +167,12 @@ int main(int argc, char *argv[])
 		std::cout << "Classifying tracks..." << std::endl;
 
 		SOM::Matrix< std::vector<Database::Track::pointer> > tracksMap(width, height);
-		for (auto track : tracks)
+		for (Database::IdType trackId : trackIds)
 		{
+			Database::Track::pointer track {Database::Track::getById(db.getSession(), trackId)};
+			if (!track)
+				continue;
+
 			SOM::InputVector features {nbDims};
 			if (!getTrackFeatures(db.getSession(), track, featuresSettings, features))
 				continue;
@@ -194,8 +202,12 @@ int main(int argc, char *argv[])
 		}
 
 		// For each track, get the nearest tracks
-		for (const auto& track : tracks)
+		for (Database::IdType trackId : trackIds)
 		{
+			Database::Track::pointer track {Database::Track::getById(db.getSession(), trackId)};
+			if (!track)
+				continue;
+
 			SOM::InputVector features {nbDims};
 			if (!getTrackFeatures(db.getSession(), track, featuresSettings, features))
 				continue;