From e609b239f3b91e6c1d8ebaaafca4cc7b5f2b1728 Mon Sep 17 00:00:00 2001 From: emeric Date: Sun, 10 Mar 2019 14:13:59 +0100 Subject: [PATCH 1/2] Do not lock the database while retrieving all the track features --- src/database/Track.cpp | 10 +++--- src/database/Track.hpp | 12 +++---- .../features/AcousticBrainzUtils.cpp | 6 ++++ .../SimilarityFeaturesScannerAddon.cpp | 4 +-- .../features/SimilarityFeaturesSearcher.cpp | 36 +++++++++++-------- tools/similarity/LmsSimilarity.cpp | 22 +++++++++--- 6 files changed, 58 insertions(+), 32 deletions(-) diff --git a/src/database/Track.cpp b/src/database/Track.cpp index 5e60049a..db75b0f9 100644 --- a/src/database/Track.cpp +++ b/src/database/Track.cpp @@ -118,17 +118,17 @@ Track::getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session) return std::vector(res.begin(), res.end()); } -std::vector -Track::getAllWithFeatures(Wt::Dbo::Session& session, boost::optional limit) +std::vector +Track::getAllIdsWithFeatures(Wt::Dbo::Session& session, boost::optional limit) { int size {limit ? static_cast(*limit) : -1}; - Wt::Dbo::collection res = session.query - ("SELECT t FROM track t") + Wt::Dbo::collection res = session.query + ("SELECT t.id FROM track t") .where("EXISTS (SELECT * from track_features t_f WHERE t_f.track_id = t.id)") .limit(size); - return std::vector(res.begin(), res.end()); + return std::vector(res.begin(), res.end()); } std::vector diff --git a/src/database/Track.hpp b/src/database/Track.hpp index 663ef9cc..11e7f976 100644 --- a/src/database/Track.hpp +++ b/src/database/Track.hpp @@ -64,13 +64,13 @@ class Track : public Wt::Dbo::Dbo bool& moreExpected); static Wt::Dbo::collection< pointer > getAll(Wt::Dbo::Session& session); - static std::vector getAllIds(Wt::Dbo::Session& session); // nested transaction + static std::vector getAllIds(Wt::Dbo::Session& session); // nested transaction static std::vector getAllPaths(Wt::Dbo::Session& session); // nested transaction - static std::vector getMBIDDuplicates(Wt::Dbo::Session& session); - static std::vector getChecksumDuplicates(Wt::Dbo::Session& session); - static std::vector getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1); - static std::vector getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction - static std::vector getAllWithFeatures(Wt::Dbo::Session& session, boost::optional limit = {}); // nested transaction + static std::vector getMBIDDuplicates(Wt::Dbo::Session& session); + static std::vector getChecksumDuplicates(Wt::Dbo::Session& session); + static std::vector getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1); + static std::vector getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction + static std::vector getAllIdsWithFeatures(Wt::Dbo::Session& session, boost::optional limit = {}); // nested transaction // Create utility static pointer create(Wt::Dbo::Session& session, const boost::filesystem::path& p); diff --git a/src/similarity/features/AcousticBrainzUtils.cpp b/src/similarity/features/AcousticBrainzUtils.cpp index 9580f163..7dfa5eae 100644 --- a/src/similarity/features/AcousticBrainzUtils.cpp +++ b/src/similarity/features/AcousticBrainzUtils.cpp @@ -81,6 +81,12 @@ getJsonData(const std::string& mbid) std::string extractLowLevelFeatures(const std::string& mbid) { + if (boost::filesystem::exists("/storage/emeric/lms-dev/features/" + mbid)) + { + std::ifstream ifs{std::string{"/storage/emeric/lms-dev/features/" + mbid}.c_str()}; + return std::string {std::istreambuf_iterator{ifs}, std::istreambuf_iterator{}}; + } + return getJsonData(mbid); } diff --git a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp b/src/similarity/features/SimilarityFeaturesScannerAddon.cpp index 9c458d52..cda88d8e 100644 --- a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp +++ b/src/similarity/features/SimilarityFeaturesScannerAddon.cpp @@ -106,10 +106,10 @@ void FeaturesScannerAddon::updateSearcher() { Wt::Dbo::Transaction transaction {_db.getSession()}; - auto tracks {Database::Track::getAllWithFeatures(_db.getSession())}; + std::vector trackIds {Database::Track::getAllIdsWithFeatures(_db.getSession())}; transaction.commit(); - if (tracks.empty()) + if (trackIds.empty()) { LMS_LOG(DBUPDATER, INFO) << "No track suitable for features similarity clustering"; std::atomic_store(&_searcher, std::shared_ptr{}); diff --git a/src/similarity/features/SimilarityFeaturesSearcher.cpp b/src/similarity/features/SimilarityFeaturesSearcher.cpp index 8b92738e..93b7a805 100644 --- a/src/similarity/features/SimilarityFeaturesSearcher.cpp +++ b/src/similarity/features/SimilarityFeaturesSearcher.cpp @@ -68,7 +68,7 @@ getFeatureInfoMapNbDimensions(const FeatureInfoMap& featureInfoMap) static boost::optional -getInputVectorFromTrack(const Database::Track::pointer& track, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions) +getInputVectorFromTrack(Wt::Dbo::Session& session, Database::IdType trackId, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions) { boost::optional res {SOM::InputVector {nbDimensions}}; @@ -76,6 +76,12 @@ getInputVectorFromTrack(const Database::Track::pointer& track, const FeatureInfo for (auto itFeatureInfo : featuresInfo) features[itFeatureInfo.first] = {}; + Wt::Dbo::Transaction transaction {session}; + + Database::Track::pointer track {Database::Track::getById(session, trackId)}; + if (!track) + return res; + if (!track->getTrackFeatures()->getFeatures(features)) return res; @@ -115,7 +121,7 @@ getInputVectorWeights(const FeatureInfoMap& featuresInfo, std::size_t nbDimensio FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested) { - Wt::Dbo::Transaction transaction{session}; + Wt::Dbo::Transaction transaction {session}; FeatureInfoMap featuresInfo {getFeatureInfoMap(session)}; std::size_t nbDimensions {getFeatureInfoMapNbDimensions(featuresInfo)}; @@ -123,31 +129,33 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste LMS_LOG(SIMILARITY, DEBUG) << "Features dimension = " << nbDimensions; LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features..."; - auto tracks {Database::Track::getAllWithFeatures(session)}; + std::vector trackIds {Database::Track::getAllIdsWithFeatures(session)}; LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features DONE"; - std::vector samples; - std::vector tracksIds; + transaction.commit(); - samples.reserve(tracks.size()); - tracksIds.reserve(tracks.size()); + std::vector samples; + std::vector samplesTrackIds; + + samples.reserve(trackIds.size()); + samplesTrackIds.reserve(trackIds.size()); LMS_LOG(SIMILARITY, DEBUG) << "Extracting features..."; - for (const Database::Track::pointer& track : tracks) + for (Database::IdType trackId : trackIds) { if (stopRequested) return; - boost::optional inputVector {getInputVectorFromTrack(track, featuresInfo, nbDimensions)}; + boost::optional inputVector {getInputVectorFromTrack(session, trackId, featuresInfo, nbDimensions)}; + if (!inputVector) + continue; samples.emplace_back(std::move(*inputVector)); - tracksIds.emplace_back(track.id()); + samplesTrackIds.emplace_back(trackId); } LMS_LOG(SIMILARITY, DEBUG) << "Extracting features DONE"; - transaction.commit(); - - if (tracksIds.empty()) + if (samples.empty()) { LMS_LOG(SIMILARITY, INFO) << "Nothing to classify!"; return; @@ -191,7 +199,7 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste const SOM::Position position {network.getClosestRefVectorPosition(samples[i])}; - trackPositions[tracksIds[i]].insert(position); + trackPositions[samplesTrackIds[i]].insert(position); } LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE"; diff --git a/tools/similarity/LmsSimilarity.cpp b/tools/similarity/LmsSimilarity.cpp index 011c0124..9c4beec5 100644 --- a/tools/similarity/LmsSimilarity.cpp +++ b/tools/similarity/LmsSimilarity.cpp @@ -100,16 +100,20 @@ int main(int argc, char *argv[]) std::cout << "Getting all features..." << std::endl; Wt::Dbo::Transaction transaction(db.getSession()); - auto tracks = Database::Track::getAllWithFeatures(db.getSession(), nbTracks); + std::vector trackIds {Database::Track::getAllIdsWithFeatures(db.getSession(), nbTracks)}; - nbTracks = tracks.size(); + nbTracks = trackIds.size(); std::cout << "Getting features DONE (" << nbTracks << " tracks)" << std::endl; std::cout << "Reading features..." << std::endl; std::vector tracksFeatures; - for (const auto& track : tracks) + for (Database::IdType trackId : trackIds) { + Database::Track::pointer track {Database::Track::getById(db.getSession(), trackId)}; + if (!track) + continue; + SOM::InputVector features {nbDims}; if (!getTrackFeatures(db.getSession(), track, featuresSettings, features)) continue; @@ -163,8 +167,12 @@ int main(int argc, char *argv[]) std::cout << "Classifying tracks..." << std::endl; SOM::Matrix< std::vector > tracksMap(width, height); - for (auto track : tracks) + for (Database::IdType trackId : trackIds) { + Database::Track::pointer track {Database::Track::getById(db.getSession(), trackId)}; + if (!track) + continue; + SOM::InputVector features {nbDims}; if (!getTrackFeatures(db.getSession(), track, featuresSettings, features)) continue; @@ -194,8 +202,12 @@ int main(int argc, char *argv[]) } // For each track, get the nearest tracks - for (const auto& track : tracks) + for (Database::IdType trackId : trackIds) { + Database::Track::pointer track {Database::Track::getById(db.getSession(), trackId)}; + if (!track) + continue; + SOM::InputVector features {nbDims}; if (!getTrackFeatures(db.getSession(), track, featuresSettings, features)) continue; From 080fa856f4a2671f8683b5d7ccc09da84311b124 Mon Sep 17 00:00:00 2001 From: emeric Date: Sun, 10 Mar 2019 14:21:40 +0100 Subject: [PATCH 2/2] Better debug dump --- src/similarity/features/SimilarityFeaturesSearcher.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/similarity/features/SimilarityFeaturesSearcher.cpp b/src/similarity/features/SimilarityFeaturesSearcher.cpp index 93b7a805..a9c86a1a 100644 --- a/src/similarity/features/SimilarityFeaturesSearcher.cpp +++ b/src/similarity/features/SimilarityFeaturesSearcher.cpp @@ -205,6 +205,7 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE"; init(session, std::move(network), std::move(trackPositions)); + } FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache) @@ -277,12 +278,12 @@ FeaturesSearcher::dump(Wt::Dbo::Session& session, std::ostream& os) const if (!track) continue; - os << "\t - " << track->getName() << " - "; + os << "\t"; if (track->getArtist()) os << track->getArtist()->getName() << " - "; if (track->getRelease()) - os << track->getRelease()->getName(); - os << std::endl; + os << track->getRelease()->getName() << " - "; + os << track->getName() << std::endl; } }