From 08988b17a2444412e302afb964743a9b7fc16515 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 13 Jun 2019 14:29:39 +0200 Subject: [PATCH] Prevent the media scanner from never quitting --- src/scanner/MediaScanner.cpp | 12 ++++++ src/scanner/MediaScanner.hpp | 1 + .../features/SimilarityFeaturesCache.cpp | 8 +++- .../SimilarityFeaturesScannerAddon.cpp | 25 +++++++---- .../features/SimilarityFeaturesSearcher.cpp | 41 +++++++++++-------- .../features/SimilarityFeaturesSearcher.hpp | 7 ++-- 6 files changed, 66 insertions(+), 28 deletions(-) diff --git a/src/scanner/MediaScanner.cpp b/src/scanner/MediaScanner.cpp index 59d6f498..fa8a2cf3 100644 --- a/src/scanner/MediaScanner.cpp +++ b/src/scanner/MediaScanner.cpp @@ -420,7 +420,10 @@ MediaScanner::scan(boost::system::error_code err) { for (auto& addon : _addons) addon->preScanComplete(); + } + if (_running) + { stats.stopTime = Wt::WLocalDateTime::currentDateTime().toUTC(); { std::unique_lock lock {_statusMutex}; @@ -471,6 +474,13 @@ MediaScanner::refreshScanSettings() addon->refreshSettings(); } +void MediaScanner::notifyInProgress(Stats& stats) +{ + std::chrono::system_clock::time_point now {std::chrono::system_clock::now()}; + _sigScanInProgress(stats); + _lastScanInProgressEmit = now; +} + void MediaScanner::notifyInProgressIfNeeded(Stats& stats) { std::chrono::system_clock::time_point now {std::chrono::system_clock::now()}; @@ -674,6 +684,8 @@ MediaScanner::scanMediaDirectory(boost::filesystem::path mediaDirectory, bool fo itPath.increment(ec); } + + notifyInProgress(stats); } // Check if a file exists and is still in a media directory diff --git a/src/scanner/MediaScanner.hpp b/src/scanner/MediaScanner.hpp index 904fca4c..9a732c86 100644 --- a/src/scanner/MediaScanner.hpp +++ b/src/scanner/MediaScanner.hpp @@ -121,6 +121,7 @@ class MediaScanner void checkDuplicatedAudioFiles(Stats& stats); void scanAudioFile(const boost::filesystem::path& file, bool forceScan, Stats& stats); void notifyInProgressIfNeeded(Stats& stats); + void notifyInProgress(Stats& stats); bool _running {false}; Wt::WIOService _ioService; diff --git a/src/similarity/features/SimilarityFeaturesCache.cpp b/src/similarity/features/SimilarityFeaturesCache.cpp index 8bfea439..3dc191ee 100644 --- a/src/similarity/features/SimilarityFeaturesCache.cpp +++ b/src/similarity/features/SimilarityFeaturesCache.cpp @@ -95,6 +95,8 @@ createNetworkFromCacheFile(boost::filesystem::path path) { try { + LMS_LOG(SIMILARITY, INFO) << "Reading network from cache..."; + boost::property_tree::ptree root; boost::property_tree::read_xml(path.string(), root); @@ -127,7 +129,7 @@ createNetworkFromCacheFile(boost::filesystem::path path) res.setRefVector({x, y}, refVector); } - LMS_LOG(SIMILARITY, DEBUG) << "Successfully read network from cache"; + LMS_LOG(SIMILARITY, INFO) << "Successfully read network from cache"; return res; } @@ -180,6 +182,8 @@ createObjectPositionsFromCacheFile(boost::filesystem::path path) { try { + LMS_LOG(SIMILARITY, INFO) << "Reading object position from cache..."; + boost::property_tree::ptree root; boost::property_tree::read_xml(path.string(), root); @@ -198,7 +202,7 @@ createObjectPositionsFromCacheFile(boost::filesystem::path path) } } - LMS_LOG(SIMILARITY, DEBUG) << "Successfully read object position from cache"; + LMS_LOG(SIMILARITY, INFO) << "Successfully read object position from cache"; return res; } diff --git a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp b/src/similarity/features/SimilarityFeaturesScannerAddon.cpp index bf29c757..2a51e543 100644 --- a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp +++ b/src/similarity/features/SimilarityFeaturesScannerAddon.cpp @@ -59,7 +59,7 @@ FeaturesScannerAddon::FeaturesScannerAddon(Wt::Dbo::SqlConnectionPool& connectio boost::optional cache {Similarity::FeaturesCache::read()}; if (cache) { - auto searcher {std::make_shared(_db.getSession(), *cache)}; + auto searcher {std::make_shared(_db.getSession(), *cache, [&]() { return _stopRequested; })}; if (searcher->isValid()) std::atomic_store(&_searcher, searcher); } @@ -107,7 +107,12 @@ FeaturesScannerAddon::preScanComplete() LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features DONE (found " << tracksInfo.size() << ")"; for (const TrackInfo& trackInfo : tracksInfo) + { + if (_stopRequested) + return; + fetchFeatures(trackInfo.id, trackInfo.mbid); + } Similarity::FeaturesCache::invalidate(); updateSearcher(); @@ -116,9 +121,13 @@ FeaturesScannerAddon::preScanComplete() void FeaturesScannerAddon::updateSearcher() { - Wt::Dbo::Transaction transaction {_db.getSession()}; - std::vector trackIds {Database::Track::getAllIdsWithFeatures(_db.getSession())}; - transaction.commit(); + LMS_LOG(SIMILARITY, INFO) << "Updating searcher..."; + + std::vector trackIds; + { + Wt::Dbo::Transaction transaction {_db.getSession()}; + trackIds = Database::Track::getAllIdsWithFeatures(_db.getSession()); + } if (trackIds.empty()) { @@ -127,7 +136,7 @@ FeaturesScannerAddon::updateSearcher() return; } - auto searcher {std::make_shared(_db.getSession(), _stopRequested)}; + auto searcher {std::make_shared(_db.getSession(), [&]() { return _stopRequested; })}; if (searcher->isValid()) { std::atomic_store(&_searcher, searcher); @@ -137,8 +146,10 @@ FeaturesScannerAddon::updateSearcher() LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated"; } else + { + LMS_LOG(DBUPDATER, ERROR) << "Cannot set up a valid features similarity searcher!"; std::atomic_store(&_searcher, std::shared_ptr{}); - + } } bool @@ -150,7 +161,7 @@ FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const std::string& std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)}; if (data.empty()) { - LMS_LOG(DBUPDATER, ERROR) << "Cannot extract features using AcousticBrainz!"; + LMS_LOG(DBUPDATER, ERROR) << "Track " << trackId << ", MBID = '" << MBID << "': cannot extract features using AcousticBrainz"; return false; } diff --git a/src/similarity/features/SimilarityFeaturesSearcher.cpp b/src/similarity/features/SimilarityFeaturesSearcher.cpp index f2aeeb7d..941e58f7 100644 --- a/src/similarity/features/SimilarityFeaturesSearcher.cpp +++ b/src/similarity/features/SimilarityFeaturesSearcher.cpp @@ -119,8 +119,10 @@ getInputVectorWeights(const FeatureInfoMap& featuresInfo, std::size_t nbDimensio return weights; } -FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested) +FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, std::function stopRequested) { + LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher..."; + Wt::Dbo::Transaction transaction {session}; FeatureInfoMap featuresInfo {getFeatureInfoMap(session)}; @@ -143,7 +145,7 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste LMS_LOG(SIMILARITY, DEBUG) << "Extracting features..."; for (Database::IdType trackId : trackIds) { - if (stopRequested) + if (stopRequested()) return; boost::optional inputVector {getInputVectorFromTrack(session, trackId, featuresInfo, nbDimensions)}; @@ -181,20 +183,18 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste LMS_LOG(SIMILARITY, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount; }}; - auto stopper{[&]() { return stopRequested; }}; - LMS_LOG(SIMILARITY, DEBUG) << "Training network..."; - network.train(samples, 10, progressIndicator, stopper); + network.train(samples, 10, progressIndicator, stopRequested); LMS_LOG(SIMILARITY, DEBUG) << "Training network DONE"; - if (stopRequested) + if (stopRequested()) return; LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks..."; std::map> trackPositions; for (std::size_t i {}; i < samples.size(); ++i) { - if (stopRequested) + if (stopRequested()) return; const SOM::Position position {network.getClosestRefVectorPosition(samples[i])}; @@ -204,15 +204,18 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE"; - init(session, std::move(network), std::move(trackPositions)); + init(session, std::move(network), std::move(trackPositions), stopRequested); + LMS_LOG(SIMILARITY, INFO) << "Successfully constructed features searcher"; } -FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache) +FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache, std::function stopRequested) { - init(session, std::move(cache._network), std::move(cache._trackPositions)); + LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher from cache..."; - LMS_LOG(SIMILARITY, DEBUG) << "Init from cache DONE"; + init(session, std::move(cache._network), std::move(cache._trackPositions), stopRequested); + + LMS_LOG(SIMILARITY, INFO) << "Successfully constructed features searcher from cache"; } bool @@ -318,14 +321,15 @@ FeaturesSearcher::toCache() const void FeaturesSearcher::init(Wt::Dbo::Session& session, SOM::Network network, - std::map> tracksPosition) + std::map> tracksPosition, + std::function stopRequested) { - _network = std::make_unique(std::move(network)); - _networkRefVectorsDistanceMedian = _network->computeRefVectorsDistanceMedian(); + _networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian(); LMS_LOG(SIMILARITY, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian; - SOM::Coordinate width {_network->getWidth()}; - SOM::Coordinate height {_network->getHeight()}; + SOM::Coordinate width {network.getWidth()}; + SOM::Coordinate height {network.getHeight()}; _artistsMap = SOM::Matrix>{width, height}; _releasesMap = SOM::Matrix>{width, height}; @@ -335,6 +339,9 @@ FeaturesSearcher::init(Wt::Dbo::Session& session, for (auto itTrackCoord : tracksPosition) { + if (stopRequested()) + return; + Wt::Dbo::Transaction transaction {session}; Database::IdType trackId {itTrackCoord.first}; @@ -362,6 +369,8 @@ FeaturesSearcher::init(Wt::Dbo::Session& session, } } + _network = std::make_unique(std::move(network)); + LMS_LOG(SIMILARITY, DEBUG) << "Constructing maps... DONE"; } diff --git a/src/similarity/features/SimilarityFeaturesSearcher.hpp b/src/similarity/features/SimilarityFeaturesSearcher.hpp index 386f3496..9325ec20 100644 --- a/src/similarity/features/SimilarityFeaturesSearcher.hpp +++ b/src/similarity/features/SimilarityFeaturesSearcher.hpp @@ -36,10 +36,10 @@ class FeaturesSearcher public: // Use cache - FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache); + FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache, std::function stopRequested); // Use training (may be very slow) - FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested); + FeaturesSearcher(Wt::Dbo::Session& session, std::function stopRequested); bool isValid() const; @@ -61,7 +61,8 @@ class FeaturesSearcher void init(Wt::Dbo::Session& session, SOM::Network network, - ObjectPositions tracksPosition); + ObjectPositions tracksPosition, + std::function stopRequested); std::vector getSimilarObjects(const std::set& ids, const SOM::Matrix>& objectsMap,