From 8f236c16cae07716a45271798a3a6e6dc2b5fa8a Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 25 Jul 2020 16:42:02 +0200 Subject: [PATCH] Added more details during scan. fixed #32 --- approot/admin-scannercontroller.xml | 1 + approot/messages.xml | 6 +- approot/messages_fr.xml | 6 +- src/libs/scanner/impl/MediaScanner.cpp | 78 +++++++++++++------ src/libs/scanner/impl/MediaScanner.hpp | 13 ++-- src/libs/scanner/impl/MediaScannerStats.cpp | 10 +-- .../scanner/include/scanner/IMediaScanner.hpp | 4 +- .../include/scanner/MediaScannerStats.hpp | 30 ++++--- src/libs/subsonic/impl/Scan.cpp | 11 ++- src/lms/ui/LmsApplication.cpp | 4 +- src/lms/ui/LmsApplication.hpp | 2 +- src/lms/ui/admin/ScannerController.cpp | 35 +++++++-- 12 files changed, 136 insertions(+), 64 deletions(-) diff --git a/approot/admin-scannercontroller.xml b/approot/admin-scannercontroller.xml index 234e298c..484fc64a 100644 --- a/approot/admin-scannercontroller.xml +++ b/approot/admin-scannercontroller.xml @@ -13,6 +13,7 @@
${status} +
${step-status}
diff --git a/approot/messages.xml b/approot/messages.xml index eb64a2f4..e155a61e 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -72,7 +72,11 @@ Status Not scheduled Scheduled on {1} -Scanning {1} of {2} files ({3} %) +Scanning: step {1}/{2} +Checking removed files... {1}% +Discovering files: {1} files +Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)... +Scanning files: {1}/{2} files ({3}%)... New user diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index fa3ffb37..4182355f 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -72,7 +72,11 @@ Statut Non planifié Planifié le {1} -Scan de {1} fichiers sur {2} ({3} %) +En cours de scan : étape {1}/{2} +Vérification des fichiers... {1}% +Découverte des fichiers: {1} fichiers +Récupération des métadonnées AcousticBrainz: {1}/{2} fichiers ({3}%)... +Scan des fichiers: {1}/{2} fichiers ({3}%)... Ajouter diff --git a/src/libs/scanner/impl/MediaScanner.cpp b/src/libs/scanner/impl/MediaScanner.cpp index ccc83e34..70e478dc 100644 --- a/src/libs/scanner/impl/MediaScanner.cpp +++ b/src/libs/scanner/impl/MediaScanner.cpp @@ -264,6 +264,8 @@ MediaScanner::~MediaScanner() void MediaScanner::start() { + std::scoped_lock lock {_controlMutex}; + scheduleNextScan(); _ioService.start(); @@ -272,6 +274,8 @@ MediaScanner::start() void MediaScanner::stop() { + std::scoped_lock lock {_controlMutex}; + _abortScan = true; _scheduleTimer.cancel(); @@ -281,10 +285,15 @@ MediaScanner::stop() void MediaScanner::abortScan() { - _abortScan = true; + LMS_LOG(DBUPDATER, DEBUG) << "Aborting scan..."; + std::scoped_lock lock {_controlMutex}; + LMS_LOG(DBUPDATER, DEBUG) << "Waiting for the scan to abort..."; + + _abortScan = true; _scheduleTimer.cancel(); _ioService.stop(); + LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!"; _abortScan = false; _ioService.start(); @@ -303,6 +312,7 @@ MediaScanner::requestImmediateScan(bool force) void MediaScanner::requestReload() { + abortScan(); _ioService.post([=]() { scheduleNextScan(); @@ -319,7 +329,7 @@ MediaScanner::getStatus() const res.currentState = _curState; res.nextScheduledScan = _nextScheduledScan; res.lastCompleteScanStats = _lastCompleteScanStats; - res.inProgressScanStats = _inProgressScanStats; + res.currentScanStepStats = _currentScanStepStats; return res; } @@ -382,7 +392,10 @@ MediaScanner::scheduleNextScan() void MediaScanner::countAllFiles(ScanStats& stats) { - stats.filesToScan = 0; + ScanStepStats stepStats{stats.startTime, ScanProgressStep::DiscoveringFiles}; + + stats.filesScanned = 0; + notifyInProgress(stepStats); exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) { @@ -391,8 +404,9 @@ MediaScanner::countAllFiles(ScanStats& stats) if (!ec && isFileSupported(path, _fileExtensions)) { - stats.filesToScan++; - notifyInProgressIfNeeded(stats); + stats.filesScanned++; + stepStats.processedFiles++; + notifyInProgressIfNeeded(stepStats); } return true; @@ -445,11 +459,11 @@ MediaScanner::scan(bool forceScan) refreshScanSettings(); + removeMissingTracks(stats); + LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'..."; countAllFiles(stats); - LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesToScan; - - removeMissingTracks(stats); + LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesScanned; LMS_LOG(UI, INFO) << "Checks complete, force scan = " << forceScan; @@ -465,7 +479,7 @@ MediaScanner::scan(bool forceScan) // Now update all the track features if needed fetchTrackFeatures(stats); - LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << "/" << stats.featuresToFetch <<", duplicates = " << stats.duplicates.size(); + LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size(); _dbSession.optimize(); @@ -476,19 +490,22 @@ MediaScanner::scan(bool forceScan) std::unique_lock lock {_statusMutex}; _lastCompleteScanStats = std::move(stats); - _inProgressScanStats.reset(); + _currentScanStepStats.reset(); } + LMS_LOG(DBUPDATER, DEBUG) << "Scan not aborted, scheduling next scan!"; scheduleNextScan(); scanComplete().emit(); } else { + LMS_LOG(DBUPDATER, DEBUG) << "Scan aborted, not scheduling next scan!"; + std::unique_lock lock {_statusMutex}; _curState = State::NotScheduled; - _inProgressScanStats.reset(); + _currentScanStepStats.reset(); } } @@ -524,6 +541,8 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats) if (_recommendationEngineType != ScanSettings::RecommendationEngineType::Features) return; + ScanStepStats stepStats{stats.startTime, ScanProgressStep::FetchingTrackFeatures}; + LMS_LOG(DBUPDATER, INFO) << "Fetching missing track features..."; struct TrackInfo @@ -545,7 +564,8 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats) return res; }()}; - stats.featuresToFetch = tracksToFetch.size(); + stepStats.filesToProcess = tracksToFetch.size(); + notifyInProgress(stepStats); LMS_LOG(DBUPDATER, INFO) << "Found " << tracksToFetch.size() << " track(s) to fetch!"; @@ -557,6 +577,8 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats) if (fetchTrackFeatures(trackToFetch.id, trackToFetch.mbid)) stats.featuresFetched++; + stepStats.processedFiles++; + notifyInProgressIfNeeded(stepStats); } LMS_LOG(DBUPDATER, INFO) << "Track features fetched!"; @@ -596,27 +618,25 @@ MediaScanner::refreshScanSettings() } void -MediaScanner::notifyInProgress(const ScanStats& stats) +MediaScanner::notifyInProgress(const ScanStepStats& stepStats) { - const ScanProgressStats progressStats {stats.toProgressStats()}; - { std::unique_lock lock {_statusMutex}; - _inProgressScanStats = progressStats; + _currentScanStepStats = stepStats; } const std::chrono::system_clock::time_point now {std::chrono::system_clock::now()}; - _sigScanInProgress(progressStats); + _sigScanInProgress(stepStats); _lastScanInProgressEmit = now; } void -MediaScanner::notifyInProgressIfNeeded(const ScanStats& stats) +MediaScanner::notifyInProgressIfNeeded(const ScanStepStats& stepStats) { std::chrono::system_clock::time_point now {std::chrono::system_clock::now()}; - if (std::chrono::duration_cast(now - _lastScanInProgressEmit).count() > 2) - notifyInProgress(stats); + if (std::chrono::duration_cast(now - _lastScanInProgressEmit).count() > 1) + notifyInProgress(stepStats); } void @@ -777,6 +797,10 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S void MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats) { + ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles}; + stepStats.filesToProcess = stats.filesScanned; + notifyInProgress(stepStats); + exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) { if (_abortScan) @@ -791,13 +815,12 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo { scanAudioFile(path, forceScan, stats ); - notifyInProgressIfNeeded(stats); + stepStats.processedFiles++; + notifyInProgressIfNeeded(stepStats); } return true; }); - - notifyInProgress(stats); } // Check if a file exists and is still in a media directory @@ -842,6 +865,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats) { static constexpr std::size_t batchSize {50}; + ScanStepStats stepStats{stats.startTime, ScanProgressStep::CheckingRemovedFiles}; + LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed..."; std::size_t trackCount {}; @@ -851,6 +876,9 @@ MediaScanner::removeMissingTracks(ScanStats& stats) } LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked..."; + stepStats.filesToProcess = trackCount; + notifyInProgress(stepStats); + std::vector> trackPaths; std::vector tracksToRemove; @@ -859,6 +887,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats) trackPaths.clear(); tracksToRemove.clear(); + stepStats.processedFiles++; + { auto transaction {_dbSession.createSharedTransaction()}; trackPaths = Track::getAllPaths(_dbSession, i, batchSize); @@ -888,7 +918,7 @@ MediaScanner::removeMissingTracks(ScanStats& stats) } } - notifyInProgressIfNeeded(stats); + notifyInProgressIfNeeded(stepStats); if (i == 0) break; diff --git a/src/libs/scanner/impl/MediaScanner.hpp b/src/libs/scanner/impl/MediaScanner.hpp index 3a2c737e..0ff66730 100644 --- a/src/libs/scanner/impl/MediaScanner.hpp +++ b/src/libs/scanner/impl/MediaScanner.hpp @@ -59,7 +59,7 @@ class MediaScanner : public IMediaScanner Wt::Signal<>& scanStarted() override { return _sigScanStarted; } Wt::Signal<>& scanComplete() override { return _sigScanComplete; } - Wt::Signal& scanInProgress() override { return _sigScanInProgress; } + Wt::Signal& scanInProgress() override { return _sigScanInProgress; } Wt::Signal& scheduled() override { return _sigScheduled; } private: @@ -86,17 +86,16 @@ class MediaScanner : public IMediaScanner void checkDuplicatedAudioFiles(ScanStats& stats); void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats); Database::IdType doScanAudioFile(const std::filesystem::path& file, ScanStats& stats); - void notifyInProgressIfNeeded(const ScanStats& stats); - void notifyInProgress(const ScanStats& stats); + void notifyInProgressIfNeeded(const ScanStepStats& stats); + void notifyInProgress(const ScanStepStats& stats); - std::mutex _scanInProgress {}; + std::mutex _controlMutex; std::atomic _abortScan {}; - Wt::WIOService _ioService; boost::asio::system_timer _scheduleTimer {_ioService}; Wt::Signal<> _sigScanStarted; Wt::Signal<> _sigScanComplete; - Wt::Signal _sigScanInProgress; + Wt::Signal _sigScanInProgress; std::chrono::system_clock::time_point _lastScanInProgressEmit {}; Wt::Signal _sigScheduled; Database::Session _dbSession; @@ -105,7 +104,7 @@ class MediaScanner : public IMediaScanner mutable std::shared_mutex _statusMutex; State _curState {State::NotScheduled}; std::optional _lastCompleteScanStats; - std::optional _inProgressScanStats; + std::optional _currentScanStepStats; Wt::WDateTime _nextScheduledScan; // Current scan settings diff --git a/src/libs/scanner/impl/MediaScannerStats.cpp b/src/libs/scanner/impl/MediaScannerStats.cpp index a2a984da..29af4235 100644 --- a/src/libs/scanner/impl/MediaScannerStats.cpp +++ b/src/libs/scanner/impl/MediaScannerStats.cpp @@ -40,16 +40,10 @@ ScanStats::nbChanges() const return additions + deletions + updates; } -ScanProgressStats -ScanStats::toProgressStats() const -{ - return ScanProgressStats {startTime, filesToScan, nbFiles()}; -} - unsigned -ScanProgressStats::progress() const +ScanStepStats::progress() const { - return (processedFiles / static_cast(filesToScan ? filesToScan : 1)) * 100; + return (processedFiles / static_cast(filesToProcess ? filesToProcess : 1)) * 100; } } // namespace Scanner diff --git a/src/libs/scanner/include/scanner/IMediaScanner.hpp b/src/libs/scanner/include/scanner/IMediaScanner.hpp index c70358c4..83e675fb 100644 --- a/src/libs/scanner/include/scanner/IMediaScanner.hpp +++ b/src/libs/scanner/include/scanner/IMediaScanner.hpp @@ -58,7 +58,7 @@ class IMediaScanner State currentState {State::NotScheduled}; Wt::WDateTime nextScheduledScan; std::optional lastCompleteScanStats; - std::optional inProgressScanStats; + std::optional currentScanStepStats; }; virtual Status getStatus() const = 0; @@ -70,7 +70,7 @@ class IMediaScanner virtual Wt::Signal<>& scanComplete() = 0; // Called during scan in progress - virtual Wt::Signal& scanInProgress() = 0; + virtual Wt::Signal& scanInProgress() = 0; // Called after a schedule virtual Wt::Signal& scheduled() = 0; diff --git a/src/libs/scanner/include/scanner/MediaScannerStats.hpp b/src/libs/scanner/include/scanner/MediaScannerStats.hpp index 99c898b6..6b05bd95 100644 --- a/src/libs/scanner/include/scanner/MediaScannerStats.hpp +++ b/src/libs/scanner/include/scanner/MediaScannerStats.hpp @@ -56,15 +56,26 @@ namespace Scanner { }; + enum class ScanProgressStep : unsigned + { + CheckingRemovedFiles = 0, + DiscoveringFiles, + ScanningFiles, + FetchingTrackFeatures, + }; + static inline constexpr unsigned ScanProgressStepCount {4}; + // reduced scan stats - struct ScanProgressStats + struct ScanStepStats { Wt::WDateTime startTime; - std::size_t filesToScan {}; - std::size_t processedFiles {}; + ScanProgressStep currentStep; - unsigned progress() const; + std::size_t filesToProcess {}; + std::size_t processedFiles {}; + + unsigned progress() const; }; struct ScanStats @@ -72,25 +83,22 @@ namespace Scanner { Wt::WDateTime startTime; Wt::WDateTime stopTime; - std::size_t filesToScan {}; // Total number of files to be scanned (estimated) + std::size_t filesScanned {}; // Total number of files scanned (estimated) - std::size_t skips {}; // no change since last scan - std::size_t scans {}; // actually scanned filed + std::size_t skips {}; // no change since last scan + std::size_t scans {}; // actually scanned filed std::size_t additions {}; // added in DB std::size_t deletions {}; // removed from DB - std::size_t updates {}; // updated file in DB + std::size_t updates {}; // updated file in DB std::size_t featuresFetched {}; // features fetched in DB - std::size_t featuresToFetch {}; // features to be fetched in DB std::vector errors; std::vector duplicates; std::size_t nbFiles() const; std::size_t nbChanges() const; - - ScanProgressStats toProgressStats() const; }; } diff --git a/src/libs/subsonic/impl/Scan.cpp b/src/libs/subsonic/impl/Scan.cpp index 46231758..b663aed2 100644 --- a/src/libs/subsonic/impl/Scan.cpp +++ b/src/libs/subsonic/impl/Scan.cpp @@ -35,8 +35,15 @@ namespace API::Subsonic::Scan const IMediaScanner::Status scanStatus {ServiceProvider::get()->getStatus()}; statusResponse.setAttribute("scanning", scanStatus.currentState == IMediaScanner::State::InProgress); - if (scanStatus.currentState == IMediaScanner::State::InProgress && scanStatus.inProgressScanStats) - statusResponse.setAttribute("count", scanStatus.inProgressScanStats->processedFiles); + if (scanStatus.currentState == IMediaScanner::State::InProgress) + { + std::size_t count{}; + + if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanProgressStep::ScanningFiles) + count = scanStatus.currentScanStepStats->processedFiles; + + statusResponse.setAttribute("count", count); + } return statusResponse; } diff --git a/src/lms/ui/LmsApplication.cpp b/src/lms/ui/LmsApplication.cpp index c9efc903..3b6edf29 100644 --- a/src/lms/ui/LmsApplication.cpp +++ b/src/lms/ui/LmsApplication.cpp @@ -544,11 +544,11 @@ LmsApplication::createHome() }); }); - ServiceProvider::get()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats) + ServiceProvider::get()->scanInProgress().connect(this, [=] (Scanner::ScanStepStats stepStats) { Wt::WServer::instance()->post(sessionId, [=] { - _events.dbScanInProgress.emit(stats); + _events.dbScanInProgress.emit(stepStats); triggerUpdate(); }); }); diff --git a/src/lms/ui/LmsApplication.hpp b/src/lms/ui/LmsApplication.hpp index 2b3b4534..ad0945ef 100644 --- a/src/lms/ui/LmsApplication.hpp +++ b/src/lms/ui/LmsApplication.hpp @@ -57,7 +57,7 @@ struct Events // Database events Wt::Signal<> dbScanStarted; Wt::Signal<> dbScanned; - Wt::Signal dbScanInProgress; + Wt::Signal dbScanInProgress; Wt::Signal dbScanScheduled; }; diff --git a/src/lms/ui/admin/ScannerController.cpp b/src/lms/ui/admin/ScannerController.cpp index 5bc6cb35..77b9ffb3 100644 --- a/src/lms/ui/admin/ScannerController.cpp +++ b/src/lms/ui/admin/ScannerController.cpp @@ -185,18 +185,43 @@ ScannerController::refreshContents() { case IMediaScanner::State::NotScheduled: bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-not-scheduled")); + bindEmpty("step-status"); break; case IMediaScanner::State::Scheduled: bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-scheduled") .arg(status.nextScheduledScan.toString())); + bindEmpty("step-status"); break; case IMediaScanner::State::InProgress: + bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress") + .arg(static_cast(status.currentScanStepStats->currentStep) + 1) + .arg(Scanner::ScanProgressStepCount)); + + switch (status.currentScanStepStats->currentStep) { - std::ostringstream oss; - bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress") - .arg(status.inProgressScanStats->processedFiles) - .arg(status.inProgressScanStats->filesToScan) - .arg(status.inProgressScanStats->progress())); + case Scanner::ScanProgressStep::CheckingRemovedFiles: + bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-checking-removed-files") + .arg(status.currentScanStepStats->progress())); + break; + + case Scanner::ScanProgressStep::DiscoveringFiles: + bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files") + .arg(status.currentScanStepStats->processedFiles)); + break; + + case Scanner::ScanProgressStep::ScanningFiles: + bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files") + .arg(status.currentScanStepStats->processedFiles) + .arg(status.currentScanStepStats->filesToProcess) + .arg(status.currentScanStepStats->progress())); + break; + + case Scanner::ScanProgressStep::FetchingTrackFeatures: + bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features") + .arg(status.currentScanStepStats->processedFiles) + .arg(status.currentScanStepStats->filesToProcess) + .arg(status.currentScanStepStats->progress())); + break; } break; }