Added more details during scan. fixed #32

approot/admin-scannercontroller.xml

@@ -13,6 +13,7 @@
 					<div class="col-lg-9">
 						<div class="well well-sm">
 							${status}
+							<div>${step-status}</div>
 						</div>
 					</div>
 				</div>

approot/messages.xml

@@ -72,7 +72,11 @@
 <message id="Lms.Admin.ScannerController.status">Status</message>
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Not scheduled</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Scheduled on {1}</message>
-<message id="Lms.Admin.ScannerController.status-in-progress">Scanning {1} of {2} files ({3} %)</message>
+<message id="Lms.Admin.ScannerController.status-in-progress">Scanning: step {1}/{2}</message>
+<message id="Lms.Admin.ScannerController.step-checking-removed-files">Checking removed files... {1}%</message>
+<message id="Lms.Admin.ScannerController.step-discovering-files">Discovering files: {1} files</message>
+<message id="Lms.Admin.ScannerController.step-fetching-track-features">Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-scanning-files">Scanning files: {1}/{2} files ({3}%)...</message>
 
 <!--Users-->
 <message id="Lms.Admin.Users.add">New user</message>

approot/messages_fr.xml

@@ -72,7 +72,11 @@
 <message id="Lms.Admin.ScannerController.status">Statut</message>
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Non planifié</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Planifié le {1}</message>
-<message id="Lms.Admin.ScannerController.status-in-progress">Scan de {1} fichiers sur {2} ({3} %)</message>
+<message id="Lms.Admin.ScannerController.status-in-progress">En cours de scan : étape {1}/{2}</message>
+<message id="Lms.Admin.ScannerController.step-checking-removed-files">Vérification des fichiers... {1}%</message>
+<message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers: {1} fichiers</message>
+<message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz: {1}/{2} fichiers ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-scanning-files">Scan des fichiers: {1}/{2} fichiers ({3}%)...</message>
 
 <!--Users-->
 <message id="Lms.Admin.Users.add">Ajouter</message>

src/libs/scanner/impl/MediaScanner.cpp

@@ -264,6 +264,8 @@ MediaScanner::~MediaScanner()
 void
 MediaScanner::start()
 {
+	std::scoped_lock lock {_controlMutex};
+
 	scheduleNextScan();
 
 	_ioService.start();
@@ -272,6 +274,8 @@ MediaScanner::start()
 void
 MediaScanner::stop()
 {
+	std::scoped_lock lock {_controlMutex};
+
 	_abortScan = true;
 
 	_scheduleTimer.cancel();
@@ -281,10 +285,15 @@ MediaScanner::stop()
 void
 MediaScanner::abortScan()
 {
-	_abortScan = true;
+	LMS_LOG(DBUPDATER, DEBUG) << "Aborting scan...";
+	std::scoped_lock lock {_controlMutex};
 
+	LMS_LOG(DBUPDATER, DEBUG) << "Waiting for the scan to abort...";
+
+	_abortScan = true;
 	_scheduleTimer.cancel();
 	_ioService.stop();
+	LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!";
 
 	_abortScan = false;
 	_ioService.start();
@@ -303,6 +312,7 @@ MediaScanner::requestImmediateScan(bool force)
 void
 MediaScanner::requestReload()
 {
+	abortScan();
 	_ioService.post([=]()
 	{
 		scheduleNextScan();
@@ -319,7 +329,7 @@ MediaScanner::getStatus() const
 	res.currentState = _curState;
 	res.nextScheduledScan = _nextScheduledScan;
 	res.lastCompleteScanStats = _lastCompleteScanStats;
-	res.inProgressScanStats = _inProgressScanStats;
+	res.currentScanStepStats = _currentScanStepStats;
 
 	return res;
 }
@@ -382,7 +392,10 @@ MediaScanner::scheduleNextScan()
 void
 MediaScanner::countAllFiles(ScanStats& stats)
 {
-	stats.filesToScan = 0;
+	ScanStepStats stepStats{stats.startTime, ScanProgressStep::DiscoveringFiles};
+
+	stats.filesScanned = 0;
+	notifyInProgress(stepStats);
 
 	exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
 	{
@@ -391,8 +404,9 @@ MediaScanner::countAllFiles(ScanStats& stats)
 
 		if (!ec && isFileSupported(path, _fileExtensions))
 		{
-			stats.filesToScan++;
+			stats.filesScanned++;
-			notifyInProgressIfNeeded(stats);
+			stepStats.processedFiles++;
+			notifyInProgressIfNeeded(stepStats);
 		}
 
 		return true;
@@ -445,11 +459,11 @@ MediaScanner::scan(bool forceScan)
 
 	refreshScanSettings();
 
+	removeMissingTracks(stats);
+
 	LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'...";
 	countAllFiles(stats);
-	LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesToScan;
+	LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesScanned;
-
-	removeMissingTracks(stats);
 
 	LMS_LOG(UI, INFO) << "Checks complete, force scan = " << forceScan;
 
@@ -465,7 +479,7 @@ MediaScanner::scan(bool forceScan)
 	// Now update all the track features if needed
 	fetchTrackFeatures(stats);
 
-	LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << "/" << stats.featuresToFetch <<",  duplicates = " << stats.duplicates.size();
+	LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ",  duplicates = " << stats.duplicates.size();
 
 	_dbSession.optimize();
 
@@ -476,19 +490,22 @@ MediaScanner::scan(bool forceScan)
 			std::unique_lock lock {_statusMutex};
 
 			_lastCompleteScanStats = std::move(stats);
-			_inProgressScanStats.reset();
+			_currentScanStepStats.reset();
 		}
 
+		LMS_LOG(DBUPDATER, DEBUG) << "Scan not aborted, scheduling next scan!";
 		scheduleNextScan();
 
 		scanComplete().emit();
 	}
 	else
 	{
+		LMS_LOG(DBUPDATER, DEBUG) << "Scan aborted, not scheduling next scan!";
+
 		std::unique_lock lock {_statusMutex};
 
 		_curState = State::NotScheduled;
-		_inProgressScanStats.reset();
+		_currentScanStepStats.reset();
 	}
 }
 
@@ -524,6 +541,8 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats)
 	if (_recommendationEngineType != ScanSettings::RecommendationEngineType::Features)
 		return;
 
+	ScanStepStats stepStats{stats.startTime, ScanProgressStep::FetchingTrackFeatures};
+
 	LMS_LOG(DBUPDATER, INFO) << "Fetching missing track features...";
 
 	struct TrackInfo
@@ -545,7 +564,8 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats)
 		return res;
 	}()};
 
-	stats.featuresToFetch = tracksToFetch.size();
+	stepStats.filesToProcess = tracksToFetch.size();
+	notifyInProgress(stepStats);
 
 	LMS_LOG(DBUPDATER, INFO) << "Found " << tracksToFetch.size() << " track(s) to fetch!";
 
@@ -557,6 +577,8 @@ MediaScanner::fetchTrackFeatures(ScanStats& stats)
 		if (fetchTrackFeatures(trackToFetch.id, trackToFetch.mbid))
 			stats.featuresFetched++;
 
+		stepStats.processedFiles++;
+		notifyInProgressIfNeeded(stepStats);
 	}
 
 	LMS_LOG(DBUPDATER, INFO) << "Track features fetched!";
@@ -596,27 +618,25 @@ MediaScanner::refreshScanSettings()
 }
 
 void
-MediaScanner::notifyInProgress(const ScanStats& stats)
+MediaScanner::notifyInProgress(const ScanStepStats& stepStats)
 {
-	const ScanProgressStats progressStats {stats.toProgressStats()};
-
 	{
 		std::unique_lock lock {_statusMutex};
-		_inProgressScanStats = progressStats;
+		_currentScanStepStats = stepStats;
 	}
 
 	const std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
-	_sigScanInProgress(progressStats);
+	_sigScanInProgress(stepStats);
 	_lastScanInProgressEmit = now;
 }
 
 void
-MediaScanner::notifyInProgressIfNeeded(const ScanStats& stats)
+MediaScanner::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
 {
 	std::chrono::system_clock::time_point now {std::chrono::system_clock::now()};
 
-	if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 2)
+	if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 1)
-		notifyInProgress(stats);
+		notifyInProgress(stepStats);
 }
 
 void
@@ -777,6 +797,10 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S
 void
 MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
 {
+	ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles};
+	stepStats.filesToProcess = stats.filesScanned;
+	notifyInProgress(stepStats);
+
 	exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
 	{
 		if (_abortScan)
@@ -791,13 +815,12 @@ MediaScanner::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bo
 		{
 			scanAudioFile(path, forceScan, stats );
 
-			notifyInProgressIfNeeded(stats);
+			stepStats.processedFiles++;
+			notifyInProgressIfNeeded(stepStats);
 		}
 
 		return true;
 	});
-
-	notifyInProgress(stats);
 }
 
 // Check if a file exists and is still in a media directory
@@ -842,6 +865,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
 {
 	static constexpr std::size_t batchSize {50};
 
+	ScanStepStats stepStats{stats.startTime, ScanProgressStep::CheckingRemovedFiles};
+
 	LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
 	std::size_t trackCount {};
 
@@ -851,6 +876,9 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
 	}
 	LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked...";
 
+	stepStats.filesToProcess = trackCount;
+	notifyInProgress(stepStats);
+
 	std::vector<std::pair<Database::IdType, std::filesystem::path>> trackPaths;
 	std::vector<IdType> tracksToRemove;
 
@@ -859,6 +887,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
 		trackPaths.clear();
 		tracksToRemove.clear();
 
+		stepStats.processedFiles++;
+
 		{
 			auto transaction {_dbSession.createSharedTransaction()};
 			trackPaths = Track::getAllPaths(_dbSession, i, batchSize);
@@ -888,7 +918,7 @@ MediaScanner::removeMissingTracks(ScanStats& stats)
 			}
 		}
 
-		notifyInProgressIfNeeded(stats);
+		notifyInProgressIfNeeded(stepStats);
 
 		if (i == 0)
 			break;

src/libs/scanner/impl/MediaScanner.hpp

@@ -59,7 +59,7 @@ class MediaScanner : public IMediaScanner
 
 		Wt::Signal<>& scanStarted() override { return _sigScanStarted; }
 		Wt::Signal<>& scanComplete() override { return _sigScanComplete; }
-		Wt::Signal<ScanProgressStats>& scanInProgress() override { return _sigScanInProgress; }
+		Wt::Signal<ScanStepStats>& scanInProgress() override { return _sigScanInProgress; }
 		Wt::Signal<Wt::WDateTime>& scheduled() override { return _sigScheduled; }
 
 	private:
@@ -86,17 +86,16 @@ class MediaScanner : public IMediaScanner
 		void checkDuplicatedAudioFiles(ScanStats& stats);
 		void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats);
 		Database::IdType doScanAudioFile(const std::filesystem::path& file, ScanStats& stats);
-		void notifyInProgressIfNeeded(const ScanStats& stats);
+		void notifyInProgressIfNeeded(const ScanStepStats& stats);
-		void notifyInProgress(const ScanStats& stats);
+		void notifyInProgress(const ScanStepStats& stats);
 
-		std::mutex								_scanInProgress {};
+		std::mutex								_controlMutex;
 		std::atomic<bool>						_abortScan {};
-
 		Wt::WIOService							_ioService;
 		boost::asio::system_timer				_scheduleTimer {_ioService};
 		Wt::Signal<>							_sigScanStarted;
 		Wt::Signal<>							_sigScanComplete;
-		Wt::Signal<ScanProgressStats>			_sigScanInProgress;
+		Wt::Signal<ScanStepStats>		_sigScanInProgress;
 		std::chrono::system_clock::time_point	_lastScanInProgressEmit {};
 		Wt::Signal<Wt::WDateTime>				_sigScheduled;
 		Database::Session						_dbSession;
@@ -105,7 +104,7 @@ class MediaScanner : public IMediaScanner
 		mutable std::shared_mutex			_statusMutex;
 		State								_curState {State::NotScheduled};
 		std::optional<ScanStats> 			_lastCompleteScanStats;
-		std::optional<ScanProgressStats> 	_inProgressScanStats;
+		std::optional<ScanStepStats> 		_currentScanStepStats;
 		Wt::WDateTime						_nextScheduledScan;
 
 		// Current scan settings

src/libs/scanner/impl/MediaScannerStats.cpp

@@ -40,16 +40,10 @@ ScanStats::nbChanges() const
 	return additions + deletions + updates;
 }
 
-ScanProgressStats
-ScanStats::toProgressStats() const
-{
-	return ScanProgressStats {startTime, filesToScan, nbFiles()};
-}
-
 unsigned
-ScanProgressStats::progress() const
+ScanStepStats::progress() const
 {
-	return (processedFiles / static_cast<float>(filesToScan ? filesToScan : 1)) * 100;
+	return (processedFiles / static_cast<float>(filesToProcess ? filesToProcess : 1)) * 100;
 }
 
 } // namespace Scanner

src/libs/scanner/include/scanner/IMediaScanner.hpp

@@ -58,7 +58,7 @@ class IMediaScanner
 			State								currentState {State::NotScheduled};
 			Wt::WDateTime						nextScheduledScan;
 			std::optional<ScanStats>			lastCompleteScanStats;
-			std::optional<ScanProgressStats>	inProgressScanStats;
+			std::optional<ScanStepStats> 		currentScanStepStats;
 		};
 
 		virtual Status getStatus() const = 0;
@@ -70,7 +70,7 @@ class IMediaScanner
 		virtual Wt::Signal<>& scanComplete() = 0;
 
 		// Called during scan in progress
-		virtual Wt::Signal<ScanProgressStats>& scanInProgress() = 0;
+		virtual Wt::Signal<ScanStepStats>& scanInProgress() = 0;
 
 		// Called after a schedule
 		virtual Wt::Signal<Wt::WDateTime>& scheduled() = 0;

src/libs/scanner/include/scanner/MediaScannerStats.hpp

@@ -56,15 +56,26 @@ namespace Scanner {
 
 	};
 
+	enum class ScanProgressStep : unsigned
+	{
+		CheckingRemovedFiles = 0,
+		DiscoveringFiles,
+		ScanningFiles,
+		FetchingTrackFeatures,
+	};
+	static inline constexpr unsigned ScanProgressStepCount {4};
+
 	// reduced scan stats
-	struct ScanProgressStats
+	struct ScanStepStats
 	{
 		Wt::WDateTime   startTime;
 
-		std::size_t     filesToScan {};
+		ScanProgressStep currentStep;
-		std::size_t	processedFiles {};
 
-		unsigned	progress() const;
+		std::size_t     filesToProcess {};
+		std::size_t		processedFiles {};
+
+		unsigned		progress() const;
 	};
 
 	struct ScanStats
@@ -72,25 +83,22 @@ namespace Scanner {
 		Wt::WDateTime	startTime;
 		Wt::WDateTime	stopTime;
 
-		std::size_t	filesToScan {};		// Total number of files to be scanned (estimated)
+		std::size_t	filesScanned {};	// Total number of files scanned (estimated)
 
-		std::size_t	skips {};		// no change since last scan
+		std::size_t	skips {};			// no change since last scan
-		std::size_t	scans {};		// actually scanned filed
+		std::size_t	scans {};			// actually scanned filed
 
 		std::size_t	additions {};		// added in DB
 		std::size_t	deletions {};		// removed from DB
-		std::size_t	updates {};		// updated file in DB
+		std::size_t	updates {};			// updated file in DB
 
 		std::size_t	featuresFetched {};	// features fetched in DB
-		std::size_t	featuresToFetch {};	// features to be fetched in DB
 
 		std::vector<ScanError>		errors;
 		std::vector<ScanDuplicate>	duplicates;
 
 		std::size_t	nbFiles() const;
 		std::size_t	nbChanges() const;
-
-		ScanProgressStats toProgressStats() const;
 	};
 
 }

src/libs/subsonic/impl/Scan.cpp

@@ -35,8 +35,15 @@ namespace API::Subsonic::Scan
 		const IMediaScanner::Status scanStatus {ServiceProvider<IMediaScanner>::get()->getStatus()};
 
 		statusResponse.setAttribute("scanning", scanStatus.currentState == IMediaScanner::State::InProgress);
-		if (scanStatus.currentState == IMediaScanner::State::InProgress && scanStatus.inProgressScanStats)
+		if (scanStatus.currentState == IMediaScanner::State::InProgress)
-			statusResponse.setAttribute("count", scanStatus.inProgressScanStats->processedFiles);
+		{
+			std::size_t count{};
+
+			if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanProgressStep::ScanningFiles)
+				count = scanStatus.currentScanStepStats->processedFiles;
+
+			statusResponse.setAttribute("count", count);
+		}
 
 		return statusResponse;
 	}

src/lms/ui/LmsApplication.cpp

@@ -544,11 +544,11 @@ LmsApplication::createHome()
 			});
 		});
 
-		ServiceProvider<Scanner::IMediaScanner>::get()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats)
+		ServiceProvider<Scanner::IMediaScanner>::get()->scanInProgress().connect(this, [=] (Scanner::ScanStepStats stepStats)
 		{
 			Wt::WServer::instance()->post(sessionId, [=]
 			{
-				_events.dbScanInProgress.emit(stats);
+				_events.dbScanInProgress.emit(stepStats);
 				triggerUpdate();
 			});
 		});

src/lms/ui/LmsApplication.hpp

@@ -57,7 +57,7 @@ struct Events
 	// Database events
 	Wt::Signal<> dbScanStarted;
 	Wt::Signal<> dbScanned;
-	Wt::Signal<Scanner::ScanProgressStats> dbScanInProgress;
+	Wt::Signal<Scanner::ScanStepStats> dbScanInProgress;
 	Wt::Signal<Wt::WDateTime> dbScanScheduled;
 };
 

src/lms/ui/admin/ScannerController.cpp

@@ -185,18 +185,43 @@ ScannerController::refreshContents()
 	{
 		case IMediaScanner::State::NotScheduled:
 			bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-not-scheduled"));
+			bindEmpty("step-status");
 			break;
 		case IMediaScanner::State::Scheduled:
 			bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-scheduled")
 					.arg(status.nextScheduledScan.toString()));
+			bindEmpty("step-status");
 			break;
 		case IMediaScanner::State::InProgress:
+			bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress")
+					.arg(static_cast<int>(status.currentScanStepStats->currentStep) + 1)
+					.arg(Scanner::ScanProgressStepCount));
+
+			switch (status.currentScanStepStats->currentStep)
 			{
-				std::ostringstream oss;
+				case Scanner::ScanProgressStep::CheckingRemovedFiles:
-				bindString("status", Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress")
+					bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-checking-removed-files")
-						.arg(status.inProgressScanStats->processedFiles)
+						.arg(status.currentScanStepStats->progress()));
-						.arg(status.inProgressScanStats->filesToScan)
+					break;
-						.arg(status.inProgressScanStats->progress()));
+
+				case Scanner::ScanProgressStep::DiscoveringFiles:
+					bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
+						.arg(status.currentScanStepStats->processedFiles));
+						break;
+
+				case Scanner::ScanProgressStep::ScanningFiles:
+					bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
+						.arg(status.currentScanStepStats->processedFiles)
+						.arg(status.currentScanStepStats->filesToProcess)
+						.arg(status.currentScanStepStats->progress()));
+					break;
+
+				case Scanner::ScanProgressStep::FetchingTrackFeatures:
+					bindString("step-status", Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
+						.arg(status.currentScanStepStats->processedFiles)
+						.arg(status.currentScanStepStats->filesToProcess)
+						.arg(status.currentScanStepStats->progress()));
+					break;
 			}
 			break;
 	}