Refactored scanner to ease further improvements

approot/messages.xml

@@ -83,6 +83,7 @@
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Not scheduled</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Scheduled on {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">Scanning: step {1}/{2}</message>
+<message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Checking for duplicate files... {1} files</message>
 <message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Checking files... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-discovering-files">Discovering files: {1} files</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)...</message>

approot/messages_fr.xml

@@ -83,6 +83,7 @@
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Non planifié</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Planifié le {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">En cours de scan : étape {1}/{2}</message>
+<message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Vérification des fichiers dupliqués... {1} fichiers</message>
 <message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Vérification des fichiers... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers : {1} fichiers</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)...</message>

approot/messages_it.xml

@@ -83,6 +83,7 @@
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Non pianificato</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Pianificato il {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">Scansione: passo {1}/{2}</message>
+
 <message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Controllo file... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-discovering-files">File trovati: {1} files</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)...</message>

approot/messages_zh.xml

@@ -83,6 +83,7 @@
 <message id="Lms.Admin.ScannerController.status-not-scheduled">无计划</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">计划于 {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">扫描中: 阶段 {1}/{2}</message>
+
 <message id="Lms.Admin.ScannerController.step-checking-for-missing-files">检查文件中... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-discovering-files">检索文件中: {1} 文件</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)...</message>

src/libs/services/scanner/CMakeLists.txt

@@ -2,6 +2,10 @@
 add_library(lmsscanner SHARED
 	impl/ScannerService.cpp
 	impl/ScannerStats.cpp
+	impl/ScanStepCheckDuplicatedDbFiles.cpp
+	impl/ScanStepDiscoverFiles.cpp
+	impl/ScanStepRemoveOrphanDbFiles.cpp
+	impl/ScanStepScanFiles.cpp
 	)
 
 target_include_directories(lmsscanner INTERFACE

src/libs/services/scanner/impl/IScanStep.hpp

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <string_view>
+
+#include "services/scanner/ScannerStats.hpp"
+
+namespace Scanner
+{
+	class IScanStep
+	{
+		public:
+			virtual ~IScanStep() = default;
+
+			virtual ScanStep getStep() const = 0;
+			virtual std::string_view getStepName() const = 0;
+
+			struct ScanContext
+			{
+				const std::filesystem::path directory;
+				const bool forceScan;
+				ScanStats stats;
+				ScanStepStats currentStepStats;
+			};
+			virtual void process(ScanContext& context) = 0;
+	};
+}

src/libs/services/scanner/impl/ScanStepBase.hpp

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <functional>
+
+#include "services/scanner/ScannerStats.hpp"
+#include "IScanStep.hpp"
+#include "ScannerSettings.hpp"
+
+namespace Database
+{
+	class Db;
+}
+
+namespace Scanner
+{
+	class ScanStepBase : public IScanStep
+	{
+		public:
+			static inline const std::filesystem::path excludeDirFileName {".lmsignore"};
+			using ProgressCallback = std::function<void(const ScanStepStats& stats)>;
+
+			struct InitParams
+			{
+				const ScannerSettings& settings;
+				ProgressCallback progressCallback;
+				bool& abortScan;
+				Database::Db& db;
+			};
+			ScanStepBase(InitParams& initParams)
+				: _settings {initParams.settings}
+			, _progressCallback {initParams.progressCallback}
+			, _abortScan {initParams.abortScan}
+			, _db {initParams.db}
+			{}
+
+		protected:
+			const ScannerSettings&	_settings;
+			ProgressCallback		_progressCallback;
+			bool&					_abortScan;
+			Database::Db&			_db;
+	};
+}

src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp

@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepCheckDuplicatedDbFiles.hpp"
+
+#include "services/database/Db.hpp"
+#include "services/database/Session.hpp"
+#include "services/database/Track.hpp"
+#include "utils/Logger.hpp"
+
+namespace Scanner
+{
+	void
+	ScanStepCheckDuplicatedDbFiles::process(ScanContext& context)
+	{
+		using namespace Database;
+
+		if (_abortScan)
+			return;
+
+		Session& session {_db.getTLSSession()};
+		auto transaction {session.createSharedTransaction()};
+
+		const RangeResults<TrackId> tracks = Track::findTrackMBIDDuplicates(session, Range {});
+		for (const TrackId trackId : tracks.results)
+		{
+			const Track::pointer track {Track::find(session, trackId)};
+			if (auto trackMBID {track->getTrackMBID()})
+			{
+				LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
+				context.stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID});
+				context.currentStepStats.processedElems++;
+				_progressCallback(context.currentStepStats);
+			}
+		}
+
+		LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files";
+	}
+}

src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace Scanner
+{
+	class ScanStepCheckDuplicatedDbFiles : public ScanStepBase
+	{
+		public:
+			using ScanStepBase::ScanStepBase;
+
+		private:
+			std::string_view getStepName() const override { return "Checking for duplicated files"; }
+			ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; }
+			void process(ScanContext& context) override;
+	};
+}

src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepDiscoverFiles.hpp"
+#include "utils/Logger.hpp"
+#include "utils/Path.hpp"
+
+namespace Scanner
+{
+	void
+	ScanStepDiscoverFiles::process(ScanContext& context)
+	{
+		context.stats.filesScanned = 0;
+		PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path)
+		{
+			if (_abortScan)
+				return false;
+
+			if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
+			{
+				context.currentStepStats.processedElems++;
+				_progressCallback(context.currentStepStats);
+			}
+
+			return true;
+		}, &excludeDirFileName);
+
+		context.stats.filesScanned = context.currentStepStats.processedElems;
+
+		LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'";
+	}
+}

src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace Scanner
+{
+	class ScanStepDiscoverFiles : public ScanStepBase
+	{
+		public:
+			using ScanStepBase::ScanStepBase;
+
+		private:
+			ScanStep getStep() const override { return ScanStep::DiscoveringFiles; }
+			std::string_view getStepName() const override { return "DiscoveringFiles"; }
+			void process(ScanContext& context) override;
+	};
+}

src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp

@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepRemoveOrphanDbFiles.hpp"
+
+#include "services/database/Artist.hpp"
+#include "services/database/Cluster.hpp"
+#include "services/database/Db.hpp"
+#include "services/database/Release.hpp"
+#include "services/database/Session.hpp"
+#include "services/database/Track.hpp"
+#include "utils/Logger.hpp"
+#include "utils/Path.hpp"
+
+namespace Scanner
+{
+	void
+	ScanStepRemoveOrphanDbFiles::process(ScanContext& context)
+	{
+		removeOrphanTracks(context);
+		removeOrphanClusters();
+		removeOrphanArtists();
+		removeOrphanReleases();
+	}
+
+	void ScanStepRemoveOrphanDbFiles::removeOrphanTracks(ScanContext& context)
+	{
+		using namespace Database;
+
+		if (_abortScan)
+			return;
+
+		static constexpr std::size_t batchSize {50};
+		Session& session {_db.getTLSSession()};
+
+		LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
+		std::size_t trackCount {};
+
+		{
+			auto transaction {session.createSharedTransaction()};
+			trackCount = Track::getCount(session);
+		}
+		LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked...";
+
+		context.currentStepStats.totalElems = trackCount;
+
+		RangeResults<Track::PathResult> trackPaths;
+		std::vector<TrackId> tracksToRemove;
+
+		// TODO handle only files in context.directory
+		for (std::size_t i {trackCount < batchSize ? 0 : trackCount - batchSize}; ; i -= (i > batchSize ? batchSize : i))
+		{
+			tracksToRemove.clear();
+
+			{
+				auto transaction {session.createSharedTransaction()};
+				trackPaths = Track::findPaths(session, Range {i, batchSize});
+			}
+
+			for (const Track::PathResult& trackPath : trackPaths.results)
+			{
+				if (_abortScan)
+					return;
+
+				if (!checkFile(trackPath.path, _settings.mediaDirectory))
+					tracksToRemove.push_back(trackPath.trackId);
+
+				context.currentStepStats.processedElems++;
+			}
+
+			if (!tracksToRemove.empty())
+			{
+				auto transaction {session.createSharedTransaction()};
+
+				for (const TrackId trackId : tracksToRemove)
+				{
+					Track::pointer track {Track::find(session, trackId)};
+					if (track)
+					{
+						track.remove();
+						context.stats.deletions++;
+					}
+				}
+			}
+
+			_progressCallback(context.currentStepStats);
+
+			if (i == 0)
+				break;
+		}
+
+		LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks checked!";
+	}
+
+	void
+	ScanStepRemoveOrphanDbFiles::removeOrphanClusters()
+	{
+		using namespace Database;
+
+		LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
+		Session& session {_db.getTLSSession()};
+		auto transaction {session.createUniqueTransaction()};
+
+		// Now process orphan Cluster (no track)
+		auto clusterIds {Cluster::findOrphans(session, Range {})};
+		for (ClusterId clusterId : clusterIds.results)
+		{
+			Cluster::pointer cluster {Cluster::find(session, clusterId)};
+			LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'";
+			cluster.remove();
+		}
+	}
+
+	void
+	ScanStepRemoveOrphanDbFiles::removeOrphanArtists()
+	{
+		using namespace Database;
+
+		LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists...";
+
+		Session& session {_db.getTLSSession()};
+		auto transaction {session.createUniqueTransaction()};
+
+		auto artistIds {Artist::findAllOrphans(session, Range {})};
+		for (const ArtistId artistId : artistIds.results)
+		{
+			Artist::pointer artist {Artist::find(session, artistId)};
+			LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'";
+			artist.remove();
+		}
+	}
+
+	void
+	ScanStepRemoveOrphanDbFiles::removeOrphanReleases()
+	{
+		using namespace Database;
+
+		LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases...";
+
+		Session& session {_db.getTLSSession()};
+		auto transaction {session.createUniqueTransaction()};
+
+		auto releases {Release::findOrphans(session, Range {})};
+		for (const ReleaseId releaseId : releases.results)
+		{
+			Release::pointer release {Release::find(session, releaseId)};
+			LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'";
+			release.remove();
+		}
+	}
+
+	bool
+	ScanStepRemoveOrphanDbFiles::checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory)
+	{
+		try
+		{
+			// For each track, make sure the the file still exists
+			// and still belongs to a media directory
+			if (!std::filesystem::exists( p )
+					|| !std::filesystem::is_regular_file( p ) )
+			{
+				LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing";
+				return false;
+			}
+
+			if (!PathUtils::isPathInRootPath(p, mediaDirectory, &excludeDirFileName))
+			{
+				LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory";
+				return false;
+			}
+
+			if (!PathUtils::hasFileAnyExtension(p, _settings.supportedExtensions))
+			{
+				LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled";
+				return false;
+			}
+
+			return true;
+		}
+		catch (std::filesystem::filesystem_error& e)
+		{
+			LMS_LOG(DBUPDATER, ERROR) << "Caught exception while checking file '" << p.string() << "': " << e.what();
+			return false;
+		}
+	}
+}

src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <filesystem>
+
+#include "ScanStepBase.hpp"
+
+namespace Scanner
+{
+	class ScanStepRemoveOrphanDbFiles : public ScanStepBase
+	{
+		public:
+			using ScanStepBase::ScanStepBase;
+
+		private:
+			std::string_view getStepName() const override { return "Checking orphaned entries"; }
+			ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; }
+			void process(ScanContext& context) override;
+
+			void removeOrphanTracks(ScanContext& context);
+			void removeOrphanClusters();
+			void removeOrphanArtists();
+			void removeOrphanReleases();
+			bool checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory);
+	};
+}

src/libs/services/scanner/impl/ScanStepScanFiles.cpp

@@ -0,0 +1,430 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepScanFiles.hpp"
+
+#include "metadata/IParser.hpp"
+#include "services/database/Artist.hpp"
+#include "services/database/Cluster.hpp"
+#include "services/database/Db.hpp"
+#include "services/database/Release.hpp"
+#include "services/database/Session.hpp"
+#include "services/database/Track.hpp"
+#include "services/database/TrackFeatures.hpp"
+#include "services/database/TrackArtistLink.hpp"
+#include "utils/Exception.hpp"
+#include "utils/IConfig.hpp"
+#include "utils/Logger.hpp"
+#include "utils/Path.hpp"
+
+using namespace Database;
+
+namespace
+{
+	Artist::pointer
+	createArtist(Session& session, const MetaData::Artist& artistInfo)
+	{
+		Artist::pointer artist {session.create<Artist>(artistInfo.name)};
+
+		if (artistInfo.musicBrainzArtistID)
+			artist.modify()->setMBID(*artistInfo.musicBrainzArtistID);
+		if (artistInfo.sortName)
+			artist.modify()->setSortName(*artistInfo.sortName);
+
+		return artist;
+	}
+
+	void
+	updateArtistIfNeeded(Artist::pointer artist, const MetaData::Artist& artistInfo)
+	{
+		// Name may have been updated
+		if (artist->getName() != artistInfo.name)
+		{
+			artist.modify()->setName(artistInfo.name);
+		}
+
+		// Sortname may have been updated
+		if (artistInfo.sortName && *artistInfo.sortName != artist->getSortName() )
+		{
+			artist.modify()->setSortName(*artistInfo.sortName);
+		}
+	}
+
+	std::vector<Artist::pointer>
+	getOrCreateArtists(Session& session, const std::vector<MetaData::Artist>& artistsInfo, bool allowFallbackOnMBIDEntries)
+	{
+		std::vector<Artist::pointer> artists;
+
+		for (const MetaData::Artist& artistInfo : artistsInfo)
+		{
+			Artist::pointer artist;
+
+			// First try to get by MBID
+			if (artistInfo.musicBrainzArtistID)
+			{
+				artist = Artist::find(session, *artistInfo.musicBrainzArtistID);
+				if (!artist)
+					artist = createArtist(session, artistInfo);
+				else
+					updateArtistIfNeeded(artist, artistInfo);
+
+				artists.emplace_back(std::move(artist));
+				continue;
+			}
+
+			// Fall back on artist name (collisions may occur)
+			if (!artistInfo.name.empty())
+			{
+				for (const Artist::pointer& sameNamedArtist : Artist::find(session, artistInfo.name))
+				{
+					// Do not fallback on artist that is correctly tagged
+					if (!allowFallbackOnMBIDEntries && sameNamedArtist->getMBID())
+						continue;
+
+					artist = sameNamedArtist;
+					break;
+				}
+
+				// No Artist found with the same name and without MBID -> creating
+				if (!artist)
+					artist = createArtist(session, artistInfo);
+				else
+					updateArtistIfNeeded(artist, artistInfo);
+
+				artists.emplace_back(std::move(artist));
+				continue;
+			}
+		}
+
+		return artists;
+	}
+
+	Release::pointer
+	getOrCreateRelease(Session& session, const MetaData::Album& album)
+	{
+		Release::pointer release;
+
+		// First try to get by MBID
+		if (album.musicBrainzAlbumID)
+		{
+			release = Release::find(session, *album.musicBrainzAlbumID);
+			if (!release)
+			{
+				release = session.create<Release>(album.name, album.musicBrainzAlbumID);
+			}
+			else if (release->getName() != album.name)
+			{
+				// Name may have been updated
+				release.modify()->setName(album.name);
+			}
+
+			return release;
+		}
+
+		// Fall back on release name (collisions may occur)
+		if (!album.name.empty())
+		{
+			for (const Release::pointer& sameNamedRelease : Release::find(session, album.name))
+			{
+				// do not fallback on properly tagged releases
+				if (!sameNamedRelease->getMBID())
+				{
+					release = sameNamedRelease;
+					break;
+				}
+			}
+
+			// No release found with the same name and without MBID -> creating
+			if (!release)
+				release = session.create<Release>(album.name);
+
+			return release;
+		}
+
+		return Release::pointer{};
+	}
+
+	std::vector<Cluster::pointer>
+	getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames)
+	{
+		std::vector< Cluster::pointer > clusters;
+
+		for (auto clusterNames : clustersNames)
+		{
+			auto clusterType = ClusterType::find(session, clusterNames.first);
+			if (!clusterType)
+				continue;
+
+			for (auto clusterName : clusterNames.second)
+			{
+				auto cluster = clusterType->getCluster(clusterName);
+				if (!cluster)
+					cluster = session.create<Cluster>(clusterType, clusterName);
+
+				clusters.push_back(cluster);
+			}
+		}
+
+		return clusters;
+	}
+
+	MetaData::ParserReadStyle
+	getParserReadStyle()
+	{
+		std::string_view readStyle {Service<IConfig>::get()->getString("scanner-parser-read-style", "accurate")};
+
+		if (readStyle == "fast")
+			return MetaData::ParserReadStyle::Fast;
+		else if (readStyle == "average")
+			return MetaData::ParserReadStyle::Average;
+		else if (readStyle == "accurate")
+			return MetaData::ParserReadStyle::Accurate;
+
+		throw LmsException {"Invalid value for 'scanner-parser-read-style'"};
+	}
+} // namespace
+
+namespace Scanner
+{
+	ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
+		: ScanStepBase {initParams}
+	, _metadataParser {MetaData::createParser(MetaData::ParserType::TagLib, getParserReadStyle())} // For now, always use TagLib
+	{
+	}
+
+	void
+	ScanStepScanFiles::process(ScanContext& context)
+	{
+		_metadataParser->setClusterTypeNames(_settings.clusterTypeNames);
+
+		context.currentStepStats.totalElems = context.stats.filesScanned;
+
+		PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path)
+		{
+			if (_abortScan)
+				return false;
+
+			if (ec)
+			{
+				LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message();
+				context.stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()});
+			}
+			else if (PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
+			{
+				scanAudioFile(path, context);
+
+				context.currentStepStats.processedElems++;
+				_progressCallback(context.currentStepStats);
+			}
+
+			return true;
+		}, &excludeDirFileName);
+	}
+
+	void
+	ScanStepScanFiles::scanAudioFile(const std::filesystem::path& file, ScanContext& context)
+	{
+		ScanStats& stats {context.stats};
+		Wt::WDateTime lastWriteTime;
+		try
+		{
+			lastWriteTime = PathUtils::getLastWriteTime(file);
+		}
+		catch (LmsException& e)
+		{
+			LMS_LOG(DBUPDATER, ERROR) << e.what();
+			stats.skips++;
+			return;
+		}
+
+		if (!context.forceScan)
+		{
+			// Skip file if last write is the same
+			Database::Session& dbSession {_db.getTLSSession()};
+			auto transaction {_db.getTLSSession().createSharedTransaction()};
+
+			const Track::pointer track {Track::findByPath(dbSession, file)};
+
+			if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t()
+					&& track->getScanVersion() == _settings.scanVersion)
+			{
+				stats.skips++;
+				return;
+			}
+		}
+
+		std::optional<MetaData::Track> trackInfo {_metadataParser->parse(file)};
+		if (!trackInfo)
+		{
+			context.stats.errors.emplace_back(file, ScanErrorType::CannotParseFile);
+			return;
+		}
+
+		stats.scans++;
+
+		Database::Session& dbSession {_db.getTLSSession()};
+		auto uniqueTransaction {dbSession.createUniqueTransaction()};
+
+		Track::pointer track {Track::findByPath(dbSession, file) };
+
+		// Skip duplicate recording MBID
+		if (trackInfo->recordingMBID && _settings.skipDuplicateRecordingMBID)
+		{
+			for (Track::pointer otherTrack : Track::findByRecordingMBID(dbSession, *trackInfo->recordingMBID))
+			{
+				if (track && track->getId() == otherTrack->getId())
+					continue;
+
+				LMS_LOG(DBUPDATER, DEBUG) << "Skipped '" << file.string() << "' (similar recording MBID in '" << otherTrack->getPath().string() << "')";
+				// This recording MBID already exists, just remove what we just scanned
+				if (track)
+				{
+					track.remove();
+					stats.deletions++;
+				}
+				return;
+			}
+		}
+
+		// We estimate this is an audio file if:
+		// - we found a least one audio stream
+		// - the duration is not null
+		if (trackInfo->audioStreams.empty())
+		{
+			LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (no audio stream found)";
+
+			// If Track exists here, delete it!
+			if (track)
+			{
+				track.remove();
+				stats.deletions++;
+			}
+			stats.errors.emplace_back(ScanError {file, ScanErrorType::NoAudioTrack});
+			return;
+		}
+		if (trackInfo->duration == std::chrono::milliseconds::zero())
+		{
+			LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (duration is 0)";
+
+			// If Track exists here, delete it!
+			if (track)
+			{
+				track.remove();
+				stats.deletions++;
+			}
+			stats.errors.emplace_back(ScanError {file, ScanErrorType::BadDuration});
+			return;
+		}
+
+		// ***** Title
+		std::string title;
+		if (!trackInfo->title.empty())
+			title = trackInfo->title;
+		else
+		{
+			// TODO parse file name guess track etc.
+			// For now juste use file name as title
+			title = file.filename().string();
+		}
+
+		// If file already exists, update its data
+		// Otherwise, create it
+		if (!track)
+		{
+			track = dbSession.create<Track>(file);
+			LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'";
+			stats.additions++;
+		}
+		else
+		{
+			LMS_LOG(DBUPDATER, INFO) << "Updating '" << file.string() << "'";
+
+			stats.updates++;
+		}
+
+		// Track related data
+		assert(track);
+
+		track.modify()->clearArtistLinks();
+		// Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files
+		for (const Artist::pointer& artist : getOrCreateArtists(dbSession, trackInfo->artists, false))
+			track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, artist, TrackArtistLinkType::Artist));
+
+		for (const Artist::pointer& releaseArtist : getOrCreateArtists(dbSession, trackInfo->albumArtists, false))
+			track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, releaseArtist, TrackArtistLinkType::ReleaseArtist));
+
+		// Allow fallbacks on artists with the same name even if they have MBID, since there is no tag to indicate the MBID of these artists
+		// We could ask MusicBrainz to get all the information, but that would heavily slow down the import process
+		for (const Artist::pointer& conductor : getOrCreateArtists(dbSession, trackInfo->conductorArtists, true))
+			track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, conductor, TrackArtistLinkType::Conductor));
+
+		for (const Artist::pointer& composer : getOrCreateArtists(dbSession, trackInfo->composerArtists, true))
+			track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, composer, TrackArtistLinkType::Composer));
+
+		for (const Artist::pointer& lyricist : getOrCreateArtists(dbSession, trackInfo->lyricistArtists, true))
+			track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, lyricist, TrackArtistLinkType::Lyricist));
+
+		for (const Artist::pointer& mixer : getOrCreateArtists(dbSession, trackInfo->mixerArtists, true))
+			track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, mixer, TrackArtistLinkType::Mixer));
+
+		for (const auto& [role, performers] : trackInfo->performerArtists)
+		{
+			for (const Artist::pointer& performer : getOrCreateArtists(dbSession, performers, true))
+				track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, performer, TrackArtistLinkType::Performer, role));
+		}
+
+		for (const Artist::pointer& producer : getOrCreateArtists(dbSession, trackInfo->producerArtists, true))
+			track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, producer, TrackArtistLinkType::Producer));
+
+		for (const Artist::pointer& remixer : getOrCreateArtists(dbSession, trackInfo->remixerArtists, true))
+			track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, remixer, TrackArtistLinkType::Remixer));
+
+		track.modify()->setScanVersion(_settings.scanVersion);
+		if (trackInfo->album)
+			track.modify()->setRelease(getOrCreateRelease(dbSession, *trackInfo->album));
+		else
+			track.modify()->setRelease({});
+		track.modify()->setClusters(getOrCreateClusters(dbSession, trackInfo->clusters));
+		track.modify()->setLastWriteTime(lastWriteTime);
+		track.modify()->setName(title);
+		track.modify()->setDuration(trackInfo->duration);
+		track.modify()->setAddedTime(Wt::WDateTime::currentDateTime());
+		track.modify()->setTrackNumber(trackInfo->trackNumber ? *trackInfo->trackNumber : 0);
+		track.modify()->setDiscNumber(trackInfo->discNumber ? *trackInfo->discNumber : 0);
+		track.modify()->setTotalTrack(trackInfo->totalTrack);
+		track.modify()->setTotalDisc(trackInfo->totalDisc);
+		track.modify()->setDiscSubtitle(trackInfo->discSubtitle);
+		track.modify()->setDate(trackInfo->date);
+		track.modify()->setOriginalDate(trackInfo->originalDate);
+
+		// If a file has an OriginalYear but no Year, set it to ease filtering
+		if (!trackInfo->date.isValid() && trackInfo->originalDate.isValid())
+			track.modify()->setDate(trackInfo->originalDate);
+
+		track.modify()->setRecordingMBID(trackInfo->recordingMBID);
+		track.modify()->setTrackMBID(trackInfo->trackMBID);
+		if (auto trackFeatures {TrackFeatures::find(dbSession, track->getId())})
+			trackFeatures.remove(); // TODO: only if MBID changed?
+		track.modify()->setHasCover(trackInfo->hasCover);
+		track.modify()->setCopyright(trackInfo->copyright);
+		track.modify()->setCopyrightURL(trackInfo->copyrightURL);
+		track.modify()->setTrackReplayGain(trackInfo->trackReplayGain);
+		track.modify()->setReleaseReplayGain(trackInfo->albumReplayGain);
+	}
+}

src/libs/services/scanner/impl/ScanStepScanFiles.hpp

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <filesystem>
+
+#include "metadata/IParser.hpp"
+#include "ScanStepBase.hpp"
+
+namespace Scanner
+{
+	class ScanStepScanFiles : public ScanStepBase
+	{
+		public:
+			ScanStepScanFiles(InitParams& initParams);
+
+		private:
+			ScanStep getStep() const override { return ScanStep::ScanningFiles; }
+			std::string_view getStepName() const override { return "Scanning files"; }
+			void process(ScanContext& context) override;
+
+			void scanAudioFile(const std::filesystem::path& file, ScanContext& context);
+
+			std::unique_ptr<MetaData::IParser>			_metadataParser;
+	};
+}

src/libs/services/scanner/impl/ScannerService.cpp

@@ -22,27 +22,25 @@
 #include <ctime>
 #include <boost/asio/placeholders.hpp>
 
-#include "services/database/Artist.hpp"
 #include "services/database/Cluster.hpp"
-#include "services/database/Release.hpp"
-#include "services/database/ScanSettings.hpp"
-#include "services/database/Track.hpp"
-#include "services/database/TrackArtistLink.hpp"
 #include "services/database/TrackFeatures.hpp"
-#include "metadata/IParser.hpp"
+#include "services/database/ScanSettings.hpp"
 #include "services/recommendation/IRecommendationService.hpp"
 #include "utils/Exception.hpp"
 #include "utils/IConfig.hpp"
 #include "utils/Logger.hpp"
 #include "utils/Path.hpp"
-#include "utils/UUID.hpp"
+#include "utils/Tuple.hpp"
+
+#include "ScanStepCheckDuplicatedDbFiles.hpp"
+#include "ScanStepDiscoverFiles.hpp"
+#include "ScanStepRemoveOrphanDbFiles.hpp"
+#include "ScanStepScanFiles.hpp"
 
 using namespace Database;
 
 namespace {
 
-const std::filesystem::path excludeDirFileName {".lmsignore"};
-
 Wt::WDate
 getNextMonday(Wt::WDate current)
 {
@@ -65,183 +63,6 @@ getNextFirstOfMonth(Wt::WDate current)
 	return current;
 }
 
-bool
-isFileSupported(const std::filesystem::path& file, const std::vector<std::filesystem::path>& extensions)
-{
-	const std::filesystem::path extension {StringUtils::stringToLower(file.extension().string())};
-
-	return (std::find(std::cbegin(extensions), std::cend(extensions), extension) != std::cend(extensions));
-}
-
-bool
-isPathInMediaDirectory(const std::filesystem::path& path, const std::filesystem::path& rootPath)
-{
-	std::filesystem::path curPath = path;
-
-	while (curPath.parent_path() != curPath)
-	{
-		curPath = curPath.parent_path();
-
-		std::error_code ec;
-		if (std::filesystem::exists(curPath / excludeDirFileName, ec))
-			return false;
-
-		if (curPath == rootPath)
-			return true;
-	}
-
-	return false;
-}
-
-static
-Artist::pointer
-createArtist(Session& session, const MetaData::Artist& artistInfo)
-{
-	Artist::pointer artist {session.create<Artist>(artistInfo.name)};
-
-	if (artistInfo.musicBrainzArtistID)
-		artist.modify()->setMBID(*artistInfo.musicBrainzArtistID);
-	if (artistInfo.sortName)
-		artist.modify()->setSortName(*artistInfo.sortName);
-
-	return artist;
-}
-
-static
-void
-updateArtistIfNeeded(Artist::pointer artist, const MetaData::Artist& artistInfo)
-{
-	// Name may have been updated
-	if (artist->getName() != artistInfo.name)
-	{
-		artist.modify()->setName(artistInfo.name);
-	}
-
-	// Sortname may have been updated
-	if (artistInfo.sortName && *artistInfo.sortName != artist->getSortName() )
-	{
-		artist.modify()->setSortName(*artistInfo.sortName);
-	}
-}
-
-std::vector<Artist::pointer>
-getOrCreateArtists(Session& session, const std::vector<MetaData::Artist>& artistsInfo, bool allowFallbackOnMBIDEntries)
-{
-	std::vector<Artist::pointer> artists;
-
-	for (const MetaData::Artist& artistInfo : artistsInfo)
-	{
-		Artist::pointer artist;
-
-		// First try to get by MBID
-		if (artistInfo.musicBrainzArtistID)
-		{
-			artist = Artist::find(session, *artistInfo.musicBrainzArtistID);
-			if (!artist)
-				artist = createArtist(session, artistInfo);
-			else
-				updateArtistIfNeeded(artist, artistInfo);
-
-			artists.emplace_back(std::move(artist));
-			continue;
-		}
-
-		// Fall back on artist name (collisions may occur)
-		if (!artistInfo.name.empty())
-		{
-			for (const Artist::pointer& sameNamedArtist : Artist::find(session, artistInfo.name))
-			{
-				// Do not fallback on artist that is correctly tagged
-				if (!allowFallbackOnMBIDEntries && sameNamedArtist->getMBID())
-					continue;
-
-				artist = sameNamedArtist;
-				break;
-			}
-
-			// No Artist found with the same name and without MBID -> creating
-			if (!artist)
-				artist = createArtist(session, artistInfo);
-			else
-				updateArtistIfNeeded(artist, artistInfo);
-
-			artists.emplace_back(std::move(artist));
-			continue;
-		}
-	}
-
-	return artists;
-}
-
-Release::pointer
-getOrCreateRelease(Session& session, const MetaData::Album& album)
-{
-	Release::pointer release;
-
-	// First try to get by MBID
-	if (album.musicBrainzAlbumID)
-	{
-		release = Release::find(session, *album.musicBrainzAlbumID);
-		if (!release)
-		{
-			release = session.create<Release>(album.name, album.musicBrainzAlbumID);
-		}
-		else if (release->getName() != album.name)
-		{
-			// Name may have been updated
-			release.modify()->setName(album.name);
-		}
-
-		return release;
-	}
-
-	// Fall back on release name (collisions may occur)
-	if (!album.name.empty())
-	{
-		for (const Release::pointer& sameNamedRelease : Release::find(session, album.name))
-		{
-			// do not fallback on properly tagged releases
-			if (!sameNamedRelease->getMBID())
-			{
-				release = sameNamedRelease;
-				break;
-			}
-		}
-
-		// No release found with the same name and without MBID -> creating
-		if (!release)
-			release = session.create<Release>(album.name);
-
-		return release;
-	}
-
-	return Release::pointer{};
-}
-
-std::vector<Cluster::pointer>
-getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames)
-{
-	std::vector< Cluster::pointer > clusters;
-
-	for (auto clusterNames : clustersNames)
-	{
-		auto clusterType = ClusterType::find(session, clusterNames.first);
-		if (!clusterType)
-			continue;
-
-		for (auto clusterName : clusterNames.second)
-		{
-			auto cluster = clusterType->getCluster(clusterName);
-			if (!cluster)
-				cluster = session.create<Cluster>(clusterType, clusterName);
-
-			clusters.push_back(cluster);
-		}
-	}
-
-	return clusters;
-}
-
 } // namespace
 
 namespace Scanner {
@@ -252,29 +73,11 @@ createScannerService(Db& db, Recommendation::IRecommendationService& recommendat
 	return std::make_unique<ScannerService>(db, recommendationService);
 }
 
-MetaData::ParserReadStyle
-getParserReadStyle()
-{
-	std::string_view readStyle {Service<IConfig>::get()->getString("scanner-parser-read-style", "accurate")};
-
-	if (readStyle == "fast")
-		return MetaData::ParserReadStyle::Fast;
-	else if (readStyle == "average")
-		return MetaData::ParserReadStyle::Average;
-	else if (readStyle == "accurate")
-		return MetaData::ParserReadStyle::Accurate;
-
-	throw LmsException {"Invalid value for 'scanner-parser-read-style'"};
-}
-
 ScannerService::ScannerService(Db& db, Recommendation::IRecommendationService& recommendationService)
 : _recommendationService {recommendationService}
-, _skipDuplicateRecordingMBID {Service<IConfig>::get()->getBool("scanner-skip-duplicate-recording-mbid", false)}
+, _db {db}
 , _dbSession {db}
-, _metadataParser {MetaData::createParser(MetaData::ParserType::TagLib, getParserReadStyle())} // For now, always use TagLib
 {
-	LMS_LOG(DBUPDATER, INFO) << "skipDuplicateRecordingMBID = " << _skipDuplicateRecordingMBID;
-
 	_ioService.setThreadCount(1);
 
 	refreshScanSettings();
@@ -383,34 +186,34 @@ ScannerService::getStatus() const
 void
 ScannerService::scheduleNextScan()
 {
-	LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan";
+	LMS_LOG(DBUPDATER, DEBUG) << "Scheduling next scan";
 
 	refreshScanSettings();
 
 	const Wt::WDateTime now {Wt::WDateTime::currentDateTime()};
 
 	Wt::WDateTime nextScanDateTime;
-	switch (_updatePeriod)
+	switch (_settings.updatePeriod)
 	{
 		case ScanSettings::UpdatePeriod::Daily:
-			if (now.time() < _startTime)
+			if (now.time() < _settings.startTime)
-				nextScanDateTime = {now.date(), _startTime};
+				nextScanDateTime = {now.date(), _settings.startTime};
 			else
-				nextScanDateTime = {now.date().addDays(1), _startTime};
+				nextScanDateTime = {now.date().addDays(1), _settings.startTime};
 			break;
 
 		case ScanSettings::UpdatePeriod::Weekly:
-			if (now.time() < _startTime && now.date().dayOfWeek() == 1)
+			if (now.time() < _settings.startTime && now.date().dayOfWeek() == 1)
-				nextScanDateTime = {now.date(), _startTime};
+				nextScanDateTime = {now.date(), _settings.startTime};
 			else
-				nextScanDateTime = {getNextMonday(now.date()), _startTime};
+				nextScanDateTime = {getNextMonday(now.date()), _settings.startTime};
 			break;
 
 		case ScanSettings::UpdatePeriod::Monthly:
-			if (now.time() < _startTime && now.date().day() == 1)
+			if (now.time() < _settings.startTime && now.date().day() == 1)
-				nextScanDateTime = {now.date(), _startTime};
+				nextScanDateTime = {now.date(), _settings.startTime};
 			else
-				nextScanDateTime = {getNextFirstOfMonth(now.date()), _startTime};
+				nextScanDateTime = {getNextFirstOfMonth(now.date()), _settings.startTime};
 			break;
 
 		case ScanSettings::UpdatePeriod::Hourly:
@@ -434,31 +237,6 @@ ScannerService::scheduleNextScan()
 	_events.scanScheduled.emit(_nextScheduledScan);
 }
 
-void
-ScannerService::countAllFiles(ScanStats& stats)
-{
-	ScanStepStats stepStats{stats.startTime, ScanProgressStep::DiscoveringFiles};
-
-	stats.filesScanned = 0;
-	notifyInProgress(stepStats);
-
-	exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
-	{
-		if (_abortScan)
-			return false;
-
-		if (!ec && isFileSupported(path, _fileExtensions))
-		{
-			stats.filesScanned++;
-			stepStats.processedElems++;
-			notifyInProgressIfNeeded(stepStats);
-		}
-
-		return true;
-	}, excludeDirFileName);
-	notifyInProgress(stepStats);
-}
-
 void
 ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime)
 {
@@ -499,35 +277,29 @@ ScannerService::scan(bool forceScan)
 		_nextScheduledScan = {};
 	}
 
-	ScanStats stats;
-	stats.startTime = Wt::WDateTime::currentDateTime();
 
 	LMS_LOG(UI, INFO) << "New scan started!";
 
 	refreshScanSettings();
 
-	removeMissingTracks(stats);
+	IScanStep::ScanContext scanContext {_settings.mediaDirectory, forceScan, ScanStats {}, ScanStepStats {}};
+	ScanStats& stats {scanContext.stats};
+	stats.startTime = Wt::WDateTime::currentDateTime();
 
-	LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'...";
+	for (auto& scanStep : _scanSteps)
-	countAllFiles(stats);
-	LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesScanned;
-
-	LMS_LOG(UI, INFO) << "Checks complete, force scan = " << forceScan;
-
-	LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "'...";
-	scanMediaDirectory(_mediaDirectory, forceScan, stats);
-	LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "' DONE";
-
-	removeOrphanEntries();
-
-	if (!_abortScan)
 	{
-		checkDuplicatedAudioFiles(stats);
+		LMS_LOG(DBUPDATER, DEBUG) << "Starting scan step '" << scanStep->getStepName() << "'";
-		reloadSimilarityEngine(stats);
+		scanContext.currentStepStats = ScanStepStats {Wt::WDateTime::currentDateTime(), scanStep->getStep()};
+
+		notifyInProgress(scanContext.currentStepStats);
+		scanStep->process(scanContext);
+		notifyInProgress(scanContext.currentStepStats);
+		LMS_LOG(DBUPDATER, DEBUG) << "Completed scan step '" << scanStep->getStepName() << "'";
 	}
 
 	LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ",  duplicates = " << stats.duplicates.size();
 
+	// TODO make it a scan step
 	_dbSession.optimize();
 
 	if (!_abortScan)
@@ -559,33 +331,71 @@ ScannerService::scan(bool forceScan)
 void
 ScannerService::refreshScanSettings()
 {
-	auto transaction {_dbSession.createSharedTransaction()};
+	ScannerSettings newSettings {readSettings()};
+	if (_settings == newSettings)
+		return;
 
-	const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
+	LMS_LOG(DBUPDATER, DEBUG) << "Scanner settings updated";
+	LMS_LOG(DBUPDATER, DEBUG) << "skipDuplicateRecordingMBID = " << newSettings.skipDuplicateRecordingMBID;
+	LMS_LOG(DBUPDATER, DEBUG) << "Using scan settings version " << newSettings.scanVersion;
 
-	LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion();
+	_settings = std::move(newSettings);
 
-	_scanVersion = scanSettings->getScanVersion();
+	auto cbFunc {[this](const ScanStepStats& stats)
-	_startTime = scanSettings->getUpdateStartTime();
+		{
-	_updatePeriod = scanSettings->getUpdatePeriod();
+			notifyInProgressIfNeeded(stats);
+		}};
 
+	ScanStepBase::InitParams params
 	{
-		const auto fileExtensions {scanSettings->getAudioFileExtensions()};
+		_settings,
-		_fileExtensions.clear();
+		cbFunc,
-		std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(_fileExtensions),
+		_abortScan,
+		_db
+	};
+
+	_scanSteps.clear();
+	_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
+	_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
+	_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
+	_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
+}
+
+ScannerSettings
+ScannerService::readSettings()
+{
+	ScannerSettings newSettings;
+
+	newSettings.skipDuplicateRecordingMBID = Service<IConfig>::get()->getBool("scanner-skip-duplicate-recording-mbid", false);
+	{
+		auto transaction {_dbSession.createSharedTransaction()};
+
+		const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
+
+		newSettings.scanVersion = scanSettings->getScanVersion();
+		newSettings.startTime = scanSettings->getUpdateStartTime();
+		newSettings.updatePeriod = scanSettings->getUpdatePeriod();
+
+		{
+			const auto fileExtensions {scanSettings->getAudioFileExtensions()};
+			newSettings.supportedExtensions.reserve(fileExtensions.size());
+			std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions),
 				[](const std::filesystem::path& extension) { return std::filesystem::path{ StringUtils::stringToLower(extension.string()) }; });
+		}
+		newSettings.recommendationServiceType = scanSettings->getRecommendationEngineType();
+		newSettings.mediaDirectory = scanSettings->getMediaDirectory();
+
+		const auto clusterTypes = scanSettings->getClusterTypes();
+		std::set<std::string> clusterTypeNames;
+
+		std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
+				std::inserter(clusterTypeNames, clusterTypeNames.begin()),
+				[](ClusterType::pointer clusterType) { return clusterType->getName(); });
+
+		newSettings.clusterTypeNames = std::move(clusterTypeNames);
 	}
-	_mediaDirectory = scanSettings->getMediaDirectory();
-	_recommendationServiceType = scanSettings->getRecommendationEngineType();
 
-	const auto clusterTypes = scanSettings->getClusterTypes();
+	return newSettings;
-	std::set<std::string> clusterTypeNames;
-
-	std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
-			std::inserter(clusterTypeNames, clusterTypeNames.begin()),
-			[](ClusterType::pointer clusterType) { return clusterType->getName(); });
-
-	_metadataParser->setClusterTypeNames(clusterTypeNames);
 }
 
 void
@@ -610,410 +420,4 @@ ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
 		notifyInProgress(stepStats);
 }
 
-void
-ScannerService::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats)
-{
-	Wt::WDateTime lastWriteTime;
-	try
-	{
-		lastWriteTime = getLastWriteTime(file);
-	}
-	catch (LmsException& e)
-	{
-		LMS_LOG(DBUPDATER, ERROR) << e.what();
-		stats.skips++;
-		return;
-	}
-
-	if (!forceScan)
-	{
-		// Skip file if last write is the same
-		auto transaction {_dbSession.createSharedTransaction()};
-
-		const Track::pointer track {Track::findByPath(_dbSession, file)};
-
-		if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t()
-				&& track->getScanVersion() == _scanVersion)
-		{
-			stats.skips++;
-			return;
-		}
-	}
-
-	std::optional<MetaData::Track> trackInfo {_metadataParser->parse(file)};
-	if (!trackInfo)
-	{
-		stats.errors.emplace_back(file, ScanErrorType::CannotParseFile);
-		return;
-	}
-
-	stats.scans++;
-
-	auto uniqueTransaction {_dbSession.createUniqueTransaction()};
-
-	Track::pointer track {Track::findByPath(_dbSession, file) };
-
-	// Skip duplicate recording MBID
-	if (trackInfo->recordingMBID && _skipDuplicateRecordingMBID)
-	{
-		for (Track::pointer otherTrack : Track::findByRecordingMBID(_dbSession, *trackInfo->recordingMBID))
-		{
-			if (track && track->getId() == otherTrack->getId())
-				continue;
-
-			LMS_LOG(DBUPDATER, DEBUG) << "Skipped '" << file.string() << "' (similar recording MBID in '" << otherTrack->getPath().string() << "')";
-			// This recording MBID already exists, just remove what we just scanned
-			if (track)
-			{
-				track.remove();
-				stats.deletions++;
-			}
-			return;
-		}
-	}
-
-	// We estimate this is an audio file if:
-	// - we found a least one audio stream
-	// - the duration is not null
-	if (trackInfo->audioStreams.empty())
-	{
-		LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (no audio stream found)";
-
-		// If Track exists here, delete it!
-		if (track)
-		{
-			track.remove();
-			stats.deletions++;
-		}
-		stats.errors.emplace_back(ScanError {file, ScanErrorType::NoAudioTrack});
-		return;
-	}
-	if (trackInfo->duration == std::chrono::milliseconds::zero())
-	{
-		LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (duration is 0)";
-
-		// If Track exists here, delete it!
-		if (track)
-		{
-			track.remove();
-			stats.deletions++;
-		}
-		stats.errors.emplace_back(ScanError {file, ScanErrorType::BadDuration});
-		return;
-	}
-
-	// ***** Title
-	std::string title;
-	if (!trackInfo->title.empty())
-		title = trackInfo->title;
-	else
-	{
-		// TODO parse file name guess track etc.
-		// For now juste use file name as title
-		title = file.filename().string();
-	}
-
-	// If file already exist, update data
-	// Otherwise, create it
-	if (!track)
-	{
-		// Create a new song
-		track = _dbSession.create<Track>(file);
-		LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'";
-		stats.additions++;
-	}
-	else
-	{
-		LMS_LOG(DBUPDATER, INFO) << "Updating '" << file.string() << "'";
-
-		stats.updates++;
-	}
-
-	// Track related data
-	assert(track);
-
-	track.modify()->clearArtistLinks();
-	// Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files
-	for (const Artist::pointer& artist : getOrCreateArtists(_dbSession, trackInfo->artists, false))
-		track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, artist, TrackArtistLinkType::Artist));
-
-	for (const Artist::pointer& releaseArtist : getOrCreateArtists(_dbSession, trackInfo->albumArtists, false))
-		track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, releaseArtist, TrackArtistLinkType::ReleaseArtist));
-
-	// Allow fallbacks on artists with the same name even if they have MBID, since there is no tag to indicate the MBID of these artists
-	// We could ask MusicBrainz to get all the information, but that would heavily slow down the import process
-	for (const Artist::pointer& conductor : getOrCreateArtists(_dbSession, trackInfo->conductorArtists, true))
-		track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, conductor, TrackArtistLinkType::Conductor));
-
-	for (const Artist::pointer& composer : getOrCreateArtists(_dbSession, trackInfo->composerArtists, true))
-		track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, composer, TrackArtistLinkType::Composer));
-
-	for (const Artist::pointer& lyricist : getOrCreateArtists(_dbSession, trackInfo->lyricistArtists, true))
-		track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, lyricist, TrackArtistLinkType::Lyricist));
-
-	for (const Artist::pointer& mixer : getOrCreateArtists(_dbSession, trackInfo->mixerArtists, true))
-		track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, mixer, TrackArtistLinkType::Mixer));
-
-	for (const auto& [role, performers] : trackInfo->performerArtists)
-	{
-		for (const Artist::pointer& performer : getOrCreateArtists(_dbSession, performers, true))
-			track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, performer, TrackArtistLinkType::Performer, role));
-	}
-
-	for (const Artist::pointer& producer : getOrCreateArtists(_dbSession, trackInfo->producerArtists, true))
-		track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, producer, TrackArtistLinkType::Producer));
-
-	for (const Artist::pointer& remixer : getOrCreateArtists(_dbSession, trackInfo->remixerArtists, true))
-		track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, remixer, TrackArtistLinkType::Remixer));
-
-	track.modify()->setScanVersion(_scanVersion);
-	if (trackInfo->album)
-		track.modify()->setRelease(getOrCreateRelease(_dbSession, *trackInfo->album));
-	else
-		track.modify()->setRelease({});
-	track.modify()->setClusters(getOrCreateClusters(_dbSession, trackInfo->clusters));
-	track.modify()->setLastWriteTime(lastWriteTime);
-	track.modify()->setName(title);
-	track.modify()->setDuration(trackInfo->duration);
-	track.modify()->setAddedTime(Wt::WDateTime::currentDateTime());
-	track.modify()->setTrackNumber(trackInfo->trackNumber ? *trackInfo->trackNumber : 0);
-	track.modify()->setDiscNumber(trackInfo->discNumber ? *trackInfo->discNumber : 0);
-	track.modify()->setTotalTrack(trackInfo->totalTrack);
-	track.modify()->setTotalDisc(trackInfo->totalDisc);
-	track.modify()->setDiscSubtitle(trackInfo->discSubtitle);
-	track.modify()->setDate(trackInfo->date);
-	track.modify()->setOriginalDate(trackInfo->originalDate);
-
-	// If a file has an OriginalYear but no Year, set it to ease filtering
-	if (!trackInfo->date.isValid() && trackInfo->originalDate.isValid())
-		track.modify()->setDate(trackInfo->originalDate);
-
-	track.modify()->setRecordingMBID(trackInfo->recordingMBID);
-	track.modify()->setTrackMBID(trackInfo->trackMBID);
-	if (auto trackFeatures {TrackFeatures::find(_dbSession, track->getId())})
-		trackFeatures.remove(); // TODO: only if MBID changed?
-	track.modify()->setHasCover(trackInfo->hasCover);
-	track.modify()->setCopyright(trackInfo->copyright);
-	track.modify()->setCopyrightURL(trackInfo->copyrightURL);
-	track.modify()->setTrackReplayGain(trackInfo->trackReplayGain);
-	track.modify()->setReleaseReplayGain(trackInfo->albumReplayGain);
-}
-
-void
-ScannerService::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
-{
-	ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles};
-	stepStats.totalElems = stats.filesScanned;
-	notifyInProgress(stepStats);
-
-	exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
-	{
-		if (_abortScan)
-			return false;
-
-		if (ec)
-		{
-			LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message();
-			stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()});
-		}
-		else if (isFileSupported(path, _fileExtensions))
-		{
-			scanAudioFile(path, forceScan, stats );
-
-			stepStats.processedElems++;
-			notifyInProgressIfNeeded(stepStats);
-		}
-
-		return true;
-	}, excludeDirFileName);
-
-	notifyInProgress(stepStats);
-}
-
-// Check if a file exists and is still in a media directory
-static bool
-checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::vector<std::filesystem::path>& extensions)
-{
-	try
-	{
-		// For each track, make sure the the file still exists
-		// and still belongs to a media directory
-		if (!std::filesystem::exists( p )
-			|| !std::filesystem::is_regular_file( p ) )
-		{
-			LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing";
-			return false;
-		}
-
-		if (!isPathInMediaDirectory(p, mediaDirectory))
-		{
-			LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory";
-			return false;
-		}
-
-		if (!isFileSupported(p, extensions))
-		{
-			LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled";
-			return false;
-		}
-
-		return true;
-	}
-	catch (std::filesystem::filesystem_error& e)
-	{
-		LMS_LOG(DBUPDATER, ERROR) << "Caught exception while checking file '" << p.string() << "': " << e.what();
-		return false;
-	}
-}
-
-void
-ScannerService::removeMissingTracks(ScanStats& stats)
-{
-	static constexpr std::size_t batchSize {50};
-
-	ScanStepStats stepStats{stats.startTime, ScanProgressStep::ChekingForMissingFiles};
-
-	LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
-	std::size_t trackCount {};
-
-	{
-		auto transaction {_dbSession.createSharedTransaction()};
-		trackCount = Track::getCount(_dbSession);
-	}
-	LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked...";
-
-	stepStats.totalElems = trackCount;
-	notifyInProgress(stepStats);
-
-	RangeResults<Track::PathResult> trackPaths;
-	std::vector<TrackId> tracksToRemove;
-
-	for (std::size_t i {trackCount < batchSize ? 0 : trackCount - batchSize}; ; i -= (i > batchSize ? batchSize : i))
-	{
-		tracksToRemove.clear();
-
-		{
-			auto transaction {_dbSession.createSharedTransaction()};
-			trackPaths = Track::findPaths(_dbSession, Range {i, batchSize});
-		}
-
-		for (const Track::PathResult& trackPath : trackPaths.results)
-		{
-			if (_abortScan)
-				return;
-
-			if (!checkFile(trackPath.path, _mediaDirectory, _fileExtensions))
-				tracksToRemove.push_back(trackPath.trackId);
-
-			stepStats.processedElems++;
-		}
-
-		if (!tracksToRemove.empty())
-		{
-			auto transaction {_dbSession.createUniqueTransaction()};
-
-			for (const TrackId trackId : tracksToRemove)
-			{
-				Track::pointer track {Track::find(_dbSession, trackId)};
-				if (track)
-				{
-					track.remove();
-					stats.deletions++;
-				}
-			}
-		}
-
-		notifyInProgressIfNeeded(stepStats);
-
-		if (i == 0)
-			break;
-	}
-
-	LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks checked!";
-}
-
-void
-ScannerService::removeOrphanEntries()
-{
-	LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
-	{
-		auto transaction {_dbSession.createUniqueTransaction()};
-
-		// Now process orphan Cluster (no track)
-		auto clusterIds {Cluster::findOrphans(_dbSession, Range {})};
-		for (ClusterId clusterId : clusterIds.results)
-		{
-			Cluster::pointer cluster {Cluster::find(_dbSession, clusterId)};
-			LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'";
-			cluster.remove();
-		}
-	}
-
-	LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists...";
-	{
-		auto transaction {_dbSession.createUniqueTransaction()};
-
-		auto artistIds {Artist::findAllOrphans(_dbSession, Range {})};
-		for (const ArtistId artistId : artistIds.results)
-		{
-			Artist::pointer artist {Artist::find(_dbSession, artistId)};
-			LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'";
-			artist.remove();
-		}
-	}
-
-	LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases...";
-	{
-		auto transaction {_dbSession.createUniqueTransaction()};
-
-		auto releases {Release::findOrphans(_dbSession, Range {})};
-		for (const ReleaseId releaseId : releases.results)
-		{
-			Release::pointer release {Release::find(_dbSession, releaseId)};
-			LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'";
-			release.remove();
-		}
-	}
-
-	LMS_LOG(DBUPDATER, INFO) << "Check audio files done!";
-}
-
-void
-ScannerService::checkDuplicatedAudioFiles(ScanStats& stats)
-{
-	LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files";
-
-	auto transaction {_dbSession.createSharedTransaction()};
-
-	const RangeResults<TrackId> tracks = Track::findTrackMBIDDuplicates(_dbSession, Range {});
-	for (const TrackId trackId : tracks.results)
-	{
-		const Track::pointer track {Track::find(_dbSession, trackId)};
-		if (auto trackMBID {track->getTrackMBID()})
-		{
-			LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
-			stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID});
-		}
-	}
-
-	LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!";
-}
-
-void
-ScannerService::reloadSimilarityEngine(ScanStats& stats)
-{
-	ScanStepStats stepStats {stats.startTime, ScanProgressStep::ReloadingSimilarityEngine};
-
-	auto progressCallback {[&](const Recommendation::Progress& progress)
-	{
-		stepStats.totalElems = progress.totalElems;
-		stepStats.processedElems = progress.processedElems;
-		notifyInProgressIfNeeded(stepStats);
-	}};
-
-	notifyInProgress(stepStats);
-	_recommendationService.load(stats.nbChanges() > 0, progressCallback);
-	notifyInProgress(stepStats);
-}
-
 } // namespace Scanner

src/libs/services/scanner/impl/ScannerService.hpp

@@ -30,14 +30,13 @@
 
 #include <boost/asio/system_timer.hpp>
 
-#include "services/database/Types.hpp"
+#include "services/database/Db.hpp"
-#include "services/database/ScanSettings.hpp"
 #include "services/database/Session.hpp"
-#include "metadata/IParser.hpp"
+#include "services/database/Types.hpp"
 #include "services/scanner/IScannerService.hpp"
 #include "utils/Path.hpp"
-
+#include "IScanStep.hpp"
-class UUID;
+#include "ScannerSettings.hpp"
 
 namespace Recommendation
 {
@@ -80,27 +79,24 @@ namespace Scanner
 
 			// Helpers
 			void refreshScanSettings();
+			ScannerSettings readSettings();
 
-			void countAllFiles(ScanStats& stats);
-			void removeMissingTracks(ScanStats& stats);
-			void removeOrphanEntries();
-			void checkDuplicatedAudioFiles(ScanStats& stats);
-			void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats);
 			void notifyInProgressIfNeeded(const ScanStepStats& stats);
 			void notifyInProgress(const ScanStepStats& stats);
 			void reloadSimilarityEngine(ScanStats& stats);
 
 			Recommendation::IRecommendationService&	_recommendationService;
 
+			std::vector<std::unique_ptr<IScanStep>>	_scanSteps;
+
 			std::mutex								_controlMutex;
-			std::atomic<bool>						_abortScan {};
+			bool									_abortScan {};
 			Wt::WIOService							_ioService;
 			boost::asio::system_timer				_scheduleTimer {_ioService};
-			const bool								_skipDuplicateRecordingMBID {};
 			Events									_events;
 			std::chrono::system_clock::time_point	_lastScanInProgressEmit {};
+			Database::Db&							_db;
 			Database::Session						_dbSession;
-			std::unique_ptr<MetaData::IParser>		_metadataParser;
 
 			mutable std::shared_mutex			_statusMutex;
 			State								_curState {State::NotScheduled};
@@ -108,13 +104,7 @@ namespace Scanner
 			std::optional<ScanStepStats> 		_currentScanStepStats;
 			Wt::WDateTime						_nextScheduledScan;
 
-			// Current scan settings
+			ScannerSettings						_settings;
-			std::size_t				_scanVersion {};
-			Wt::WTime				_startTime;
-			Database::ScanSettings::UpdatePeriod 	_updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
-			std::vector<std::filesystem::path>		_fileExtensions;
-			std::filesystem::path					_mediaDirectory;
-			Database::ScanSettings::RecommendationEngineType _recommendationServiceType;
 	};
 } // Scanner
 

src/libs/services/scanner/impl/ScannerSettings.hpp

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <set>
+#include <string>
+#include <vector>
+#include <Wt/WDateTime.h>
+#include "services/database/ScanSettings.hpp"
+
+namespace Scanner
+{
+	struct ScannerSettings
+	{
+		std::size_t											scanVersion {};
+		Wt::WTime											startTime;
+		Database::ScanSettings::UpdatePeriod 				updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
+		std::vector<std::filesystem::path>					supportedExtensions;
+		Database::ScanSettings::RecommendationEngineType	recommendationServiceType;
+		std::filesystem::path								mediaDirectory;
+		bool												skipDuplicateRecordingMBID {};
+		std::set<std::string>								clusterTypeNames;
+
+		bool operator==(const ScannerSettings& rhs) const
+		{
+			return scanVersion == rhs.scanVersion
+				&& startTime == rhs.startTime
+				&& updatePeriod == rhs.updatePeriod
+				&& supportedExtensions == rhs.supportedExtensions
+				&& recommendationServiceType == rhs.recommendationServiceType
+				&& mediaDirectory == rhs.mediaDirectory
+				&& skipDuplicateRecordingMBID == rhs.skipDuplicateRecordingMBID
+				&& clusterTypeNames == rhs.clusterTypeNames;
+		}
+	};
+}

src/libs/services/scanner/include/services/scanner/ScannerStats.hpp

@@ -57,11 +57,12 @@ namespace Scanner
 		DuplicateReason		reason;
 	};
 
-	enum class ScanProgressStep : unsigned
+	enum class ScanStep
 	{
-		ChekingForMissingFiles = 0,
 		DiscoveringFiles,
 		ScanningFiles,
+		ChekingForMissingFiles,
+		CheckingForDuplicateFiles,
 		FetchingTrackFeatures,
 		ReloadingSimilarityEngine,
 	};
@@ -72,7 +73,7 @@ namespace Scanner
 	{
 		Wt::WDateTime   startTime;
 
-		ScanProgressStep currentStep;
+		ScanStep currentStep;
 
 		std::size_t	totalElems {};
 		std::size_t	processedElems {};

src/libs/subsonic/impl/Scan.cpp

@@ -39,7 +39,7 @@ namespace API::Subsonic::Scan
 		{
 			std::size_t count{};
 
-			if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanProgressStep::ScanningFiles)
+			if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanningFiles)
 				count = scanStatus.currentScanStepStats->processedElems;
 
 			statusResponse.setAttribute("count", count);

src/libs/utils/impl/Path.cpp

@@ -31,106 +31,145 @@
 #include "utils/Crc32Calculator.hpp"
 #include "utils/Exception.hpp"
 #include "utils/Logger.hpp"
+#include "utils/String.hpp"
 
-std::uint32_t
+namespace PathUtils
-computeCrc32(const std::filesystem::path& p)
 {
-	Utils::Crc32Calculator crc32;
 
-	std::ifstream ifs {p.string().c_str(), std::ios_base::binary};
+	std::uint32_t
-	if (ifs)
+	computeCrc32(const std::filesystem::path& p)
 	{
-		do
+		Utils::Crc32Calculator crc32;
+
+		std::ifstream ifs {p.string().c_str(), std::ios_base::binary};
+		if (ifs)
 		{
-			std::array<char,1024>	buffer;
+			do
+			{
+				std::array<char,1024>	buffer;
 
-			ifs.read( buffer.data(), buffer.size() );
+				ifs.read( buffer.data(), buffer.size() );
-			crc32.processBytes( reinterpret_cast<const std::byte*>(buffer.data()), ifs.gcount() );
+				crc32.processBytes( reinterpret_cast<const std::byte*>(buffer.data()), ifs.gcount() );
-		}
+			}
-		while (ifs);
+			while (ifs);
-	}
-	else
-	{
-		LMS_LOG(DBUPDATER, ERROR) << "Failed to open file '" << p.string() << "'";
-		throw LmsException("Failed to open file '" + p.string() + "'" );
-	}
-
-	return crc32.getResult();
-}
-
-bool
-ensureDirectory(const std::filesystem::path& dir)
-{
-	if (std::filesystem::exists(dir))
-		return std::filesystem::is_directory(dir);
-	else
-		return std::filesystem::create_directory(dir);
-}
-
-Wt::WDateTime
-getLastWriteTime(const std::filesystem::path& file)
-{
-	struct stat sb {};
-
-	if (stat(file.string().c_str(), &sb) == -1)
-		throw LmsException("Failed to get stats on file '" + file.string() + "'" );
-
-	return Wt::WDateTime::fromTime_t(sb.st_mtime);
-}
-
-bool
-exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path& excludeDirFileName)
-{
-	std::error_code ec;
-	std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec};
-
-	if (ec)
-	{
-		cb(ec, directory);
-		return true; // try to continue exploring anyway
-	}
-
-	if (!excludeDirFileName.empty())
-	{
-		const std::filesystem::path excludePath {directory / excludeDirFileName};
-
-		if (std::filesystem::exists(excludePath, ec))
-		{
-			LMS_LOG(DBUPDATER, DEBUG) << "Found '" << excludePath.string() << "': skipping directory";
-			return true;
-		}
-	}
-
-	std::filesystem::directory_iterator itEnd;
-	while (itPath != itEnd)
-	{
-		bool continueExploring {true};
-
-		if (ec)
-		{
-			continueExploring = cb(ec, *itPath);
 		}
 		else
 		{
-			if (std::filesystem::is_regular_file(*itPath, ec))
+			LMS_LOG(DBUPDATER, ERROR) << "Failed to open file '" << p.string() << "'";
+			throw LmsException("Failed to open file '" + p.string() + "'" );
+		}
+
+		return crc32.getResult();
+	}
+
+	bool
+	ensureDirectory(const std::filesystem::path& dir)
+	{
+		if (std::filesystem::exists(dir))
+			return std::filesystem::is_directory(dir);
+		else
+			return std::filesystem::create_directory(dir);
+	}
+
+	Wt::WDateTime
+	getLastWriteTime(const std::filesystem::path& file)
+	{
+		struct stat sb {};
+
+		if (stat(file.string().c_str(), &sb) == -1)
+			throw LmsException("Failed to get stats on file '" + file.string() + "'" );
+
+		return Wt::WDateTime::fromTime_t(sb.st_mtime);
+	}
+
+	bool
+	exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path* excludeDirFileName)
+	{
+		std::error_code ec;
+		std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec};
+
+		if (ec)
+		{
+			cb(ec, directory);
+			return true; // try to continue exploring anyway
+		}
+
+		if (excludeDirFileName && !excludeDirFileName->empty())
+		{
+			const std::filesystem::path excludePath {directory / *excludeDirFileName};
+
+			if (std::filesystem::exists(excludePath, ec))
 			{
-				continueExploring = cb(ec, *itPath);
+				LMS_LOG(DBUPDATER, DEBUG) << "Found '" << excludePath.string() << "': skipping directory";
-			}
+				return true;
-			else if (std::filesystem::is_directory(*itPath, ec))
-			{
-				if (!ec)
-					continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName);
-				else
-					continueExploring = cb(ec, *itPath);
 			}
 		}
 
-		if (!continueExploring)
+		std::filesystem::directory_iterator itEnd;
-			return false;
+		while (itPath != itEnd)
+		{
+			bool continueExploring {true};
 
-		itPath.increment(ec);
+			if (ec)
+			{
+				continueExploring = cb(ec, *itPath);
+			}
+			else
+			{
+				if (std::filesystem::is_regular_file(*itPath, ec))
+				{
+					continueExploring = cb(ec, *itPath);
+				}
+				else if (std::filesystem::is_directory(*itPath, ec))
+				{
+					if (!ec)
+						continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName);
+					else
+						continueExploring = cb(ec, *itPath);
+				}
+			}
+
+			if (!continueExploring)
+				return false;
+
+			itPath.increment(ec);
+		}
+
+		return true;
 	}
 
-	return true;
+	bool
-}
+	hasFileAnyExtension(const std::filesystem::path& file, const std::vector<std::filesystem::path>& supportedExtensions)
+	{
+		const std::filesystem::path extension {StringUtils::stringToLower(file.extension().string())};
 
+		return (std::find(std::cbegin(supportedExtensions), std::cend(supportedExtensions), extension) != std::cend(supportedExtensions));
+	}
+
+	bool
+	isPathInRootPath(const std::filesystem::path& path, const std::filesystem::path& rootPath, const std::filesystem::path* excludeDirFileName)
+	{
+		std::filesystem::path curPath = path;
+
+		while (curPath.parent_path() != curPath)
+		{
+			curPath = curPath.parent_path();
+
+			if (excludeDirFileName && !excludeDirFileName->empty())
+			{
+				assert(!excludeDirFileName->has_parent_path());
+
+				std::error_code ec;
+				if (std::filesystem::exists(curPath / *excludeDirFileName, ec))
+					return false;
+			}
+
+			if (curPath == rootPath)
+				return true;
+		}
+
+		return false;
+	}
+
+
+} // ns PathUtils

src/libs/utils/impl/Zipper.cpp

@@ -266,7 +266,7 @@ namespace Zip
 			if (lastModifiedTime.isValid())
 				fileContext.lastModifiedTime = lastModifiedTime;
 			else
-				fileContext.lastModifiedTime = getLastWriteTime(filePath);
+				fileContext.lastModifiedTime = PathUtils::getLastWriteTime(filePath);
 
 			_files[filename] = std::move(fileContext);
 

src/libs/utils/include/utils/Path.hpp

@@ -26,15 +26,25 @@
 
 #include <Wt/WDateTime.h>
 
-std::uint32_t computeCrc32(const std::filesystem::path& p);
+namespace PathUtils
+{
+	std::uint32_t computeCrc32(const std::filesystem::path& p);
 
-// Make sure the given path is a directory
+	// Make sure the given path is a directory
-// Create it if needed
+	// Create it if needed
-bool ensureDirectory(const std::filesystem::path& dir);
+	bool ensureDirectory(const std::filesystem::path& dir);
 
-// Get the last write time since Epoch
+	// Get the last write time since Epoch
-Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir);
+	Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir);
 
-// returns false if aborted by user
+	// returns false if aborted by user
-bool exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path& excludeDirFileName = {});
+	bool exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path* excludeDirFileName = {});
 
+	// Check if file's extension is one of provided extensions
+	bool hasFileAnyExtension(const std::filesystem::path& file, const std::vector<std::filesystem::path>& extensions);
+
+	// Check if a path is within a directory (excludeDirFileName is a relative can be used to exclude a whole directory and its subdirectory, must not have parent_path)
+	bool isPathInRootPath(const std::filesystem::path& path, const std::filesystem::path& rootPath, const std::filesystem::path* excludeDirFileName = {});
+
+
+}

src/libs/utils/include/utils/Tuple.hpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2015 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <tuple>
+
+namespace Utils
+{
+	namespace Details
+	{
+		template<int... Is>
+		struct Seq { };
+
+		template<int N, int... Is>
+		struct GenSeq : GenSeq<N - 1, N - 1, Is...> { };
+
+		template<int... Is>
+		struct GenSeq<0, Is...> : Seq<Is...> { };
+
+	    template<typename T, typename Func, int... Is>
+	    void forEachTypeInTuple(T&& t, Func f, Seq<Is...>)
+	    {
+	        auto l = { (f(std::get<Is>(t)), 0)... };
+	    }
+	}
+
+	template<typename... Ts, typename Func>
+	void forEachTypeInTuple(std::tuple<Ts...> const& t, Func f)
+	{
+	    Details::forEachTypeInTuple(t, f, Details::GenSeq<sizeof...(Ts)>());
+	}
+}
+
+

src/lms/ui/admin/ScannerController.cpp

@@ -237,30 +237,35 @@ ScannerController::refreshContents()
 
 			switch (status.currentScanStepStats->currentStep)
 			{
-				case Scanner::ScanProgressStep::ChekingForMissingFiles:
+				case Scanner::ScanStep::CheckingForDuplicateFiles:
+					_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files")
+						.arg(status.currentScanStepStats->processedElems));
+					break;
+
+				case Scanner::ScanStep::ChekingForMissingFiles:
 					_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files")
 						.arg(status.currentScanStepStats->progress()));
 					break;
 
-				case Scanner::ScanProgressStep::DiscoveringFiles:
+				case Scanner::ScanStep::DiscoveringFiles:
 					_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
 						.arg(status.currentScanStepStats->processedElems));
 						break;
 
-				case Scanner::ScanProgressStep::ScanningFiles:
+				case Scanner::ScanStep::ScanningFiles:
 					_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
 						.arg(status.currentScanStepStats->processedElems)
 						.arg(status.currentScanStepStats->totalElems)
 						.arg(status.currentScanStepStats->progress()));
 					break;
 
-				case Scanner::ScanProgressStep::FetchingTrackFeatures:
+				case Scanner::ScanStep::FetchingTrackFeatures:
 					_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
 						.arg(status.currentScanStepStats->processedElems)
 						.arg(status.currentScanStepStats->totalElems)
 						.arg(status.currentScanStepStats->progress()));
 					break;
-				case Scanner::ScanProgressStep::ReloadingSimilarityEngine:
+				case Scanner::ScanStep::ReloadingSimilarityEngine:
 					_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine")
 						.arg(status.currentScanStepStats->progress()));
 					break;