Added playlist import (and sync)from m3u files, ref #391

src/libs/services/scanner/impl/steps/FileScanQueue.cpp

@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "FileScanQueue.hpp"
+
+#include <boost/asio/post.hpp>
+
+#include "core/ITraceLogger.hpp"
+
+#include "scanners/IFileScanOperation.hpp"
+
+namespace lms::scanner
+{
+    FileScanQueue::FileScanQueue(std::size_t threadCount, bool& abort)
+        : _scanContextRunner{ _scanIoContext, threadCount, "FileScan" }
+        , _abort{ abort }
+    {
+    }
+
+    FileScanQueue::~FileScanQueue() = default;
+
+    void FileScanQueue::pushScanRequest(std::unique_ptr<IFileScanOperation> operation)
+    {
+        {
+            std::scoped_lock lock{ _mutex };
+            _ongoingScanCount += 1;
+        }
+
+        auto operationHandler{ [operation = std::move(operation), this]() mutable {
+            if (_abort)
+            {
+                std::scoped_lock lock{ _mutex };
+                _ongoingScanCount -= 1;
+            }
+            else
+            {
+                {
+                    LMS_SCOPED_TRACE_OVERVIEW("Scanner", operation->getName());
+                    operation->scan();
+                }
+
+                {
+                    std::scoped_lock lock{ _mutex };
+
+                    _scanResults.emplace_back(std::move(operation));
+                    _ongoingScanCount -= 1;
+                }
+            }
+
+            _condVar.notify_all();
+        } };
+
+        boost::asio::post(_scanIoContext, std::move(operationHandler));
+    }
+
+    std::size_t FileScanQueue::getResultsCount() const
+    {
+        std::scoped_lock lock{ _mutex };
+        return _scanResults.size();
+    }
+
+    size_t FileScanQueue::popResults(std::vector<std::unique_ptr<IFileScanOperation>>& results, std::size_t maxCount)
+    {
+        results.clear();
+        results.reserve(maxCount);
+
+        {
+            std::scoped_lock lock{ _mutex };
+
+            while (results.size() < maxCount && !_scanResults.empty())
+            {
+                results.push_back(std::move(_scanResults.front()));
+                _scanResults.pop_front();
+            }
+        }
+
+        return results.size();
+    }
+
+    void FileScanQueue::wait(std::size_t maxScanRequestCount)
+    {
+        if (_ongoingScanCount <= maxScanRequestCount)
+            return;
+
+        {
+            LMS_SCOPED_TRACE_OVERVIEW("Scanner", "WaitParseResults");
+
+            std::unique_lock lock{ _mutex };
+            _condVar.wait(lock, [=, this] { return _ongoingScanCount <= maxScanRequestCount; });
+        }
+    }
+
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/FileScanQueue.hpp

@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <atomic>
+#include <condition_variable>
+#include <deque>
+#include <memory>
+#include <mutex>
+#include <vector>
+
+#include "core/IOContextRunner.hpp"
+
+namespace lms::scanner
+{
+    class IFileScanOperation;
+
+    class FileScanQueue
+    {
+    public:
+        FileScanQueue(std::size_t threadCount, bool& abort);
+        ~FileScanQueue();
+        FileScanQueue(const FileScanQueue&) = delete;
+        FileScanQueue& operator=(const FileScanQueue&) = delete;
+
+        std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); }
+
+        void pushScanRequest(std::unique_ptr<IFileScanOperation> operation);
+
+        std::size_t getResultsCount() const;
+        size_t popResults(std::vector<std::unique_ptr<IFileScanOperation>>& results, std::size_t maxCount);
+
+        void wait(std::size_t maxScanRequestCount = 0); // wait until ongoing scan request count <= maxScanRequestCount
+
+    private:
+        boost::asio::io_context _scanIoContext;
+        core::IOContextRunner _scanContextRunner;
+
+        mutable std::mutex _mutex;
+        std::atomic<std::size_t> _ongoingScanCount{};
+        std::deque<std::unique_ptr<IFileScanOperation>> _scanResults;
+        std::condition_variable _condVar;
+        bool& _abort;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/IScanStep.hpp

@@ -0,0 +1,37 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "core/LiteralString.hpp"
+
+#include "ScanContext.hpp"
+
+namespace lms::scanner
+{
+    class IScanStep
+    {
+    public:
+        virtual ~IScanStep() = default;
+
+        virtual ScanStep getStep() const = 0;
+        virtual core::LiteralString getStepName() const = 0;
+        virtual void process(ScanContext& context) = 0;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp

@@ -0,0 +1,256 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepAssociateArtistImages.hpp"
+
+#include <array>
+#include <cassert>
+#include <deque>
+#include <set>
+#include <span>
+
+#include "core/IConfig.hpp"
+#include "core/ILogger.hpp"
+#include "core/Path.hpp"
+#include "database/Artist.hpp"
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/Image.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+
+namespace lms::scanner
+{
+    namespace
+    {
+        constexpr std::size_t readBatchSize{ 100 };
+        constexpr std::size_t writeBatchSize{ 20 };
+
+        struct ArtistImageAssociation
+        {
+            db::ArtistId artistId;
+            db::ImageId imageId;
+        };
+        using ArtistImageAssociationContainer = std::deque<ArtistImageAssociation>;
+
+        struct SearchImageContext
+        {
+            db::Session& session;
+            db::ArtistId lastRetrievedArtistId;
+            std::size_t processedArtistCount{};
+            std::span<const std::string> artistFileNames;
+        };
+
+        db::Image::pointer findImageInDirectory(SearchImageContext& searchContext, const std::filesystem::path& directoryPath)
+        {
+            db::Image::pointer image;
+
+            const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) };
+            if (directory) // may not exist for artists that are split on different media libraries
+            {
+                for (std::string_view fileStem : searchContext.artistFileNames)
+                {
+                    db::Image::FindParameters params;
+                    params.setDirectory(directory->getId());
+                    params.setFileStem(fileStem);
+
+                    db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) {
+                        if (!image)
+                            image = foundImg;
+                    });
+
+                    if (image)
+                        break;
+                }
+            }
+
+            return image;
+        }
+
+        db::Image::pointer computeBestArtistImage(SearchImageContext& searchContext, const db::Artist::pointer& artist)
+        {
+            db::Image::pointer image;
+
+            const auto mbid{ artist->getMBID() };
+            if (mbid)
+            {
+                // Find anywhere, since it is suppoed to be unique!
+                db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer foundImg) {
+                    if (!image)
+                        image = foundImg;
+                });
+            }
+
+            if (!image)
+            {
+                std::set<std::filesystem::path> releasePaths;
+                db::Directory::FindParameters params;
+                params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist });
+
+                db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) {
+                    releasePaths.insert(directory->getAbsolutePath());
+                });
+
+                if (!releasePaths.empty())
+                {
+                    // Expect layout like this:
+                    // ReleaseArtist/Release/Tracks'
+                    //              /artist.jpg
+                    //              /someOtherUserConfiguredArtistFile.jpg
+                    //
+                    // Or:
+                    // ReleaseArtist/SomeGrouping/Release/Tracks'
+                    //              /artist.jpg
+                    //              /someOtherUserConfiguredArtistFile.jpg
+                    //
+                    std::filesystem::path directoryToInspect{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
+                    while (true)
+                    {
+                        image = findImageInDirectory(searchContext, directoryToInspect);
+                        if (image)
+                            break;
+
+                        std::filesystem::path parentPath{ directoryToInspect.parent_path() };
+                        if (parentPath == directoryToInspect)
+                            break;
+
+                        directoryToInspect = parentPath;
+                    }
+
+                    if (!image)
+                    {
+                        // Expect layout like this:
+                        // ReleaseArtist/Release/Tracks'
+                        //                      /artist.jpg
+                        //                      /someOtherUserConfiguredArtistFile.jpg
+                        for (const std::filesystem::path& releasePath : releasePaths)
+                        {
+                            image = findImageInDirectory(searchContext, releasePath);
+                            if (image)
+                                break;
+                        }
+                    }
+                }
+            }
+
+            return image;
+        }
+
+        bool fetchNextArtistImagesToUpdate(SearchImageContext& searchContext, ArtistImageAssociationContainer& artistImageAssociations)
+        {
+            const db::ArtistId artistId{ searchContext.lastRetrievedArtistId };
+
+            {
+                auto transaction{ searchContext.session.createReadTransaction() };
+
+                db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) {
+                    db::Image::pointer image{ computeBestArtistImage(searchContext, artist) };
+
+                    if (image != artist->getImage())
+                    {
+                        LMS_LOG(DBUPDATER, DEBUG, "Updating artist image for artist '" << artist->getName() << "', using '" << (image ? image->getAbsoluteFilePath().c_str() : "<none>") << "'");
+                        artistImageAssociations.push_back(ArtistImageAssociation{ artist->getId(), image ? image->getId() : db::ImageId{} });
+                    }
+                    searchContext.processedArtistCount++;
+                });
+            }
+
+            return artistId != searchContext.lastRetrievedArtistId;
+        }
+
+        void updateArtistImage(db::Session& session, const ArtistImageAssociation& artistImageAssociation)
+        {
+            db::Artist::pointer artist{ db::Artist::find(session, artistImageAssociation.artistId) };
+            assert(artist);
+
+            db::Image::pointer image;
+            if (artistImageAssociation.imageId.isValid())
+                image = db::Image::find(session, artistImageAssociation.imageId);
+
+            artist.modify()->setImage(image);
+        }
+
+        void updateArtistImages(db::Session& session, ArtistImageAssociationContainer& imageAssociations)
+        {
+            while (!imageAssociations.empty())
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                for (std::size_t i{}; !imageAssociations.empty() && i < writeBatchSize; ++i)
+                {
+                    updateArtistImage(session, imageAssociations.front());
+                    imageAssociations.pop_front();
+                }
+            }
+        }
+
+        std::vector<std::string> constructArtistFileNames()
+        {
+            std::vector<std::string> res;
+
+            core::Service<core::IConfig>::get()->visitStrings("artist-image-file-names",
+                [&res](std::string_view fileName) {
+                    res.emplace_back(fileName);
+                },
+                { "artist" });
+
+            return res;
+        }
+
+    } // namespace
+
+    ScanStepAssociateArtistImages::ScanStepAssociateArtistImages(InitParams& initParams)
+        : ScanStepBase{ initParams }
+        , _artistFileNames{ constructArtistFileNames() }
+    {
+    }
+
+    void ScanStepAssociateArtistImages::process(ScanContext& context)
+    {
+        if (_abortScan)
+            return;
+
+        if (context.stats.nbChanges() == 0)
+            return;
+
+        auto& session{ _db.getTLSSession() };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+            context.currentStepStats.totalElems = db::Artist::getCount(session);
+        }
+
+        SearchImageContext searchContext{
+            .session = session,
+            .lastRetrievedArtistId = {},
+            .artistFileNames = _artistFileNames,
+        };
+
+        ArtistImageAssociationContainer artistImageAssociations;
+        while (fetchNextArtistImagesToUpdate(searchContext, artistImageAssociations))
+        {
+            if (_abortScan)
+                return;
+
+            updateArtistImages(session, artistImageAssociations);
+            context.currentStepStats.processedElems = searchContext.processedArtistCount;
+            _progressCallback(context.currentStepStats);
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.hpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepAssociateArtistImages : public ScanStepBase
+    {
+    public:
+        ScanStepAssociateArtistImages(InitParams& initParams);
+        ~ScanStepAssociateArtistImages() override = default;
+        ScanStepAssociateArtistImages(const ScanStepAssociateArtistImages&) = delete;
+        ScanStepAssociateArtistImages& operator=(const ScanStepAssociateArtistImages&) = delete;
+
+    private:
+        ScanStep getStep() const override { return ScanStep::AssociateArtistImages; }
+        core::LiteralString getStepName() const override { return "Associate artist images"; }
+        void process(ScanContext& context) override;
+
+        const std::vector<std::string> _artistFileNames;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateExternalLyrics.cpp

@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepAssociateExternalLyrics.hpp"
+
+#include <deque>
+
+#include "core/ILogger.hpp"
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+#include "database/TrackLyrics.hpp"
+
+namespace lms::scanner
+{
+    namespace
+    {
+        constexpr std::size_t readBatchSize{ 100 };
+        constexpr std::size_t writeBatchSize{ 20 };
+
+        struct TrackLyricsAssociation
+        {
+            db::TrackLyricsId trackLyricsId;
+            db::TrackId trackId;
+        };
+        using TrackLyricsAssociationContainer = std::deque<TrackLyricsAssociation>;
+
+        struct SearchTrackLyricsContext
+        {
+            db::Session& session;
+            db::TrackLyricsId lastRetrievedTrackLyricsId;
+            std::size_t processedLyricsCount{};
+        };
+
+        db::Track::pointer getMatchingTrack(db::Session& session, const db::TrackLyrics::pointer& lyrics)
+        {
+            db::Track::pointer matchingTrack;
+
+            auto tryMatch = [&](std::string_view stem) {
+                db::Track::FindParameters params;
+                assert(lyrics->getDirectory()->getId().isValid());
+                assert(!lyrics->getFileStem().empty());
+
+                params.setDirectory(lyrics->getDirectory()->getId());
+                params.setFileStem(stem);
+
+                db::Track::find(session, params, [&](const db::Track::pointer& track) {
+                    if (matchingTrack)
+                        LMS_LOG(DBUPDATER, DEBUG, "External lyrics '" << lyrics->getAbsoluteFilePath() << "' already matched with '" << matchingTrack->getAbsoluteFilePath() << "', replaced by '" << track->getAbsoluteFilePath() << "'");
+
+                    matchingTrack = track;
+                });
+            };
+
+            // First try with the stem. If it does not match, try again with the parent steam, if it exists, to handle the file.laguagecode.lrc case
+            tryMatch(lyrics->getFileStem());
+            if (!matchingTrack)
+            {
+                std::filesystem::path stem{ lyrics->getFileStem() };
+                if (stem.has_extension())
+                    tryMatch(stem.stem().string());
+            }
+
+            return matchingTrack;
+        }
+
+        bool fetchNextTrackLyricsToUpdate(SearchTrackLyricsContext& searchContext, TrackLyricsAssociationContainer& trackLyricsAssociations)
+        {
+            const db::TrackLyricsId trackLyricsId{ searchContext.lastRetrievedTrackLyricsId };
+
+            {
+                auto transaction{ searchContext.session.createReadTransaction() };
+
+                db::TrackLyrics::find(searchContext.session, searchContext.lastRetrievedTrackLyricsId, readBatchSize, [&](const db::TrackLyrics::pointer& trackLyrics) {
+                    // Only iterate over external lyrics
+                    if (trackLyrics->getAbsoluteFilePath().empty())
+                        return;
+
+                    db::Track::pointer track{ getMatchingTrack(searchContext.session, trackLyrics) };
+                    if (track != trackLyrics->getTrack())
+                    {
+                        LMS_LOG(DBUPDATER, DEBUG, "Updating track for external lyrics " << trackLyrics->getAbsoluteFilePath() << ", using " << (track ? track->getAbsoluteFilePath() : "<none>"));
+                        trackLyricsAssociations.push_back(TrackLyricsAssociation{ .trackLyricsId = trackLyrics->getId(), .trackId = (track ? track->getId() : db::TrackId{}) });
+                    }
+                    else if (!track)
+                    {
+                        LMS_LOG(DBUPDATER, DEBUG, "No track found for external lyrics " << trackLyrics->getAbsoluteFilePath() << "'");
+                    }
+
+                    searchContext.processedLyricsCount++;
+                });
+            }
+
+            return trackLyricsId != searchContext.lastRetrievedTrackLyricsId;
+        }
+
+        void updateTrackLyrics(db::Session& session, const TrackLyricsAssociation& trackLyricsAssociation)
+        {
+            db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(session, trackLyricsAssociation.trackLyricsId) };
+            assert(lyrics);
+
+            db::Track::pointer track;
+            if (trackLyricsAssociation.trackId.isValid())
+                track = db::Track::find(session, trackLyricsAssociation.trackId);
+
+            lyrics.modify()->setTrack(track);
+        }
+
+        void updateTrackLyrics(db::Session& session, TrackLyricsAssociationContainer& lyricsAssociations)
+        {
+            while (!lyricsAssociations.empty())
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                for (std::size_t i{}; !lyricsAssociations.empty() && i < writeBatchSize; ++i)
+                {
+                    updateTrackLyrics(session, lyricsAssociations.front());
+                    lyricsAssociations.pop_front();
+                }
+            }
+        }
+    } // namespace
+
+    void ScanStepAssociateExternalLyrics::process(ScanContext& context)
+    {
+        if (_abortScan)
+            return;
+
+        if (context.stats.nbChanges() == 0)
+            return;
+
+        auto& session{ _db.getTLSSession() };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+            context.currentStepStats.totalElems = db::TrackLyrics::getExternalLyricsCount(session);
+        }
+
+        SearchTrackLyricsContext searchContext{
+            .session = session,
+            .lastRetrievedTrackLyricsId = {},
+        };
+
+        TrackLyricsAssociationContainer trackLyricsAssociations;
+        while (fetchNextTrackLyricsToUpdate(searchContext, trackLyricsAssociations))
+        {
+            if (_abortScan)
+                return;
+
+            updateTrackLyrics(session, trackLyricsAssociations);
+            context.currentStepStats.processedElems = searchContext.processedLyricsCount;
+            _progressCallback(context.currentStepStats);
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateExternalLyrics.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepAssociateExternalLyrics : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        ScanStep getStep() const override { return ScanStep::AssociateExternalLyrics; }
+        core::LiteralString getStepName() const override { return "Associate external lyrics"; }
+        void process(ScanContext& context) override;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp

@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepAssociatePlayListTracks.hpp"
+
+#include <deque>
+#include <filesystem>
+
+#include "core/ILogger.hpp"
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/PlayListFile.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+#include "database/TrackList.hpp"
+
+namespace lms::scanner
+{
+    namespace
+    {
+        constexpr std::size_t readBatchSize{ 20 };
+        constexpr std::size_t writeBatchSize{ 5 };
+
+        struct PlayListFileAssociation
+        {
+            db::PlayListFileId playListFileIdId;
+            std::vector<db::TrackId> trackIds;
+        };
+        using PlayListFileAssociationContainer = std::deque<PlayListFileAssociation>;
+
+        struct SearchPlayListFileContext
+        {
+            db::Session& session;
+            db::PlayListFileId lastRetrievedPlayListFileId;
+            std::size_t processedPlayListFileCount{};
+        };
+
+        db::Track::pointer getMatchingTrack(db::Session& session, const std::filesystem::path& filePath, const db::Directory::pointer& playListDirectory)
+        {
+            db::Track::pointer matchingTrack;
+            if (filePath.is_absolute())
+            {
+                matchingTrack = db::Track::findByPath(session, filePath);
+            }
+            else
+            {
+                const std::filesystem::path absolutePath{ playListDirectory->getAbsolutePath() / filePath };
+                matchingTrack = db::Track::findByPath(session, absolutePath.lexically_normal());
+            }
+
+            return matchingTrack;
+        }
+
+        bool trackListNeedsUpdate(db::Session& session, std::string_view name, std::span<const db::TrackId> trackIds, const db::TrackList::pointer& trackList)
+        {
+            if (trackList->getName() != name)
+                return true;
+
+            db::TrackListEntry::FindParameters params;
+            params.setTrackList(trackList->getId());
+
+            bool needUpdate{};
+            std::size_t currentIndex{};
+            db::TrackListEntry::find(session, params, [&](const db::TrackListEntry::pointer& entry) {
+                if (currentIndex > trackIds.size() || trackIds[currentIndex] != entry->getTrackId())
+                    needUpdate = true;
+
+                currentIndex += 1;
+            });
+
+            if (currentIndex != trackIds.size())
+                needUpdate = true;
+
+            return needUpdate;
+        }
+
+        bool fetchNextPlayListFilesToUpdate(SearchPlayListFileContext& searchContext, PlayListFileAssociationContainer& playListFileAssociations)
+        {
+            const db::PlayListFileId playListFileIdId{ searchContext.lastRetrievedPlayListFileId };
+
+            {
+                auto transaction{ searchContext.session.createReadTransaction() };
+
+                db::PlayListFile::find(searchContext.session, searchContext.lastRetrievedPlayListFileId, readBatchSize, [&](const db::PlayListFile::pointer& playListFile) {
+                    PlayListFileAssociation playListAssociation;
+
+                    playListAssociation.playListFileIdId = playListFile->getId();
+
+                    const auto files{ playListFile->getFiles() };
+                    for (const std::filesystem::path& file : files)
+                    {
+                        // TODO optim: no need to fetch the whole track
+                        db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) };
+                        if (track)
+                            playListAssociation.trackIds.push_back(track->getId());
+                        else
+                            LMS_LOG(DBUPDATER, DEBUG, "Track '" << file.string() << "' not found in playlist '" << playListFile->getAbsoluteFilePath().string() << "'");
+                    }
+
+                    bool needUpdate{ true };
+                    if (const db::TrackList::pointer trackList{ playListFile->getTrackList() })
+                        needUpdate = trackListNeedsUpdate(searchContext.session, playListFile->getName(), playListAssociation.trackIds, trackList);
+
+                    if (needUpdate)
+                    {
+                        LMS_LOG(DBUPDATER, DEBUG, "Updating PlayList '" << playListFile->getAbsoluteFilePath().string() << "' (" << playListAssociation.trackIds.size() << " files)");
+                        playListFileAssociations.emplace_back(std::move(playListAssociation));
+                    }
+                    searchContext.processedPlayListFileCount++;
+                });
+            }
+
+            return playListFileIdId != searchContext.lastRetrievedPlayListFileId;
+        }
+
+        void updatePlayListFile(db::Session& session, const PlayListFileAssociation& playListFileAssociation)
+        {
+            db::PlayListFile::pointer playListFile{ db::PlayListFile::find(session, playListFileAssociation.playListFileIdId) };
+            assert(playListFile);
+
+            db::TrackList::pointer trackList{ playListFile->getTrackList() };
+            if (!trackList)
+            {
+                trackList = session.create<db::TrackList>(playListFile->getName(), db::TrackListType::PlayList);
+                playListFile.modify()->setTrackList(trackList);
+            }
+
+            trackList.modify()->setVisibility(db::TrackList::Visibility::Public);
+            trackList.modify()->setLastModifiedDateTime(playListFile->getLastWriteTime());
+            trackList.modify()->setName(playListFile->getName());
+
+            trackList.modify()->clear();
+            for (const db::TrackId trackId : playListFileAssociation.trackIds)
+            {
+                if (db::Track::pointer track{ db::Track::find(session, trackId) })
+                    session.create<db::TrackListEntry>(track, trackList, playListFile->getLastWriteTime());
+            }
+        }
+
+        void updatePlayListFiles(db::Session& session, PlayListFileAssociationContainer& playListFileAssociations)
+        {
+            while (!playListFileAssociations.empty())
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                for (std::size_t i{}; !playListFileAssociations.empty() && i < writeBatchSize; ++i)
+                {
+                    updatePlayListFile(session, playListFileAssociations.front());
+                    playListFileAssociations.pop_front();
+                }
+            }
+        }
+    } // namespace
+
+    void ScanStepAssociatePlayListTracks::process(ScanContext& context)
+    {
+        if (_abortScan)
+            return;
+
+        if (context.stats.nbChanges() == 0)
+            return;
+
+        auto& session{ _db.getTLSSession() };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+            context.currentStepStats.totalElems = db::PlayListFile::getCount(session);
+        }
+
+        SearchPlayListFileContext searchContext{
+            .session = session,
+            .lastRetrievedPlayListFileId = {},
+        };
+
+        PlayListFileAssociationContainer playListFileAssociations;
+        while (fetchNextPlayListFilesToUpdate(searchContext, playListFileAssociations))
+        {
+            if (_abortScan)
+                return;
+
+            updatePlayListFiles(session, playListFileAssociations);
+            context.currentStepStats.processedElems = searchContext.processedPlayListFileCount;
+            _progressCallback(context.currentStepStats);
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepAssociatePlayListTracks : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        ScanStep getStep() const override { return ScanStep::AssociatePlayListTracks; }
+        core::LiteralString getStepName() const override { return "Associate playlist tracks"; }
+        void process(ScanContext& context) override;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp

@@ -0,0 +1,234 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepAssociateReleaseImages.hpp"
+
+#include <array>
+#include <cassert>
+#include <deque>
+#include <set>
+
+#include "core/IConfig.hpp"
+#include "core/ILogger.hpp"
+#include "core/Path.hpp"
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/Image.hpp"
+#include "database/Release.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+
+namespace lms::scanner
+{
+    namespace
+    {
+        constexpr std::size_t readBatchSize{ 100 };
+        constexpr std::size_t writeBatchSize{ 20 };
+
+        struct ReleaseImageAssociation
+        {
+            db::ReleaseId releaseId;
+            db::ImageId imageId;
+        };
+        using ReleaseImageAssociationContainer = std::deque<ReleaseImageAssociation>;
+
+        struct SearchImageContext
+        {
+            db::Session& session;
+            db::ReleaseId lastRetrievedReleaseId;
+            std::size_t processedReleaseCount{};
+            const std::vector<std::string>& releaseFileNames;
+        };
+
+        db::Image::pointer findImageInDirectory(SearchImageContext& searchContext, const std::filesystem::path& directoryPath)
+        {
+            db::Image::pointer image;
+
+            const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) };
+            if (directory) // may not exist for releases that are split on different media libraries
+            {
+                for (std::string_view fileStem : searchContext.releaseFileNames)
+                {
+                    db::Image::FindParameters params;
+                    params.setDirectory(directory->getId());
+                    params.setFileStem(fileStem);
+
+                    db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) {
+                        if (!image)
+                            image = foundImg;
+                    });
+
+                    if (image)
+                        break;
+                }
+            }
+
+            return image;
+        }
+
+        db::Image::pointer computeBestReleaseImage(SearchImageContext& searchContext, const db::Release::pointer& release)
+        {
+            db::Image::pointer image;
+
+            const auto mbid{ release->getMBID() };
+            if (mbid)
+            {
+                // Find anywhere, since it is suppoed to be unique!
+                db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer foundImg) {
+                    if (!image)
+                        image = foundImg;
+                });
+            }
+
+            if (!image)
+            {
+                std::set<std::filesystem::path> releasePaths;
+                db::Directory::FindParameters params;
+                params.setRelease(release->getId());
+
+                db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) {
+                    releasePaths.insert(directory->getAbsolutePath());
+                });
+
+                // Expect layout like this:
+                // Artist/Release/CD1/...
+                //               /CD2/...
+                //               /cover.jpg
+                if (releasePaths.size() > 1)
+                {
+                    const std::filesystem::path releasePath{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
+                    image = findImageInDirectory(searchContext, releasePath);
+                }
+
+                if (!image)
+                {
+                    for (const std::filesystem::path& releasePath : releasePaths)
+                    {
+                        image = findImageInDirectory(searchContext, releasePath);
+                        if (image)
+                            break;
+                    }
+                }
+            }
+
+            return image;
+        }
+
+        bool fetchNextReleaseImagesToUpdate(SearchImageContext& searchContext, ReleaseImageAssociationContainer& releaseImageAssociations)
+        {
+            const db::ReleaseId releaseId{ searchContext.lastRetrievedReleaseId };
+
+            {
+                auto transaction{ searchContext.session.createReadTransaction() };
+
+                db::Release::find(searchContext.session, searchContext.lastRetrievedReleaseId, readBatchSize, [&](const db::Release::pointer& release) {
+                    db::Image::pointer image{ computeBestReleaseImage(searchContext, release) };
+
+                    if (image != release->getImage())
+                    {
+                        LMS_LOG(DBUPDATER, DEBUG, "Updating release image for release '" << release->getName() << "', using '" << (image ? image->getAbsoluteFilePath().c_str() : "<none>") << "'");
+                        releaseImageAssociations.push_back(ReleaseImageAssociation{ release->getId(), image ? image->getId() : db::ImageId{} });
+                    }
+                    searchContext.processedReleaseCount++;
+                });
+            }
+
+            return releaseId != searchContext.lastRetrievedReleaseId;
+        }
+
+        void updateReleaseImage(db::Session& session, const ReleaseImageAssociation& releaseImageAssociation)
+        {
+            db::Release::pointer release{ db::Release::find(session, releaseImageAssociation.releaseId) };
+            assert(release);
+
+            db::Image::pointer image;
+            if (releaseImageAssociation.imageId.isValid())
+                image = db::Image::find(session, releaseImageAssociation.imageId);
+
+            release.modify()->setImage(image);
+        }
+
+        void updateReleaseImages(db::Session& session, ReleaseImageAssociationContainer& imageAssociations)
+        {
+            while (!imageAssociations.empty())
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                for (std::size_t i{}; !imageAssociations.empty() && i < writeBatchSize; ++i)
+                {
+                    updateReleaseImage(session, imageAssociations.front());
+                    imageAssociations.pop_front();
+                }
+            }
+        }
+
+        std::vector<std::string> constructReleaseFileNames()
+        {
+            std::vector<std::string> res;
+
+            core::Service<core::IConfig>::get()->visitStrings("cover-preferred-file-names",
+                [&res](std::string_view fileName) {
+                    res.emplace_back(fileName);
+                },
+                { "cover", "front", "folder", "default" });
+
+            return res;
+        }
+
+    } // namespace
+
+    ScanStepAssociateReleaseImages::ScanStepAssociateReleaseImages(InitParams& initParams)
+        : ScanStepBase{ initParams }
+        , _releaseFileNames{ constructReleaseFileNames() }
+    {
+    }
+
+    void ScanStepAssociateReleaseImages::process(ScanContext& context)
+    {
+        if (_abortScan)
+            return;
+
+        if (context.stats.nbChanges() == 0)
+            return;
+
+        auto& session{ _db.getTLSSession() };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+            context.currentStepStats.totalElems = db::Release::getCount(session);
+        }
+
+        SearchImageContext searchContext{
+            .session = session,
+            .lastRetrievedReleaseId = {},
+            .releaseFileNames = _releaseFileNames,
+        };
+
+        ReleaseImageAssociationContainer releaseImageAssociations;
+        while (fetchNextReleaseImagesToUpdate(searchContext, releaseImageAssociations))
+        {
+            if (_abortScan)
+                return;
+
+            updateReleaseImages(session, releaseImageAssociations);
+            context.currentStepStats.processedElems = searchContext.processedReleaseCount;
+            _progressCallback(context.currentStepStats);
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.hpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepAssociateReleaseImages : public ScanStepBase
+    {
+    public:
+        ScanStepAssociateReleaseImages(InitParams& initParams);
+        ~ScanStepAssociateReleaseImages() override = default;
+        ScanStepAssociateReleaseImages(const ScanStepAssociateReleaseImages&) = delete;
+        ScanStepAssociateReleaseImages& operator=(const ScanStepAssociateReleaseImages&) = delete;
+
+    private:
+        ScanStep getStep() const override { return ScanStep::AssociateReleaseImages; }
+        core::LiteralString getStepName() const override { return "Associate release images"; }
+        void process(ScanContext& context) override;
+
+        const std::vector<std::string> _releaseFileNames;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepBase.hpp

@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <functional>
+#include <span>
+#include <vector>
+
+#include "IScanStep.hpp"
+
+namespace lms::db
+{
+    class Db;
+}
+
+namespace lms::scanner
+{
+    class IFileScanner;
+    struct ScannerSettings;
+    struct ScanStepStats;
+
+    class ScanStepBase : public IScanStep
+    {
+    public:
+        using ProgressCallback = std::function<void(const ScanStepStats& stats)>;
+
+        struct InitParams
+        {
+            const ScannerSettings& settings;
+            ProgressCallback progressCallback;
+            bool& abortScan;
+            db::Db& db;
+            std::span<IFileScanner*> fileScanners;
+        };
+        ScanStepBase(InitParams& initParams)
+            : _settings{ initParams.settings }
+            , _progressCallback{ initParams.progressCallback }
+            , _abortScan{ initParams.abortScan }
+            , _db{ initParams.db }
+            , _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners))
+        {
+        }
+
+    protected:
+        ~ScanStepBase() override = default;
+        ScanStepBase(const ScanStepBase&) = delete;
+        ScanStepBase& operator=(const ScanStepBase&) = delete;
+
+        const ScannerSettings& _settings;
+        ProgressCallback _progressCallback;
+        bool& _abortScan;
+        db::Db& _db;
+        std::vector<IFileScanner*> _fileScanners;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepCheckForDuplicatedFiles.cpp

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepCheckForDuplicatedFiles.hpp"
+
+#include "core/ILogger.hpp"
+#include "database/Db.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+
+namespace lms::scanner
+{
+    void ScanStepCheckForDuplicatedFiles::process(ScanContext& context)
+    {
+        using namespace db;
+
+        if (_abortScan)
+            return;
+
+        Session& session{ _db.getTLSSession() };
+        auto transaction{ session.createReadTransaction() };
+
+        const RangeResults<TrackId> tracks = Track::findIdsTrackMBIDDuplicates(session);
+        for (const TrackId trackId : tracks.results)
+        {
+            if (_abortScan)
+                break;
+
+            const Track::pointer track{ Track::find(session, trackId) };
+            if (auto trackMBID{ track->getTrackMBID() })
+            {
+                LMS_LOG(DBUPDATER, INFO, "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getAbsoluteFilePath().string() << " - " << track->getName());
+                context.stats.duplicates.emplace_back(ScanDuplicate{ track->getId(), DuplicateReason::SameTrackMBID });
+                context.currentStepStats.processedElems++;
+                _progressCallback(context.currentStepStats);
+            }
+        }
+
+        LMS_LOG(DBUPDATER, DEBUG, "Found " << context.currentStepStats.processedElems << " duplicated audio files");
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepCheckForDuplicatedFiles.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepCheckForDuplicatedFiles : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        core::LiteralString getStepName() const override { return "Check for duplicated files"; }
+        ScanStep getStep() const override { return ScanStep::CheckForDuplicatedFiles; }
+        void process(ScanContext& context) override;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp

@@ -0,0 +1,163 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepCheckForRemovedFiles.hpp"
+
+#include <vector>
+
+#include "ScannerSettings.hpp"
+#include "core/ILogger.hpp"
+#include "core/Path.hpp"
+#include "database/Db.hpp"
+#include "database/Image.hpp"
+#include "database/PlayListFile.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+#include "database/TrackLyrics.hpp"
+#include "scanners/IFileScanner.hpp"
+
+namespace lms::scanner
+{
+    namespace
+    {
+        constexpr std::size_t batchSize = 100;
+    }
+
+    void ScanStepCheckForRemovedFiles::process(ScanContext& context)
+    {
+        if (_abortScan)
+            return;
+
+        db::Session& session{ _db.getTLSSession() };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+            context.currentStepStats.totalElems = 0;
+            context.currentStepStats.totalElems += db::Track::getCount(session);
+            context.currentStepStats.totalElems += db::Image::getCount(session);
+            context.currentStepStats.totalElems += db::TrackLyrics::getExternalLyricsCount(session);
+            context.currentStepStats.totalElems += db::PlayListFile::getCount(session);
+        }
+        LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " files to be checked...");
+
+        std::vector<std::filesystem::path> supportedFileExtensions;
+        for (IFileScanner* scanner : _fileScanners)
+        {
+            for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
+                supportedFileExtensions.emplace_back(extension);
+        }
+
+        checkForRemovedFiles<db::Track>(context, supportedFileExtensions);
+        checkForRemovedFiles<db::Image>(context, supportedFileExtensions);
+        checkForRemovedFiles<db::TrackLyrics>(context, supportedFileExtensions);
+        checkForRemovedFiles<db::PlayListFile>(context, supportedFileExtensions);
+    }
+
+    template<typename Object>
+    void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context, std::span<const std::filesystem::path> supportedFileExtensions)
+    {
+        using namespace db;
+
+        if (_abortScan)
+            return;
+
+        Session& session{ _db.getTLSSession() };
+
+        std::vector<typename Object::pointer> objectsToRemove;
+
+        typename Object::IdType lastCheckedId;
+        bool endReached{};
+        while (!endReached)
+        {
+            if (_abortScan)
+                break;
+
+            objectsToRemove.clear();
+            {
+                auto transaction{ session.createReadTransaction() };
+
+                endReached = true;
+                Object::find(session, lastCheckedId, batchSize, [&](const typename Object::pointer& object) {
+                    endReached = false;
+
+                    // special case for track lyrics, only check external lyrics
+                    if constexpr (std::is_same_v<Object, TrackLyrics>)
+                    {
+                        if (object->getAbsoluteFilePath().empty())
+                            return;
+                    }
+
+                    if (!checkFile(object->getAbsoluteFilePath(), supportedFileExtensions))
+                        objectsToRemove.push_back(object);
+
+                    context.currentStepStats.processedElems++;
+                });
+            }
+
+            if (!objectsToRemove.empty())
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                for (typename Object::pointer& object : objectsToRemove)
+                {
+                    object.remove();
+                    context.stats.deletions++;
+                }
+            }
+
+            _progressCallback(context.currentStepStats);
+        }
+    }
+
+    bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p, std::span<const std::filesystem::path> allowedExtensions)
+    {
+        try
+        {
+            // For each track, make sure the the file still exists
+            // and still belongs to a media directory
+            if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
+            {
+                LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': missing");
+                return false;
+            }
+
+            if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
+                    [&](const MediaLibraryInfo& libraryInfo) {
+                        return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
+                    }))
+            {
+                LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': out of media directory");
+                return false;
+            }
+
+            if (!core::pathUtils::hasFileAnyExtension(p, allowedExtensions))
+            {
+                LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': file format no longer handled");
+                return false;
+            }
+
+            return true;
+        }
+        catch (std::filesystem::filesystem_error& e)
+        {
+            LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file '" << p.string() << "': " << e.what());
+            return false;
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.hpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <span>
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepCheckForRemovedFiles : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        core::LiteralString getStepName() const override { return "Check for removed files"; }
+        ScanStep getStep() const override { return ScanStep::CheckForRemovedFiles; }
+        void process(ScanContext& context) override;
+
+        template<typename Object>
+        void checkForRemovedFiles(ScanContext& context, std::span<const std::filesystem::path> supportedFileExtensions);
+
+        bool checkFile(const std::filesystem::path& p, std::span<const std::filesystem::path> allowedExtensions);
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepCompact.cpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepCompact.hpp"
+
+#include "database/Db.hpp"
+#include "database/Session.hpp"
+
+namespace lms::scanner
+{
+    void ScanStepCompact::process(ScanContext& context)
+    {
+        // Don't auto compact as it may be too annoying to block the whole application for very large databases
+        if (context.scanOptions.compact)
+            _db.getTLSSession().vacuum();
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepCompact.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepCompact : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        ScanStep getStep() const override { return ScanStep::Compact; }
+        core::LiteralString getStepName() const override { return "Compact"; }
+        void process(ScanContext& context) override;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.cpp

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepComputeClusterStats.hpp"
+#include "core/ILogger.hpp"
+#include "database/Cluster.hpp"
+#include "database/Db.hpp"
+#include "database/Session.hpp"
+
+namespace lms::scanner
+{
+    void ScanStepComputeClusterStats::process(ScanContext& context)
+    {
+        using namespace db;
+
+        if (context.stats.nbChanges() == 0)
+            return;
+
+        Session& dbSession{ _db.getTLSSession() };
+
+        const std::size_t clusterCount{ [&] {
+            auto transaction{ dbSession.createReadTransaction() };
+            return Cluster::getCount(dbSession);
+        }() };
+
+        context.currentStepStats.totalElems = clusterCount;
+
+        foreachSubRange(Range{ 0, clusterCount }, 100, [&](Range range) {
+            const std::vector<ClusterId> clusterIds{ [&] {
+                Cluster::FindParameters params;
+                params.setRange(range);
+
+                {
+                    auto transaction{ dbSession.createReadTransaction() };
+                    return std::move(Cluster::findIds(dbSession, params).results);
+                }
+            }() };
+
+            for (const ClusterId clusterId : clusterIds)
+            {
+                if (_abortScan)
+                    break;
+
+                std::size_t trackCount;
+                std::size_t releaseCount;
+
+                {
+                    auto transaction{ dbSession.createReadTransaction() };
+
+                    trackCount = Cluster::computeTrackCount(dbSession, clusterId);
+                    releaseCount = Cluster::computeReleaseCount(dbSession, clusterId);
+                }
+
+                {
+                    auto transaction{ dbSession.createWriteTransaction() };
+
+                    auto cluster{ Cluster::find(dbSession, clusterId) };
+                    cluster.modify()->setTrackCount(trackCount);
+                    cluster.modify()->setReleaseCount(releaseCount);
+                }
+
+                context.currentStepStats.processedElems++;
+                _progressCallback(context.currentStepStats);
+            }
+
+            return true;
+        });
+
+        LMS_LOG(DBUPDATER, DEBUG, "Recomputed stats for " << context.currentStepStats.processedElems << " clusters!");
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepComputeClusterStats : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        ScanStep getStep() const override { return ScanStep::ComputeClusterStats; }
+        core::LiteralString getStepName() const override { return "Compute cluster stats"; }
+        void process(ScanContext& context) override;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.cpp

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepDiscoverFiles.hpp"
+
+#include "core/ILogger.hpp"
+#include "core/Path.hpp"
+
+#include "MediaLibraryInfo.hpp"
+#include "ScannerSettings.hpp"
+#include "scanners/IFileScanner.hpp"
+
+namespace lms::scanner
+{
+    void ScanStepDiscoverFiles::process(ScanContext& context)
+    {
+        context.stats.totalFileCount = 0;
+
+        std::vector<std::filesystem::path> supportedFileExtensions;
+        for (IFileScanner* scanner : _fileScanners)
+        {
+            for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
+                supportedFileExtensions.emplace_back(extension);
+        }
+
+        for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
+        {
+            std::size_t currentDirectoryProcessElemsCount{};
+            core::pathUtils::exploreFilesRecursive(
+                mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) {
+                    if (_abortScan)
+                        return false;
+
+                    if (!ec && core::pathUtils::hasFileAnyExtension(path, supportedFileExtensions))
+                    {
+                        context.currentStepStats.processedElems++;
+                        currentDirectoryProcessElemsCount++;
+                        _progressCallback(context.currentStepStats);
+                    }
+
+                    return true;
+                },
+                &excludeDirFileName);
+
+            LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in '" << mediaLibrary.rootDirectory << "'");
+        }
+
+        context.stats.totalFileCount = context.currentStepStats.processedElems;
+
+        LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.totalFileCount << " files in all directories");
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepDiscoverFiles : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        ScanStep getStep() const override { return ScanStep::DiscoverFiles; }
+        core::LiteralString getStepName() const override { return "Discover files"; }
+        void process(ScanContext& context) override;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepOptimize.cpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepOptimize.hpp"
+
+#include "core/ILogger.hpp"
+#include "database/Db.hpp"
+#include "database/Session.hpp"
+
+namespace lms::scanner
+{
+    void ScanStepOptimize::process(ScanContext& context)
+    {
+        ScanStats& stats{ context.stats };
+
+        if (context.scanOptions.forceOptimize || (stats.nbChanges() > (stats.nbFiles() / 10)))
+        {
+            LMS_LOG(DBUPDATER, INFO, "Database analyze started");
+
+            auto& session{ _db.getTLSSession() };
+
+            std::vector<std::string> entries;
+            session.retrieveEntriesToAnalyze(entries);
+            context.currentStepStats.totalElems = entries.size();
+            _progressCallback(context.currentStepStats);
+
+            for (const std::string& entry : entries)
+            {
+                if (_abortScan)
+                    break;
+
+                _db.getTLSSession().analyzeEntry(entry);
+                context.currentStepStats.processedElems++;
+                _progressCallback(context.currentStepStats);
+            }
+
+            LMS_LOG(DBUPDATER, INFO, "Database analyze complete");
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepOptimize.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepOptimize : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        ScanStep getStep() const override { return ScanStep::Optimize; }
+        core::LiteralString getStepName() const override { return "Optimize"; }
+        void process(ScanContext& context) override;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.cpp

@@ -0,0 +1,124 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepRemoveOrphanedDbEntries.hpp"
+
+#include "core/ILogger.hpp"
+#include "database/Artist.hpp"
+#include "database/Cluster.hpp"
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/Release.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+
+namespace lms::scanner
+{
+    void ScanStepRemoveOrphanedDbEntries::process(ScanContext& context)
+    {
+        removeOrphanedClusters(context);
+        removeOrphanedClusterTypes(context);
+        removeOrphanedArtists(context);
+        removeOrphanedReleases(context);
+        removeOrphanedReleaseTypes(context);
+        removeOrphanedLabels(context);
+        removeOrphanedDirectories(context);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusters(ScanContext& context)
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned clusters...");
+        removeOrphanedEntries<db::Cluster>(context);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusterTypes(ScanContext& context)
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned cluster types...");
+        removeOrphanedEntries<db::ClusterType>(context);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedArtists(ScanContext& context)
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned artists...");
+        removeOrphanedEntries<db::Artist>(context);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedReleases(ScanContext& context)
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned releases...");
+        removeOrphanedEntries<db::Release>(context);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedReleaseTypes(ScanContext& context)
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned release types...");
+        removeOrphanedEntries<db::ReleaseType>(context);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedLabels(ScanContext& context)
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned labels...");
+        removeOrphanedEntries<db::Label>(context);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedDirectories(ScanContext& context)
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned directories...");
+        removeOrphanedEntries<db::Directory>(context);
+    }
+
+    template<typename T>
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedEntries(ScanContext& context)
+    {
+        constexpr std::size_t batchSize = 100;
+
+        using IdType = typename T::IdType;
+
+        db::Session& session{ _db.getTLSSession() };
+
+        db::RangeResults<IdType> entries;
+        while (!_abortScan)
+        {
+            {
+                auto transaction{ session.createReadTransaction() };
+
+                entries = T::findOrphanIds(session, db::Range{ 0, batchSize });
+            };
+
+            if (entries.results.empty())
+                break;
+
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                for (const IdType objectId : entries.results)
+                {
+                    if (_abortScan)
+                        break;
+
+                    typename T::pointer entry{ T::find(session, objectId) };
+                    entry.remove();
+                }
+            }
+
+            context.currentStepStats.processedElems += entries.results.size();
+            _progressCallback(context.currentStepStats);
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.hpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepRemoveOrphanedDbEntries : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        core::LiteralString getStepName() const override { return "Remove orphaned DB entries"; }
+        ScanStep getStep() const override { return ScanStep::RemoveOrphanedDbEntries; }
+        void process(ScanContext& context) override;
+
+        void removeOrphanedClusters(ScanContext& context);
+        void removeOrphanedClusterTypes(ScanContext& context);
+        void removeOrphanedArtists(ScanContext& context);
+        void removeOrphanedReleases(ScanContext& context);
+        void removeOrphanedReleaseTypes(ScanContext& context);
+        void removeOrphanedLabels(ScanContext& context);
+        void removeOrphanedDirectories(ScanContext& context);
+
+        template<typename T>
+        void removeOrphanedEntries(ScanContext& context);
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp

@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepScanFiles.hpp"
+
+#include "FileScanQueue.hpp"
+#include "ScannerSettings.hpp"
+#include "core/IConfig.hpp"
+#include "core/ILogger.hpp"
+#include "core/ITraceLogger.hpp"
+#include "core/Path.hpp"
+#include "database/Db.hpp"
+#include "database/Session.hpp"
+#include "scanners/IFileScanOperation.hpp"
+#include "scanners/IFileScanner.hpp"
+
+namespace lms::scanner
+{
+    using namespace db;
+
+    namespace
+    {
+        std::size_t getScanMetaDataThreadCount()
+        {
+            std::size_t threadCount{ core::Service<core::IConfig>::get()->getULong("scanner-metadata-thread-count", 0) };
+
+            if (threadCount == 0)
+                threadCount = std::max<std::size_t>(std::thread::hardware_concurrency() / 2, 1);
+
+            return threadCount;
+        }
+    } // namespace
+
+    ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
+        : ScanStepBase{ initParams }
+        , _fileScanQueue{ getScanMetaDataThreadCount(), _abortScan }
+    {
+        for (IFileScanner* scanner : _fileScanners)
+        {
+            for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
+            {
+                [[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) };
+                assert(inserted);
+            }
+        }
+
+        LMS_LOG(DBUPDATER, INFO, "Using " << _fileScanQueue.getThreadCount() << " thread(s) for scanning file metadata");
+    }
+
+    void ScanStepScanFiles::process(ScanContext& context)
+    {
+        context.currentStepStats.totalElems = context.stats.totalFileCount;
+
+        for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
+            process(context, mediaLibrary);
+    }
+
+    void ScanStepScanFiles::process(ScanContext& context, const MediaLibraryInfo& mediaLibrary)
+    {
+        const std::size_t scanQueueMaxScanRequestCount{ 100 * _fileScanQueue.getThreadCount() };
+        const std::size_t processFileResultsBatchSize{ 5 };
+
+        std::vector<std::unique_ptr<IFileScanOperation>> scanOperations;
+
+        core::pathUtils::exploreFilesRecursive(
+            mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) {
+                LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile");
+
+                if (_abortScan)
+                    return false; // stop iterating
+
+                if (ec)
+                {
+                    LMS_LOG(DBUPDATER, ERROR, "Cannot scan file '" << path.string() << "': " << ec.message());
+                    context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
+                }
+                else
+                {
+                    auto itScanner{ _scannerByExtension.find(core::stringUtils::stringToLower(path.extension().string())) };
+                    if (itScanner != std::cend(_scannerByExtension))
+                    {
+                        IFileScanner& scanner{ *itScanner->second };
+
+                        FileToScan fileToScan{ .file = path, .mediaLibrary = mediaLibrary };
+                        if (scanner.needsScan(context, fileToScan))
+                        {
+                            auto scanOperation{ scanner.createScanOperation(fileToScan) };
+                            _fileScanQueue.pushScanRequest(std::move(scanOperation));
+                        }
+
+                        context.currentStepStats.processedElems++;
+                        _progressCallback(context.currentStepStats);
+                    }
+                }
+
+                while (_fileScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2))
+                {
+                    _fileScanQueue.popResults(scanOperations, processFileResultsBatchSize);
+                    processFileScanResults(context, scanOperations);
+                }
+
+                _fileScanQueue.wait(scanQueueMaxScanRequestCount);
+
+                return true;
+            },
+            &excludeDirFileName);
+
+        _fileScanQueue.wait();
+
+        while (!_abortScan && _fileScanQueue.popResults(scanOperations, processFileResultsBatchSize) > 0)
+            processFileScanResults(context, scanOperations);
+    }
+
+    void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<std::unique_ptr<IFileScanOperation>> scanOperations)
+    {
+        LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ProcessScanResults");
+
+        db::Session& dbSession{ _db.getTLSSession() };
+        auto transaction{ dbSession.createWriteTransaction() };
+
+        for (auto& scanOperation : scanOperations)
+        {
+            if (_abortScan)
+                return;
+
+            scanOperation->processResult(context);
+            context.stats.scans++;
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <span>
+
+#include "FileScanQueue.hpp"
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class IFileScanner;
+    class MediaLibraryInfo;
+
+    class ScanStepScanFiles : public ScanStepBase
+    {
+    public:
+        ScanStepScanFiles(InitParams& initParams);
+
+    private:
+        ScanStep getStep() const override { return ScanStep::ScanFiles; }
+        core::LiteralString getStepName() const override { return "Scan files"; }
+        void process(ScanContext& context) override;
+        void process(ScanContext& context, const MediaLibraryInfo& mediaLibrary);
+        void processFileScanResults(ScanContext& context, std::span<std::unique_ptr<IFileScanOperation>> scanOperations);
+
+        FileScanQueue _fileScanQueue;
+        std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepUpdateLibraryFields.cpp

@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepUpdateLibraryFields.hpp"
+
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/MediaLibrary.hpp"
+#include "database/Session.hpp"
+
+#include "MediaLibraryInfo.hpp"
+#include "ScannerSettings.hpp"
+
+namespace lms::scanner
+{
+
+    void ScanStepUpdateLibraryFields::process(ScanContext& context)
+    {
+        processDirectories(context);
+    }
+
+    void ScanStepUpdateLibraryFields::processDirectories(ScanContext& context)
+    {
+        for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
+        {
+            if (_abortScan)
+                break;
+
+            processDirectory(context, mediaLibrary);
+        }
+    }
+
+    void ScanStepUpdateLibraryFields::processDirectory(ScanContext& context, const MediaLibraryInfo& mediaLibrary)
+    {
+        db::Session& session{ _db.getTLSSession() };
+
+        constexpr std::size_t batchSize = 100;
+
+        db::RangeResults<db::DirectoryId> entries;
+        while (!_abortScan)
+        {
+            {
+                auto transaction{ session.createReadTransaction() };
+
+                entries = db::Directory::findMismatchedLibrary(session, db::Range{ 0, batchSize }, mediaLibrary.rootDirectory, mediaLibrary.id);
+            };
+
+            if (entries.results.empty())
+                break;
+
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                db::MediaLibrary::pointer library{ db::MediaLibrary::find(session, mediaLibrary.id) };
+                if (!library) // may be legit
+                    break;
+
+                for (const db::DirectoryId directoryId : entries.results)
+                {
+                    if (_abortScan)
+                        break;
+
+                    db::Directory::pointer directory{ db::Directory::find(session, directoryId) };
+                    directory.modify()->setMediaLibrary(library);
+                }
+            }
+
+            context.currentStepStats.processedElems += entries.results.size();
+            _progressCallback(context.currentStepStats);
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepUpdateLibraryFields.hpp

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class MediaLibraryInfo;
+
+    class ScanStepUpdateLibraryFields : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        core::LiteralString getStepName() const override { return "Update Library fields"; }
+        ScanStep getStep() const override { return ScanStep::UpdateLibraryFields; }
+        void process(ScanContext& context) override;
+
+        void processDirectories(ScanContext& context);
+        void processDirectory(ScanContext& context, const MediaLibraryInfo& mediaLibrary);
+    };
+} // namespace lms::scanner