Put all directories and images in database, use the info to associate an image to each artist

src/libs/services/scanner/CMakeLists.txt

@@ -1,15 +1,17 @@
 
 add_library(lmsscanner SHARED
+	impl/FileScanQueue.cpp
 	impl/ScannerService.cpp
 	impl/ScannerStats.cpp
-	impl/ScanStepCheckDuplicatedDbFiles.cpp
+	impl/ScanStepAssociateArtistImages.cpp
+	impl/ScanStepCheckForDuplicatedFiles.cpp
+	impl/ScanStepCheckForRemovedFiles.cpp
 	impl/ScanStepCompact.cpp
 	impl/ScanStepComputeClusterStats.cpp
 	impl/ScanStepDiscoverFiles.cpp
 	impl/ScanStepOptimize.cpp
-	impl/ScanStepRemoveOrphanDbFiles.cpp
-	impl/ScanStepScanArtistImages.cpp
-	impl/ScanStepScanAudioFiles.cpp
+	impl/ScanStepRemoveOrphanedDbEntries.cpp
+	impl/ScanStepScanFiles.cpp
 	)
 
 target_include_directories(lmsscanner INTERFACE

src/libs/services/scanner/impl/FileScanQueue.cpp

@@ -0,0 +1,149 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "FileScanQueue.hpp"
+
+#include "core/Exception.hpp"
+#include "core/IConfig.hpp"
+#include "core/ILogger.hpp"
+#include "core/ITraceLogger.hpp"
+#include "core/Path.hpp"
+#include "image/Exception.hpp"
+#include "image/Image.hpp"
+#include "metadata/Exception.hpp"
+
+namespace lms::scanner
+{
+    FileScanQueue::FileScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort)
+        : _metadataParser{ parser }
+        , _scanContextRunner{ _scanContext, threadCount, "FileScan" }
+        , _abort{ abort }
+    {
+    }
+
+    void FileScanQueue::pushScanRequest(const std::filesystem::path& path, ScanRequestType type)
+    {
+        {
+            std::scoped_lock lock{ _mutex };
+            _ongoingScanCount += 1;
+        }
+
+        _scanContext.post([=, this] {
+            if (_abort)
+            {
+                std::scoped_lock lock{ _mutex };
+                _ongoingScanCount -= 1;
+            }
+            else
+            {
+                FileScanResult result;
+                result.path = path;
+
+                switch (type)
+                {
+                case ScanRequestType::AudioFile:
+                    result.scanData = scanAudioFile(path);
+                    break;
+                case ScanRequestType::ImageFile:
+                    result.scanData = scanImageFile(path);
+                }
+
+                {
+                    std::scoped_lock lock{ _mutex };
+
+                    _scanResults.emplace_back(std::move(result));
+                    _ongoingScanCount -= 1;
+                }
+            }
+
+            _condVar.notify_all();
+        });
+    }
+
+    AudioFileScanData FileScanQueue::scanAudioFile(const std::filesystem::path& path)
+    {
+        LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanAudioFile");
+        std::unique_ptr<metadata::Track> track;
+
+        try
+        {
+            track = _metadataParser.parse(path);
+        }
+        catch (const metadata::Exception& e)
+        {
+            LMS_LOG(DBUPDATER, INFO, "Failed to parse audio file '" << path.string() << "'");
+        }
+
+        return track;
+    }
+
+    ImageFileScanData FileScanQueue::scanImageFile(const std::filesystem::path& path)
+    {
+        LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanImageFile");
+
+        std::optional<ImageInfo> optInfo;
+
+        try
+        {
+            std::unique_ptr<image::IRawImage> rawImage{ image::decodeImage(path) };
+            ImageInfo& imageInfo{ optInfo.emplace() };
+            imageInfo.width = rawImage->getWidth();
+            imageInfo.height = rawImage->getHeight();
+        }
+        catch (const image::Exception& e)
+        {
+            LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file '" << path.string() << "': " << e.what());
+        }
+
+        return optInfo;
+    }
+
+    std::size_t FileScanQueue::getResultsCount() const
+    {
+        std::scoped_lock lock{ _mutex };
+        return _scanResults.size();
+    }
+
+    size_t FileScanQueue::popResults(std::vector<FileScanResult>& results, std::size_t maxCount)
+    {
+        results.clear();
+        results.reserve(maxCount);
+
+        {
+            std::scoped_lock lock{ _mutex };
+
+            while (results.size() < maxCount && !_scanResults.empty())
+            {
+                results.push_back(std::move(_scanResults.front()));
+                _scanResults.pop_front();
+            }
+        }
+
+        return results.size();
+    }
+
+    void FileScanQueue::wait(std::size_t maxScanRequestCount)
+    {
+        LMS_SCOPED_TRACE_OVERVIEW("Scanner", "WaitParseResults");
+
+        std::unique_lock lock{ _mutex };
+        _condVar.wait(lock, [=, this] { return _ongoingScanCount <= maxScanRequestCount; });
+    }
+
+} // namespace lms::scanner

src/libs/services/scanner/impl/FileScanQueue.hpp

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <condition_variable>
+#include <deque>
+#include <filesystem>
+#include <mutex>
+#include <span>
+#include <variant>
+#include <vector>
+
+#include "core/IOContextRunner.hpp"
+#include "metadata/IParser.hpp"
+
+namespace lms::scanner
+{
+    struct ImageInfo
+    {
+        std::size_t height{};
+        std::size_t width{};
+    };
+
+    using AudioFileScanData = std::unique_ptr<metadata::Track>;
+    using ImageFileScanData = std::optional<ImageInfo>;
+    struct FileScanResult
+    {
+        std::filesystem::path path;
+        std::variant<std::monostate, AudioFileScanData, ImageFileScanData> scanData;
+    };
+
+    class FileScanQueue
+    {
+    public:
+        FileScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort);
+
+        std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); }
+
+        enum ScanRequestType
+        {
+            AudioFile,
+            ImageFile,
+        };
+        void pushScanRequest(const std::filesystem::path& path, ScanRequestType type);
+
+        std::size_t getResultsCount() const;
+        size_t popResults(std::vector<FileScanResult>& results, std::size_t maxCount);
+
+        void wait(std::size_t maxScanRequestCount = 0); // wait until ongoing scan request count <= maxScanRequestCount
+
+    private:
+        AudioFileScanData scanAudioFile(const std::filesystem::path& path);
+        ImageFileScanData scanImageFile(const std::filesystem::path& path);
+
+        metadata::IParser& _metadataParser;
+        boost::asio::io_context _scanContext;
+        core::IOContextRunner _scanContextRunner;
+
+        mutable std::mutex _mutex;
+        std::size_t _ongoingScanCount{};
+        std::deque<FileScanResult> _scanResults;
+        std::condition_variable _condVar;
+        bool& _abort;
+    };
+
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepAssociateArtistImages.cpp

@@ -0,0 +1,232 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepAssociateArtistImages.hpp"
+
+#include <array>
+#include <cassert>
+#include <deque>
+#include <set>
+
+#include "core/IConfig.hpp"
+#include "core/ILogger.hpp"
+#include "core/Path.hpp"
+#include "database/Artist.hpp"
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/Image.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+#include "image/Exception.hpp"
+#include "image/Image.hpp"
+
+namespace lms::scanner
+{
+    namespace
+    {
+        constexpr std::size_t readBatchSize{ 100 };
+        constexpr std::size_t writeBatchSize{ 10 };
+
+        struct ArtistImageAssociation
+        {
+            db::ArtistId artistId;
+            db::ImageId imageId;
+        };
+        using ArtistImageAssociationContainer = std::deque<ArtistImageAssociation>;
+
+        struct SearchImageContext
+        {
+            db::Session& session;
+            db::ArtistId lastRetrievedArtistId;
+            const std::vector<std::string>& artistFileNames;
+        };
+
+        db::Image::pointer findImageInDirectory(SearchImageContext& searchContext, const std::filesystem::path& directoryPath)
+        {
+            db::Image::pointer image;
+
+            const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) };
+            if (directory) // may not exist for artists that are split on different media libraries
+            {
+                for (std::string_view fileStem : searchContext.artistFileNames)
+                {
+                    db::Image::FindParameters params;
+                    params.setDirectory(directory->getId());
+                    params.setFileStem(fileStem);
+
+                    db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) {
+                        if (!image)
+                            image = foundImg;
+                    });
+
+                    if (image)
+                        break;
+                }
+            }
+
+            return image;
+        }
+
+        db::Image::pointer computeBestArtistImage(SearchImageContext& searchContext, const db::Artist::pointer& artist)
+        {
+            db::Image::pointer image;
+
+            const auto mbid{ artist->getMBID() };
+            if (mbid)
+            {
+                // Find anywhere, since it is suppoed to be unique!
+                db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer foundImg) {
+                    if (!image)
+                        image = foundImg;
+                });
+            }
+
+            if (!image)
+            {
+                std::set<std::filesystem::path> releasePaths;
+                db::Directory::FindParameters params;
+                params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist });
+
+                db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) {
+                    releasePaths.insert(directory->getAbsolutePath());
+                });
+
+                // Expect layout like this:
+                // ReleaseArtist/Release/Tracks'
+                //              /artist.jpg
+                //              /someOtherUserConfiguredArtistFile.jpg
+                if (!releasePaths.empty())
+                {
+                    const std::filesystem::path artistPath{ releasePaths.size() == 1 ? releasePaths.begin()->parent_path() : core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
+                    image = findImageInDirectory(searchContext, artistPath);
+                }
+
+                if (!image)
+                {
+                    // Expect layout like this:
+                    // ReleaseArtist/Release/Tracks'
+                    //                      /artist.jpg
+                    //                      /someOtherUserConfiguredArtistFile.jpg
+                    for (const std::filesystem::path& releasePath : releasePaths)
+                    {
+                        image = findImageInDirectory(searchContext, releasePath);
+                        if (image)
+                            break;
+                    }
+                }
+            }
+
+            return image;
+        }
+
+        bool fetchNextArtistImagesToUpdate(SearchImageContext& searchContext, ArtistImageAssociationContainer& artistImageAssociations)
+        {
+            const db::ArtistId artistId{ searchContext.lastRetrievedArtistId };
+
+            {
+                auto transaction{ searchContext.session.createReadTransaction() };
+
+                db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) {
+                    db::Image::pointer image{ computeBestArtistImage(searchContext, artist) };
+
+                    if (image != artist->getImage())
+                    {
+                        LMS_LOG(DBUPDATER, DEBUG, "Updating artist image for artist '" << artist->getName() << "', using '" << (image ? image->getAbsoluteFilePath().c_str() : "<none>") << "'");
+                        artistImageAssociations.push_back(ArtistImageAssociation{ artist->getId(), image ? image->getId() : db::ImageId{} });
+                    }
+                });
+            }
+
+            return artistId != searchContext.lastRetrievedArtistId;
+        }
+
+        void updateArtistImage(db::Session& session, const ArtistImageAssociation& artistImageAssociation)
+        {
+            db::Artist::pointer artist{ db::Artist::find(session, artistImageAssociation.artistId) };
+            assert(artist);
+
+            db::Image::pointer image;
+            if (artistImageAssociation.imageId.isValid())
+                image = db::Image::find(session, artistImageAssociation.imageId);
+
+            artist.modify()->setImage(image);
+        }
+
+        void updateArtistImages(db::Session& session, ArtistImageAssociationContainer& imageAssociations)
+        {
+            if (imageAssociations.empty())
+                return;
+
+            auto transaction{ session.createWriteTransaction() };
+
+            for (std::size_t i{}; !imageAssociations.empty() && i < writeBatchSize; ++i)
+            {
+                updateArtistImage(session, imageAssociations.front());
+                imageAssociations.pop_front();
+            }
+        }
+
+        std::vector<std::string> constructArtistFileNames()
+        {
+            std::vector<std::string> res;
+
+            core::Service<core::IConfig>::get()->visitStrings("artist-image-file-names",
+                [&res](std::string_view fileName) {
+                    res.emplace_back(fileName);
+                },
+                { "artist" });
+
+            return res;
+        }
+
+    } // namespace
+
+    ScanStepAssociateArtistImages::ScanStepAssociateArtistImages(InitParams& initParams)
+        : ScanStepBase{ initParams }
+        , _artistFileNames{ constructArtistFileNames() }
+    {
+    }
+
+    void ScanStepAssociateArtistImages::process(ScanContext& context)
+    {
+        if (context.stats.nbChanges() == 0)
+            return;
+
+        auto& session{ _db.getTLSSession() };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+            context.currentStepStats.totalElems = db::Artist::getCount(session);
+        }
+
+        SearchImageContext searchContext{
+            .session = session,
+            .lastRetrievedArtistId = {},
+            .artistFileNames = _artistFileNames,
+        };
+
+        ArtistImageAssociationContainer artistImageAssociations;
+        while (fetchNextArtistImagesToUpdate(searchContext, artistImageAssociations))
+        {
+            updateArtistImages(session, artistImageAssociations);
+            context.currentStepStats.processedElems += readBatchSize;
+            _progressCallback(context.currentStepStats);
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepAssociateArtistImages.hpp

@@ -26,14 +26,14 @@
 
 namespace lms::scanner
 {
-    class ScanStepScanArtistImages : public ScanStepBase
+    class ScanStepAssociateArtistImages : public ScanStepBase
     {
     public:
-        ScanStepScanArtistImages(InitParams& initParams);
+        ScanStepAssociateArtistImages(InitParams& initParams);
 
     private:
-        ScanStep getStep() const override { return ScanStep::ScanArtistImages; }
-        core::LiteralString getStepName() const override { return "Scan artist images"; }
+        ScanStep getStep() const override { return ScanStep::AssociateArtistImages; }
+        core::LiteralString getStepName() const override { return "Associate artist images"; }
         void process(ScanContext& context) override;
 
         const std::vector<std::string> _artistFileNames;

src/libs/services/scanner/impl/ScanStepCheckForDuplicatedFiles.cpp

@@ -17,7 +17,7 @@
  * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "ScanStepCheckDuplicatedDbFiles.hpp"
+#include "ScanStepCheckForDuplicatedFiles.hpp"
 
 #include "core/ILogger.hpp"
 #include "database/Db.hpp"
@@ -26,7 +26,7 @@
 
 namespace lms::scanner
 {
-    void ScanStepCheckDuplicatedDbFiles::process(ScanContext& context)
+    void ScanStepCheckForDuplicatedFiles::process(ScanContext& context)
     {
         using namespace db;
 

src/libs/services/scanner/impl/ScanStepCheckForDuplicatedFiles.hpp

@@ -23,14 +23,14 @@
 
 namespace lms::scanner
 {
-    class ScanStepCheckDuplicatedDbFiles : public ScanStepBase
+    class ScanStepCheckForDuplicatedFiles : public ScanStepBase
     {
     public:
         using ScanStepBase::ScanStepBase;
 
     private:
         core::LiteralString getStepName() const override { return "Check for duplicated files"; }
-        ScanStep getStep() const override { return ScanStep::CheckForDuplicateFiles; }
+        ScanStep getStep() const override { return ScanStep::CheckForDuplicatedFiles; }
         void process(ScanContext& context) override;
     };
 } // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.cpp

@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepCheckForRemovedFiles.hpp"
+
+#include "core/ILogger.hpp"
+#include "core/Path.hpp"
+#include "database/Db.hpp"
+#include "database/Image.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+
+namespace lms::scanner
+{
+    namespace
+    {
+        constexpr std::size_t batchSize = 100;
+    }
+
+    void ScanStepCheckForRemovedFiles::process(ScanContext& context)
+    {
+        if (_abortScan)
+            return;
+
+        db::Session& session{ _db.getTLSSession() };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+            context.currentStepStats.totalElems = 0;
+            context.currentStepStats.totalElems += db::Track::getCount(session);
+            context.currentStepStats.totalElems += db::Image::getCount(session);
+        }
+        LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " files to be checked...");
+
+        checkForRemovedFiles<db::Track>(context, _settings.supportedAudioFileExtensions);
+        checkForRemovedFiles<db::Image>(context, _settings.supportedImageFileExtensions);
+    }
+
+    template<typename Object>
+    void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context, const std::vector<std::filesystem::path>& supportedFileExtensions)
+    {
+        using namespace db;
+
+        if (_abortScan)
+            return;
+
+        Session& session{ _db.getTLSSession() };
+
+        std::vector<typename Object::pointer> objectsToRemove;
+
+        typename Object::IdType lastCheckedId;
+        bool endReached{};
+        while (!endReached)
+        {
+            if (_abortScan)
+                break;
+
+            objectsToRemove.clear();
+            {
+                auto transaction{ session.createReadTransaction() };
+
+                endReached = true;
+                Object::find(session, lastCheckedId, batchSize, [&](const typename Object::pointer& object) {
+                    endReached = false;
+
+                    if (!checkFile(object->getAbsoluteFilePath(), supportedFileExtensions))
+                        objectsToRemove.push_back(object);
+
+                    context.currentStepStats.processedElems++;
+                });
+            }
+
+            if (!objectsToRemove.empty())
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                for (typename Object::pointer& object : objectsToRemove)
+                {
+                    object.remove();
+                    context.stats.deletions++;
+                }
+            }
+
+            _progressCallback(context.currentStepStats);
+        }
+    }
+
+    bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p, const std::vector<std::filesystem::path>& allowedExtensions)
+    {
+        try
+        {
+            // For each track, make sure the the file still exists
+            // and still belongs to a media directory
+            if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
+            {
+                LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': missing");
+                return false;
+            }
+
+            if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
+                    [&](const ScannerSettings::MediaLibraryInfo& libraryInfo) {
+                        return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
+                    }))
+            {
+                LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': out of media directory");
+                return false;
+            }
+
+            if (!core::pathUtils::hasFileAnyExtension(p, allowedExtensions))
+            {
+                LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': file format no longer handled");
+                return false;
+            }
+
+            return true;
+        }
+        catch (std::filesystem::filesystem_error& e)
+        {
+            LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file '" << p.string() << "': " << e.what());
+            return false;
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.hpp

@@ -25,21 +25,19 @@
 
 namespace lms::scanner
 {
-    class ScanStepRemoveOrphanDbFiles : public ScanStepBase
+    class ScanStepCheckForRemovedFiles : public ScanStepBase
     {
     public:
         using ScanStepBase::ScanStepBase;
 
     private:
-        core::LiteralString getStepName() const override { return "Check orphaned entries"; }
-        ScanStep getStep() const override { return ScanStep::CheckForMissingFiles; }
+        core::LiteralString getStepName() const override { return "Check for removed files"; }
+        ScanStep getStep() const override { return ScanStep::CheckForRemovedFiles; }
         void process(ScanContext& context) override;
 
-        void removeOrphanTracks(ScanContext& context);
-        void removeOrphanClusters();
-        void removeOrphanClusterTypes();
-        void removeOrphanArtists();
-        void removeOrphanReleases();
-        bool checkFile(const std::filesystem::path& p);
+        template<typename Object>
+        void checkForRemovedFiles(ScanContext& context, const std::vector<std::filesystem::path>& supportedFileExtensions);
+
+        bool checkFile(const std::filesystem::path& p, const std::vector<std::filesystem::path>& allowedExtensions);
     };
 } // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp

@@ -26,7 +26,7 @@ namespace lms::scanner
 {
     void ScanStepDiscoverFiles::process(ScanContext& context)
     {
-        context.stats.filesScanned = 0;
+        context.stats.totalFileCount = 0;
 
         for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
         {
@@ -36,7 +36,7 @@ namespace lms::scanner
                     if (_abortScan)
                         return false;
 
-                    if (!ec && core::pathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
+                    if (!ec && (core::pathUtils::hasFileAnyExtension(path, _settings.supportedAudioFileExtensions) || core::pathUtils::hasFileAnyExtension(path, _settings.supportedImageFileExtensions)))
                     {
                         context.currentStepStats.processedElems++;
                         currentDirectoryProcessElemsCount++;
@@ -50,8 +50,8 @@ namespace lms::scanner
             LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in '" << mediaLibrary.rootDirectory << "'");
         }
 
-        context.stats.filesScanned = context.currentStepStats.processedElems;
+        context.stats.totalFileCount = context.currentStepStats.processedElems;
 
-        LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.filesScanned << " files in all directories");
+        LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.totalFileCount << " files in all directories");
     }
 } // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp

@@ -1,199 +0,0 @@
-/*
- * Copyright (C) 2023 Emeric Poupon
- *
- * This file is part of LMS.
- *
- * LMS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LMS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "ScanStepRemoveOrphanDbFiles.hpp"
-
-#include "core/ILogger.hpp"
-#include "core/Path.hpp"
-#include "database/Artist.hpp"
-#include "database/Cluster.hpp"
-#include "database/Db.hpp"
-#include "database/Release.hpp"
-#include "database/Session.hpp"
-#include "database/Track.hpp"
-
-namespace lms::scanner
-{
-    using namespace db;
-
-    namespace
-    {
-        constexpr std::size_t batchSize = 100;
-
-        template<typename T>
-        void removeOrphanEntries(Session& session, bool& abortScan)
-        {
-            using IdType = typename T::IdType;
-
-            RangeResults<IdType> entries;
-            while (!abortScan)
-            {
-                {
-                    auto transaction{ session.createReadTransaction() };
-
-                    entries = T::findOrphanIds(session, Range{ 0, batchSize });
-                };
-
-                {
-                    auto transaction{ session.createWriteTransaction() };
-
-                    for (const IdType objectId : entries.results)
-                    {
-                        if (abortScan)
-                            break;
-
-                        typename T::pointer entry{ T::find(session, objectId) };
-
-                        entry.remove();
-                    }
-                }
-
-                if (!entries.moreResults)
-                    break;
-            }
-        }
-    } // namespace
-
-    void ScanStepRemoveOrphanDbFiles::process(ScanContext& context)
-    {
-        removeOrphanTracks(context);
-        removeOrphanClusters();
-        removeOrphanClusterTypes();
-        removeOrphanArtists();
-        removeOrphanReleases();
-    }
-
-    void ScanStepRemoveOrphanDbFiles::removeOrphanTracks(ScanContext& context)
-    {
-        using namespace db;
-
-        if (_abortScan)
-            return;
-
-        Session& session{ _db.getTLSSession() };
-
-        LMS_LOG(DBUPDATER, DEBUG, "Checking tracks to be removed...");
-        {
-            auto transaction{ session.createReadTransaction() };
-            context.currentStepStats.totalElems = Track::getCount(session);
-        }
-        LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " tracks to be checked...");
-
-        // TODO handle only files in context.directory?
-        std::vector<Track::pointer> tracksToRemove;
-
-        TrackId lastCheckedTrackID;
-        bool endReached{};
-        while (!endReached)
-        {
-            if (_abortScan)
-                break;
-
-            tracksToRemove.clear();
-            {
-                auto transaction{ session.createReadTransaction() };
-
-                endReached = true;
-                Track::find(session, lastCheckedTrackID, batchSize, [&](const Track::pointer& track) {
-                    endReached = false;
-
-                    if (!checkFile(track->getAbsoluteFilePath()))
-                        tracksToRemove.push_back(track);
-
-                    context.currentStepStats.processedElems++;
-                });
-            }
-
-            if (!tracksToRemove.empty())
-            {
-                auto transaction{ session.createWriteTransaction() };
-
-                for (Track::pointer& track : tracksToRemove)
-                {
-                    track.remove();
-                    context.stats.deletions++;
-                }
-            }
-
-            _progressCallback(context.currentStepStats);
-        }
-
-        LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.processedElems << " tracks checked!");
-    }
-
-    void ScanStepRemoveOrphanDbFiles::removeOrphanClusters()
-    {
-        LMS_LOG(DBUPDATER, DEBUG, "Checking orphan clusters...");
-        removeOrphanEntries<db::Cluster>(_db.getTLSSession(), _abortScan);
-    }
-
-    void ScanStepRemoveOrphanDbFiles::removeOrphanClusterTypes()
-    {
-        LMS_LOG(DBUPDATER, DEBUG, "Checking orphan cluster types...");
-        removeOrphanEntries<db::ClusterType>(_db.getTLSSession(), _abortScan);
-    }
-
-    void ScanStepRemoveOrphanDbFiles::removeOrphanArtists()
-    {
-        LMS_LOG(DBUPDATER, DEBUG, "Checking orphan artists...");
-        removeOrphanEntries<db::Artist>(_db.getTLSSession(), _abortScan);
-    }
-
-    void ScanStepRemoveOrphanDbFiles::removeOrphanReleases()
-    {
-        LMS_LOG(DBUPDATER, DEBUG, "Checking orphan releases...");
-        removeOrphanEntries<db::Release>(_db.getTLSSession(), _abortScan);
-    }
-
-    bool ScanStepRemoveOrphanDbFiles::checkFile(const std::filesystem::path& p)
-    {
-        try
-        {
-            // For each track, make sure the the file still exists
-            // and still belongs to a media directory
-            if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
-            {
-                LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': missing");
-                return false;
-            }
-
-            if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
-                    [&](const ScannerSettings::MediaLibraryInfo& libraryInfo) {
-                        return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
-                    }))
-            {
-                LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': out of media directory");
-                return false;
-            }
-
-            if (!core::pathUtils::hasFileAnyExtension(p, _settings.supportedExtensions))
-            {
-                LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': file format no longer handled");
-                return false;
-            }
-
-            return true;
-        }
-        catch (std::filesystem::filesystem_error& e)
-        {
-            LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file '" << p.string() << "': " << e.what());
-            return false;
-        }
-    }
-} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp

@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepRemoveOrphanedDbEntries.hpp"
+
+#include "core/ILogger.hpp"
+#include "core/Path.hpp"
+#include "database/Artist.hpp"
+#include "database/Cluster.hpp"
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/Release.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+
+namespace lms::scanner
+{
+    using namespace db;
+
+    namespace
+    {
+        constexpr std::size_t batchSize = 100;
+
+        template<typename T>
+        void removeOrphanedEntries(Session& session, bool& abortScan)
+        {
+            using IdType = typename T::IdType;
+
+            RangeResults<IdType> entries;
+            while (!abortScan)
+            {
+                {
+                    auto transaction{ session.createReadTransaction() };
+
+                    entries = T::findOrphanIds(session, Range{ 0, batchSize });
+                };
+
+                {
+                    auto transaction{ session.createWriteTransaction() };
+
+                    for (const IdType objectId : entries.results)
+                    {
+                        if (abortScan)
+                            break;
+
+                        typename T::pointer entry{ T::find(session, objectId) };
+
+                        entry.remove();
+                    }
+                }
+
+                if (!entries.moreResults)
+                    break;
+            }
+        }
+    } // namespace
+
+    void ScanStepRemoveOrphanedDbEntries::process(ScanContext& context)
+    {
+        auto& session{ _db.getTLSSession() };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+            context.currentStepStats.totalElems = 0;
+            context.currentStepStats.totalElems += Cluster::getCount(session);
+            context.currentStepStats.totalElems += ClusterType::getCount(session);
+            context.currentStepStats.totalElems += Artist::getCount(session);
+            context.currentStepStats.totalElems += Release::getCount(session);
+            context.currentStepStats.totalElems += Directory::getCount(session);
+        }
+        LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " database entries to be checked...");
+
+        removeOrphanedClusters();
+        removeOrphanedClusterTypes();
+        removeOrphanedArtists();
+        removeOrphanedReleases();
+        removeOrphanedDirectories();
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusters()
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned clusters...");
+        removeOrphanedEntries<db::Cluster>(_db.getTLSSession(), _abortScan);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusterTypes()
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned cluster types...");
+        removeOrphanedEntries<db::ClusterType>(_db.getTLSSession(), _abortScan);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedArtists()
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned artists...");
+        removeOrphanedEntries<db::Artist>(_db.getTLSSession(), _abortScan);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedReleases()
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned releases...");
+        removeOrphanedEntries<db::Release>(_db.getTLSSession(), _abortScan);
+    }
+
+    void ScanStepRemoveOrphanedDbEntries::removeOrphanedDirectories()
+    {
+        LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned releases...");
+        removeOrphanedEntries<db::Directory>(_db.getTLSSession(), _abortScan);
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.hpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <filesystem>
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepRemoveOrphanedDbEntries : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        core::LiteralString getStepName() const override { return "Remove orphaned DB entries"; }
+        ScanStep getStep() const override { return ScanStep::RemoveOrphanedDbEntries; }
+        void process(ScanContext& context) override;
+
+        void removeOrphanedClusters();
+        void removeOrphanedClusterTypes();
+        void removeOrphanedArtists();
+        void removeOrphanedReleases();
+        void removeOrphanedDirectories();
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepScanArtistImages.cpp

@@ -1,347 +0,0 @@
-/*
- * Copyright (C) 2024 Emeric Poupon
- *
- * This file is part of LMS.
- *
- * LMS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LMS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "ScanStepScanArtistImages.hpp"
-
-#include <array>
-#include <cassert>
-#include <deque>
-#include <set>
-
-#include "core/IConfig.hpp"
-#include "core/ILogger.hpp"
-#include "core/Path.hpp"
-#include "database/Artist.hpp"
-#include "database/Db.hpp"
-#include "database/Image.hpp"
-#include "database/Session.hpp"
-#include "database/Track.hpp"
-#include "image/Exception.hpp"
-#include "image/Image.hpp"
-
-namespace lms::scanner
-{
-    namespace
-    {
-        constexpr std::size_t readBatchSize{ 10 };
-        constexpr std::size_t writeBatchSize{ 5 };
-
-        struct ImageInfo
-        {
-            operator bool() const { return !imagePath.empty(); }
-            void clear()
-            {
-                imagePath.clear();
-                lastWriteTime = {};
-                fileSize = {};
-                height = {};
-                width = {};
-            }
-
-            std::filesystem::path imagePath;
-            Wt::WDateTime lastWriteTime;
-            std::size_t fileSize{};
-            std::size_t height{};
-            std::size_t width{};
-        };
-
-        bool tryDecodeImage(const std::filesystem::path& imagePath, ImageInfo& imageInfo)
-        {
-            assert(!imageInfo);
-
-            try
-            {
-                std::unique_ptr<image::IRawImage> rawImage{ image::decodeImage(imagePath) };
-                imageInfo.imagePath = imagePath;
-                imageInfo.fileSize = std::filesystem::file_size(imagePath);
-                imageInfo.width = rawImage->getWidth();
-                imageInfo.height = rawImage->getHeight();
-                imageInfo.lastWriteTime = core::pathUtils::getLastWriteTime(imagePath);
-            }
-            catch (const image::Exception& e)
-            {
-                LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file '" << imagePath.string() << "': " << e.what());
-                return false;
-            }
-
-            return true;
-        }
-
-        struct ArtistImageInfo
-        {
-            db::ArtistId artistId;
-            ImageInfo imageInfo;
-        };
-
-        using ArtistImageInfoContainer = std::deque<ArtistImageInfo>;
-
-        bool isFileSupported(const std::filesystem::path& file)
-        {
-            static const std::array<std::filesystem::path, 4> fileExtensions{ ".jpg", ".jpeg", ".png", ".bmp" }; // TODO parametrize
-
-            return (std::find(std::cbegin(fileExtensions), std::cend(fileExtensions), file.extension()) != std::cend(fileExtensions));
-        }
-
-        std::multimap<std::string, std::filesystem::path> getImagePaths(const std::filesystem::path& directoryPath, const std::vector<std::string>& fileNames)
-        {
-            std::multimap<std::string, std::filesystem::path> res;
-            std::error_code ec;
-
-            std::filesystem::directory_iterator itPath(directoryPath, ec);
-            const std::filesystem::directory_iterator itEnd;
-            while (!ec && itPath != itEnd)
-            {
-                const std::filesystem::path& path{ *itPath };
-                const std::string stem{ path.stem().string() };
-                if (isFileSupported(path)
-                    && std::any_of(std::cbegin(fileNames), std::cend(fileNames), [&](const std::string& fileName) { return core::stringUtils::stringCaseInsensitiveEqual(stem, fileName); }))
-                {
-                    res.emplace(stem, path);
-                }
-
-                itPath.increment(ec);
-            }
-
-            return res;
-        }
-
-        bool findImageInDirectory(const std::filesystem::path& directory, const std::vector<std::string>& fileNames, ImageInfo& imageInfo)
-        {
-            assert(!imageInfo);
-
-            const std::multimap<std::string, std::filesystem::path> coverPaths{ getImagePaths(directory, fileNames) };
-
-            for (const std::string_view fileName : fileNames)
-            {
-                const auto range{ coverPaths.equal_range(std::string{ fileName }) };
-                for (auto it{ range.first }; it != range.second; ++it)
-                {
-                    if (tryDecodeImage(it->second, imageInfo))
-                        return true;
-                }
-            }
-
-            return false;
-        }
-
-        void fetchArtistImageInfo(db::Session& session, const std::vector<std::string>& genericArtistFileNames, const db::Artist::pointer& artist, ImageInfo& imageInfo)
-        {
-            const std::string artistMBID{ [&] {
-                std::string artistMBID;
-                if (auto mbid{ artist->getMBID() })
-                    artistMBID = mbid->getAsString();
-                return artistMBID;
-            }() };
-
-            std::set<std::filesystem::path> releasePaths;
-            std::set<std::filesystem::path> multiArtistReleasePaths;
-
-            db::Track::FindParameters params;
-            params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist });
-
-            db::Track::find(session, params, [&](const db::Track::pointer& track) {
-                db::Artist::FindParameters artistFindParams;
-                artistFindParams.setTrack(track->getId());
-                artistFindParams.setLinkType(db::TrackArtistLinkType::ReleaseArtist);
-
-                const auto releaseArtists{ db::Artist::findIds(session, artistFindParams) };
-                if (releaseArtists.results.size() == 1)
-                    releasePaths.insert(track->getAbsoluteFilePath().parent_path());
-                else
-                    multiArtistReleasePaths.insert(track->getAbsoluteFilePath().parent_path());
-            });
-
-            std::vector<std::string> artistFileNames;
-            if (!artistMBID.empty())
-                artistFileNames.push_back(artistMBID);
-            artistFileNames.push_back(artist->getName());
-
-            std::vector<std::string> artistFileNamesWithGenericNames{ artistFileNames };
-            artistFileNamesWithGenericNames.insert(artistFileNamesWithGenericNames.end(), std::cbegin(genericArtistFileNames), std::cend(genericArtistFileNames));
-
-            // Expect layout like this:
-            // ReleaseArtist/Release/Tracks'
-            //              /artist-mbid.jpg
-            //              /artist-name.jpg
-            //              /artist.jpg
-            if (!releasePaths.empty())
-            {
-                const std::filesystem::path artistPath{ releasePaths.size() == 1 ? releasePaths.begin()->parent_path() : core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
-                if (findImageInDirectory(artistPath, artistFileNamesWithGenericNames, imageInfo))
-                    return;
-            }
-
-            // Expect layout like this:
-            // ReleaseArtist/Release/Tracks'
-            //                      /artist-mbid.jpg
-            //                      /artist-name.jpg
-            //                      /artist.jpg
-            for (const std::filesystem::path& releasePath : releasePaths)
-            {
-                // TODO: what if an artist has released an album that bears their name?
-                if (findImageInDirectory(releasePath, artistFileNamesWithGenericNames, imageInfo))
-                    return;
-            }
-
-            // Expect layout like this:
-            // Only search for the artist's name in the release path, as we can't map a generic name to several artists
-            // ReleaseArtist/Release/Tracks'
-            //                      /artist-name.jpg
-            //                      /artist-mbid.jpg
-            for (const std::filesystem::path& releasePath : multiArtistReleasePaths)
-            {
-                if (findImageInDirectory(releasePath, artistFileNames, imageInfo))
-                    return;
-            }
-        }
-
-        bool artistImageNeedsUpdate(const db::Image::pointer& image, const ImageInfo& imageInfo)
-        {
-            if (!imageInfo && !image) // no image as before
-                return false;
-            else if (!imageInfo && image) // no longer has image
-                return true;
-            else if (imageInfo && !image) // image has been added
-                return true;
-
-            assert(imageInfo);
-            // artist image still here, consider it is the same only if the last modified time is the same
-            return imageInfo.lastWriteTime != image->getLastWriteTime();
-        }
-
-        struct SearchImageContext
-        {
-            db::Session& session;
-            db::ArtistId lastRetrievedArtistId;
-            const std::vector<std::string>& artistFileNames;
-            bool fullScan;
-        };
-
-        bool fetchNextArtistImagesToUpdate(SearchImageContext& searchContext, ArtistImageInfoContainer& artistImageInfoList)
-        {
-            const db::ArtistId artistId{ searchContext.lastRetrievedArtistId };
-            ImageInfo imageInfo;
-
-            {
-                auto transaction{ searchContext.session.createReadTransaction() };
-
-                db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) {
-                    imageInfo.clear();
-
-                    fetchArtistImageInfo(searchContext.session, searchContext.artistFileNames, artist, imageInfo);
-                    if (imageInfo)
-                        LMS_LOG(DBUPDATER, DEBUG, "Found artist image for artist '" << artist->getName() << "' at '" << imageInfo.imagePath << "'");
-
-                    if (searchContext.fullScan || artistImageNeedsUpdate(artist->getImage(), imageInfo))
-                        artistImageInfoList.push_back(ArtistImageInfo{ artist->getId(), imageInfo });
-                });
-            }
-
-            return artistId != searchContext.lastRetrievedArtistId;
-        }
-
-        void updateArtistImage(db::Session& session, const ArtistImageInfo& artistImageInfo)
-        {
-            db::Artist::pointer artist{ db::Artist::find(session, artistImageInfo.artistId) };
-            assert(artist);
-
-            db::Image::pointer image{ artist->getImage() };
-            const ImageInfo& imageInfo{ artistImageInfo.imageInfo };
-
-            if (!imageInfo)
-            {
-                if (image)
-                    image.remove();
-                return;
-            }
-
-            if (!image)
-            {
-                image = session.create<db::Image>(imageInfo.imagePath);
-                image.modify()->setArtist(artist);
-            }
-            else
-                image.modify()->setPath(imageInfo.imagePath);
-
-            image.modify()->setLastWriteTime(imageInfo.lastWriteTime);
-            image.modify()->setFileSize(imageInfo.fileSize);
-            image.modify()->setHeight(imageInfo.height);
-            image.modify()->setWidth(imageInfo.width);
-        }
-
-        void updateArtistImages(db::Session& session, ArtistImageInfoContainer& imageInfoList)
-        {
-            if (imageInfoList.empty())
-                return;
-
-            auto transaction{ session.createWriteTransaction() };
-
-            for (std::size_t i{}; !imageInfoList.empty() && i < writeBatchSize; ++i)
-            {
-                updateArtistImage(session, imageInfoList.front());
-                imageInfoList.pop_front();
-            }
-        }
-
-        std::vector<std::string> constructArtistFileNames()
-        {
-            std::vector<std::string> res;
-
-            core::Service<core::IConfig>::get()->visitStrings("artist-image-file-names",
-                [&res](std::string_view fileName) {
-                    res.emplace_back(fileName);
-                },
-                { "artist" });
-
-            return res;
-        }
-
-    } // namespace
-
-    ScanStepScanArtistImages::ScanStepScanArtistImages(InitParams& initParams)
-        : ScanStepBase{ initParams }
-        , _artistFileNames{ constructArtistFileNames() }
-    {
-    }
-
-    void ScanStepScanArtistImages::process(ScanContext& context)
-    {
-        auto& session{ _db.getTLSSession() };
-
-        {
-            auto transaction{ session.createReadTransaction() };
-            context.currentStepStats.totalElems = db::Artist::getCount(session);
-        }
-
-        SearchImageContext searchContext{
-            .session = session,
-            .lastRetrievedArtistId = {},
-            .artistFileNames = _artistFileNames,
-            .fullScan = context.scanOptions.fullScan
-        };
-
-        ArtistImageInfoContainer imageInfoList;
-        while (fetchNextArtistImagesToUpdate(searchContext, imageInfoList))
-        {
-            updateArtistImages(session, imageInfoList);
-            context.currentStepStats.processedElems += readBatchSize;
-            _progressCallback(context.currentStepStats);
-        }
-    }
-} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepScanAudioFiles.hpp

@@ -1,88 +0,0 @@
-/*
- * Copyright (C) 2023 Emeric Poupon
- *
- * This file is part of LMS.
- *
- * LMS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LMS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#pragma once
-
-#include <condition_variable>
-#include <deque>
-#include <filesystem>
-#include <mutex>
-#include <span>
-#include <string>
-#include <vector>
-
-#include "core/IOContextRunner.hpp"
-#include "metadata/IParser.hpp"
-
-#include "ScanStepBase.hpp"
-
-namespace lms::scanner
-{
-    class ScanStepScanAudioFiles : public ScanStepBase
-    {
-    public:
-        ScanStepScanAudioFiles(InitParams& initParams);
-
-    private:
-        ScanStep getStep() const override { return ScanStep::ScanAudioFiles; }
-        core::LiteralString getStepName() const override { return "Scan audio files"; }
-        void process(ScanContext& context) override;
-
-        bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);
-        struct MetaDataScanResult
-        {
-            std::filesystem::path path;
-            std::unique_ptr<metadata::Track> trackMetaData;
-        };
-        void processMetaDataScanResults(ScanContext& context, std::span<const MetaDataScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo);
-        void processFileMetaData(ScanContext& context, const std::filesystem::path& file, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo);
-
-        std::unique_ptr<metadata::IParser> _metadataParser;
-        const std::vector<std::string> _extraTagsToParse;
-
-        class MetadataScanQueue
-        {
-        public:
-            MetadataScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort);
-
-            std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); }
-
-            void pushScanRequest(const std::filesystem::path& path);
-
-            std::size_t getResultsCount() const;
-            size_t popResults(std::vector<MetaDataScanResult>& results, std::size_t maxCount);
-
-            void wait(std::size_t maxScanRequestCount = 0); // wait until ongoing scan request count <= maxScanRequestCount
-
-        private:
-            metadata::IParser& _metadataParser;
-            boost::asio::io_context _scanContext;
-            core::IOContextRunner _scanContextRunner;
-
-            mutable std::mutex _mutex;
-            std::size_t _ongoingScanCount{};
-            std::deque<MetaDataScanResult> _scanResults;
-            std::condition_variable _condVar;
-            bool& _abort;
-        };
-        MetadataScanQueue _metadataScanQueue;
-
-        std::deque<MetaDataScanResult> _metaDataScanResults;
-    };
-} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepScanFiles.cpp

@@ -17,7 +17,7 @@
  * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "ScanStepScanAudioFiles.hpp"
+#include "ScanStepScanFiles.hpp"
 
 #include "core/Exception.hpp"
 #include "core/IConfig.hpp"
@@ -27,6 +27,8 @@
 #include "database/Artist.hpp"
 #include "database/Cluster.hpp"
 #include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/Image.hpp"
 #include "database/MediaLibrary.hpp"
 #include "database/Release.hpp"
 #include "database/Session.hpp"
@@ -102,6 +104,22 @@ namespace lms::scanner
             return res;
         }
 
+        Directory::pointer getOrCreateDirectory(Session& session, const std::filesystem::path& path, const std::filesystem::path& rootPath)
+        {
+            Directory::pointer directory{ Directory::find(session, path) };
+            if (!directory)
+            {
+                Directory::pointer parentDirectory;
+                if (path != rootPath)
+                    parentDirectory = getOrCreateDirectory(session, path.parent_path(), rootPath);
+
+                directory = session.create<Directory>(path);
+                directory.modify()->setParent(parentDirectory);
+            }
+
+            return directory;
+        }
+
         Artist::pointer createArtist(Session& session, const metadata::Artist& artistInfo)
         {
             Artist::pointer artist{ session.create<Artist>(artistInfo.name) };
@@ -301,98 +319,16 @@ namespace lms::scanner
         }
     } // namespace
 
-    ScanStepScanAudioFiles::MetadataScanQueue::MetadataScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort)
-        : _metadataParser{ parser }
-        , _scanContextRunner{ _scanContext, threadCount, "ScannerMetadata" }
-        , _abort{ abort }
-    {
-    }
-
-    void ScanStepScanAudioFiles::MetadataScanQueue::pushScanRequest(const std::filesystem::path& path)
-    {
-        {
-            std::scoped_lock lock{ _mutex };
-            _ongoingScanCount += 1;
-        }
-
-        _scanContext.post([=, this] {
-            LMS_SCOPED_TRACE_OVERVIEW("Scanner", "AudioFileParseJob");
-
-            std::unique_ptr<metadata::Track> track;
-
-            if (_abort)
-            {
-                std::scoped_lock lock{ _mutex };
-                _ongoingScanCount -= 1;
-            }
-            else
-            {
-                try
-                {
-                    track = _metadataParser.parse(path);
-                }
-                catch (const metadata::Exception& e)
-                {
-                    LMS_LOG(DBUPDATER, INFO, "Failed to parse '" << path.string() << "'");
-                }
-
-                {
-                    std::scoped_lock lock{ _mutex };
-
-                    _scanResults.emplace_back(MetaDataScanResult{ std::move(path), std::move(track) });
-                    _ongoingScanCount -= 1;
-                }
-            }
-
-            _condVar.notify_all();
-        });
-    }
-
-    std::size_t ScanStepScanAudioFiles::MetadataScanQueue::getResultsCount() const
-    {
-        std::scoped_lock lock{ _mutex };
-        return _scanResults.size();
-    }
-
-    size_t ScanStepScanAudioFiles::MetadataScanQueue::popResults(std::vector<MetaDataScanResult>& results, std::size_t maxCount)
-    {
-        results.clear();
-        results.reserve(maxCount);
-
-        {
-            std::scoped_lock lock{ _mutex };
-
-            while (results.size() < maxCount && !_scanResults.empty())
-            {
-                results.push_back(std::move(_scanResults.front()));
-                _scanResults.pop_front();
-            }
-        }
-
-        return results.size();
-    }
-
-    void ScanStepScanAudioFiles::MetadataScanQueue::wait(std::size_t maxScanRequestCount)
-    {
-        LMS_SCOPED_TRACE_OVERVIEW("Scanner", "WaitParseResults");
-
-        std::unique_lock lock{ _mutex };
-        _condVar.wait(lock, [=, this] { return _ongoingScanCount <= maxScanRequestCount; });
-    }
-
-    ScanStepScanAudioFiles::ScanStepScanAudioFiles(InitParams& initParams)
+    ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
         : ScanStepBase{ initParams }
         , _metadataParser{ metadata::createParser(metadata::ParserBackend::TagLib, getParserReadStyle()) } // For now, always use TagLib
-        , _metadataScanQueue{ *_metadataParser, getScanMetaDataThreadCount(), _abortScan }
+        , _fileScanQueue{ *_metadataParser, getScanMetaDataThreadCount(), _abortScan }
     {
-        LMS_LOG(DBUPDATER, INFO, "Using " << _metadataScanQueue.getThreadCount() << " thread(s) for scanning file metadata");
+        LMS_LOG(DBUPDATER, INFO, "Using " << _fileScanQueue.getThreadCount() << " thread(s) for scanning file metadata");
     }
 
-    void ScanStepScanAudioFiles::process(ScanContext& context)
+    void ScanStepScanFiles::process(ScanContext& context)
     {
-        const std::size_t scanQueueMaxScanRequestCount{ 100 * _metadataScanQueue.getThreadCount() };
-        const std::size_t processMetaDataBatchSize{ 5 };
-
         {
             std::vector<std::string> tagsToParse{ _extraTagsToParse };
             tagsToParse.insert(std::end(tagsToParse), std::cbegin(_settings.extraTags), std::cend(_settings.extraTags));
@@ -401,56 +337,77 @@ namespace lms::scanner
             _metadataParser->setDefaultTagDelimiters(_settings.defaultTagDelimiters);
         }
 
-        std::vector<MetaDataScanResult> scanResults;
-        context.currentStepStats.totalElems = context.stats.filesScanned;
+        context.currentStepStats.totalElems = context.stats.totalFileCount;
 
         for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
-        {
-            core::pathUtils::exploreFilesRecursive(
-                mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) {
-                    LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile");
+            process(context, mediaLibrary);
+    }
 
-                    if (_abortScan)
-                        return false;
+    void ScanStepScanFiles::process(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary)
+    {
+        const std::size_t scanQueueMaxScanRequestCount{ 100 * _fileScanQueue.getThreadCount() };
+        const std::size_t processFileResultsBatchSize{ 5 };
 
-                    if (ec)
+        std::vector<FileScanResult> scanResults;
+
+        core::pathUtils::exploreFilesRecursive(
+            mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) {
+                LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile");
+
+                if (_abortScan)
+                    return false;
+
+                if (ec)
+                {
+                    LMS_LOG(DBUPDATER, ERROR, "Cannot scan file '" << path.string() << "': " << ec.message());
+                    context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
+                }
+                else
+                {
+                    bool fileToProcess{};
+                    if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedAudioFileExtensions))
                     {
-                        LMS_LOG(DBUPDATER, ERROR, "Cannot process entry '" << path.string() << "': " << ec.message());
-                        context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
+                        fileToProcess = true;
+                        if (checkAudioFileNeedScan(context, path, mediaLibrary))
+                            _fileScanQueue.pushScanRequest(path, FileScanQueue::ScanRequestType::AudioFile);
                     }
-                    else if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
+                    else if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedImageFileExtensions))
                     {
-                        if (checkFileNeedScan(context, path, mediaLibrary))
-                            _metadataScanQueue.pushScanRequest(path);
+                        fileToProcess = true;
+                        if (checkImageFileNeedScan(context, path))
+                            _fileScanQueue.pushScanRequest(path, FileScanQueue::ScanRequestType::ImageFile);
+                    }
 
+                    if (fileToProcess)
+                    {
                         context.currentStepStats.processedElems++;
                         _progressCallback(context.currentStepStats);
                     }
+                }
 
-                    while (_metadataScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2))
-                    {
-                        _metadataScanQueue.popResults(scanResults, processMetaDataBatchSize);
-                        processMetaDataScanResults(context, scanResults, mediaLibrary);
-                    }
+                while (_fileScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2))
+                {
+                    _fileScanQueue.popResults(scanResults, processFileResultsBatchSize);
+                    processFileScanResults(context, scanResults, mediaLibrary);
+                }
 
-                    _metadataScanQueue.wait(scanQueueMaxScanRequestCount);
+                _fileScanQueue.wait(scanQueueMaxScanRequestCount);
 
-                    return true;
-                },
-                &excludeDirFileName);
+                return true;
+            },
+            &excludeDirFileName);
 
-            _metadataScanQueue.wait();
+        _fileScanQueue.wait();
 
-            while (!_abortScan && _metadataScanQueue.popResults(scanResults, processMetaDataBatchSize) > 0)
-                processMetaDataScanResults(context, scanResults, mediaLibrary);
-        }
+        while (!_abortScan && _fileScanQueue.popResults(scanResults, processFileResultsBatchSize) > 0)
+            processFileScanResults(context, scanResults, mediaLibrary);
     }
 
-    bool ScanStepScanAudioFiles::checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo)
+    bool ScanStepScanFiles::checkAudioFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo)
     {
         ScanStats& stats{ context.stats };
 
-        Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) };
+        const Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) };
         // Should rarely fail as we are currently iterating it
         if (!lastWriteTime.isValid())
         {
@@ -498,35 +455,77 @@ namespace lms::scanner
         return true; // need to scan
     }
 
-    void ScanStepScanAudioFiles::processMetaDataScanResults(ScanContext& context, std::span<const MetaDataScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo)
+    bool ScanStepScanFiles::checkImageFileNeedScan(ScanContext& context, const std::filesystem::path& file)
+    {
+        ScanStats& stats{ context.stats };
+
+        const Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) };
+        // Should rarely fail as we are currently iterating it
+        if (!lastWriteTime.isValid())
+        {
+            stats.skips++;
+            return false;
+        }
+
+        if (!context.scanOptions.fullScan)
+        {
+            db::Session& dbSession{ _db.getTLSSession() };
+            auto transaction{ _db.getTLSSession().createReadTransaction() };
+
+            const db::Image::pointer image{ db::Image::find(dbSession, file) };
+            if (image && image->getLastWriteTime() == lastWriteTime)
+            {
+                stats.skips++;
+                return false;
+            }
+        }
+
+        return true; // need to scan
+    }
+
+    void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<const FileScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo)
     {
         LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ProcessScanResults");
 
         db::Session& dbSession{ _db.getTLSSession() };
         auto transaction{ dbSession.createWriteTransaction() };
 
-        for (const MetaDataScanResult& scanResult : scanResults)
+        for (const FileScanResult& scanResult : scanResults)
         {
-            LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessScanResult");
-
             if (_abortScan)
                 return;
 
-            if (scanResult.trackMetaData)
+            if (const AudioFileScanData * scanData{ std::get_if<AudioFileScanData>(&scanResult.scanData) })
             {
-                context.stats.scans++;
-
-                processFileMetaData(context, scanResult.path, *scanResult.trackMetaData, libraryInfo);
+                if (metadata::Track * track{ scanData->get() })
+                {
+                    context.stats.scans++;
+                    processAudioFileScanData(context, scanResult.path, *track, libraryInfo);
+                }
+                else
+                {
+                    context.stats.errors.emplace_back(scanResult.path, ScanErrorType::CannotReadAudioFile);
+                }
             }
-            else
+            else if (const ImageFileScanData * scanData{ std::get_if<ImageFileScanData>(&scanResult.scanData) })
             {
-                context.stats.errors.emplace_back(scanResult.path, ScanErrorType::CannotParseFile);
+                if (scanData->has_value())
+                {
+                    context.stats.scans++;
+                    processImageFileScanData(context, scanResult.path, scanData->value(), libraryInfo);
+                }
+                else
+                {
+                    context.stats.errors.emplace_back(scanResult.path, ScanErrorType::CannotReadImageFile);
+                }
             }
         }
     }
 
-    void ScanStepScanAudioFiles::processFileMetaData(ScanContext& context, const std::filesystem::path& file, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo)
+    void ScanStepScanFiles::processAudioFileScanData(ScanContext& context, const std::filesystem::path& file, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo)
     {
+        LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessAudioScanData");
+
         ScanStats& stats{ context.stats };
 
         const std::optional<FileInfo> fileInfo{ retrieveFileInfo(file, libraryInfo.rootDirectory) };
@@ -637,6 +636,8 @@ namespace lms::scanner
         track.modify()->setLastWriteTime(fileInfo->lastWriteTime);
 
         track.modify()->setMediaLibrary(MediaLibrary::find(dbSession, libraryInfo.id)); // may be null if settings are updated in // => next scan will correct this
+        track.modify()->setDirectory(getOrCreateDirectory(dbSession, file.parent_path(), libraryInfo.rootDirectory));
+
         track.modify()->clearArtistLinks();
         // Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files
         for (const Artist::pointer& artist : getOrCreateArtists(dbSession, trackMetadata.artists, false))
@@ -712,12 +713,57 @@ namespace lms::scanner
 
         if (added)
         {
-            LMS_LOG(DBUPDATER, DEBUG, "Added '" << file.string() << "'");
+            LMS_LOG(DBUPDATER, DEBUG, "Added audio file '" << file.string() << "'");
             stats.additions++;
         }
         else
         {
-            LMS_LOG(DBUPDATER, DEBUG, "Updated '" << file.string() << "'");
+            LMS_LOG(DBUPDATER, DEBUG, "Updated audio file '" << file.string() << "'");
+            stats.updates++;
+        }
+    }
+
+    void ScanStepScanFiles::processImageFileScanData(ScanContext& context, const std::filesystem::path& file, const ImageInfo& imageInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo)
+    {
+        LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessImageScanData");
+
+        ScanStats& stats{ context.stats };
+
+        const std::optional<FileInfo> fileInfo{ retrieveFileInfo(file, libraryInfo.rootDirectory) };
+        if (!fileInfo)
+        {
+            stats.skips++;
+            return;
+        }
+
+        db::Session& dbSession{ _db.getTLSSession() };
+        db::Image::pointer image{ db::Image::find(dbSession, file) };
+
+        bool added;
+        if (!image)
+        {
+            image = dbSession.create<db::Image>(file);
+            added = true;
+        }
+        else
+        {
+            added = false;
+        }
+
+        image.modify()->setLastWriteTime(fileInfo->lastWriteTime);
+        image.modify()->setFileSize(fileInfo->fileSize);
+        image.modify()->setHeight(imageInfo.height);
+        image.modify()->setWidth(imageInfo.width);
+        image.modify()->setDirectory(getOrCreateDirectory(dbSession, file.parent_path(), libraryInfo.rootDirectory));
+
+        if (added)
+        {
+            LMS_LOG(DBUPDATER, DEBUG, "Added image '" << file.string() << "'");
+            stats.additions++;
+        }
+        else
+        {
+            LMS_LOG(DBUPDATER, DEBUG, "Updated image '" << file.string() << "'");
             stats.updates++;
         }
     }

src/libs/services/scanner/impl/ScanStepScanFiles.hpp

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <filesystem>
+#include <span>
+#include <string>
+#include <vector>
+
+#include "metadata/IParser.hpp"
+
+#include "FileScanQueue.hpp"
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepScanFiles : public ScanStepBase
+    {
+    public:
+        ScanStepScanFiles(InitParams& initParams);
+
+    private:
+        ScanStep getStep() const override { return ScanStep::ScanFiles; }
+        core::LiteralString getStepName() const override { return "Scan files"; }
+        void process(ScanContext& context) override;
+        void process(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary);
+
+        bool checkAudioFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);
+        bool checkImageFileNeedScan(ScanContext& context, const std::filesystem::path& file);
+
+        void processFileScanResults(ScanContext& context, std::span<const FileScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo);
+        void processAudioFileScanData(ScanContext& context, const std::filesystem::path& path, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo);
+        void processImageFileScanData(ScanContext& context, const std::filesystem::path& path, const ImageInfo& imageInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo);
+
+        std::unique_ptr<metadata::IParser> _metadataParser;
+        const std::vector<std::string> _extraTagsToParse;
+
+        FileScanQueue _fileScanQueue;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScannerService.cpp

@@ -29,15 +29,17 @@
 #include "database/MediaLibrary.hpp"
 #include "database/ScanSettings.hpp"
 #include "database/TrackFeatures.hpp"
+#include "image/Image.hpp"
 
-#include "ScanStepCheckDuplicatedDbFiles.hpp"
+#include "ScanStepAssociateArtistImages.hpp"
+#include "ScanStepCheckForDuplicatedFiles.hpp"
+#include "ScanStepCheckForRemovedFiles.hpp"
 #include "ScanStepCompact.hpp"
 #include "ScanStepComputeClusterStats.hpp"
 #include "ScanStepDiscoverFiles.hpp"
 #include "ScanStepOptimize.hpp"
-#include "ScanStepRemoveOrphanDbFiles.hpp"
-#include "ScanStepScanArtistImages.hpp"
-#include "ScanStepScanAudioFiles.hpp"
+#include "ScanStepRemoveOrphanedDbEntries.hpp"
+#include "ScanStepScanFiles.hpp"
 
 namespace lms::scanner
 {
@@ -340,13 +342,14 @@ namespace lms::scanner
         // Order is important
         _scanSteps.clear();
         _scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
-        _scanSteps.push_back(std::make_unique<ScanStepScanAudioFiles>(params));
-        _scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
-        _scanSteps.push_back(std::make_unique<ScanStepScanArtistImages>(params));
+        _scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
+        _scanSteps.push_back(std::make_unique<ScanStepCheckForRemovedFiles>(params));
+        _scanSteps.push_back(std::make_unique<ScanStepAssociateArtistImages>(params));
+        _scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanedDbEntries>(params));
         _scanSteps.push_back(std::make_unique<ScanStepCompact>(params));
         _scanSteps.push_back(std::make_unique<ScanStepOptimize>(params));
         _scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
-        _scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
+        _scanSteps.push_back(std::make_unique<ScanStepCheckForDuplicatedFiles>(params));
     }
 
     ScannerSettings ScannerService::readSettings()
@@ -364,9 +367,16 @@ namespace lms::scanner
             newSettings.updatePeriod = scanSettings->getUpdatePeriod();
 
             {
-                const auto fileExtensions{ scanSettings->getAudioFileExtensions() };
-                newSettings.supportedExtensions.reserve(fileExtensions.size());
-                std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions),
+                const auto audioFileExtensions{ scanSettings->getAudioFileExtensions() };
+                newSettings.supportedAudioFileExtensions.reserve(audioFileExtensions.size());
+                std::transform(std::cbegin(audioFileExtensions), std::end(audioFileExtensions), std::back_inserter(newSettings.supportedAudioFileExtensions),
+                    [](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; });
+            }
+
+            {
+                const auto imageFileExtensions{ image::getSupportedFileExtensions() };
+                newSettings.supportedImageFileExtensions.reserve(imageFileExtensions.size());
+                std::transform(std::cbegin(imageFileExtensions), std::end(imageFileExtensions), std::back_inserter(newSettings.supportedImageFileExtensions),
                     [](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; });
             }
 

src/libs/services/scanner/impl/ScannerSettings.hpp

@@ -35,7 +35,8 @@ namespace lms::scanner
         std::size_t scanVersion{};
         Wt::WTime startTime;
         db::ScanSettings::UpdatePeriod updatePeriod{ db::ScanSettings::UpdatePeriod::Never };
-        std::vector<std::filesystem::path> supportedExtensions;
+        std::vector<std::filesystem::path> supportedAudioFileExtensions;
+        std::vector<std::filesystem::path> supportedImageFileExtensions;
         bool skipDuplicateMBID{};
         std::vector<std::string> extraTags;
         std::vector<std::string> artistTagDelimiters;

src/libs/services/scanner/include/services/scanner/ScannerStats.hpp

@@ -30,10 +30,11 @@ namespace lms::scanner
 {
     enum class ScanErrorType
     {
-        CannotReadFile,  // cannot read file
-        CannotParseFile, // cannot parse file
-        NoAudioTrack,    // no audio track found
-        BadDuration,     // bad duration
+        CannotReadFile,      // cannot read file
+        CannotReadAudioFile, // cannot parse audio file
+        CannotReadImageFile, // cannot parse image file
+        NoAudioTrack,        // no audio track found
+        BadDuration,         // bad duration
     };
 
     enum class DuplicateReason
@@ -60,18 +61,19 @@ namespace lms::scanner
     // Alphabetical order
     enum class ScanStep
     {
-        CheckForMissingFiles,
-        CheckForDuplicateFiles,
+        AssociateArtistImages,
+        CheckForDuplicatedFiles,
+        CheckForRemovedFiles,
         ComputeClusterStats,
         Compact,
         DiscoverFiles,
         FetchTrackFeatures,
         Optimize,
         ReloadSimilarityEngine,
-        ScanArtistImages,
-        ScanAudioFiles,
+        RemoveOrphanedDbEntries,
+        ScanFiles,
     };
-    static inline constexpr unsigned ScanProgressStepCount{ 9 };
+    static inline constexpr unsigned ScanProgressStepCount{ 11 };
 
     // reduced scan stats
     struct ScanStepStats
@@ -92,7 +94,7 @@ namespace lms::scanner
         Wt::WDateTime startTime;
         Wt::WDateTime stopTime;
 
-        std::size_t filesScanned{}; // Total number of files scanned (estimated)
+        std::size_t totalFileCount{}; // Total number of files (estimated)
 
         std::size_t skips{}; // no change since last scan
         std::size_t scans{}; // actually scanned filed