Scanner: parallelized processing within the playlist/track association step

src/libs/database/impl/Artist.cpp

@@ -291,7 +291,7 @@ namespace lms::db
         utils::forEachQueryRangeResult(query, params.range, func);
     }
 
-    IdRange<ArtistId> Artist::findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count)
+    IdRange<ArtistId> Artist::findNextIdRange(Session& session, ArtistId lastRetrievedId, std::size_t count)
     {
         auto query{ session.getDboSession()->query<std::tuple<ArtistId, ArtistId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT a.id FROM artist a WHERE a.id > ? ORDER BY a.id LIMIT ?) sub") };
         query.bind(lastRetrievedId);

src/libs/database/impl/PlayListFile.cpp

@@ -83,6 +83,27 @@ namespace lms::db
         });
     }
 
+    void PlayListFile::find(Session& session, const IdRange<PlayListFileId>& idRange, const std::function<void(const PlayListFile::pointer&)>& func)
+    {
+        assert(idRange.isValid());
+
+        auto query{ session.getDboSession()->query<Wt::Dbo::ptr<PlayListFile>>("SELECT pl_f from playlist_file pl_f").orderBy("pl_f.id").where("pl_f.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
+
+        utils::forEachQueryResult(query, [&](const PlayListFile::pointer& release) {
+            func(release);
+        });
+    }
+
+    IdRange<PlayListFileId> PlayListFile::findNextIdRange(Session& session, PlayListFileId lastRetrievedId, std::size_t count)
+    {
+        auto query{ session.getDboSession()->query<std::tuple<PlayListFileId, PlayListFileId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT pl_f.id FROM playlist_file pl_f WHERE pl_f.id > ? ORDER BY pl_f.id LIMIT ?) sub") };
+        query.bind(lastRetrievedId);
+        query.bind(static_cast<int>(count));
+
+        auto res{ utils::fetchQuerySingleResult(query) };
+        return IdRange<PlayListFileId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
+    }
+
     PlayListFile::pointer PlayListFile::find(Session& session, PlayListFileId id)
     {
         session.checkReadTransaction();

src/libs/database/impl/Release.cpp

@@ -515,7 +515,7 @@ namespace lms::db
         });
     }
 
-    IdRange<ReleaseId> Release::findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count)
+    IdRange<ReleaseId> Release::findNextIdRange(Session& session, ReleaseId lastRetrievedId, std::size_t count)
     {
         auto query{ session.getDboSession()->query<std::tuple<ReleaseId, ReleaseId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT r.id FROM release r WHERE r.id > ? ORDER BY r.id LIMIT ?) sub") };
         query.bind(lastRetrievedId);

src/libs/database/impl/Track.cpp

@@ -303,7 +303,7 @@ namespace lms::db
         });
     }
 
-    IdRange<TrackId> Track::findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count)
+    IdRange<TrackId> Track::findNextIdRange(Session& session, TrackId lastRetrievedId, std::size_t count)
     {
         auto query{ session.getDboSession()->query<std::tuple<TrackId, TrackId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT t.id FROM track t WHERE t.id > ? ORDER BY t.id LIMIT ?) sub") };
         query.bind(lastRetrievedId);

src/libs/database/include/database/Artist.hpp

@@ -129,7 +129,7 @@ namespace lms::db
         static void find(Session& session, const IdRange<ArtistId>& idRange, const std::function<void(const Artist::pointer&)>& func);
         static RangeResults<pointer> find(Session& session, const FindParameters& params);
         static void find(Session& session, const FindParameters& params, std::function<void(const pointer&)> func);
-        static IdRange<ArtistId> findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count);
+        static IdRange<ArtistId> findNextIdRange(Session& session, ArtistId lastRetrievedId, std::size_t count);
         static RangeResults<ArtistId> findIds(Session& session, const FindParameters& params);
         static RangeResults<ArtistId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt); // No track related
         static bool exists(Session& session, ArtistId id);

src/libs/database/include/database/PlayListFile.hpp

@@ -28,6 +28,7 @@
 #include <Wt/WDateTime.h>
 
 #include "database/DirectoryId.hpp"
+#include "database/IdRange.hpp"
 #include "database/Object.hpp"
 
 LMS_DECLARE_IDTYPE(PlayListFileId)
@@ -50,6 +51,8 @@ namespace lms::db
         static pointer find(Session& session, const std::filesystem::path& path);
         static void find(Session& session, PlayListFileId& lastRetrievedId, std::size_t count, const std::function<void(const pointer&)>& func);
         static void findAbsoluteFilePath(Session& session, PlayListFileId& lastRetrievedId, std::size_t count, const std::function<void(PlayListFileId playListFileId, const std::filesystem::path& absoluteFilePath)>& func);
+        static void find(Session& session, const IdRange<PlayListFileId>& idRange, const std::function<void(const PlayListFile::pointer&)>& func);
+        static IdRange<PlayListFileId> findNextIdRange(Session& session, PlayListFileId lastRetrievedId, std::size_t count);
 
         // getters
         const std::filesystem::path& getAbsoluteFilePath() const { return _absoluteFilePath; }

src/libs/database/include/database/Release.hpp

@@ -251,7 +251,7 @@ namespace lms::db
         static pointer find(Session& session, ReleaseId id);
         static void find(Session& session, ReleaseId& lastRetrievedRelease, std::size_t count, const std::function<void(const Release::pointer&)>& func, MediaLibraryId library = {});
         static void find(Session& session, const IdRange<ReleaseId>& idRange, const std::function<void(const Release::pointer&)>& func);
-        static IdRange<ReleaseId> findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count);
+        static IdRange<ReleaseId> findNextIdRange(Session& session, ReleaseId lastRetrievedId, std::size_t count);
         static RangeResults<pointer> find(Session& session, const FindParameters& parameters);
         static void find(Session& session, const FindParameters& parameters, const std::function<void(const pointer&)>& func);
         static RangeResults<ReleaseId> findIds(Session& session, const FindParameters& parameters);

src/libs/database/include/database/Track.hpp

@@ -194,7 +194,7 @@ namespace lms::db
         static pointer find(Session& session, TrackId id);
         static void find(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(const Track::pointer&)>& func, MediaLibraryId library = {});
         static void find(Session& session, const IdRange<TrackId>& idRange, const std::function<void(const Track::pointer&)>& func);
-        static IdRange<TrackId> findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count);
+        static IdRange<TrackId> findNextIdRange(Session& session, TrackId lastRetrievedId, std::size_t count);
         static void findAbsoluteFilePath(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(TrackId trackId, const std::filesystem::path& absoluteFilePath)>& func);
 
         static bool exists(Session& session, TrackId id);

src/libs/database/test/Artist.cpp

@@ -608,17 +608,17 @@ namespace lms::db::tests
         }
     }
 
-    TEST_F(DatabaseFixture, Artist_findNextRange)
+    TEST_F(DatabaseFixture, Artist_findNextIdRange)
     {
         {
             auto transaction{ session.createReadTransaction() };
 
-            auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
+            auto range{ Artist::findNextIdRange(session, ArtistId{}, 0) };
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, ArtistId{});
             EXPECT_EQ(range.last, ArtistId{});
 
-            range = Artist::findNextRange(session, ArtistId{}, 100);
+            range = Artist::findNextIdRange(session, ArtistId{}, 100);
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, ArtistId{});
             EXPECT_EQ(range.last, ArtistId{});
@@ -628,22 +628,22 @@ namespace lms::db::tests
         {
             auto transaction{ session.createReadTransaction() };
 
-            auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
+            auto range{ Artist::findNextIdRange(session, ArtistId{}, 0) };
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, ArtistId{});
             EXPECT_EQ(range.last, ArtistId{});
 
-            range = Artist::findNextRange(session, ArtistId{}, 1);
+            range = Artist::findNextIdRange(session, ArtistId{}, 1);
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, artist1.getId());
             EXPECT_EQ(range.last, artist1.getId());
 
-            range = Artist::findNextRange(session, range.last, 1);
+            range = Artist::findNextIdRange(session, range.last, 1);
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, ArtistId{});
             EXPECT_EQ(range.last, ArtistId{});
 
-            range = Artist::findNextRange(session, ArtistId{}, 100);
+            range = Artist::findNextIdRange(session, ArtistId{}, 100);
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, artist1.getId());
             EXPECT_EQ(range.last, artist1.getId());
@@ -655,12 +655,12 @@ namespace lms::db::tests
         {
             auto transaction{ session.createReadTransaction() };
 
-            auto range{ Artist::findNextRange(session, ArtistId{}, 2) };
+            auto range{ Artist::findNextIdRange(session, ArtistId{}, 2) };
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, artist1.getId());
             EXPECT_EQ(range.last, artist2.getId());
 
-            range = Artist::findNextRange(session, artist2.getId(), 2);
+            range = Artist::findNextIdRange(session, artist2.getId(), 2);
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, artist3.getId());
             EXPECT_EQ(range.last, artist3.getId());

src/libs/database/test/Release.cpp

@@ -173,17 +173,17 @@ namespace lms::db::tests
         }
     }
 
-    TEST_F(DatabaseFixture, Release_findNextRange)
+    TEST_F(DatabaseFixture, Release_findNextIdRange)
     {
         {
             auto transaction{ session.createReadTransaction() };
 
-            auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
+            auto range{ Release::findNextIdRange(session, ReleaseId{}, 0) };
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, ReleaseId{});
             EXPECT_EQ(range.last, ReleaseId{});
 
-            range = Release::findNextRange(session, ReleaseId{}, 100);
+            range = Release::findNextIdRange(session, ReleaseId{}, 100);
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, ReleaseId{});
             EXPECT_EQ(range.last, ReleaseId{});
@@ -193,22 +193,22 @@ namespace lms::db::tests
         {
             auto transaction{ session.createReadTransaction() };
 
-            auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
+            auto range{ Release::findNextIdRange(session, ReleaseId{}, 0) };
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, ReleaseId{});
             EXPECT_EQ(range.last, ReleaseId{});
 
-            range = Release::findNextRange(session, ReleaseId{}, 1);
+            range = Release::findNextIdRange(session, ReleaseId{}, 1);
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, release1.getId());
             EXPECT_EQ(range.last, release1.getId());
 
-            range = Release::findNextRange(session, range.last, 1);
+            range = Release::findNextIdRange(session, range.last, 1);
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, ReleaseId{});
             EXPECT_EQ(range.last, ReleaseId{});
 
-            range = Release::findNextRange(session, ReleaseId{}, 100);
+            range = Release::findNextIdRange(session, ReleaseId{}, 100);
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, release1.getId());
             EXPECT_EQ(range.last, release1.getId());
@@ -220,12 +220,12 @@ namespace lms::db::tests
         {
             auto transaction{ session.createReadTransaction() };
 
-            auto range{ Release::findNextRange(session, ReleaseId{}, 2) };
+            auto range{ Release::findNextIdRange(session, ReleaseId{}, 2) };
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, release1.getId());
             EXPECT_EQ(range.last, release2.getId());
 
-            range = Release::findNextRange(session, release2.getId(), 2);
+            range = Release::findNextIdRange(session, release2.getId(), 2);
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, release3.getId());
             EXPECT_EQ(range.last, release3.getId());

src/libs/database/test/Track.cpp

@@ -151,17 +151,17 @@ namespace lms::db::tests
         }
     }
 
-    TEST_F(DatabaseFixture, Track_findNextRange)
+    TEST_F(DatabaseFixture, Track_findNextIdRange)
     {
         {
             auto transaction{ session.createReadTransaction() };
 
-            auto range{ Track::findNextRange(session, TrackId{}, 0) };
+            auto range{ Track::findNextIdRange(session, TrackId{}, 0) };
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, TrackId{});
             EXPECT_EQ(range.last, TrackId{});
 
-            range = Track::findNextRange(session, TrackId{}, 100);
+            range = Track::findNextIdRange(session, TrackId{}, 100);
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, TrackId{});
             EXPECT_EQ(range.last, TrackId{});
@@ -171,22 +171,22 @@ namespace lms::db::tests
         {
             auto transaction{ session.createReadTransaction() };
 
-            auto range{ Track::findNextRange(session, TrackId{}, 0) };
+            auto range{ Track::findNextIdRange(session, TrackId{}, 0) };
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, TrackId{});
             EXPECT_EQ(range.last, TrackId{});
 
-            range = Track::findNextRange(session, TrackId{}, 1);
+            range = Track::findNextIdRange(session, TrackId{}, 1);
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, track1.getId());
             EXPECT_EQ(range.last, track1.getId());
 
-            range = Track::findNextRange(session, range.last, 1);
+            range = Track::findNextIdRange(session, range.last, 1);
             EXPECT_FALSE(range.isValid());
             EXPECT_EQ(range.first, TrackId{});
             EXPECT_EQ(range.last, TrackId{});
 
-            range = Track::findNextRange(session, TrackId{}, 100);
+            range = Track::findNextIdRange(session, TrackId{}, 100);
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, track1.getId());
             EXPECT_EQ(range.last, track1.getId());
@@ -198,12 +198,12 @@ namespace lms::db::tests
         {
             auto transaction{ session.createReadTransaction() };
 
-            auto range{ Track::findNextRange(session, TrackId{}, 2) };
+            auto range{ Track::findNextIdRange(session, TrackId{}, 2) };
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, track1.getId());
             EXPECT_EQ(range.last, track2.getId());
 
-            range = Track::findNextRange(session, track2.getId(), 2);
+            range = Track::findNextIdRange(session, track2.getId(), 2);
             EXPECT_TRUE(range.isValid());
             EXPECT_EQ(range.first, track3.getId());
             EXPECT_EQ(range.last, track3.getId());

src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp

@@ -261,7 +261,7 @@ namespace lms::scanner
 
             auto transaction{ session.createReadTransaction() };
 
-            idRange = db::Artist::findNextRange(session, lastRetrievedId, readBatchSize);
+            idRange = db::Artist::findNextIdRange(session, lastRetrievedId, readBatchSize);
             lastRetrievedId = idRange.last;
 
             return idRange.isValid();
@@ -281,7 +281,7 @@ namespace lms::scanner
             std::size_t getProcessedArtistCount() const { return _processedArtistCount; }
 
         private:
-            core::LiteralString getName() const override { return "Associate Track Artworks"; }
+            core::LiteralString getName() const override { return "Associate Artist Artworks"; }
             void run() override
             {
                 auto& session{ _db.getTLSSession() };

src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp

@@ -21,7 +21,9 @@
 
 #include <deque>
 #include <filesystem>
+#include <span>
 
+#include "core/IJob.hpp"
 #include "core/ILogger.hpp"
 #include "database/Db.hpp"
 #include "database/Directory.hpp"
@@ -32,6 +34,7 @@
 #include "database/TrackList.hpp"
 #include "services/scanner/ScanErrors.hpp"
 
+#include "JobQueue.hpp"
 #include "ScanContext.hpp"
 #include "ScannerSettings.hpp"
 
@@ -53,15 +56,6 @@ namespace lms::scanner
         };
         using PlayListFileAssociationContainer = std::deque<PlayListFileAssociation>;
 
-        struct SearchPlayListFileContext
-        {
-            db::Session& session;
-            db::PlayListFileId lastRetrievedPlayListFileId;
-            std::size_t processedPlayListFileCount{};
-            const ScannerSettings& settings;
-            std::vector<std::shared_ptr<ScanError>> errors;
-        };
-
         db::Track::pointer getMatchingTrack(db::Session& session, const std::filesystem::path& filePath, const db::Directory::pointer& playListDirectory)
         {
             db::Track::pointer matchingTrack;
@@ -110,64 +104,7 @@ namespace lms::scanner
             return needUpdate;
         }
 
-        bool fetchNextPlayListFilesToUpdate(SearchPlayListFileContext& searchContext, PlayListFileAssociationContainer& playListFileAssociations)
+        void updatePlayList(db::Session& session, const PlayListFileAssociation& playListFileAssociation)
-        {
-            const db::PlayListFileId playListFileIdId{ searchContext.lastRetrievedPlayListFileId };
-
-            {
-                constexpr std::size_t readBatchSize{ 20 };
-
-                auto transaction{ searchContext.session.createReadTransaction() };
-
-                db::PlayListFile::find(searchContext.session, searchContext.lastRetrievedPlayListFileId, readBatchSize, [&](const db::PlayListFile::pointer& playListFile) {
-                    PlayListFileAssociation playListAssociation;
-
-                    playListAssociation.playListFileIdId = playListFile->getId();
-
-                    std::vector<std::shared_ptr<ScanError>> pendingErrors;
-
-                    const auto files{ playListFile->getFiles() };
-                    for (const std::filesystem::path& file : files)
-                    {
-                        // TODO optim: no need to fetch the whole track
-                        const db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) };
-                        if (track)
-                            playListAssociation.tracks.push_back(TrackInfo{ .trackId = track->getId(), .releaseId = track->getReleaseId() });
-                        else
-                            pendingErrors.emplace_back(std::make_shared<PlayListFilePathMissingError>(playListFile->getAbsoluteFilePath(), file));
-                    }
-
-                    if (pendingErrors.size() == files.size())
-                    {
-                        pendingErrors.clear();
-                        pendingErrors.emplace_back(std::make_shared<PlayListFileAllPathesMissingError>(playListFile->getAbsoluteFilePath()));
-                    }
-                    searchContext.errors.insert(std::end(searchContext.errors), std::begin(pendingErrors), std::end(pendingErrors));
-
-                    if (playListAssociation.tracks.empty()
-                        || (searchContext.settings.skipSingleReleasePlayLists && isSingleReleasePlayList(playListAssociation.tracks)))
-                    {
-                        playListAssociation.tracks.clear();
-                    }
-
-                    bool needUpdate{ true };
-                    if (const db::TrackList::pointer trackList{ playListFile->getTrackList() })
-                    {
-                        if (!playListAssociation.tracks.empty())
-                            needUpdate = trackListNeedsUpdate(searchContext.session, playListFile->getName(), playListAssociation.tracks, trackList);
-                    }
-
-                    if (needUpdate)
-                        playListFileAssociations.emplace_back(std::move(playListAssociation));
-
-                    searchContext.processedPlayListFileCount++;
-                });
-            }
-
-            return playListFileIdId != searchContext.lastRetrievedPlayListFileId;
-        }
-
-        void updatePlayListFile(db::Session& session, const PlayListFileAssociation& playListFileAssociation)
         {
             db::PlayListFile::pointer playListFile{ db::PlayListFile::find(session, playListFileAssociation.playListFileIdId) };
             assert(playListFile);
@@ -198,6 +135,7 @@ namespace lms::scanner
             trackList.modify()->clear();
             for (const TrackInfo trackInfo : playListFileAssociation.tracks)
             {
+                // TODO no need to fetch the whole track for that
                 if (db::Track::pointer track{ db::Track::find(session, trackInfo.trackId) })
                     session.create<db::TrackListEntry>(track, trackList, playListFile->getLastWriteTime());
             }
@@ -205,21 +143,110 @@ namespace lms::scanner
             LMS_LOG(DBUPDATER, DEBUG, std::string_view{ createTrackList ? "Created" : "Updated" } << " associated tracklist for " << playListFile->getAbsoluteFilePath() << " (" << playListFileAssociation.tracks.size() << " tracks)");
         }
 
-        void updatePlayListFiles(db::Session& session, PlayListFileAssociationContainer& playListFileAssociations)
+        void updatePlayLists(db::Session& session, PlayListFileAssociationContainer& playListFileAssociations, bool forceFullBatch)
         {
             constexpr std::size_t writeBatchSize{ 5 };
 
-            while (!playListFileAssociations.empty())
+            while ((forceFullBatch && playListFileAssociations.size() >= writeBatchSize) || !playListFileAssociations.empty())
             {
                 auto transaction{ session.createWriteTransaction() };
 
                 for (std::size_t i{}; !playListFileAssociations.empty() && i < writeBatchSize; ++i)
                 {
-                    updatePlayListFile(session, playListFileAssociations.front());
+                    updatePlayList(session, playListFileAssociations.front());
                     playListFileAssociations.pop_front();
                 }
             }
         }
+
+        bool fetchNextPlayListFileIdRange(db::Session& session, db::PlayListFileId& lastPlayListFileId, db::IdRange<db::PlayListFileId>& idRange)
+        {
+            constexpr std::size_t readBatchSize{ 100 };
+
+            auto transaction{ session.createReadTransaction() };
+
+            idRange = db::PlayListFile::findNextIdRange(session, lastPlayListFileId, readBatchSize);
+            lastPlayListFileId = idRange.last;
+
+            return idRange.isValid();
+        }
+
+        class ComputePlayListFileAssociationsJob : public core::IJob
+        {
+        public:
+            ComputePlayListFileAssociationsJob(db::Db& db, const ScannerSettings& settings, db::IdRange<db::PlayListFileId> playListFileIdRange)
+                : _db{ db }
+                , _settings{ settings }
+                , _playListFileIdRange{ playListFileIdRange }
+            {
+            }
+
+            std::span<const PlayListFileAssociation> getAssociations() const { return _associations; }
+            std::size_t getProcessedCount() const { return _processedCount; }
+            std::span<const std::shared_ptr<ScanError>> getErrors() const { return _errors; }
+
+        private:
+            core::LiteralString getName() const override { return "Associate PlayList Tracks"; }
+            void run() override
+            {
+                auto& session{ _db.getTLSSession() };
+                auto transaction{ session.createReadTransaction() };
+
+                db::PlayListFile::find(session, _playListFileIdRange, [&](const db::PlayListFile::pointer& playListFile) {
+                    PlayListFileAssociation playListAssociation;
+
+                    playListAssociation.playListFileIdId = playListFile->getId();
+
+                    std::vector<std::shared_ptr<ScanError>> pendingErrors;
+
+                    const auto& files{ playListFile->getFiles() };
+                    for (const std::filesystem::path& file : files)
+                    {
+                        // TODO optim: no need to fetch the whole track
+                        const db::Track::pointer track{ getMatchingTrack(session, file, playListFile->getDirectory()) };
+                        if (track)
+                            playListAssociation.tracks.push_back(TrackInfo{ .trackId = track->getId(), .releaseId = track->getReleaseId() });
+                        else
+                            pendingErrors.emplace_back(std::make_shared<PlayListFilePathMissingError>(playListFile->getAbsoluteFilePath(), file));
+                    }
+
+                    if (pendingErrors.size() == files.size())
+                    {
+                        pendingErrors.clear();
+                        pendingErrors.emplace_back(std::make_shared<PlayListFileAllPathesMissingError>(playListFile->getAbsoluteFilePath()));
+                    }
+                    _errors.insert(std::end(_errors), std::begin(pendingErrors), std::end(pendingErrors));
+
+                    if (playListAssociation.tracks.empty()
+                        || (_settings.skipSingleReleasePlayLists && isSingleReleasePlayList(playListAssociation.tracks)))
+                    {
+                        playListAssociation.tracks.clear();
+                    }
+
+                    bool needUpdate{ true };
+                    if (const db::TrackList::pointer trackList{ playListFile->getTrackList() })
+                    {
+                        if (!playListAssociation.tracks.empty())
+                            needUpdate = trackListNeedsUpdate(session, playListFile->getName(), playListAssociation.tracks, trackList);
+                    }
+                    else if (playListAssociation.tracks.empty())
+                        needUpdate = false;
+
+                    if (needUpdate)
+                        _associations.emplace_back(std::move(playListAssociation));
+
+                    _processedCount++;
+                });
+            }
+
+            db::Db& _db;
+            const ScannerSettings& _settings;
+            db::IdRange<db::PlayListFileId> _playListFileIdRange;
+            std::vector<PlayListFileAssociation> _associations;
+            std::vector<std::shared_ptr<ScanError>> _errors;
+            std::size_t _processedCount{};
+        };
+
     } // namespace
 
     bool ScanStepAssociatePlayListTracks::needProcess(const ScanContext& context) const
@@ -242,27 +269,37 @@ namespace lms::scanner
             context.currentStepStats.totalElems = db::PlayListFile::getCount(session);
         }
 
-        SearchPlayListFileContext searchContext{
+        PlayListFileAssociationContainer playListTrackAssociations;
-            .session = session,
+        auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
-            .lastRetrievedPlayListFileId = {},
-            .settings = _settings,
-            .errors = context.stats.errors
-        };
-
-        PlayListFileAssociationContainer playListFileAssociations;
-        while (fetchNextPlayListFilesToUpdate(searchContext, playListFileAssociations))
-        {
             if (_abortScan)
                 return;
 
-            updatePlayListFiles(session, playListFileAssociations);
+            for (const auto& job : jobs)
+            {
+                const auto& associationJob{ static_cast<const ComputePlayListFileAssociationsJob&>(*job) };
+                const auto& associations{ associationJob.getAssociations() };
 
-            context.currentStepStats.processedElems = searchContext.processedPlayListFileCount;
+                playListTrackAssociations.insert(std::end(playListTrackAssociations), std::cbegin(associations), std::cend(associations));
-            for (const std::shared_ptr<ScanError>& error : searchContext.errors)
+                for (const std::shared_ptr<ScanError>& error : associationJob.getErrors())
-                addError(context, error);
+                    addError(context, error);
-            searchContext.errors.clear();
 
+                context.currentStepStats.processedElems += associationJob.getProcessedCount();
+            }
+
+            updatePlayLists(session, playListTrackAssociations, true);
             _progressCallback(context.currentStepStats);
+        };
+
+        {
+            JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
+
+            db::PlayListFileId lastPlayListFileId;
+            db::IdRange<db::PlayListFileId> playListFileIdRange;
+            while (fetchNextPlayListFileIdRange(session, lastPlayListFileId, playListFileIdRange))
+                queue.push(std::make_unique<ComputePlayListFileAssociationsJob>(_db, _settings, playListFileIdRange));
         }
+
+        // process all remaining associations
+        updatePlayLists(session, playListTrackAssociations, false);
     }
 } // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp

@@ -204,7 +204,7 @@ namespace lms::scanner
 
             auto transaction{ session.createReadTransaction() };
 
-            idRange = db::Release::findNextRange(session, lastRetrievedId, readBatchSize);
+            idRange = db::Release::findNextIdRange(session, lastRetrievedId, readBatchSize);
             lastRetrievedId = idRange.last;
 
             return idRange.isValid();
@@ -224,7 +224,7 @@ namespace lms::scanner
             std::size_t getProcessedReleaseCount() const { return _processedReleaseCount; }
 
         private:
-            core::LiteralString getName() const override { return "Associate Track Artworks"; }
+            core::LiteralString getName() const override { return "Associate Release Artworks"; }
             void run() override
             {
                 auto& session{ _db.getTLSSession() };

src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp

@@ -53,13 +53,6 @@ namespace lms::scanner
         };
         using TrackArtworksAssociationContainer = std::deque<TrackArtworksAssociation>;
 
-        struct SearchTrackArtworkContext
-        {
-            db::Session& session;
-            db::TrackId lastRetrievedTrackId;
-            std::size_t processedTrackCount{};
-        };
-
         db::Artwork::pointer computePreferredTrackArtwork(db::Session& session, const db::Track::pointer& track, const db::Artwork::pointer& preferredMediaArtwork)
         {
             db::Artwork::pointer res{ preferredMediaArtwork };
@@ -152,14 +145,14 @@ namespace lms::scanner
             }
         }
 
-        bool fetchNextTrackIdRange(SearchTrackArtworkContext& searchContext, db::IdRange<db::TrackId>& trackIdRange)
+        bool fetchNextTrackIdRange(db::Session& session, db::TrackId& lastRetrievedTrackId, db::IdRange<db::TrackId>& trackIdRange)
         {
             constexpr std::size_t readBatchSize{ 100 };
 
-            auto transaction{ searchContext.session.createReadTransaction() };
+            auto transaction{ session.createReadTransaction() };
 
-            trackIdRange = db::Track::findNextRange(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize);
+            trackIdRange = db::Track::findNextIdRange(session, lastRetrievedTrackId, readBatchSize);
-            searchContext.lastRetrievedTrackId = trackIdRange.last;
+            lastRetrievedTrackId = trackIdRange.last;
 
             return trackIdRange.isValid();
         }
@@ -242,11 +235,6 @@ namespace lms::scanner
             context.currentStepStats.totalElems = db::Track::getCount(session);
         }
 
-        SearchTrackArtworkContext searchContext{
-            .session = session,
-            .lastRetrievedTrackId = {},
-        };
-
         TrackArtworksAssociationContainer trackArtworksAssociations;
         auto processTracks = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
             if (_abortScan)
@@ -266,13 +254,14 @@ namespace lms::scanner
             _progressCallback(context.currentStepStats);
         };
 
-        JobQueue queue{ getJobScheduler(), 20, processTracks, 1, 0.85F };
+        {
+            JobQueue queue{ getJobScheduler(), 20, processTracks, 1, 0.85F };
 
-        db::IdRange<db::TrackId> trackIdRange;
+            db::TrackId lastRetrievedTrackId;
-        while (fetchNextTrackIdRange(searchContext, trackIdRange))
+            db::IdRange<db::TrackId> trackIdRange;
-            queue.push(std::make_unique<ComputeTrackArtworkAssociationsJob>(_db, trackIdRange));
+            while (fetchNextTrackIdRange(session, lastRetrievedTrackId, trackIdRange))
-
+                queue.push(std::make_unique<ComputeTrackArtworkAssociationsJob>(_db, trackIdRange));
-        queue.finish();
+        }
 
         // process all remaining associations
         updateTrackPreferredArtworks(session, trackArtworksAssociations, false);