Scanner: parallelized processing within artwork association steps

src/libs/database/impl/Artist.cpp

@@ -237,6 +237,17 @@ namespace lms::db
         });
     }
 
+    void Artist::find(Session& session, const IdRange<ArtistId>& idRange, const std::function<void(const Artist::pointer&)>& func)
+    {
+        assert(idRange.isValid());
+
+        auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Artist>>("SELECT a from artist a").orderBy("a.id").where("a.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
+
+        utils::forEachQueryResult(query, [&](const Artist::pointer& artist) {
+            func(artist);
+        });
+    }
+
     std::vector<Artist::pointer> Artist::find(Session& session, std::string_view name)
     {
         session.checkReadTransaction();
@@ -280,6 +291,16 @@ namespace lms::db
         utils::forEachQueryRangeResult(query, params.range, func);
     }
 
+    IdRange<ArtistId> Artist::findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count)
+    {
+        auto query{ session.getDboSession()->query<std::tuple<ArtistId, ArtistId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT a.id FROM artist a WHERE a.id > ? ORDER BY a.id LIMIT ?) sub") };
+        query.bind(lastRetrievedId);
+        query.bind(static_cast<int>(count));
+
+        auto res{ utils::fetchQuerySingleResult(query) };
+        return IdRange<ArtistId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
+    }
+
     RangeResults<ArtistId> Artist::findOrphanIds(Session& session, std::optional<Range> range)
     {
         session.checkReadTransaction();
@@ -304,6 +325,16 @@ AND NOT EXISTS (
         return utils::fetchQuerySingleResult(session.getDboSession()->query<int>("SELECT 1 FROM artist").where("id = ?").bind(id)) == 1;
     }
 
+    void Artist::updatePreferredArtwork(Session& session, ArtistId artistId, ArtworkId artworkId)
+    {
+        session.checkWriteTransaction();
+
+        if (artworkId.isValid())
+            utils::executeCommand(*session.getDboSession(), "UPDATE artist SET preferred_artwork_id = ? WHERE id = ?", artworkId, artistId);
+        else
+            utils::executeCommand(*session.getDboSession(), "UPDATE artist SET preferred_artwork_id = NULL WHERE id = ?", artistId);
+    }
+
     std::optional<core::UUID> Artist::getMBID() const
     {
         return core::UUID::fromString(_mbid);

src/libs/database/impl/Migration.cpp

@@ -35,7 +35,7 @@ namespace lms::db
 {
     namespace
     {
-        static constexpr Version LMS_DATABASE_VERSION{ 95 };
+        static constexpr Version LMS_DATABASE_VERSION{ 96 };
     }
 
     VersionInfo::VersionInfo()
@@ -1396,6 +1396,22 @@ FROM artist)");
         utils::executeCommand(*session.getDboSession(), "ALTER TABLE track DROP COLUMN file_name");
     }
 
+    void migrateFromV95(Session& session)
+    {
+        // Make sure each image and each embessed image has an artwork object
+        utils::executeCommand(*session.getDboSession(), R"(INSERT INTO artwork (version, track_embedded_image_id)
+SELECT 1 AS version, tei.id
+FROM track_embedded_image tei
+LEFT JOIN artwork art ON tei.id = art.track_embedded_image_id
+WHERE art.track_embedded_image_id IS NULL)");
+
+        utils::executeCommand(*session.getDboSession(), R"(INSERT INTO artwork (version, image_id)
+SELECT 1 AS version, img.id
+FROM image img
+LEFT JOIN artwork art ON img.id = art.image_id
+WHERE art.image_id IS NULL)");
+    }
+
     bool doDbMigration(Session& session)
     {
         constexpr std::string_view outdatedMsg{ "Outdated database, please rebuild it (delete the .db file and restart)" };
@@ -1467,6 +1483,7 @@ FROM artist)");
             { 92, migrateFromV92 },
             { 93, migrateFromV93 },
             { 94, migrateFromV94 },
+            { 95, migrateFromV95 },
         };
 
         bool migrationPerformed{};

src/libs/database/impl/Release.cpp

@@ -504,6 +504,27 @@ namespace lms::db
         });
     }
 
+    void Release::find(Session& session, const IdRange<ReleaseId>& idRange, const std::function<void(const Release::pointer&)>& func)
+    {
+        assert(idRange.isValid());
+
+        auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Release>>("SELECT r from release r").orderBy("r.id").where("r.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
+
+        utils::forEachQueryResult(query, [&](const Release::pointer& release) {
+            func(release);
+        });
+    }
+
+    IdRange<ReleaseId> Release::findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count)
+    {
+        auto query{ session.getDboSession()->query<std::tuple<ReleaseId, ReleaseId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT r.id FROM release r WHERE r.id > ? ORDER BY r.id LIMIT ?) sub") };
+        query.bind(lastRetrievedId);
+        query.bind(static_cast<int>(count));
+
+        auto res{ utils::fetchQuerySingleResult(query) };
+        return IdRange<ReleaseId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
+    }
+
     RangeResults<Release::pointer> Release::find(Session& session, const FindParameters& params)
     {
         session.checkReadTransaction();
@@ -528,6 +549,16 @@ namespace lms::db
         return utils::execRangeQuery<ReleaseId>(query, params.range);
     }
 
+    void Release::updatePreferredArtwork(Session& session, ReleaseId releaseId, ArtworkId artworkId)
+    {
+        session.checkWriteTransaction();
+
+        if (artworkId.isValid())
+            utils::executeCommand(*session.getDboSession(), "UPDATE release SET preferred_artwork_id = ? WHERE id = ?", artworkId, releaseId);
+        else
+            utils::executeCommand(*session.getDboSession(), "UPDATE release SET preferred_artwork_id = NULL WHERE id = ?", releaseId);
+    }
+
     std::size_t Release::getCount(Session& session, const FindParameters& params)
     {
         session.checkReadTransaction();

src/libs/database/impl/Track.cpp

@@ -292,6 +292,27 @@ namespace lms::db
         });
     }
 
+    void Track::find(Session& session, const IdRange<TrackId>& idRange, const std::function<void(const Track::pointer&)>& func)
+    {
+        assert(idRange.isValid());
+
+        auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Track>>("SELECT t from track t").orderBy("t.id").where("t.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
+
+        utils::forEachQueryResult(query, [&](const Track::pointer& track) {
+            func(track);
+        });
+    }
+
+    IdRange<TrackId> Track::findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count)
+    {
+        auto query{ session.getDboSession()->query<std::tuple<TrackId, TrackId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT t.id FROM track t WHERE t.id > ? ORDER BY t.id LIMIT ?) sub") };
+        query.bind(lastRetrievedId);
+        query.bind(static_cast<int>(count));
+
+        auto res{ utils::fetchQuerySingleResult(query) };
+        return IdRange<TrackId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
+    }
+
     bool Track::exists(Session& session, TrackId id)
     {
         session.checkReadTransaction();

src/libs/database/include/database/Artist.hpp

@@ -34,6 +34,7 @@
 #include "database/ArtworkId.hpp"
 #include "database/ClusterId.hpp"
 #include "database/Filters.hpp"
+#include "database/IdRange.hpp"
 #include "database/MediaLibraryId.hpp"
 #include "database/Object.hpp"
 #include "database/ReleaseId.hpp"
@@ -125,12 +126,17 @@ namespace lms::db
         static pointer find(Session& session, ArtistId id);
         static std::vector<pointer> find(Session& session, std::string_view name); // exact match on name field
         static void find(Session& session, ArtistId& lastRetrievedArtist, std::size_t count, const std::function<void(const Artist::pointer&)>& func, MediaLibraryId library = {});
+        static void find(Session& session, const IdRange<ArtistId>& idRange, const std::function<void(const Artist::pointer&)>& func);
         static RangeResults<pointer> find(Session& session, const FindParameters& params);
         static void find(Session& session, const FindParameters& params, std::function<void(const pointer&)> func);
+        static IdRange<ArtistId> findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count);
         static RangeResults<ArtistId> findIds(Session& session, const FindParameters& params);
         static RangeResults<ArtistId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt); // No track related
         static bool exists(Session& session, ArtistId id);
 
+        // Updates
+        static void updatePreferredArtwork(Session& session, ArtistId artistId, ArtworkId artworkId);
+
         // Accessors
         const std::string& getName() const { return _name; }
         const std::string& getSortName() const { return _sortName; }

src/libs/database/include/database/IdRange.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2024 Emeric Poupon
+ * Copyright (C) 2025 Emeric Poupon
  *
  * This file is part of LMS.
  *
@@ -19,21 +19,15 @@
 
 #pragma once
 
-#include <filesystem>
-
-#include "database/ArtworkId.hpp"
-#include "database/ImageId.hpp"
-#include "database/Object.hpp"
-#include "database/TrackEmbeddedImageId.hpp"
-
 namespace lms::db
 {
-    class Artwork;
+    template<typename IdType>
-    class Session;
+    struct IdRange
-} // namespace lms::db
+    {
+    public:
+        IdType first;
+        IdType last;
 
-namespace lms::scanner::utils
+        bool isValid() const { return first.isValid() && last.isValid(); }
-{
+    };
-    db::ObjectPtr<db::Artwork> getOrCreateArtworkFromTrackEmbeddedImage(db::Session& session, db::TrackEmbeddedImageId trackEmbeddedImageId);
+} // namespace lms::db
-    db::ObjectPtr<db::Artwork> getOrCreateArtworkFromImage(db::Session& session, db::ImageId imageId);
-} // namespace lms::scanner::utils

src/libs/database/include/database/Release.hpp

@@ -35,6 +35,7 @@
 #include "database/CountryId.hpp"
 #include "database/DirectoryId.hpp"
 #include "database/Filters.hpp"
+#include "database/IdRange.hpp"
 #include "database/LabelId.hpp"
 #include "database/MediaLibraryId.hpp"
 #include "database/Object.hpp"
@@ -249,12 +250,17 @@ namespace lms::db
         static pointer find(Session& session, const core::UUID& MBID);
         static pointer find(Session& session, ReleaseId id);
         static void find(Session& session, ReleaseId& lastRetrievedRelease, std::size_t count, const std::function<void(const Release::pointer&)>& func, MediaLibraryId library = {});
+        static void find(Session& session, const IdRange<ReleaseId>& idRange, const std::function<void(const Release::pointer&)>& func);
+        static IdRange<ReleaseId> findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count);
         static RangeResults<pointer> find(Session& session, const FindParameters& parameters);
         static void find(Session& session, const FindParameters& parameters, const std::function<void(const pointer&)>& func);
         static RangeResults<ReleaseId> findIds(Session& session, const FindParameters& parameters);
         static std::size_t getCount(Session& session, const FindParameters& parameters);
         static RangeResults<ReleaseId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt); // not track related
 
+        // Updates
+        static void updatePreferredArtwork(Session& session, ReleaseId id, ArtworkId artworkId);
+
         // Get the cluster of the tracks that belong to this release
         // Each clusters are grouped by cluster type, sorted by the number of occurence (max to min)
         // size is the max number of cluster per cluster type

src/libs/database/include/database/Track.hpp

@@ -39,6 +39,7 @@
 #include "database/ClusterId.hpp"
 #include "database/DirectoryId.hpp"
 #include "database/Filters.hpp"
+#include "database/IdRange.hpp"
 #include "database/MediaLibraryId.hpp"
 #include "database/Object.hpp"
 #include "database/ReleaseId.hpp"
@@ -192,6 +193,8 @@ namespace lms::db
         static pointer findByPath(Session& session, const std::filesystem::path& p);
         static pointer find(Session& session, TrackId id);
         static void find(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(const Track::pointer&)>& func, MediaLibraryId library = {});
+        static void find(Session& session, const IdRange<TrackId>& idRange, const std::function<void(const Track::pointer&)>& func);
+        static IdRange<TrackId> findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count);
         static void findAbsoluteFilePath(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(TrackId trackId, const std::filesystem::path& absoluteFilePath)>& func);
 
         static bool exists(Session& session, TrackId id);

src/libs/database/test/Artist.cpp

@@ -608,6 +608,93 @@ namespace lms::db::tests
         }
     }
 
+    TEST_F(DatabaseFixture, Artist_findNextRange)
+    {
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, ArtistId{});
+            EXPECT_EQ(range.last, ArtistId{});
+
+            range = Artist::findNextRange(session, ArtistId{}, 100);
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, ArtistId{});
+            EXPECT_EQ(range.last, ArtistId{});
+        }
+
+        ScopedArtist artist1{ session, "Artist1" };
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, ArtistId{});
+            EXPECT_EQ(range.last, ArtistId{});
+
+            range = Artist::findNextRange(session, ArtistId{}, 1);
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, artist1.getId());
+            EXPECT_EQ(range.last, artist1.getId());
+
+            range = Artist::findNextRange(session, range.last, 1);
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, ArtistId{});
+            EXPECT_EQ(range.last, ArtistId{});
+
+            range = Artist::findNextRange(session, ArtistId{}, 100);
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, artist1.getId());
+            EXPECT_EQ(range.last, artist1.getId());
+        }
+
+        ScopedArtist artist2{ session, "Artist2" };
+        ScopedArtist artist3{ session, "Artist3" };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            auto range{ Artist::findNextRange(session, ArtistId{}, 2) };
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, artist1.getId());
+            EXPECT_EQ(range.last, artist2.getId());
+
+            range = Artist::findNextRange(session, artist2.getId(), 2);
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, artist3.getId());
+            EXPECT_EQ(range.last, artist3.getId());
+        }
+    }
+
+    TEST_F(DatabaseFixture, Artist_findByRange)
+    {
+        ScopedArtist artist1{ session, "Artist1" };
+        ScopedArtist artist2{ session, "Artist2" };
+        ScopedArtist artist3{ session, "Artist3" };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            std::size_t count{};
+            Artist::find(session, IdRange<ArtistId>{ .first = artist1.getId(), .last = artist1.getId() }, [&](const db::Artist::pointer& artist) {
+                count++;
+                EXPECT_EQ(artist->getId(), artist1.getId());
+            });
+            EXPECT_EQ(count, 1);
+        }
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            std::size_t count{};
+            Artist::find(session, IdRange<ArtistId>{ .first = artist1.getId(), .last = artist3.getId() }, [&](const db::Artist::pointer&) {
+                count++;
+            });
+            EXPECT_EQ(count, 3);
+        }
+    }
+
     TEST_F(DatabaseFixture, Artist_sortMethod)
     {
         ScopedArtist artistA{ session, "artistA" };
@@ -817,4 +904,34 @@ namespace lms::db::tests
             EXPECT_EQ(artists.results[3], artistC.getId());
         }
     }
+
+    TEST_F(DatabaseFixture, Artist_updateArtwork)
+    {
+        ScopedArtist artist{ session, "MyArtist" };
+        {
+            auto transaction{ session.createReadTransaction() };
+            EXPECT_EQ(artist->getPreferredArtwork(), Artwork::pointer{});
+        }
+
+        ScopedImage image{ session, "/image1.jpg" };
+        ScopedArtwork artwork{ session, image.lockAndGet() };
+
+        {
+            auto transaction{ session.createWriteTransaction() };
+            Artist::updatePreferredArtwork(session, artist->getId(), artwork.getId());
+        }
+        {
+            auto transaction{ session.createReadTransaction() };
+            EXPECT_EQ(artist->getPreferredArtwork()->getId(), artwork.getId());
+        }
+
+        {
+            auto transaction{ session.createWriteTransaction() };
+            Artist::updatePreferredArtwork(session, artist.getId(), ArtworkId{});
+        }
+        {
+            auto transaction{ session.createReadTransaction() };
+            EXPECT_EQ(artist->getPreferredArtwork(), Artwork::pointer{});
+        }
+    }
 } // namespace lms::db::tests

src/libs/database/test/Release.cpp

@@ -173,6 +173,93 @@ namespace lms::db::tests
         }
     }
 
+    TEST_F(DatabaseFixture, Release_findNextRange)
+    {
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, ReleaseId{});
+            EXPECT_EQ(range.last, ReleaseId{});
+
+            range = Release::findNextRange(session, ReleaseId{}, 100);
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, ReleaseId{});
+            EXPECT_EQ(range.last, ReleaseId{});
+        }
+
+        ScopedRelease release1{ session, "Artist1" };
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, ReleaseId{});
+            EXPECT_EQ(range.last, ReleaseId{});
+
+            range = Release::findNextRange(session, ReleaseId{}, 1);
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, release1.getId());
+            EXPECT_EQ(range.last, release1.getId());
+
+            range = Release::findNextRange(session, range.last, 1);
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, ReleaseId{});
+            EXPECT_EQ(range.last, ReleaseId{});
+
+            range = Release::findNextRange(session, ReleaseId{}, 100);
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, release1.getId());
+            EXPECT_EQ(range.last, release1.getId());
+        }
+
+        ScopedRelease release2{ session, "Artist2" };
+        ScopedRelease release3{ session, "Artist3" };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            auto range{ Release::findNextRange(session, ReleaseId{}, 2) };
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, release1.getId());
+            EXPECT_EQ(range.last, release2.getId());
+
+            range = Release::findNextRange(session, release2.getId(), 2);
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, release3.getId());
+            EXPECT_EQ(range.last, release3.getId());
+        }
+    }
+
+    TEST_F(DatabaseFixture, Release_findByRange)
+    {
+        ScopedRelease release1{ session, "Artist1" };
+        ScopedRelease release2{ session, "Artist2" };
+        ScopedRelease release3{ session, "Artist3" };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            std::size_t count{};
+            Release::find(session, IdRange<ReleaseId>{ .first = release1.getId(), .last = release1.getId() }, [&](const db::Release::pointer& release) {
+                count++;
+                EXPECT_EQ(release->getId(), release1.getId());
+            });
+            EXPECT_EQ(count, 1);
+        }
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            std::size_t count{};
+            Release::find(session, IdRange<ReleaseId>{ .first = release1.getId(), .last = release3.getId() }, [&](const db::Release::pointer&) {
+                count++;
+            });
+            EXPECT_EQ(count, 3);
+        }
+    }
+
     TEST_F(DatabaseFixture, Release_singleTrack)
     {
         ScopedRelease release{ session, "MyRelease" };
@@ -1431,4 +1518,35 @@ namespace lms::db::tests
             EXPECT_EQ(releases.results[1]->getId(), release1->getId());
         }
     }
+
+    TEST_F(DatabaseFixture, Release_updateArtwork)
+    {
+        ScopedRelease release{ session, "MyRelease" };
+        {
+            auto transaction{ session.createReadTransaction() };
+            EXPECT_EQ(release->getPreferredArtwork(), Artwork::pointer{});
+        }
+
+        ScopedImage image{ session, "/image1.jpg" };
+        ScopedArtwork artwork{ session, image.lockAndGet() };
+
+        {
+            auto transaction{ session.createWriteTransaction() };
+            Release::updatePreferredArtwork(session, release->getId(), artwork.getId());
+        }
+        {
+            auto transaction{ session.createReadTransaction() };
+            EXPECT_EQ(release->getPreferredArtwork()->getId(), artwork.getId());
+        }
+
+        {
+            auto transaction{ session.createWriteTransaction() };
+            Release::updatePreferredArtwork(session, release.getId(), ArtworkId{});
+        }
+        {
+            auto transaction{ session.createReadTransaction() };
+            EXPECT_EQ(release->getPreferredArtwork(), Artwork::pointer{});
+        }
+    }
+
 } // namespace lms::db::tests

src/libs/database/test/Track.cpp

@@ -151,6 +151,93 @@ namespace lms::db::tests
         }
     }
 
+    TEST_F(DatabaseFixture, Track_findNextRange)
+    {
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            auto range{ Track::findNextRange(session, TrackId{}, 0) };
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, TrackId{});
+            EXPECT_EQ(range.last, TrackId{});
+
+            range = Track::findNextRange(session, TrackId{}, 100);
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, TrackId{});
+            EXPECT_EQ(range.last, TrackId{});
+        }
+
+        ScopedTrack track1{ session };
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            auto range{ Track::findNextRange(session, TrackId{}, 0) };
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, TrackId{});
+            EXPECT_EQ(range.last, TrackId{});
+
+            range = Track::findNextRange(session, TrackId{}, 1);
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, track1.getId());
+            EXPECT_EQ(range.last, track1.getId());
+
+            range = Track::findNextRange(session, range.last, 1);
+            EXPECT_FALSE(range.isValid());
+            EXPECT_EQ(range.first, TrackId{});
+            EXPECT_EQ(range.last, TrackId{});
+
+            range = Track::findNextRange(session, TrackId{}, 100);
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, track1.getId());
+            EXPECT_EQ(range.last, track1.getId());
+        }
+
+        ScopedTrack track2{ session };
+        ScopedTrack track3{ session };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            auto range{ Track::findNextRange(session, TrackId{}, 2) };
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, track1.getId());
+            EXPECT_EQ(range.last, track2.getId());
+
+            range = Track::findNextRange(session, track2.getId(), 2);
+            EXPECT_TRUE(range.isValid());
+            EXPECT_EQ(range.first, track3.getId());
+            EXPECT_EQ(range.last, track3.getId());
+        }
+    }
+
+    TEST_F(DatabaseFixture, Track_findByRange)
+    {
+        ScopedTrack track1{ session };
+        ScopedTrack track2{ session };
+        ScopedTrack track3{ session };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            std::size_t count{};
+            Track::find(session, IdRange<TrackId>{ .first = track1.getId(), .last = track1.getId() }, [&](const db::Track::pointer& track) {
+                count++;
+                EXPECT_EQ(track->getId(), track1.getId());
+            });
+            EXPECT_EQ(count, 1);
+        }
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            std::size_t count{};
+            Track::find(session, IdRange<TrackId>{ .first = track1.getId(), .last = track3.getId() }, [&](const db::Track::pointer&) {
+                count++;
+            });
+            EXPECT_EQ(count, 3);
+        }
+    }
+
     TEST_F(DatabaseFixture, Track_findAbsoluteFilePath)
     {
         ScopedTrack track{ session };

src/libs/services/scanner/CMakeLists.txt

@@ -8,7 +8,7 @@ add_library(lmsscanner STATIC
 	impl/scanners/LyricsFileScanner.cpp
 	impl/scanners/PlayListFileScanner.cpp
 	impl/scanners/Utils.cpp
-	impl/steps/ArtworkUtils.cpp
+	impl/steps/JobQueue.cpp
 	impl/steps/ScanErrorLogger.cpp
 	impl/steps/ScanStepArtistReconciliation.cpp
 	impl/steps/ScanStepAssociateArtistImages.cpp

src/libs/services/scanner/impl/ScannerService.cpp

@@ -448,7 +448,7 @@ namespace lms::scanner
         if (!_lastScanSettings)
             _lastScanSettings = readScannerSettings(_db.getTLSSession(), lastScanSettingsName);
 
-        auto cbFunc{ [this](const ScanStepStats& stats) {
+        auto progressFunc{ [this](const ScanStepStats& stats) {
             notifyInProgressIfNeeded(stats);
         } };
 
@@ -466,7 +466,7 @@ namespace lms::scanner
             .jobScheduler = *_jobScheduler,
             .settings = _settings,
             .lastScanSettings = _lastScanSettings.has_value() ? &(_lastScanSettings.value()) : nullptr,
-            .progressCallback = cbFunc,
+            .progressCallback = progressFunc,
             .abortScan = _abortScan,
             .db = _db,
             .fileScanners = fileScanners,
@@ -500,14 +500,14 @@ namespace lms::scanner
             _currentScanStepStats = stepStats;
         }
 
-        const std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
+        const auto now{ std::chrono::steady_clock::now() };
         _events.scanInProgress.emit(stepStats);
         _lastScanInProgressEmit = now;
     }
 
     void ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
     {
-        std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
+        const auto now{ std::chrono::steady_clock::now() };
 
         if (now - _lastScanInProgressEmit >= std::chrono::seconds{ 1 })
             notifyInProgress(stepStats);

src/libs/services/scanner/impl/ScannerService.hpp

@@ -96,7 +96,7 @@ namespace lms::scanner
         Wt::WIOService _ioService;
         boost::asio::system_timer _scheduleTimer{ _ioService };
         Events _events;
-        std::chrono::system_clock::time_point _lastScanInProgressEmit;
+        std::chrono::steady_clock::time_point _lastScanInProgressEmit;
 
         mutable std::shared_mutex _statusMutex;
         State _curState{ State::NotScheduled };

src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp

@@ -25,6 +25,7 @@
 #include "core/Path.hpp"
 #include "core/XxHash3.hpp"
 #include "database/Artist.hpp"
+#include "database/Artwork.hpp"
 #include "database/Cluster.hpp"
 #include "database/Db.hpp"
 #include "database/Directory.hpp"
@@ -332,6 +333,8 @@ namespace lms::scanner
                 image.modify()->setWidth(imageInfo.properties.width);
                 image.modify()->setHeight(imageInfo.properties.height);
                 image.modify()->setMimeType(imageInfo.mimeType);
+
+                session.create<db::Artwork>(image);
             }
 
             return image;

src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp

@@ -22,6 +22,7 @@
 #include <optional>
 
 #include "core/ILogger.hpp"
+#include "database/Artwork.hpp"
 #include "database/Db.hpp"
 #include "database/Image.hpp"
 #include "database/MediaLibrary.hpp"
@@ -84,7 +85,10 @@ namespace lms::scanner
 
             const bool added{ !image };
             if (!image)
+            {
                 image = dbSession.create<db::Image>(getFilePath());
+                dbSession.create<db::Artwork>(image);
+            }
 
             image.modify()->setLastWriteTime(getLastWriteTime());
             image.modify()->setFileSize(getFileSize());

src/libs/services/scanner/impl/steps/ArtworkUtils.cpp

@@ -1,59 +0,0 @@
-/*
- * Copyright (C) 2024 Emeric Poupon
- *
- * This file is part of LMS.
- *
- * LMS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LMS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "ArtworkUtils.hpp"
-
-#include "database/Artwork.hpp"
-#include "database/Image.hpp"
-#include "database/Session.hpp"
-#include "database/Track.hpp"
-#include "database/TrackEmbeddedImage.hpp"
-
-namespace lms::scanner::utils
-{
-    db::ObjectPtr<db::Artwork> getOrCreateArtworkFromTrackEmbeddedImage(db::Session& session, db::TrackEmbeddedImageId trackEmbeddedImageId)
-    {
-        assert(trackEmbeddedImageId.isValid());
-        session.checkWriteTransaction();
-
-        db::ObjectPtr<db::Artwork> artwork{ db::Artwork::find(session, trackEmbeddedImageId) };
-        if (!artwork)
-        {
-            db::TrackEmbeddedImage::pointer trackEmbeddedImage{ db::TrackEmbeddedImage::find(session, trackEmbeddedImageId) };
-            assert(trackEmbeddedImage);
-            artwork = session.create<db::Artwork>(trackEmbeddedImage);
-        }
-        return artwork;
-    }
-
-    db::ObjectPtr<db::Artwork> getOrCreateArtworkFromImage(db::Session& session, db::ImageId imageId)
-    {
-        assert(imageId.isValid());
-        session.checkWriteTransaction();
-
-        db::ObjectPtr<db::Artwork> artwork{ db::Artwork::find(session, imageId) };
-        if (!artwork)
-        {
-            db::Image::pointer image{ db::Image::find(session, imageId) };
-            assert(image);
-            artwork = session.create<db::Artwork>(image);
-        }
-        return artwork;
-    }
-} // namespace lms::scanner::utils

src/libs/services/scanner/impl/steps/JobQueue.cpp

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2025 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "JobQueue.hpp"
+
+#include <cassert>
+
+#include "core/IJob.hpp"
+#include "core/IJobScheduler.hpp"
+
+namespace lms::scanner
+{
+    JobQueue::JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float _drainThreshold)
+        : _scheduler{ scheduler }
+        , _maxQueueSize{ maxQueueSize }
+        , _processJobsDoneFunc{ std::move(processJobsDoneFunc) }
+        , _batchSize{ batchSize }
+        , _drainThreshold{ _drainThreshold }
+    {
+        assert(_scheduler.getJobsDoneCount() == 0);
+    }
+
+    JobQueue::~JobQueue()
+    {
+        finish();
+    }
+
+    void JobQueue::push(std::unique_ptr<core::IJob> job)
+    {
+        _scheduler.scheduleJob(std::move(job));
+        drainIfNeeded();
+    }
+
+    void JobQueue::finish()
+    {
+        _scheduler.wait();
+        while (_scheduler.popJobsDone(_jobsDone, _batchSize) > 0)
+        {
+            _processJobsDoneFunc(std::span{ _jobsDone });
+            _jobsDone.clear();
+        }
+    }
+
+    void JobQueue::drainIfNeeded()
+    {
+        while (_scheduler.getJobsDoneCount() > static_cast<std::size_t>(_maxQueueSize * _drainThreshold))
+        {
+            _scheduler.popJobsDone(_jobsDone, _batchSize);
+            _processJobsDoneFunc(std::span{ _jobsDone });
+            _jobsDone.clear();
+        }
+        _scheduler.waitUntilJobCountAtMost(_maxQueueSize);
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/JobQueue.hpp

@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2025 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <functional>
+#include <memory>
+#include <span>
+#include <vector>
+
+namespace lms::core
+{
+    class IJob;
+    class IJobScheduler;
+} // namespace lms::core
+
+namespace lms::scanner
+{
+    class JobQueue
+    {
+    public:
+        using ProcessFunction = std::function<void(std::span<std::unique_ptr<core::IJob>>)>;
+
+        JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float drainThreshold);
+        ~JobQueue();
+        JobQueue(const JobQueue&) = delete;
+        JobQueue& operator=(const JobQueue&) = delete;
+
+        // push can wait and invoke the supplied ProcessFunction
+        void push(std::unique_ptr<core::IJob> job);
+        void finish();
+
+    private:
+        void drainIfNeeded();
+
+        core::IJobScheduler& _scheduler;
+        const std::size_t _maxQueueSize;
+        ProcessFunction _processJobsDoneFunc;
+        const std::size_t _batchSize;
+        const float _drainThreshold;
+        std::vector<std::unique_ptr<core::IJob>> _jobsDone;
+    };
+
+} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp

@@ -24,9 +24,9 @@
 #include <deque>
 #include <set>
 #include <span>
-#include <variant>
 
 #include "core/IConfig.hpp"
+#include "core/IJob.hpp"
 #include "core/ILogger.hpp"
 #include "core/Path.hpp"
 #include "core/String.hpp"
@@ -41,7 +41,7 @@
 #include "database/Session.hpp"
 #include "database/Track.hpp"
 
-#include "ArtworkUtils.hpp"
+#include "JobQueue.hpp"
 #include "ScanContext.hpp"
 #include "ScannerSettings.hpp"
 
@@ -49,42 +49,24 @@ namespace lms::scanner
 {
     namespace
     {
-        using ArtistArtwork = std::variant<std::monostate, db::ImageId, db::ArtworkId>; // TODO handle embedded images in tracks?
-        bool isSameArtwork(ArtistArtwork preferredArtwork, const db::ObjectPtr<db::Artwork>& artwork)
-        {
-            if (std::holds_alternative<std::monostate>(preferredArtwork))
-                return !artwork;
-
-            if (const db::ImageId* imageId = std::get_if<db::ImageId>(&preferredArtwork))
-                return artwork && *imageId == artwork->getImageId();
-
-            if (const db::ArtworkId* artworkId = std::get_if<db::ArtworkId>(&preferredArtwork))
-                return artwork && *artworkId == artwork->getId();
-
-            return false;
-        }
-
         struct ArtistArtworkAssociation
         {
-            db::Artist::pointer artist;
+            db::ArtistId artistId;
-            ArtistArtwork preferredArtwork;
+            db::ArtworkId preferredArtworkId;
         };
         using ArtistArtworkAssociationContainer = std::deque<ArtistArtworkAssociation>;
 
-        struct SearchArtistImageContext
+        struct SearchArtistArtworkParams
         {
-            db::Session& session;
-            db::ArtistId lastRetrievedArtistId;
-            std::size_t processedArtistCount{};
             std::span<const std::string> artistFileNames;
             const ScannerSettings& settings;
         };
 
-        db::Image::pointer findImageInDirectory(SearchArtistImageContext& searchContext, const std::filesystem::path& directoryPath, std::span<const std::string> fileStemsToSearch)
+        db::Image::pointer findImageInDirectory(db::Session& session, const std::filesystem::path& directoryPath, std::span<const std::string> fileStemsToSearch)
         {
             db::Image::pointer image;
 
-            const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) };
+            const db::Directory::pointer directory{ db::Directory::find(session, directoryPath) };
             if (directory) // may not exist for artists that are split on different media libraries
             {
                 for (std::string_view fileStem : fileStemsToSearch)
@@ -93,7 +75,7 @@ namespace lms::scanner
                     params.setDirectory(directory->getId());
                     params.setFileStem(fileStem);
 
-                    db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) {
+                    db::Image::find(session, params, [&](const db::Image::pointer foundImg) {
                         if (!image)
                             image = foundImg;
                     });
@@ -106,38 +88,38 @@ namespace lms::scanner
             return image;
         }
 
-        db::ImageId getImageFromMbid(SearchArtistImageContext& searchContext, const core::UUID& mbid)
+        db::Image::pointer getImageFromMbid(db::Session& session, const core::UUID& mbid)
         {
             db::Image::pointer image;
 
             // Find anywhere, since it is supposed to be unique!
-            db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid.getAsString()), [&](const db::Image::pointer foundImg) {
+            db::Image::find(session, db::Image::FindParameters{}.setFileStem(mbid.getAsString()), [&](const db::Image::pointer foundImg) {
                 if (!image)
                     image = foundImg;
             });
 
-            return image ? image->getId() : db::ImageId{};
+            return image;
         }
 
-        db::ImageId searchImageInArtistInfoDirectory(SearchArtistImageContext& searchContext, db::ArtistId artistId)
+        db::Image::pointer searchImageInArtistInfoDirectory(db::Session& session, db::ArtistId artistId)
         {
             db::Image::pointer image;
 
             std::vector<std::string> fileInfoPaths;
-            db::ArtistInfo::find(searchContext.session, artistId, [&](const db::ArtistInfo::pointer& artistInfo) {
+            db::ArtistInfo::find(session, artistId, [&](const db::ArtistInfo::pointer& artistInfo) {
                 fileInfoPaths.push_back(artistInfo->getAbsoluteFilePath());
 
                 if (!image)
-                    image = findImageInDirectory(searchContext, artistInfo->getDirectory()->getAbsolutePath(), std::array<std::string, 2>{ "thumb", "folder" });
+                    image = findImageInDirectory(session, artistInfo->getDirectory()->getAbsolutePath(), std::array<std::string, 2>{ "thumb", "folder" });
             });
 
             if (fileInfoPaths.size() > 1)
                 LMS_LOG(DBUPDATER, DEBUG, "Found " << fileInfoPaths.size() << " artist info files for same artist: " << core::stringUtils::joinStrings(fileInfoPaths, ", "));
 
-            return image ? image->getId() : db::ImageId{};
+            return image;
         }
 
-        db::ImageId searchImageInDirectories(SearchArtistImageContext& searchContext, db::ArtistId artistId)
+        db::Image::pointer searchImageInDirectories(db::Session& session, const SearchArtistArtworkParams& searchParams, db::ArtistId artistId)
         {
             db::Image::pointer image;
 
@@ -145,7 +127,7 @@ namespace lms::scanner
             db::Directory::FindParameters params;
             params.setArtist(artistId, { db::TrackArtistLinkType::ReleaseArtist });
 
-            db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) {
+            db::Directory::find(session, params, [&](const db::Directory::pointer& directory) {
                 releasePaths.insert(directory->getAbsolutePath());
             });
 
@@ -164,9 +146,9 @@ namespace lms::scanner
                 std::filesystem::path directoryToInspect{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
                 while (true)
                 {
-                    image = findImageInDirectory(searchContext, directoryToInspect, searchContext.artistFileNames);
+                    image = findImageInDirectory(session, directoryToInspect, searchParams.artistFileNames);
                     if (image)
-                        return image->getId();
+                        return image;
 
                     std::filesystem::path parentPath{ directoryToInspect.parent_path() };
                     if (parentPath == directoryToInspect)
@@ -181,106 +163,74 @@ namespace lms::scanner
                 //                      /someOtherUserConfiguredArtistFile.jpg
                 for (const std::filesystem::path& releasePath : releasePaths)
                 {
-                    image = findImageInDirectory(searchContext, releasePath, searchContext.artistFileNames);
+                    image = findImageInDirectory(session, releasePath, searchParams.artistFileNames);
                     if (image)
-                        return image->getId();
+                        return image;
                 }
             }
 
-            return image ? image->getId() : db::ImageId{};
+            return image;
         }
 
-        db::ArtworkId getFirstReleaseArtwork(db::Session& session, const db::Artist::pointer& artist)
+        db::Artwork::pointer getFirstReleaseArtwork(db::Session& session, const db::Artist::pointer& artist)
         {
-            db::ArtworkId artworkId;
+            db::Artwork::pointer artwork;
 
             db::Release::FindParameters params;
             params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist });
             params.setSortMethod(db::ReleaseSortMethod::OriginalDate);
 
             db::Release::find(session, params, [&](const db::Release::pointer& release) {
-                if (artworkId.isValid())
+                if (artwork)
                     return;
 
-                artworkId = release->getPreferredArtworkId();
+                artwork = release->getPreferredArtwork();
             });
 
-            return artworkId;
+            return artwork;
         }
 
-        ArtistArtwork computePreferredArtwork(SearchArtistImageContext& searchContext, const db::Artist::pointer& artist)
+        db::Artwork::pointer computePreferredArtistArtwork(db::Session& session, const SearchArtistArtworkParams& searchParams, const db::Artist::pointer& artist)
         {
-            db::ImageId imageId;
-
             if (const auto mbid{ artist->getMBID() })
             {
-                imageId = getImageFromMbid(searchContext, *mbid);
+                const db::Image::pointer image{ getImageFromMbid(session, *mbid) };
-                if (imageId.isValid())
+                if (image)
-                    return ArtistArtwork{ imageId };
+                    return db::Artwork::find(session, image->getId());
             }
 
-            imageId = searchImageInArtistInfoDirectory(searchContext, artist->getId());
-            if (imageId.isValid())
-                return ArtistArtwork{ imageId };
-
-            imageId = searchImageInDirectories(searchContext, artist->getId());
-            if (imageId.isValid())
-                return ArtistArtwork{ imageId };
-
-            if (searchContext.settings.artistImageFallbackToRelease)
-            {
-                db::ArtworkId artworkId{ getFirstReleaseArtwork(searchContext.session, artist) };
-                if (artworkId.isValid())
-                    return ArtistArtwork{ artworkId };
-            }
-
-            return ArtistArtwork{};
-        }
-
-        bool fetchNextArtistArtworksToUpdate(SearchArtistImageContext& searchContext, ArtistArtworkAssociationContainer& ArtistArtworkAssociations)
-        {
-            const db::ArtistId artistId{ searchContext.lastRetrievedArtistId };
-
             {
-                constexpr std::size_t readBatchSize{ 100 };
+                const db::Image::pointer image{ searchImageInArtistInfoDirectory(session, artist->getId()) };
-
+                if (image)
-                auto transaction{ searchContext.session.createReadTransaction() };
+                    return db::Artwork::find(session, image->getId());
-
-                db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) {
-                    ArtistArtwork preferredArtwork{ computePreferredArtwork(searchContext, artist) };
-
-                    if (!isSameArtwork(preferredArtwork, artist->getPreferredArtwork()))
-                        ArtistArtworkAssociations.push_back(ArtistArtworkAssociation{ artist, preferredArtwork });
-
-                    searchContext.processedArtistCount++;
-                });
             }
 
-            return artistId != searchContext.lastRetrievedArtistId;
+            {
+                const db::Image::pointer image{ searchImageInDirectories(session, searchParams, artist->getId()) };
+                if (image)
+                    return db::Artwork::find(session, image->getId());
+            }
+
+            if (searchParams.settings.artistImageFallbackToRelease)
+            {
+                const db::Artwork::pointer artwork{ getFirstReleaseArtwork(session, artist) };
+                if (artwork)
+                    return artwork;
+            }
+
+            return db::Artwork::pointer{};
         }
 
-        void updateArtistPreferredArtwork(db::Session& session, const ArtistArtworkAssociation& ArtistArtworkAssociation)
+        void updateArtistPreferredArtwork(db::Session& session, const ArtistArtworkAssociation& artistArtworkAssociation)
         {
-            db::Artist::pointer artist{ ArtistArtworkAssociation.artist };
+            db::Artist::updatePreferredArtwork(session, artistArtworkAssociation.artistId, artistArtworkAssociation.preferredArtworkId);
-
-            db::Artwork::pointer artwork;
-            if (const db::ImageId * imageId{ std::get_if<db::ImageId>(&ArtistArtworkAssociation.preferredArtwork) })
-                artwork = utils::getOrCreateArtworkFromImage(session, *imageId);
-            else if (const db::ArtworkId * artworkId{ std::get_if<db::ArtworkId>(&ArtistArtworkAssociation.preferredArtwork) })
-                artwork = db::Artwork::find(session, *artworkId);
-
-            artist.modify()->setPreferredArtwork(artwork);
-            if (artwork)
-                LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork for artist '" << artist->getName() << "' with image in " << artwork->getAbsoluteFilePath());
-            else
-                LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from artist '" << artist->getName() << "'");
         }
 
-        void updateArtistArtworks(db::Session& session, ArtistArtworkAssociationContainer& imageAssociations)
+        void updateArtistPreferredArtworks(db::Session& session, ArtistArtworkAssociationContainer& imageAssociations, bool forceFullBatch)
         {
             constexpr std::size_t writeBatchSize{ 50 };
 
-            while (!imageAssociations.empty())
+            while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
             {
                 auto transaction{ session.createWriteTransaction() };
 
@@ -305,6 +255,62 @@ namespace lms::scanner
             return res;
         }
 
+        bool fetchNextArtistIdRange(db::Session& session, db::ArtistId& lastRetrievedId, db::IdRange<db::ArtistId>& idRange)
+        {
+            constexpr std::size_t readBatchSize{ 100 };
+
+            auto transaction{ session.createReadTransaction() };
+
+            idRange = db::Artist::findNextRange(session, lastRetrievedId, readBatchSize);
+            lastRetrievedId = idRange.last;
+
+            return idRange.isValid();
+        }
+
+        class ComputeArtistArtworkAssociationsJob : public core::IJob
+        {
+        public:
+            ComputeArtistArtworkAssociationsJob(db::Db& db, const SearchArtistArtworkParams& searchParams, db::IdRange<db::ArtistId> artistIdRange)
+                : _db{ db }
+                , _searchParams{ searchParams }
+                , _artistIdRange{ artistIdRange }
+            {
+            }
+
+            std::span<const ArtistArtworkAssociation> getAssociations() const { return _associations; }
+            std::size_t getProcessedArtistCount() const { return _processedArtistCount; }
+
+        private:
+            core::LiteralString getName() const override { return "Associate Track Artworks"; }
+            void run() override
+            {
+                auto& session{ _db.getTLSSession() };
+                auto transaction{ session.createReadTransaction() };
+
+                db::Artist::find(session, _artistIdRange, [this, &session](const db::Artist::pointer& artist) {
+                    const db::Artwork::pointer preferredArtwork{ computePreferredArtistArtwork(session, _searchParams, artist) };
+
+                    if (artist->getPreferredArtwork() != preferredArtwork)
+                    {
+                        _associations.push_back(ArtistArtworkAssociation{ artist->getId(), preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{} });
+
+                        if (preferredArtwork)
+                            LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork for artist '" << artist->getName() << "' with image in " << preferredArtwork->getAbsoluteFilePath());
+                        else
+                            LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from artist '" << artist->getName() << "'");
+                    }
+
+                    _processedArtistCount++;
+                });
+            }
+
+            db::Db& _db;
+            const SearchArtistArtworkParams& _searchParams;
+            db::IdRange<db::ArtistId> _artistIdRange;
+            std::vector<ArtistArtworkAssociation> _associations;
+            std::size_t _processedArtistCount{};
+        };
+
     } // namespace
 
     ScanStepAssociateArtistImages::ScanStepAssociateArtistImages(InitParams& initParams)
@@ -333,22 +339,40 @@ namespace lms::scanner
             context.currentStepStats.totalElems = db::Artist::getCount(session);
         }
 
-        SearchArtistImageContext searchContext{
+        const SearchArtistArtworkParams searchParams{
-            .session = session,
-            .lastRetrievedArtistId = {},
             .artistFileNames = _artistFileNames,
             .settings = _settings,
         };
 
-        ArtistArtworkAssociationContainer ArtistArtworkAssociations;
+        ArtistArtworkAssociationContainer artistArtworkAssociations;
-        while (fetchNextArtistArtworksToUpdate(searchContext, ArtistArtworkAssociations))
+        auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
-        {
             if (_abortScan)
                 return;
 
-            updateArtistArtworks(session, ArtistArtworkAssociations);
+            for (const auto& job : jobs)
-            context.currentStepStats.processedElems = searchContext.processedArtistCount;
+            {
+                const auto& associationJob{ static_cast<const ComputeArtistArtworkAssociationsJob&>(*job) };
+                const auto& artistAssociations{ associationJob.getAssociations() };
+
+                artistArtworkAssociations.insert(std::end(artistArtworkAssociations), std::cbegin(artistAssociations), std::cend(artistAssociations));
+
+                context.currentStepStats.processedElems += associationJob.getProcessedArtistCount();
+            }
+
+            updateArtistPreferredArtworks(session, artistArtworkAssociations, true);
             _progressCallback(context.currentStepStats);
-        }
+        };
+
+        JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
+
+        db::ArtistId lastRetrievedArtistId{};
+        db::IdRange<db::ArtistId> artistIdRange;
+        while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange))
+            queue.push(std::make_unique<ComputeArtistArtworkAssociationsJob>(_db, searchParams, artistIdRange));
+
+        queue.finish();
+
+        // process all remaining associations
+        updateArtistPreferredArtworks(session, artistArtworkAssociations, false);
     }
 } // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp

@@ -23,9 +23,9 @@
 #include <cassert>
 #include <deque>
 #include <set>
-#include <variant>
 
 #include "core/IConfig.hpp"
+#include "core/IJob.hpp"
 #include "core/ILogger.hpp"
 #include "core/Path.hpp"
 #include "database/Artwork.hpp"
@@ -36,92 +36,73 @@
 #include "database/Session.hpp"
 #include "database/Track.hpp"
 #include "database/TrackEmbeddedImage.hpp"
-#include "database/TrackEmbeddedImageId.hpp"
 
-#include "ArtworkUtils.hpp"
+#include "JobQueue.hpp"
 #include "ScanContext.hpp"
 
 namespace lms::scanner
 {
     namespace
     {
-        using ReleaseArtwork = std::variant<std::monostate, db::TrackEmbeddedImageId, db::ImageId>;
-        bool isSameArtwork(ReleaseArtwork preferredArtwork, const db::ObjectPtr<db::Artwork>& artwork)
-        {
-            if (std::holds_alternative<std::monostate>(preferredArtwork))
-                return !artwork;
-
-            if (const db::TrackEmbeddedImageId* trackEmbeddedImageId = std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork))
-                return artwork && *trackEmbeddedImageId == artwork->getTrackEmbeddedImageId();
-
-            if (const db::ImageId* imageId = std::get_if<db::ImageId>(&preferredArtwork))
-                return artwork && *imageId == artwork->getImageId();
-
-            return false;
-        }
-
         struct ReleaseArtworkAssociation
         {
-            db::Release::pointer release;
+            db::ReleaseId releaseId;
-            ReleaseArtwork preferredArtwork;
+            db::ArtworkId preferredArtworkId;
         };
         using ReleaseArtworkAssociationContainer = std::deque<ReleaseArtworkAssociation>;
 
-        struct SearchReleaseArtworkContext
+        struct SearchReleaseArtworkParams
         {
-            db::Session& session;
-            db::ReleaseId lastRetrievedReleaseId;
-            std::size_t processedReleaseCount{};
             const std::vector<std::string>& releaseImageFileNames;
         };
 
-        db::Image::pointer findImageInDirectory(SearchReleaseArtworkContext& searchContext, const std::filesystem::path& directoryPath)
+        db::Artwork::pointer findImageInDirectory(db::Session& session, const SearchReleaseArtworkParams& searchParams, const std::filesystem::path& directoryPath)
         {
-            db::Image::pointer image;
+            db::Artwork::pointer artwork;
 
-            const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) };
+            const db::Directory::pointer directory{ db::Directory::find(session, directoryPath) };
             if (directory) // may not exist for releases that are split on different media libraries
             {
-                for (std::string_view fileStem : searchContext.releaseImageFileNames)
+                for (std::string_view fileStem : searchParams.releaseImageFileNames)
                 {
                     db::Image::FindParameters params;
                     params.setDirectory(directory->getId());
                     params.setFileStem(fileStem);
 
-                    db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) {
+                    db::Image::find(session, params, [&](const db::Image::pointer& image) {
-                        if (!image)
+                        if (!artwork)
-                            image = foundImg;
+                            artwork = db::Artwork::find(session, image->getId());
                     });
 
-                    if (image)
+                    if (artwork)
                         break;
                 }
             }
 
-            return image;
+            return artwork;
         }
 
-        db::Image::pointer computePreferredReleaseImage(SearchReleaseArtworkContext& searchContext, const db::Release::pointer& release)
+        db::Artwork::pointer computePreferredReleaseImage(db::Session& session, const SearchReleaseArtworkParams& searchParams, const db::Release::pointer& release)
         {
-            db::Image::pointer image;
+            db::Artwork::pointer artwork;
 
             const auto mbid{ release->getMBID() };
             if (mbid)
             {
                 // Find anywhere, since it is suppoed to be unique!
-                db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer foundImg) {
+                db::Image::find(session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer& image) {
-                    if (!image)
+                    if (!artwork)
-                        image = foundImg;
+                        artwork = db::Artwork::find(session, image->getId());
                 });
             }
 
-            if (!image)
+            if (!artwork)
             {
                 std::set<std::filesystem::path> releasePaths;
                 db::Directory::FindParameters params;
                 params.setRelease(release->getId());
 
-                db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) {
+                db::Directory::find(session, params, [&](const db::Directory::pointer& directory) {
                     releasePaths.insert(directory->getAbsolutePath());
                 });
 
@@ -132,31 +113,30 @@ namespace lms::scanner
                 if (releasePaths.size() > 1)
                 {
                     const std::filesystem::path releasePath{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
-                    image = findImageInDirectory(searchContext, releasePath);
+                    artwork = findImageInDirectory(session, searchParams, releasePath);
                 }
 
-                if (!image)
+                if (!artwork)
                 {
                     for (const std::filesystem::path& releasePath : releasePaths)
                     {
-                        image = findImageInDirectory(searchContext, releasePath);
+                        artwork = findImageInDirectory(session, searchParams, releasePath);
-                        if (image)
+                        if (artwork)
                             break;
                     }
                 }
             }
 
-            return image;
+            return artwork;
         }
 
-        ReleaseArtwork computePreferredReleaseArtwork(SearchReleaseArtworkContext& searchContext, const db::Release::pointer& release)
+        db::Artwork::pointer computePreferredReleaseArtwork(db::Session& session, const SearchReleaseArtworkParams& searchParams, const db::Release::pointer& release)
         {
-            const db::Image::pointer image{ computePreferredReleaseImage(searchContext, release) };
+            db::Artwork::pointer artwork{ computePreferredReleaseImage(session, searchParams, release) };
-            if (image)
+            if (artwork)
-                return ReleaseArtwork{ image->getId() };
+                return artwork;
 
             // Fallback on embedded Front image
-            db::TrackEmbeddedImageId trackEmbeddedImageId;
             {
                 db::TrackEmbeddedImage::FindParameters params;
                 params.setRelease(release->getId());
@@ -164,11 +144,11 @@ namespace lms::scanner
                 params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc);
                 params.setRange(db::Range{ .offset = 0, .size = 1 });
 
-                db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { trackEmbeddedImageId = image->getId(); });
+                db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); });
             }
 
-            if (trackEmbeddedImageId.isValid())
+            if (artwork)
-                return ReleaseArtwork{ trackEmbeddedImageId };
+                return artwork;
 
             // Fallback on embedded media image
             {
@@ -178,66 +158,28 @@ namespace lms::scanner
                 params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc);
                 params.setRange(db::Range{ .offset = 0, .size = 1 });
 
-                db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { trackEmbeddedImageId = image->getId(); });
+                db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); });
             }
 
-            if (trackEmbeddedImageId.isValid())
+            return artwork;
-                return ReleaseArtwork{ trackEmbeddedImageId };
-
-            return ReleaseArtwork{};
         }
 
-        bool fetchNextReleaseArtworksToUpdate(SearchReleaseArtworkContext& searchContext, ReleaseArtworkAssociationContainer& ReleaseArtworkAssociations)
+        void updateReleasePreferredArtwork(db::Session& session, const ReleaseArtworkAssociation& releaseArtworkAssociation)
         {
-            const db::ReleaseId releaseId{ searchContext.lastRetrievedReleaseId };
+            db::Release::updatePreferredArtwork(session, releaseArtworkAssociation.releaseId, releaseArtworkAssociation.preferredArtworkId);
-
-            {
-                constexpr std::size_t readBatchSize{ 100 };
-
-                auto transaction{ searchContext.session.createReadTransaction() };
-
-                db::Release::find(searchContext.session, searchContext.lastRetrievedReleaseId, readBatchSize, [&](const db::Release::pointer& release) {
-                    const ReleaseArtwork preferredArtwork{ computePreferredReleaseArtwork(searchContext, release) };
-                    const db::Artwork::pointer currentPreferredArtwork{ release->getPreferredArtwork() };
-
-                    if (!isSameArtwork(preferredArtwork, currentPreferredArtwork))
-                        ReleaseArtworkAssociations.push_back(ReleaseArtworkAssociation{ release, preferredArtwork });
-                    searchContext.processedReleaseCount++;
-                });
-            }
-
-            return releaseId != searchContext.lastRetrievedReleaseId;
         }
 
-        void updateReleaseArtwork(db::Session& session, const ReleaseArtworkAssociation& ReleaseArtworkAssociation)
+        void updateReleasePreferredArtworks(db::Session& session, ReleaseArtworkAssociationContainer& imageAssociations, bool forceFullBatch)
-        {
-            db::Release::pointer release{ ReleaseArtworkAssociation.release };
-
-            db::Artwork::pointer artwork;
-            if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if<db::TrackEmbeddedImageId>(&ReleaseArtworkAssociation.preferredArtwork) })
-                artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId);
-            else if (const db::ImageId * imageId{ std::get_if<db::ImageId>(&ReleaseArtworkAssociation.preferredArtwork) })
-                artwork = utils::getOrCreateArtworkFromImage(session, *imageId);
-
-            release.modify()->setPreferredArtwork(artwork);
-
-            if (artwork)
-                LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork in release '" << release->getName() << "' with image in " << artwork->getAbsoluteFilePath());
-            else
-                LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from release '" << release->getName() << "'");
-        }
-
-        void updateReleaseArtworks(db::Session& session, ReleaseArtworkAssociationContainer& imageAssociations)
         {
             constexpr std::size_t writeBatchSize{ 50 };
 
-            while (!imageAssociations.empty())
+            while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
             {
                 auto transaction{ session.createWriteTransaction() };
 
                 for (std::size_t i{}; !imageAssociations.empty() && i < writeBatchSize; ++i)
                 {
-                    updateReleaseArtwork(session, imageAssociations.front());
+                    updateReleasePreferredArtwork(session, imageAssociations.front());
                     imageAssociations.pop_front();
                 }
             }
@@ -256,6 +198,62 @@ namespace lms::scanner
             return res;
         }
 
+        bool fetchNextReleaseIdRange(db::Session& session, db::ReleaseId& lastRetrievedId, db::IdRange<db::ReleaseId>& idRange)
+        {
+            constexpr std::size_t readBatchSize{ 100 };
+
+            auto transaction{ session.createReadTransaction() };
+
+            idRange = db::Release::findNextRange(session, lastRetrievedId, readBatchSize);
+            lastRetrievedId = idRange.last;
+
+            return idRange.isValid();
+        }
+
+        class ComputeReleaseArtworkAssociationsJob : public core::IJob
+        {
+        public:
+            ComputeReleaseArtworkAssociationsJob(db::Db& db, const SearchReleaseArtworkParams& searchParams, db::IdRange<db::ReleaseId> artistIdRange)
+                : _db{ db }
+                , _searchParams{ searchParams }
+                , _artistIdRange{ artistIdRange }
+            {
+            }
+
+            std::span<const ReleaseArtworkAssociation> getAssociations() const { return _associations; }
+            std::size_t getProcessedReleaseCount() const { return _processedReleaseCount; }
+
+        private:
+            core::LiteralString getName() const override { return "Associate Track Artworks"; }
+            void run() override
+            {
+                auto& session{ _db.getTLSSession() };
+                auto transaction{ session.createReadTransaction() };
+
+                db::Release::find(session, _artistIdRange, [this, &session](const db::Release::pointer& release) {
+                    const db::Artwork::pointer preferredArtwork{ computePreferredReleaseArtwork(session, _searchParams, release) };
+
+                    if (release->getPreferredArtwork() != preferredArtwork)
+                    {
+                        _associations.push_back(ReleaseArtworkAssociation{ release->getId(), preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{} });
+
+                        if (preferredArtwork)
+                            LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork for release '" << release->getName() << "' with image in " << preferredArtwork->getAbsoluteFilePath());
+                        else
+                            LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from release '" << release->getName() << "'");
+                    }
+
+                    _processedReleaseCount++;
+                });
+            }
+
+            db::Db& _db;
+            const SearchReleaseArtworkParams& _searchParams;
+            db::IdRange<db::ReleaseId> _artistIdRange;
+            std::vector<ReleaseArtworkAssociation> _associations;
+            std::size_t _processedReleaseCount{};
+        };
+
     } // namespace
 
     ScanStepAssociateReleaseImages::ScanStepAssociateReleaseImages(InitParams& initParams)
@@ -278,21 +276,39 @@ namespace lms::scanner
             context.currentStepStats.totalElems = db::Release::getCount(session);
         }
 
-        SearchReleaseArtworkContext searchContext{
+        const SearchReleaseArtworkParams searchParams{
-            .session = session,
-            .lastRetrievedReleaseId = {},
             .releaseImageFileNames = _releaseImageFileNames,
         };
 
-        ReleaseArtworkAssociationContainer ReleaseArtworkAssociations;
+        ReleaseArtworkAssociationContainer artistArtworkAssociations;
-        while (fetchNextReleaseArtworksToUpdate(searchContext, ReleaseArtworkAssociations))
+        auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
-        {
             if (_abortScan)
                 return;
 
-            updateReleaseArtworks(session, ReleaseArtworkAssociations);
+            for (const auto& job : jobs)
-            context.currentStepStats.processedElems = searchContext.processedReleaseCount;
+            {
+                const auto& associationJob{ static_cast<const ComputeReleaseArtworkAssociationsJob&>(*job) };
+                const auto& artistAssociations{ associationJob.getAssociations() };
+
+                artistArtworkAssociations.insert(std::end(artistArtworkAssociations), std::cbegin(artistAssociations), std::cend(artistAssociations));
+
+                context.currentStepStats.processedElems += associationJob.getProcessedReleaseCount();
+            }
+
+            updateReleasePreferredArtworks(session, artistArtworkAssociations, true);
             _progressCallback(context.currentStepStats);
-        }
+        };
+
+        JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
+
+        db::ReleaseId lastRetrievedReleaseId{};
+        db::IdRange<db::ReleaseId> artistIdRange;
+        while (fetchNextReleaseIdRange(session, lastRetrievedReleaseId, artistIdRange))
+            queue.push(std::make_unique<ComputeReleaseArtworkAssociationsJob>(_db, searchParams, artistIdRange));
+
+        queue.finish();
+
+        // process all remaining associations
+        updateReleasePreferredArtworks(session, artistArtworkAssociations, false);
     }
 } // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp

@@ -21,20 +21,23 @@
 
 #include <cassert>
 #include <deque>
-#include <variant>
+#include <optional>
 
+#include "core/IJob.hpp"
+#include "core/IJobScheduler.hpp"
 #include "core/ILogger.hpp"
+#include "core/LiteralString.hpp"
 #include "database/Artwork.hpp"
 #include "database/Db.hpp"
 #include "database/Directory.hpp"
+#include "database/IdRange.hpp"
 #include "database/Image.hpp"
 #include "database/Release.hpp"
 #include "database/Session.hpp"
 #include "database/Track.hpp"
 #include "database/TrackEmbeddedImage.hpp"
-#include "database/TrackEmbeddedImageId.hpp"
 
-#include "ArtworkUtils.hpp"
+#include "JobQueue.hpp"
 #include "ScanContext.hpp"
 
 namespace lms::scanner
@@ -42,37 +45,11 @@ namespace lms::scanner
     namespace
     {
         // May come from an embedded image in a track, or from what has been previously resolved for the release
-        using TrackArtwork = std::variant<std::monostate, db::TrackEmbeddedImageId, db::ArtworkId>;
-        bool isSameArtwork(TrackArtwork preferredArtwork, const db::ObjectPtr<db::Artwork>& artwork)
-        {
-            if (std::holds_alternative<std::monostate>(preferredArtwork))
-                return !artwork;
-
-            if (const db::TrackEmbeddedImageId* trackEmbeddedImageId = std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork))
-            {
-                assert(trackEmbeddedImageId->isValid());
-                return artwork && *trackEmbeddedImageId == artwork->getTrackEmbeddedImageId();
-            }
-
-            if (const db::ArtworkId* artworkId = std::get_if<db::ArtworkId>(&preferredArtwork))
-            {
-                assert(artworkId->isValid());
-                return artwork && *artworkId == artwork->getId();
-            }
-
-            return false;
-        }
-
-        bool isValid(const TrackArtwork& res)
-        {
-            return !std::holds_alternative<std::monostate>(res);
-        }
-
         struct TrackArtworksAssociation
         {
-            db::Track::pointer track;
+            db::TrackId trackId;
-            TrackArtwork preferredArtwork;
+            std::optional<db::ArtworkId> preferredArtworkId;
-            TrackArtwork preferredMediaArtwork;
+            std::optional<db::ArtworkId> preferredMediaArtworkId;
         };
         using TrackArtworksAssociationContainer = std::deque<TrackArtworksAssociation>;
 
@@ -83,11 +60,10 @@ namespace lms::scanner
             std::size_t processedTrackCount{};
         };
 
-        TrackArtwork computePreferredTrackArtwork(SearchTrackArtworkContext& searchContext, const db::Track::pointer& track, TrackArtwork preferredMediaArtwork)
+        db::Artwork::pointer computePreferredTrackArtwork(db::Session& session, const db::Track::pointer& track, const db::Artwork::pointer& preferredMediaArtwork)
         {
-            TrackArtwork res{ preferredMediaArtwork };
+            db::Artwork::pointer res{ preferredMediaArtwork };
-
+            if (res)
-            if (isValid(res))
                 return res;
 
             // Fallback on front cover for this track
@@ -97,10 +73,10 @@ namespace lms::scanner
                 params.setImageTypes({ db::ImageType::FrontCover });
                 params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc);
                 params.setRange(db::Range{ .offset = 0, .size = 1 });
-                db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); });
+                db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); });
             }
 
-            if (isValid(res))
+            if (res)
                 return res;
 
             // Fallback on the artwork already resolved for the release
@@ -108,28 +84,26 @@ namespace lms::scanner
             if (!releaseId.isValid())
                 return res;
 
-            if (const db::Release::pointer release{ db::Release::find(searchContext.session, releaseId) })
+            if (const db::Release::pointer release{ db::Release::find(session, releaseId) })
-            {
+                res = release->getPreferredArtwork();
-                if (const db::ArtworkId releaseArtworkId{ release->getPreferredArtworkId() }; releaseArtworkId.isValid())
-                    res = releaseArtworkId;
-            }
 
             return res;
         }
 
-        TrackArtwork computePreferredTrackMediaArtwork(SearchTrackArtworkContext& searchContext, const db::Track::pointer& track)
+        db::Artwork::pointer computePreferredTrackMediaArtwork(db::Session& session, const db::Track::pointer& track)
         {
-            TrackArtwork res;
+            db::Artwork::pointer res;
+
             {
                 db::TrackEmbeddedImage::FindParameters params;
                 params.setTrack(track->getId());
                 params.setImageTypes({ db::ImageType::Media });
                 params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc);
                 params.setRange(db::Range{ .offset = 0, .size = 1 });
-                db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); });
+                db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); });
             }
 
-            if (isValid(res))
+            if (res)
                 return res;
 
             // fallback on another track of the same disc
@@ -145,95 +119,28 @@ namespace lms::scanner
                 params.setImageTypes({ db::ImageType::Media });
                 params.setSortMethod(db::TrackEmbeddedImageSortMethod::TrackNumberThenSizeDesc);
                 params.setRange(db::Range{ .offset = 0, .size = 1 });
-                db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); });
+                db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); });
             }
 
             return res;
         }
 
-        bool fetchNextTrackArtworksToUpdate(SearchTrackArtworkContext& searchContext, TrackArtworksAssociationContainer& TrackArtworksAssociations)
+        void updateTrackPreferredArtworks(db::Session& session, const TrackArtworksAssociation& trackArtworksAssociation)
         {
-            const db::TrackId trackId{ searchContext.lastRetrievedTrackId };
+            assert(trackArtworksAssociation.preferredArtworkId || trackArtworksAssociation.preferredMediaArtworkId);
 
-            {
+            if (trackArtworksAssociation.preferredArtworkId)
-                constexpr std::size_t readBatchSize{ 100 };
+                db::Track::updatePreferredArtwork(session, trackArtworksAssociation.trackId, *trackArtworksAssociation.preferredArtworkId);
 
-                auto transaction{ searchContext.session.createReadTransaction() };
+            if (trackArtworksAssociation.preferredMediaArtworkId)
-
+                db::Track::updatePreferredMediaArtwork(session, trackArtworksAssociation.trackId, *trackArtworksAssociation.preferredMediaArtworkId);
-                db::Track::find(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize, [&](const db::Track::pointer& track) {
-                    const TrackArtwork preferredMediaArtwork{ computePreferredTrackMediaArtwork(searchContext, track) };
-                    const TrackArtwork preferredArtwork{ computePreferredTrackArtwork(searchContext, track, preferredMediaArtwork) };
-
-                    const db::Artwork::pointer currentPreferredArtwork{ track->getPreferredArtwork() };
-                    const db::Artwork::pointer currentPreferredMediaArtwork{ track->getPreferredMediaArtwork() };
-
-                    if (!isSameArtwork(preferredArtwork, currentPreferredArtwork)
-                        || !isSameArtwork(preferredMediaArtwork, currentPreferredMediaArtwork))
-                    {
-                        TrackArtworksAssociations.push_back(TrackArtworksAssociation{ .track = track, .preferredArtwork = preferredArtwork, .preferredMediaArtwork = preferredMediaArtwork });
-                    }
-
-                    searchContext.processedTrackCount++;
-                });
-            }
-
-            return trackId != searchContext.lastRetrievedTrackId;
         }
 
-        void updateTrackPreferredArtwork(db::Session& session, db::Track::pointer& track, TrackArtwork preferredArtwork)
+        void updateTrackPreferredArtworks(db::Session& session, TrackArtworksAssociationContainer& imageAssociations, bool forceFullBatch)
-        {
-            db::Artwork::pointer artwork;
-            if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork) })
-                artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId);
-            else if (const db::ArtworkId * artworkId{ std::get_if<db::ArtworkId>(&preferredArtwork) })
-                artwork = db::Artwork::find(session, *artworkId);
-
-            // Using track.modify() is quite CPU intensive as the track class has too many fields
-            db::Track::updatePreferredArtwork(session, track->getId(), artwork ? artwork->getId() : db::ArtworkId{});
-            if (artwork)
-                LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork in track " << track->getAbsoluteFilePath() << " with image in " << artwork->getAbsoluteFilePath());
-            else
-                LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from track " << track->getAbsoluteFilePath());
-        }
-
-        void updateTrackPreferredMediaArtwork(db::Session& session, db::Track::pointer& track, TrackArtwork preferredArtwork)
-        {
-            db::Artwork::pointer artwork;
-            if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork) })
-                artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId);
-            else if (const db::ArtworkId * artworkId{ std::get_if<db::ArtworkId>(&preferredArtwork) })
-                artwork = db::Artwork::find(session, *artworkId);
-
-            // Using track.modify() is quite CPU intensive as the track class has too many fields
-            db::Track::updatePreferredMediaArtwork(session, track->getId(), artwork ? artwork->getId() : db::ArtworkId{});
-            if (artwork)
-                LMS_LOG(DBUPDATER, DEBUG, "Updated preferred media artwork in track " << track->getAbsoluteFilePath() << " with image in " << artwork->getAbsoluteFilePath());
-            else
-                LMS_LOG(DBUPDATER, DEBUG, "Removed preferred media artwork from track " << track->getAbsoluteFilePath());
-        }
-
-        void updateTrackPreferredArtworks(db::Session& session, const TrackArtworksAssociation& TrackArtworksAssociation)
-        {
-            db::Track::pointer track{ TrackArtworksAssociation.track };
-
-            {
-                const db::Artwork::pointer currentPreferredArtwork{ track->getPreferredArtwork() };
-                if (!isSameArtwork(TrackArtworksAssociation.preferredArtwork, currentPreferredArtwork))
-                    updateTrackPreferredArtwork(session, track, TrackArtworksAssociation.preferredArtwork);
-            }
-
-            {
-                const db::Artwork::pointer currentPreferredMediaArtwork{ track->getPreferredMediaArtwork() };
-                if (!isSameArtwork(TrackArtworksAssociation.preferredMediaArtwork, currentPreferredMediaArtwork))
-                    updateTrackPreferredMediaArtwork(session, track, TrackArtworksAssociation.preferredMediaArtwork);
-            }
-        }
-
-        void updateTrackPreferredArtworks(db::Session& session, TrackArtworksAssociationContainer& imageAssociations)
         {
             constexpr std::size_t writeBatchSize{ 50 };
 
-            while (!imageAssociations.empty())
+            while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
             {
                 auto transaction{ session.createWriteTransaction() };
 
@@ -244,6 +151,76 @@ namespace lms::scanner
                 }
             }
         }
+
+        bool fetchNextTrackIdRange(SearchTrackArtworkContext& searchContext, db::IdRange<db::TrackId>& trackIdRange)
+        {
+            constexpr std::size_t readBatchSize{ 100 };
+
+            auto transaction{ searchContext.session.createReadTransaction() };
+
+            trackIdRange = db::Track::findNextRange(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize);
+            searchContext.lastRetrievedTrackId = trackIdRange.last;
+
+            return trackIdRange.isValid();
+        }
+
+        class ComputeTrackArtworkAssociationsJob : public core::IJob
+        {
+        public:
+            ComputeTrackArtworkAssociationsJob(db::Db& db, db::IdRange<db::TrackId> trackIdRange)
+                : _db{ db }
+                , _trackIdRange{ trackIdRange }
+            {
+            }
+
+            std::span<const TrackArtworksAssociation> getTrackAssociations() const { return _trackAssociations; }
+            std::size_t getProcessedTrackCount() const { return _processedTrackCount; }
+
+        private:
+            core::LiteralString getName() const override { return "Associate Track Artworks"; }
+            void run() override
+            {
+                auto& session{ _db.getTLSSession() };
+                auto transaction{ session.createReadTransaction() };
+
+                db::Track::find(session, _trackIdRange, [&](const db::Track::pointer& track) {
+                    const db::Artwork::pointer preferredMediaArtwork{ computePreferredTrackMediaArtwork(session, track) };
+                    const db::Artwork::pointer preferredArtwork{ computePreferredTrackArtwork(session, track, preferredMediaArtwork) };
+
+                    TrackArtworksAssociation artworksAssociation{ .trackId = track->getId(), .preferredArtworkId = std::nullopt, .preferredMediaArtworkId = std::nullopt };
+
+                    if (track->getPreferredArtwork() != preferredArtwork)
+                    {
+                        artworksAssociation.preferredArtworkId = preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{};
+
+                        if (preferredArtwork)
+                            LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork in track " << track->getAbsoluteFilePath() << " with image in " << preferredArtwork->getAbsoluteFilePath());
+                        else
+                            LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from track " << track->getAbsoluteFilePath());
+                    }
+
+                    if (track->getPreferredMediaArtwork() != preferredMediaArtwork)
+                    {
+                        artworksAssociation.preferredMediaArtworkId = preferredMediaArtwork ? preferredMediaArtwork->getId() : db::ArtworkId{};
+
+                        if (preferredMediaArtwork)
+                            LMS_LOG(DBUPDATER, DEBUG, "Updating preferred media artwork in track " << track->getAbsoluteFilePath() << " with image in " << preferredMediaArtwork->getAbsoluteFilePath());
+                        else
+                            LMS_LOG(DBUPDATER, DEBUG, "Removing preferred media artwork from track " << track->getAbsoluteFilePath());
+                    }
+
+                    if (artworksAssociation.preferredArtworkId || artworksAssociation.preferredMediaArtworkId)
+                        _trackAssociations.push_back(artworksAssociation);
+
+                    _processedTrackCount++;
+                });
+            }
+
+            db::Db& _db;
+            db::IdRange<db::TrackId> _trackIdRange;
+            std::vector<TrackArtworksAssociation> _trackAssociations;
+            std::size_t _processedTrackCount{};
+        };
     } // namespace
 
     ScanStepAssociateTrackImages::ScanStepAssociateTrackImages(InitParams& initParams)
@@ -270,15 +247,34 @@ namespace lms::scanner
             .lastRetrievedTrackId = {},
         };
 
-        TrackArtworksAssociationContainer TrackArtworksAssociations;
+        TrackArtworksAssociationContainer trackArtworksAssociations;
-        while (fetchNextTrackArtworksToUpdate(searchContext, TrackArtworksAssociations))
+        auto processTracks = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
-        {
             if (_abortScan)
                 return;
 
-            updateTrackPreferredArtworks(session, TrackArtworksAssociations);
+            for (const auto& job : jobs)
-            context.currentStepStats.processedElems = searchContext.processedTrackCount;
+            {
+                const auto& associationJob{ static_cast<const ComputeTrackArtworkAssociationsJob&>(*job) };
+                const auto& trackAssociations{ associationJob.getTrackAssociations() };
+
+                trackArtworksAssociations.insert(std::end(trackArtworksAssociations), std::cbegin(trackAssociations), std::cend(trackAssociations));
+
+                context.currentStepStats.processedElems += associationJob.getProcessedTrackCount();
+            }
+
+            updateTrackPreferredArtworks(session, trackArtworksAssociations, true);
             _progressCallback(context.currentStepStats);
-        }
+        };
+
+        JobQueue queue{ getJobScheduler(), 20, processTracks, 1, 0.85F };
+
+        db::IdRange<db::TrackId> trackIdRange;
+        while (fetchNextTrackIdRange(searchContext, trackIdRange))
+            queue.push(std::make_unique<ComputeTrackArtworkAssociationsJob>(_db, trackIdRange));
+
+        queue.finish();
+
+        // process all remaining associations
+        updateTrackPreferredArtworks(session, trackArtworksAssociations, false);
     }
 } // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp

@@ -31,6 +31,7 @@
 #include "scanners/IFileScanOperation.hpp"
 #include "scanners/IFileScanner.hpp"
 
+#include "JobQueue.hpp"
 #include "ScanContext.hpp"
 
 namespace lms::scanner
@@ -114,11 +115,16 @@ namespace lms::scanner
 
     void ScanStepScanFiles::process(ScanContext& context, const MediaLibraryInfo& mediaLibrary)
     {
-        core::IJobScheduler& jobScheduler{ getJobScheduler() };
+        const std::size_t scanQueueMaxScanRequestCount{ 50 * getJobScheduler().getThreadCount() };
-        assert(jobScheduler.getJobsDoneCount() == 0);
-
-        const std::size_t scanQueueMaxScanRequestCount{ 100 * jobScheduler.getThreadCount() };
         constexpr std::size_t processFileResultsBatchSize{ 10 };
+        constexpr float drainRatio{ 0.85 };
+
+        auto processDoneJobs = [&](std::span<std::unique_ptr<core::IJob>> jobsDone) {
+            if (!_abortScan)
+                processFileScanResults(context, jobsDone);
+        };
+
+        JobQueue queue{ getJobScheduler(), scanQueueMaxScanRequestCount, processDoneJobs, processFileResultsBatchSize, drainRatio };
 
         std::vector<std::unique_ptr<core::IJob>> jobsDone;
         std::vector<std::unique_ptr<IFileScanOperation>> scanOperations;
@@ -148,7 +154,7 @@ namespace lms::scanner
                         if (context.scanOptions.fullScan || scanner->needsScan(fileToScan))
                         {
                             auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) };
-                            jobScheduler.scheduleJob(std::make_unique<FileScanJob>(std::move(scanOperation)));
+                            queue.push(std::make_unique<FileScanJob>(std::move(scanOperation)));
                         }
                     }
 
@@ -156,30 +162,9 @@ namespace lms::scanner
                     _progressCallback(context.currentStepStats);
                 }
 
-                while (jobScheduler.getJobsDoneCount() > (scanQueueMaxScanRequestCount / 2))
-                {
-                    jobScheduler.popJobsDone(jobsDone, processFileResultsBatchSize);
-                    processFileScanResults(context, jobsDone);
-                    jobsDone.clear();
-                }
-
-                jobScheduler.waitUntilJobCountAtMost(scanQueueMaxScanRequestCount);
-
                 return true;
             },
             &excludeDirFileName);
-
-        jobScheduler.wait();
-
-        while (jobScheduler.popJobsDone(jobsDone, processFileResultsBatchSize) > 0)
-        {
-            if (!_abortScan)
-                processFileScanResults(context, jobsDone);
-
-            jobsDone.clear();
-        }
-
-        assert(jobScheduler.getJobsDoneCount() == 0);
     }
 
     void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<std::unique_ptr<core::IJob>> scanJobs)
@@ -191,9 +176,6 @@ namespace lms::scanner
 
         for (auto& scanJob : scanJobs)
         {
-            if (_abortScan)
-                return;
-
             IFileScanOperation& scanOperation{ static_cast<FileScanJob&>(*scanJob).getScanOperation() };
 
             LMS_LOG(DBUPDATER, DEBUG, scanOperation.getName() << ": processing result for " << scanOperation.getFilePath());