diff --git a/src/libs/database/impl/Artist.cpp b/src/libs/database/impl/Artist.cpp index 57ff3c50..d3b106fe 100644 --- a/src/libs/database/impl/Artist.cpp +++ b/src/libs/database/impl/Artist.cpp @@ -237,6 +237,17 @@ namespace lms::db }); } + void Artist::find(Session& session, const IdRange& idRange, const std::function& func) + { + assert(idRange.isValid()); + + auto query{ session.getDboSession()->query>("SELECT a from artist a").orderBy("a.id").where("a.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) }; + + utils::forEachQueryResult(query, [&](const Artist::pointer& artist) { + func(artist); + }); + } + std::vector Artist::find(Session& session, std::string_view name) { session.checkReadTransaction(); @@ -280,6 +291,16 @@ namespace lms::db utils::forEachQueryRangeResult(query, params.range, func); } + IdRange Artist::findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count) + { + auto query{ session.getDboSession()->query>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT a.id FROM artist a WHERE a.id > ? ORDER BY a.id LIMIT ?) sub") }; + query.bind(lastRetrievedId); + query.bind(static_cast(count)); + + auto res{ utils::fetchQuerySingleResult(query) }; + return IdRange{ .first = std::get<0>(res), .last = std::get<1>(res) }; + } + RangeResults Artist::findOrphanIds(Session& session, std::optional range) { session.checkReadTransaction(); @@ -304,6 +325,16 @@ AND NOT EXISTS ( return utils::fetchQuerySingleResult(session.getDboSession()->query("SELECT 1 FROM artist").where("id = ?").bind(id)) == 1; } + void Artist::updatePreferredArtwork(Session& session, ArtistId artistId, ArtworkId artworkId) + { + session.checkWriteTransaction(); + + if (artworkId.isValid()) + utils::executeCommand(*session.getDboSession(), "UPDATE artist SET preferred_artwork_id = ? WHERE id = ?", artworkId, artistId); + else + utils::executeCommand(*session.getDboSession(), "UPDATE artist SET preferred_artwork_id = NULL WHERE id = ?", artistId); + } + std::optional Artist::getMBID() const { return core::UUID::fromString(_mbid); diff --git a/src/libs/database/impl/Migration.cpp b/src/libs/database/impl/Migration.cpp index 4d47cc69..c7c8510d 100644 --- a/src/libs/database/impl/Migration.cpp +++ b/src/libs/database/impl/Migration.cpp @@ -35,7 +35,7 @@ namespace lms::db { namespace { - static constexpr Version LMS_DATABASE_VERSION{ 95 }; + static constexpr Version LMS_DATABASE_VERSION{ 96 }; } VersionInfo::VersionInfo() @@ -1396,6 +1396,22 @@ FROM artist)"); utils::executeCommand(*session.getDboSession(), "ALTER TABLE track DROP COLUMN file_name"); } + void migrateFromV95(Session& session) + { + // Make sure each image and each embessed image has an artwork object + utils::executeCommand(*session.getDboSession(), R"(INSERT INTO artwork (version, track_embedded_image_id) +SELECT 1 AS version, tei.id +FROM track_embedded_image tei +LEFT JOIN artwork art ON tei.id = art.track_embedded_image_id +WHERE art.track_embedded_image_id IS NULL)"); + + utils::executeCommand(*session.getDboSession(), R"(INSERT INTO artwork (version, image_id) +SELECT 1 AS version, img.id +FROM image img +LEFT JOIN artwork art ON img.id = art.image_id +WHERE art.image_id IS NULL)"); + } + bool doDbMigration(Session& session) { constexpr std::string_view outdatedMsg{ "Outdated database, please rebuild it (delete the .db file and restart)" }; @@ -1467,6 +1483,7 @@ FROM artist)"); { 92, migrateFromV92 }, { 93, migrateFromV93 }, { 94, migrateFromV94 }, + { 95, migrateFromV95 }, }; bool migrationPerformed{}; diff --git a/src/libs/database/impl/Release.cpp b/src/libs/database/impl/Release.cpp index 12c85d6b..d78d6f26 100644 --- a/src/libs/database/impl/Release.cpp +++ b/src/libs/database/impl/Release.cpp @@ -504,6 +504,27 @@ namespace lms::db }); } + void Release::find(Session& session, const IdRange& idRange, const std::function& func) + { + assert(idRange.isValid()); + + auto query{ session.getDboSession()->query>("SELECT r from release r").orderBy("r.id").where("r.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) }; + + utils::forEachQueryResult(query, [&](const Release::pointer& release) { + func(release); + }); + } + + IdRange Release::findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count) + { + auto query{ session.getDboSession()->query>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT r.id FROM release r WHERE r.id > ? ORDER BY r.id LIMIT ?) sub") }; + query.bind(lastRetrievedId); + query.bind(static_cast(count)); + + auto res{ utils::fetchQuerySingleResult(query) }; + return IdRange{ .first = std::get<0>(res), .last = std::get<1>(res) }; + } + RangeResults Release::find(Session& session, const FindParameters& params) { session.checkReadTransaction(); @@ -528,6 +549,16 @@ namespace lms::db return utils::execRangeQuery(query, params.range); } + void Release::updatePreferredArtwork(Session& session, ReleaseId releaseId, ArtworkId artworkId) + { + session.checkWriteTransaction(); + + if (artworkId.isValid()) + utils::executeCommand(*session.getDboSession(), "UPDATE release SET preferred_artwork_id = ? WHERE id = ?", artworkId, releaseId); + else + utils::executeCommand(*session.getDboSession(), "UPDATE release SET preferred_artwork_id = NULL WHERE id = ?", releaseId); + } + std::size_t Release::getCount(Session& session, const FindParameters& params) { session.checkReadTransaction(); diff --git a/src/libs/database/impl/Track.cpp b/src/libs/database/impl/Track.cpp index 63220d86..9b6384fd 100644 --- a/src/libs/database/impl/Track.cpp +++ b/src/libs/database/impl/Track.cpp @@ -292,6 +292,27 @@ namespace lms::db }); } + void Track::find(Session& session, const IdRange& idRange, const std::function& func) + { + assert(idRange.isValid()); + + auto query{ session.getDboSession()->query>("SELECT t from track t").orderBy("t.id").where("t.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) }; + + utils::forEachQueryResult(query, [&](const Track::pointer& track) { + func(track); + }); + } + + IdRange Track::findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count) + { + auto query{ session.getDboSession()->query>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT t.id FROM track t WHERE t.id > ? ORDER BY t.id LIMIT ?) sub") }; + query.bind(lastRetrievedId); + query.bind(static_cast(count)); + + auto res{ utils::fetchQuerySingleResult(query) }; + return IdRange{ .first = std::get<0>(res), .last = std::get<1>(res) }; + } + bool Track::exists(Session& session, TrackId id) { session.checkReadTransaction(); diff --git a/src/libs/database/include/database/Artist.hpp b/src/libs/database/include/database/Artist.hpp index 86972a75..dd38ed0c 100644 --- a/src/libs/database/include/database/Artist.hpp +++ b/src/libs/database/include/database/Artist.hpp @@ -34,6 +34,7 @@ #include "database/ArtworkId.hpp" #include "database/ClusterId.hpp" #include "database/Filters.hpp" +#include "database/IdRange.hpp" #include "database/MediaLibraryId.hpp" #include "database/Object.hpp" #include "database/ReleaseId.hpp" @@ -125,12 +126,17 @@ namespace lms::db static pointer find(Session& session, ArtistId id); static std::vector find(Session& session, std::string_view name); // exact match on name field static void find(Session& session, ArtistId& lastRetrievedArtist, std::size_t count, const std::function& func, MediaLibraryId library = {}); + static void find(Session& session, const IdRange& idRange, const std::function& func); static RangeResults find(Session& session, const FindParameters& params); static void find(Session& session, const FindParameters& params, std::function func); + static IdRange findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count); static RangeResults findIds(Session& session, const FindParameters& params); static RangeResults findOrphanIds(Session& session, std::optional range = std::nullopt); // No track related static bool exists(Session& session, ArtistId id); + // Updates + static void updatePreferredArtwork(Session& session, ArtistId artistId, ArtworkId artworkId); + // Accessors const std::string& getName() const { return _name; } const std::string& getSortName() const { return _sortName; } diff --git a/src/libs/services/scanner/impl/steps/ArtworkUtils.hpp b/src/libs/database/include/database/IdRange.hpp similarity index 56% rename from src/libs/services/scanner/impl/steps/ArtworkUtils.hpp rename to src/libs/database/include/database/IdRange.hpp index 4dc0722b..cc7f7434 100644 --- a/src/libs/services/scanner/impl/steps/ArtworkUtils.hpp +++ b/src/libs/database/include/database/IdRange.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2024 Emeric Poupon + * Copyright (C) 2025 Emeric Poupon * * This file is part of LMS. * @@ -19,21 +19,15 @@ #pragma once -#include - -#include "database/ArtworkId.hpp" -#include "database/ImageId.hpp" -#include "database/Object.hpp" -#include "database/TrackEmbeddedImageId.hpp" - namespace lms::db { - class Artwork; - class Session; -} // namespace lms::db + template + struct IdRange + { + public: + IdType first; + IdType last; -namespace lms::scanner::utils -{ - db::ObjectPtr getOrCreateArtworkFromTrackEmbeddedImage(db::Session& session, db::TrackEmbeddedImageId trackEmbeddedImageId); - db::ObjectPtr getOrCreateArtworkFromImage(db::Session& session, db::ImageId imageId); -} // namespace lms::scanner::utils \ No newline at end of file + bool isValid() const { return first.isValid() && last.isValid(); } + }; +} // namespace lms::db diff --git a/src/libs/database/include/database/Release.hpp b/src/libs/database/include/database/Release.hpp index 1f55173a..7316b179 100644 --- a/src/libs/database/include/database/Release.hpp +++ b/src/libs/database/include/database/Release.hpp @@ -35,6 +35,7 @@ #include "database/CountryId.hpp" #include "database/DirectoryId.hpp" #include "database/Filters.hpp" +#include "database/IdRange.hpp" #include "database/LabelId.hpp" #include "database/MediaLibraryId.hpp" #include "database/Object.hpp" @@ -249,12 +250,17 @@ namespace lms::db static pointer find(Session& session, const core::UUID& MBID); static pointer find(Session& session, ReleaseId id); static void find(Session& session, ReleaseId& lastRetrievedRelease, std::size_t count, const std::function& func, MediaLibraryId library = {}); + static void find(Session& session, const IdRange& idRange, const std::function& func); + static IdRange findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count); static RangeResults find(Session& session, const FindParameters& parameters); static void find(Session& session, const FindParameters& parameters, const std::function& func); static RangeResults findIds(Session& session, const FindParameters& parameters); static std::size_t getCount(Session& session, const FindParameters& parameters); static RangeResults findOrphanIds(Session& session, std::optional range = std::nullopt); // not track related + // Updates + static void updatePreferredArtwork(Session& session, ReleaseId id, ArtworkId artworkId); + // Get the cluster of the tracks that belong to this release // Each clusters are grouped by cluster type, sorted by the number of occurence (max to min) // size is the max number of cluster per cluster type diff --git a/src/libs/database/include/database/Track.hpp b/src/libs/database/include/database/Track.hpp index 293d1055..e967596e 100644 --- a/src/libs/database/include/database/Track.hpp +++ b/src/libs/database/include/database/Track.hpp @@ -39,6 +39,7 @@ #include "database/ClusterId.hpp" #include "database/DirectoryId.hpp" #include "database/Filters.hpp" +#include "database/IdRange.hpp" #include "database/MediaLibraryId.hpp" #include "database/Object.hpp" #include "database/ReleaseId.hpp" @@ -192,6 +193,8 @@ namespace lms::db static pointer findByPath(Session& session, const std::filesystem::path& p); static pointer find(Session& session, TrackId id); static void find(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function& func, MediaLibraryId library = {}); + static void find(Session& session, const IdRange& idRange, const std::function& func); + static IdRange findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count); static void findAbsoluteFilePath(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function& func); static bool exists(Session& session, TrackId id); diff --git a/src/libs/database/test/Artist.cpp b/src/libs/database/test/Artist.cpp index 1b82997b..2e53657d 100644 --- a/src/libs/database/test/Artist.cpp +++ b/src/libs/database/test/Artist.cpp @@ -608,6 +608,93 @@ namespace lms::db::tests } } + TEST_F(DatabaseFixture, Artist_findNextRange) + { + { + auto transaction{ session.createReadTransaction() }; + + auto range{ Artist::findNextRange(session, ArtistId{}, 0) }; + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, ArtistId{}); + EXPECT_EQ(range.last, ArtistId{}); + + range = Artist::findNextRange(session, ArtistId{}, 100); + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, ArtistId{}); + EXPECT_EQ(range.last, ArtistId{}); + } + + ScopedArtist artist1{ session, "Artist1" }; + { + auto transaction{ session.createReadTransaction() }; + + auto range{ Artist::findNextRange(session, ArtistId{}, 0) }; + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, ArtistId{}); + EXPECT_EQ(range.last, ArtistId{}); + + range = Artist::findNextRange(session, ArtistId{}, 1); + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, artist1.getId()); + EXPECT_EQ(range.last, artist1.getId()); + + range = Artist::findNextRange(session, range.last, 1); + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, ArtistId{}); + EXPECT_EQ(range.last, ArtistId{}); + + range = Artist::findNextRange(session, ArtistId{}, 100); + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, artist1.getId()); + EXPECT_EQ(range.last, artist1.getId()); + } + + ScopedArtist artist2{ session, "Artist2" }; + ScopedArtist artist3{ session, "Artist3" }; + + { + auto transaction{ session.createReadTransaction() }; + + auto range{ Artist::findNextRange(session, ArtistId{}, 2) }; + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, artist1.getId()); + EXPECT_EQ(range.last, artist2.getId()); + + range = Artist::findNextRange(session, artist2.getId(), 2); + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, artist3.getId()); + EXPECT_EQ(range.last, artist3.getId()); + } + } + + TEST_F(DatabaseFixture, Artist_findByRange) + { + ScopedArtist artist1{ session, "Artist1" }; + ScopedArtist artist2{ session, "Artist2" }; + ScopedArtist artist3{ session, "Artist3" }; + + { + auto transaction{ session.createReadTransaction() }; + + std::size_t count{}; + Artist::find(session, IdRange{ .first = artist1.getId(), .last = artist1.getId() }, [&](const db::Artist::pointer& artist) { + count++; + EXPECT_EQ(artist->getId(), artist1.getId()); + }); + EXPECT_EQ(count, 1); + } + + { + auto transaction{ session.createReadTransaction() }; + + std::size_t count{}; + Artist::find(session, IdRange{ .first = artist1.getId(), .last = artist3.getId() }, [&](const db::Artist::pointer&) { + count++; + }); + EXPECT_EQ(count, 3); + } + } + TEST_F(DatabaseFixture, Artist_sortMethod) { ScopedArtist artistA{ session, "artistA" }; @@ -817,4 +904,34 @@ namespace lms::db::tests EXPECT_EQ(artists.results[3], artistC.getId()); } } + + TEST_F(DatabaseFixture, Artist_updateArtwork) + { + ScopedArtist artist{ session, "MyArtist" }; + { + auto transaction{ session.createReadTransaction() }; + EXPECT_EQ(artist->getPreferredArtwork(), Artwork::pointer{}); + } + + ScopedImage image{ session, "/image1.jpg" }; + ScopedArtwork artwork{ session, image.lockAndGet() }; + + { + auto transaction{ session.createWriteTransaction() }; + Artist::updatePreferredArtwork(session, artist->getId(), artwork.getId()); + } + { + auto transaction{ session.createReadTransaction() }; + EXPECT_EQ(artist->getPreferredArtwork()->getId(), artwork.getId()); + } + + { + auto transaction{ session.createWriteTransaction() }; + Artist::updatePreferredArtwork(session, artist.getId(), ArtworkId{}); + } + { + auto transaction{ session.createReadTransaction() }; + EXPECT_EQ(artist->getPreferredArtwork(), Artwork::pointer{}); + } + } } // namespace lms::db::tests diff --git a/src/libs/database/test/Release.cpp b/src/libs/database/test/Release.cpp index 3d3b3a11..fc7cbc44 100644 --- a/src/libs/database/test/Release.cpp +++ b/src/libs/database/test/Release.cpp @@ -173,6 +173,93 @@ namespace lms::db::tests } } + TEST_F(DatabaseFixture, Release_findNextRange) + { + { + auto transaction{ session.createReadTransaction() }; + + auto range{ Release::findNextRange(session, ReleaseId{}, 0) }; + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, ReleaseId{}); + EXPECT_EQ(range.last, ReleaseId{}); + + range = Release::findNextRange(session, ReleaseId{}, 100); + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, ReleaseId{}); + EXPECT_EQ(range.last, ReleaseId{}); + } + + ScopedRelease release1{ session, "Artist1" }; + { + auto transaction{ session.createReadTransaction() }; + + auto range{ Release::findNextRange(session, ReleaseId{}, 0) }; + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, ReleaseId{}); + EXPECT_EQ(range.last, ReleaseId{}); + + range = Release::findNextRange(session, ReleaseId{}, 1); + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, release1.getId()); + EXPECT_EQ(range.last, release1.getId()); + + range = Release::findNextRange(session, range.last, 1); + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, ReleaseId{}); + EXPECT_EQ(range.last, ReleaseId{}); + + range = Release::findNextRange(session, ReleaseId{}, 100); + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, release1.getId()); + EXPECT_EQ(range.last, release1.getId()); + } + + ScopedRelease release2{ session, "Artist2" }; + ScopedRelease release3{ session, "Artist3" }; + + { + auto transaction{ session.createReadTransaction() }; + + auto range{ Release::findNextRange(session, ReleaseId{}, 2) }; + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, release1.getId()); + EXPECT_EQ(range.last, release2.getId()); + + range = Release::findNextRange(session, release2.getId(), 2); + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, release3.getId()); + EXPECT_EQ(range.last, release3.getId()); + } + } + + TEST_F(DatabaseFixture, Release_findByRange) + { + ScopedRelease release1{ session, "Artist1" }; + ScopedRelease release2{ session, "Artist2" }; + ScopedRelease release3{ session, "Artist3" }; + + { + auto transaction{ session.createReadTransaction() }; + + std::size_t count{}; + Release::find(session, IdRange{ .first = release1.getId(), .last = release1.getId() }, [&](const db::Release::pointer& release) { + count++; + EXPECT_EQ(release->getId(), release1.getId()); + }); + EXPECT_EQ(count, 1); + } + + { + auto transaction{ session.createReadTransaction() }; + + std::size_t count{}; + Release::find(session, IdRange{ .first = release1.getId(), .last = release3.getId() }, [&](const db::Release::pointer&) { + count++; + }); + EXPECT_EQ(count, 3); + } + } + TEST_F(DatabaseFixture, Release_singleTrack) { ScopedRelease release{ session, "MyRelease" }; @@ -1431,4 +1518,35 @@ namespace lms::db::tests EXPECT_EQ(releases.results[1]->getId(), release1->getId()); } } + + TEST_F(DatabaseFixture, Release_updateArtwork) + { + ScopedRelease release{ session, "MyRelease" }; + { + auto transaction{ session.createReadTransaction() }; + EXPECT_EQ(release->getPreferredArtwork(), Artwork::pointer{}); + } + + ScopedImage image{ session, "/image1.jpg" }; + ScopedArtwork artwork{ session, image.lockAndGet() }; + + { + auto transaction{ session.createWriteTransaction() }; + Release::updatePreferredArtwork(session, release->getId(), artwork.getId()); + } + { + auto transaction{ session.createReadTransaction() }; + EXPECT_EQ(release->getPreferredArtwork()->getId(), artwork.getId()); + } + + { + auto transaction{ session.createWriteTransaction() }; + Release::updatePreferredArtwork(session, release.getId(), ArtworkId{}); + } + { + auto transaction{ session.createReadTransaction() }; + EXPECT_EQ(release->getPreferredArtwork(), Artwork::pointer{}); + } + } + } // namespace lms::db::tests \ No newline at end of file diff --git a/src/libs/database/test/Track.cpp b/src/libs/database/test/Track.cpp index 517900d9..e88b6ce2 100644 --- a/src/libs/database/test/Track.cpp +++ b/src/libs/database/test/Track.cpp @@ -151,6 +151,93 @@ namespace lms::db::tests } } + TEST_F(DatabaseFixture, Track_findNextRange) + { + { + auto transaction{ session.createReadTransaction() }; + + auto range{ Track::findNextRange(session, TrackId{}, 0) }; + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, TrackId{}); + EXPECT_EQ(range.last, TrackId{}); + + range = Track::findNextRange(session, TrackId{}, 100); + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, TrackId{}); + EXPECT_EQ(range.last, TrackId{}); + } + + ScopedTrack track1{ session }; + { + auto transaction{ session.createReadTransaction() }; + + auto range{ Track::findNextRange(session, TrackId{}, 0) }; + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, TrackId{}); + EXPECT_EQ(range.last, TrackId{}); + + range = Track::findNextRange(session, TrackId{}, 1); + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, track1.getId()); + EXPECT_EQ(range.last, track1.getId()); + + range = Track::findNextRange(session, range.last, 1); + EXPECT_FALSE(range.isValid()); + EXPECT_EQ(range.first, TrackId{}); + EXPECT_EQ(range.last, TrackId{}); + + range = Track::findNextRange(session, TrackId{}, 100); + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, track1.getId()); + EXPECT_EQ(range.last, track1.getId()); + } + + ScopedTrack track2{ session }; + ScopedTrack track3{ session }; + + { + auto transaction{ session.createReadTransaction() }; + + auto range{ Track::findNextRange(session, TrackId{}, 2) }; + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, track1.getId()); + EXPECT_EQ(range.last, track2.getId()); + + range = Track::findNextRange(session, track2.getId(), 2); + EXPECT_TRUE(range.isValid()); + EXPECT_EQ(range.first, track3.getId()); + EXPECT_EQ(range.last, track3.getId()); + } + } + + TEST_F(DatabaseFixture, Track_findByRange) + { + ScopedTrack track1{ session }; + ScopedTrack track2{ session }; + ScopedTrack track3{ session }; + + { + auto transaction{ session.createReadTransaction() }; + + std::size_t count{}; + Track::find(session, IdRange{ .first = track1.getId(), .last = track1.getId() }, [&](const db::Track::pointer& track) { + count++; + EXPECT_EQ(track->getId(), track1.getId()); + }); + EXPECT_EQ(count, 1); + } + + { + auto transaction{ session.createReadTransaction() }; + + std::size_t count{}; + Track::find(session, IdRange{ .first = track1.getId(), .last = track3.getId() }, [&](const db::Track::pointer&) { + count++; + }); + EXPECT_EQ(count, 3); + } + } + TEST_F(DatabaseFixture, Track_findAbsoluteFilePath) { ScopedTrack track{ session }; diff --git a/src/libs/services/scanner/CMakeLists.txt b/src/libs/services/scanner/CMakeLists.txt index 26976852..a09c1fbb 100644 --- a/src/libs/services/scanner/CMakeLists.txt +++ b/src/libs/services/scanner/CMakeLists.txt @@ -8,7 +8,7 @@ add_library(lmsscanner STATIC impl/scanners/LyricsFileScanner.cpp impl/scanners/PlayListFileScanner.cpp impl/scanners/Utils.cpp - impl/steps/ArtworkUtils.cpp + impl/steps/JobQueue.cpp impl/steps/ScanErrorLogger.cpp impl/steps/ScanStepArtistReconciliation.cpp impl/steps/ScanStepAssociateArtistImages.cpp diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index db70e71c..c5a002f0 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -448,7 +448,7 @@ namespace lms::scanner if (!_lastScanSettings) _lastScanSettings = readScannerSettings(_db.getTLSSession(), lastScanSettingsName); - auto cbFunc{ [this](const ScanStepStats& stats) { + auto progressFunc{ [this](const ScanStepStats& stats) { notifyInProgressIfNeeded(stats); } }; @@ -466,7 +466,7 @@ namespace lms::scanner .jobScheduler = *_jobScheduler, .settings = _settings, .lastScanSettings = _lastScanSettings.has_value() ? &(_lastScanSettings.value()) : nullptr, - .progressCallback = cbFunc, + .progressCallback = progressFunc, .abortScan = _abortScan, .db = _db, .fileScanners = fileScanners, @@ -500,14 +500,14 @@ namespace lms::scanner _currentScanStepStats = stepStats; } - const std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() }; + const auto now{ std::chrono::steady_clock::now() }; _events.scanInProgress.emit(stepStats); _lastScanInProgressEmit = now; } void ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats) { - std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() }; + const auto now{ std::chrono::steady_clock::now() }; if (now - _lastScanInProgressEmit >= std::chrono::seconds{ 1 }) notifyInProgress(stepStats); diff --git a/src/libs/services/scanner/impl/ScannerService.hpp b/src/libs/services/scanner/impl/ScannerService.hpp index d3598a36..c3705905 100644 --- a/src/libs/services/scanner/impl/ScannerService.hpp +++ b/src/libs/services/scanner/impl/ScannerService.hpp @@ -96,7 +96,7 @@ namespace lms::scanner Wt::WIOService _ioService; boost::asio::system_timer _scheduleTimer{ _ioService }; Events _events; - std::chrono::system_clock::time_point _lastScanInProgressEmit; + std::chrono::steady_clock::time_point _lastScanInProgressEmit; mutable std::shared_mutex _statusMutex; State _curState{ State::NotScheduled }; diff --git a/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp b/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp index 5be41052..ff57a281 100644 --- a/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp +++ b/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp @@ -25,6 +25,7 @@ #include "core/Path.hpp" #include "core/XxHash3.hpp" #include "database/Artist.hpp" +#include "database/Artwork.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" #include "database/Directory.hpp" @@ -332,6 +333,8 @@ namespace lms::scanner image.modify()->setWidth(imageInfo.properties.width); image.modify()->setHeight(imageInfo.properties.height); image.modify()->setMimeType(imageInfo.mimeType); + + session.create(image); } return image; diff --git a/src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp b/src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp index 7869e938..d4d512e8 100644 --- a/src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp +++ b/src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp @@ -22,6 +22,7 @@ #include #include "core/ILogger.hpp" +#include "database/Artwork.hpp" #include "database/Db.hpp" #include "database/Image.hpp" #include "database/MediaLibrary.hpp" @@ -84,7 +85,10 @@ namespace lms::scanner const bool added{ !image }; if (!image) + { image = dbSession.create(getFilePath()); + dbSession.create(image); + } image.modify()->setLastWriteTime(getLastWriteTime()); image.modify()->setFileSize(getFileSize()); diff --git a/src/libs/services/scanner/impl/steps/ArtworkUtils.cpp b/src/libs/services/scanner/impl/steps/ArtworkUtils.cpp deleted file mode 100644 index e74c6095..00000000 --- a/src/libs/services/scanner/impl/steps/ArtworkUtils.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2024 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "ArtworkUtils.hpp" - -#include "database/Artwork.hpp" -#include "database/Image.hpp" -#include "database/Session.hpp" -#include "database/Track.hpp" -#include "database/TrackEmbeddedImage.hpp" - -namespace lms::scanner::utils -{ - db::ObjectPtr getOrCreateArtworkFromTrackEmbeddedImage(db::Session& session, db::TrackEmbeddedImageId trackEmbeddedImageId) - { - assert(trackEmbeddedImageId.isValid()); - session.checkWriteTransaction(); - - db::ObjectPtr artwork{ db::Artwork::find(session, trackEmbeddedImageId) }; - if (!artwork) - { - db::TrackEmbeddedImage::pointer trackEmbeddedImage{ db::TrackEmbeddedImage::find(session, trackEmbeddedImageId) }; - assert(trackEmbeddedImage); - artwork = session.create(trackEmbeddedImage); - } - return artwork; - } - - db::ObjectPtr getOrCreateArtworkFromImage(db::Session& session, db::ImageId imageId) - { - assert(imageId.isValid()); - session.checkWriteTransaction(); - - db::ObjectPtr artwork{ db::Artwork::find(session, imageId) }; - if (!artwork) - { - db::Image::pointer image{ db::Image::find(session, imageId) }; - assert(image); - artwork = session.create(image); - } - return artwork; - } -} // namespace lms::scanner::utils \ No newline at end of file diff --git a/src/libs/services/scanner/impl/steps/JobQueue.cpp b/src/libs/services/scanner/impl/steps/JobQueue.cpp new file mode 100644 index 00000000..aa5db125 --- /dev/null +++ b/src/libs/services/scanner/impl/steps/JobQueue.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "JobQueue.hpp" + +#include + +#include "core/IJob.hpp" +#include "core/IJobScheduler.hpp" + +namespace lms::scanner +{ + JobQueue::JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float _drainThreshold) + : _scheduler{ scheduler } + , _maxQueueSize{ maxQueueSize } + , _processJobsDoneFunc{ std::move(processJobsDoneFunc) } + , _batchSize{ batchSize } + , _drainThreshold{ _drainThreshold } + { + assert(_scheduler.getJobsDoneCount() == 0); + } + + JobQueue::~JobQueue() + { + finish(); + } + + void JobQueue::push(std::unique_ptr job) + { + _scheduler.scheduleJob(std::move(job)); + drainIfNeeded(); + } + + void JobQueue::finish() + { + _scheduler.wait(); + while (_scheduler.popJobsDone(_jobsDone, _batchSize) > 0) + { + _processJobsDoneFunc(std::span{ _jobsDone }); + _jobsDone.clear(); + } + } + + void JobQueue::drainIfNeeded() + { + while (_scheduler.getJobsDoneCount() > static_cast(_maxQueueSize * _drainThreshold)) + { + _scheduler.popJobsDone(_jobsDone, _batchSize); + _processJobsDoneFunc(std::span{ _jobsDone }); + _jobsDone.clear(); + } + _scheduler.waitUntilJobCountAtMost(_maxQueueSize); + } +} // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/steps/JobQueue.hpp b/src/libs/services/scanner/impl/steps/JobQueue.hpp new file mode 100644 index 00000000..59ce792f --- /dev/null +++ b/src/libs/services/scanner/impl/steps/JobQueue.hpp @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ +#include +#include +#include +#include + +namespace lms::core +{ + class IJob; + class IJobScheduler; +} // namespace lms::core + +namespace lms::scanner +{ + class JobQueue + { + public: + using ProcessFunction = std::function>)>; + + JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float drainThreshold); + ~JobQueue(); + JobQueue(const JobQueue&) = delete; + JobQueue& operator=(const JobQueue&) = delete; + + // push can wait and invoke the supplied ProcessFunction + void push(std::unique_ptr job); + void finish(); + + private: + void drainIfNeeded(); + + core::IJobScheduler& _scheduler; + const std::size_t _maxQueueSize; + ProcessFunction _processJobsDoneFunc; + const std::size_t _batchSize; + const float _drainThreshold; + std::vector> _jobsDone; + }; + +} // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp index d3e4146f..ed9d4b30 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp @@ -24,9 +24,9 @@ #include #include #include -#include #include "core/IConfig.hpp" +#include "core/IJob.hpp" #include "core/ILogger.hpp" #include "core/Path.hpp" #include "core/String.hpp" @@ -41,7 +41,7 @@ #include "database/Session.hpp" #include "database/Track.hpp" -#include "ArtworkUtils.hpp" +#include "JobQueue.hpp" #include "ScanContext.hpp" #include "ScannerSettings.hpp" @@ -49,42 +49,24 @@ namespace lms::scanner { namespace { - using ArtistArtwork = std::variant; // TODO handle embedded images in tracks? - bool isSameArtwork(ArtistArtwork preferredArtwork, const db::ObjectPtr& artwork) - { - if (std::holds_alternative(preferredArtwork)) - return !artwork; - - if (const db::ImageId* imageId = std::get_if(&preferredArtwork)) - return artwork && *imageId == artwork->getImageId(); - - if (const db::ArtworkId* artworkId = std::get_if(&preferredArtwork)) - return artwork && *artworkId == artwork->getId(); - - return false; - } - struct ArtistArtworkAssociation { - db::Artist::pointer artist; - ArtistArtwork preferredArtwork; + db::ArtistId artistId; + db::ArtworkId preferredArtworkId; }; using ArtistArtworkAssociationContainer = std::deque; - struct SearchArtistImageContext + struct SearchArtistArtworkParams { - db::Session& session; - db::ArtistId lastRetrievedArtistId; - std::size_t processedArtistCount{}; std::span artistFileNames; const ScannerSettings& settings; }; - db::Image::pointer findImageInDirectory(SearchArtistImageContext& searchContext, const std::filesystem::path& directoryPath, std::span fileStemsToSearch) + db::Image::pointer findImageInDirectory(db::Session& session, const std::filesystem::path& directoryPath, std::span fileStemsToSearch) { db::Image::pointer image; - const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) }; + const db::Directory::pointer directory{ db::Directory::find(session, directoryPath) }; if (directory) // may not exist for artists that are split on different media libraries { for (std::string_view fileStem : fileStemsToSearch) @@ -93,7 +75,7 @@ namespace lms::scanner params.setDirectory(directory->getId()); params.setFileStem(fileStem); - db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) { + db::Image::find(session, params, [&](const db::Image::pointer foundImg) { if (!image) image = foundImg; }); @@ -106,38 +88,38 @@ namespace lms::scanner return image; } - db::ImageId getImageFromMbid(SearchArtistImageContext& searchContext, const core::UUID& mbid) + db::Image::pointer getImageFromMbid(db::Session& session, const core::UUID& mbid) { db::Image::pointer image; // Find anywhere, since it is supposed to be unique! - db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid.getAsString()), [&](const db::Image::pointer foundImg) { + db::Image::find(session, db::Image::FindParameters{}.setFileStem(mbid.getAsString()), [&](const db::Image::pointer foundImg) { if (!image) image = foundImg; }); - return image ? image->getId() : db::ImageId{}; + return image; } - db::ImageId searchImageInArtistInfoDirectory(SearchArtistImageContext& searchContext, db::ArtistId artistId) + db::Image::pointer searchImageInArtistInfoDirectory(db::Session& session, db::ArtistId artistId) { db::Image::pointer image; std::vector fileInfoPaths; - db::ArtistInfo::find(searchContext.session, artistId, [&](const db::ArtistInfo::pointer& artistInfo) { + db::ArtistInfo::find(session, artistId, [&](const db::ArtistInfo::pointer& artistInfo) { fileInfoPaths.push_back(artistInfo->getAbsoluteFilePath()); if (!image) - image = findImageInDirectory(searchContext, artistInfo->getDirectory()->getAbsolutePath(), std::array{ "thumb", "folder" }); + image = findImageInDirectory(session, artistInfo->getDirectory()->getAbsolutePath(), std::array{ "thumb", "folder" }); }); if (fileInfoPaths.size() > 1) LMS_LOG(DBUPDATER, DEBUG, "Found " << fileInfoPaths.size() << " artist info files for same artist: " << core::stringUtils::joinStrings(fileInfoPaths, ", ")); - return image ? image->getId() : db::ImageId{}; + return image; } - db::ImageId searchImageInDirectories(SearchArtistImageContext& searchContext, db::ArtistId artistId) + db::Image::pointer searchImageInDirectories(db::Session& session, const SearchArtistArtworkParams& searchParams, db::ArtistId artistId) { db::Image::pointer image; @@ -145,7 +127,7 @@ namespace lms::scanner db::Directory::FindParameters params; params.setArtist(artistId, { db::TrackArtistLinkType::ReleaseArtist }); - db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) { + db::Directory::find(session, params, [&](const db::Directory::pointer& directory) { releasePaths.insert(directory->getAbsolutePath()); }); @@ -164,9 +146,9 @@ namespace lms::scanner std::filesystem::path directoryToInspect{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) }; while (true) { - image = findImageInDirectory(searchContext, directoryToInspect, searchContext.artistFileNames); + image = findImageInDirectory(session, directoryToInspect, searchParams.artistFileNames); if (image) - return image->getId(); + return image; std::filesystem::path parentPath{ directoryToInspect.parent_path() }; if (parentPath == directoryToInspect) @@ -181,106 +163,74 @@ namespace lms::scanner // /someOtherUserConfiguredArtistFile.jpg for (const std::filesystem::path& releasePath : releasePaths) { - image = findImageInDirectory(searchContext, releasePath, searchContext.artistFileNames); + image = findImageInDirectory(session, releasePath, searchParams.artistFileNames); if (image) - return image->getId(); + return image; } } - return image ? image->getId() : db::ImageId{}; + return image; } - db::ArtworkId getFirstReleaseArtwork(db::Session& session, const db::Artist::pointer& artist) + db::Artwork::pointer getFirstReleaseArtwork(db::Session& session, const db::Artist::pointer& artist) { - db::ArtworkId artworkId; + db::Artwork::pointer artwork; db::Release::FindParameters params; params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist }); params.setSortMethod(db::ReleaseSortMethod::OriginalDate); db::Release::find(session, params, [&](const db::Release::pointer& release) { - if (artworkId.isValid()) + if (artwork) return; - artworkId = release->getPreferredArtworkId(); + artwork = release->getPreferredArtwork(); }); - return artworkId; + return artwork; } - ArtistArtwork computePreferredArtwork(SearchArtistImageContext& searchContext, const db::Artist::pointer& artist) + db::Artwork::pointer computePreferredArtistArtwork(db::Session& session, const SearchArtistArtworkParams& searchParams, const db::Artist::pointer& artist) { - db::ImageId imageId; - if (const auto mbid{ artist->getMBID() }) { - imageId = getImageFromMbid(searchContext, *mbid); - if (imageId.isValid()) - return ArtistArtwork{ imageId }; + const db::Image::pointer image{ getImageFromMbid(session, *mbid) }; + if (image) + return db::Artwork::find(session, image->getId()); } - imageId = searchImageInArtistInfoDirectory(searchContext, artist->getId()); - if (imageId.isValid()) - return ArtistArtwork{ imageId }; - - imageId = searchImageInDirectories(searchContext, artist->getId()); - if (imageId.isValid()) - return ArtistArtwork{ imageId }; - - if (searchContext.settings.artistImageFallbackToRelease) - { - db::ArtworkId artworkId{ getFirstReleaseArtwork(searchContext.session, artist) }; - if (artworkId.isValid()) - return ArtistArtwork{ artworkId }; - } - - return ArtistArtwork{}; - } - - bool fetchNextArtistArtworksToUpdate(SearchArtistImageContext& searchContext, ArtistArtworkAssociationContainer& ArtistArtworkAssociations) - { - const db::ArtistId artistId{ searchContext.lastRetrievedArtistId }; - { - constexpr std::size_t readBatchSize{ 100 }; - - auto transaction{ searchContext.session.createReadTransaction() }; - - db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) { - ArtistArtwork preferredArtwork{ computePreferredArtwork(searchContext, artist) }; - - if (!isSameArtwork(preferredArtwork, artist->getPreferredArtwork())) - ArtistArtworkAssociations.push_back(ArtistArtworkAssociation{ artist, preferredArtwork }); - - searchContext.processedArtistCount++; - }); + const db::Image::pointer image{ searchImageInArtistInfoDirectory(session, artist->getId()) }; + if (image) + return db::Artwork::find(session, image->getId()); } - return artistId != searchContext.lastRetrievedArtistId; + { + const db::Image::pointer image{ searchImageInDirectories(session, searchParams, artist->getId()) }; + if (image) + return db::Artwork::find(session, image->getId()); + } + + if (searchParams.settings.artistImageFallbackToRelease) + { + const db::Artwork::pointer artwork{ getFirstReleaseArtwork(session, artist) }; + if (artwork) + return artwork; + } + + return db::Artwork::pointer{}; } - void updateArtistPreferredArtwork(db::Session& session, const ArtistArtworkAssociation& ArtistArtworkAssociation) + void updateArtistPreferredArtwork(db::Session& session, const ArtistArtworkAssociation& artistArtworkAssociation) { - db::Artist::pointer artist{ ArtistArtworkAssociation.artist }; - - db::Artwork::pointer artwork; - if (const db::ImageId * imageId{ std::get_if(&ArtistArtworkAssociation.preferredArtwork) }) - artwork = utils::getOrCreateArtworkFromImage(session, *imageId); - else if (const db::ArtworkId * artworkId{ std::get_if(&ArtistArtworkAssociation.preferredArtwork) }) - artwork = db::Artwork::find(session, *artworkId); - - artist.modify()->setPreferredArtwork(artwork); - if (artwork) - LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork for artist '" << artist->getName() << "' with image in " << artwork->getAbsoluteFilePath()); - else - LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from artist '" << artist->getName() << "'"); + db::Artist::updatePreferredArtwork(session, artistArtworkAssociation.artistId, artistArtworkAssociation.preferredArtworkId); } - void updateArtistArtworks(db::Session& session, ArtistArtworkAssociationContainer& imageAssociations) + void updateArtistPreferredArtworks(db::Session& session, ArtistArtworkAssociationContainer& imageAssociations, bool forceFullBatch) { constexpr std::size_t writeBatchSize{ 50 }; - while (!imageAssociations.empty()) + while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty()) { auto transaction{ session.createWriteTransaction() }; @@ -305,6 +255,62 @@ namespace lms::scanner return res; } + bool fetchNextArtistIdRange(db::Session& session, db::ArtistId& lastRetrievedId, db::IdRange& idRange) + { + constexpr std::size_t readBatchSize{ 100 }; + + auto transaction{ session.createReadTransaction() }; + + idRange = db::Artist::findNextRange(session, lastRetrievedId, readBatchSize); + lastRetrievedId = idRange.last; + + return idRange.isValid(); + } + + class ComputeArtistArtworkAssociationsJob : public core::IJob + { + public: + ComputeArtistArtworkAssociationsJob(db::Db& db, const SearchArtistArtworkParams& searchParams, db::IdRange artistIdRange) + : _db{ db } + , _searchParams{ searchParams } + , _artistIdRange{ artistIdRange } + { + } + + std::span getAssociations() const { return _associations; } + std::size_t getProcessedArtistCount() const { return _processedArtistCount; } + + private: + core::LiteralString getName() const override { return "Associate Track Artworks"; } + void run() override + { + auto& session{ _db.getTLSSession() }; + auto transaction{ session.createReadTransaction() }; + + db::Artist::find(session, _artistIdRange, [this, &session](const db::Artist::pointer& artist) { + const db::Artwork::pointer preferredArtwork{ computePreferredArtistArtwork(session, _searchParams, artist) }; + + if (artist->getPreferredArtwork() != preferredArtwork) + { + _associations.push_back(ArtistArtworkAssociation{ artist->getId(), preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{} }); + + if (preferredArtwork) + LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork for artist '" << artist->getName() << "' with image in " << preferredArtwork->getAbsoluteFilePath()); + else + LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from artist '" << artist->getName() << "'"); + } + + _processedArtistCount++; + }); + } + + db::Db& _db; + const SearchArtistArtworkParams& _searchParams; + db::IdRange _artistIdRange; + std::vector _associations; + std::size_t _processedArtistCount{}; + }; + } // namespace ScanStepAssociateArtistImages::ScanStepAssociateArtistImages(InitParams& initParams) @@ -333,22 +339,40 @@ namespace lms::scanner context.currentStepStats.totalElems = db::Artist::getCount(session); } - SearchArtistImageContext searchContext{ - .session = session, - .lastRetrievedArtistId = {}, + const SearchArtistArtworkParams searchParams{ .artistFileNames = _artistFileNames, .settings = _settings, }; - ArtistArtworkAssociationContainer ArtistArtworkAssociations; - while (fetchNextArtistArtworksToUpdate(searchContext, ArtistArtworkAssociations)) - { + ArtistArtworkAssociationContainer artistArtworkAssociations; + auto processJobsDone = [&](std::span> jobs) { if (_abortScan) return; - updateArtistArtworks(session, ArtistArtworkAssociations); - context.currentStepStats.processedElems = searchContext.processedArtistCount; + for (const auto& job : jobs) + { + const auto& associationJob{ static_cast(*job) }; + const auto& artistAssociations{ associationJob.getAssociations() }; + + artistArtworkAssociations.insert(std::end(artistArtworkAssociations), std::cbegin(artistAssociations), std::cend(artistAssociations)); + + context.currentStepStats.processedElems += associationJob.getProcessedArtistCount(); + } + + updateArtistPreferredArtworks(session, artistArtworkAssociations, true); _progressCallback(context.currentStepStats); - } + }; + + JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F }; + + db::ArtistId lastRetrievedArtistId{}; + db::IdRange artistIdRange; + while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange)) + queue.push(std::make_unique(_db, searchParams, artistIdRange)); + + queue.finish(); + + // process all remaining associations + updateArtistPreferredArtworks(session, artistArtworkAssociations, false); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp index adba6876..b6a03229 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp @@ -23,9 +23,9 @@ #include #include #include -#include #include "core/IConfig.hpp" +#include "core/IJob.hpp" #include "core/ILogger.hpp" #include "core/Path.hpp" #include "database/Artwork.hpp" @@ -36,92 +36,73 @@ #include "database/Session.hpp" #include "database/Track.hpp" #include "database/TrackEmbeddedImage.hpp" -#include "database/TrackEmbeddedImageId.hpp" -#include "ArtworkUtils.hpp" +#include "JobQueue.hpp" #include "ScanContext.hpp" namespace lms::scanner { namespace { - using ReleaseArtwork = std::variant; - bool isSameArtwork(ReleaseArtwork preferredArtwork, const db::ObjectPtr& artwork) - { - if (std::holds_alternative(preferredArtwork)) - return !artwork; - - if (const db::TrackEmbeddedImageId* trackEmbeddedImageId = std::get_if(&preferredArtwork)) - return artwork && *trackEmbeddedImageId == artwork->getTrackEmbeddedImageId(); - - if (const db::ImageId* imageId = std::get_if(&preferredArtwork)) - return artwork && *imageId == artwork->getImageId(); - - return false; - } - struct ReleaseArtworkAssociation { - db::Release::pointer release; - ReleaseArtwork preferredArtwork; + db::ReleaseId releaseId; + db::ArtworkId preferredArtworkId; }; using ReleaseArtworkAssociationContainer = std::deque; - struct SearchReleaseArtworkContext + struct SearchReleaseArtworkParams { - db::Session& session; - db::ReleaseId lastRetrievedReleaseId; - std::size_t processedReleaseCount{}; const std::vector& releaseImageFileNames; }; - db::Image::pointer findImageInDirectory(SearchReleaseArtworkContext& searchContext, const std::filesystem::path& directoryPath) + db::Artwork::pointer findImageInDirectory(db::Session& session, const SearchReleaseArtworkParams& searchParams, const std::filesystem::path& directoryPath) { - db::Image::pointer image; + db::Artwork::pointer artwork; - const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) }; + const db::Directory::pointer directory{ db::Directory::find(session, directoryPath) }; if (directory) // may not exist for releases that are split on different media libraries { - for (std::string_view fileStem : searchContext.releaseImageFileNames) + for (std::string_view fileStem : searchParams.releaseImageFileNames) { db::Image::FindParameters params; params.setDirectory(directory->getId()); params.setFileStem(fileStem); - db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) { - if (!image) - image = foundImg; + db::Image::find(session, params, [&](const db::Image::pointer& image) { + if (!artwork) + artwork = db::Artwork::find(session, image->getId()); }); - if (image) + if (artwork) break; } } - return image; + return artwork; } - db::Image::pointer computePreferredReleaseImage(SearchReleaseArtworkContext& searchContext, const db::Release::pointer& release) + db::Artwork::pointer computePreferredReleaseImage(db::Session& session, const SearchReleaseArtworkParams& searchParams, const db::Release::pointer& release) { - db::Image::pointer image; + db::Artwork::pointer artwork; const auto mbid{ release->getMBID() }; if (mbid) { // Find anywhere, since it is suppoed to be unique! - db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer foundImg) { - if (!image) - image = foundImg; + db::Image::find(session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer& image) { + if (!artwork) + artwork = db::Artwork::find(session, image->getId()); }); } - if (!image) + if (!artwork) { std::set releasePaths; db::Directory::FindParameters params; params.setRelease(release->getId()); - db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) { + db::Directory::find(session, params, [&](const db::Directory::pointer& directory) { releasePaths.insert(directory->getAbsolutePath()); }); @@ -132,31 +113,30 @@ namespace lms::scanner if (releasePaths.size() > 1) { const std::filesystem::path releasePath{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) }; - image = findImageInDirectory(searchContext, releasePath); + artwork = findImageInDirectory(session, searchParams, releasePath); } - if (!image) + if (!artwork) { for (const std::filesystem::path& releasePath : releasePaths) { - image = findImageInDirectory(searchContext, releasePath); - if (image) + artwork = findImageInDirectory(session, searchParams, releasePath); + if (artwork) break; } } } - return image; + return artwork; } - ReleaseArtwork computePreferredReleaseArtwork(SearchReleaseArtworkContext& searchContext, const db::Release::pointer& release) + db::Artwork::pointer computePreferredReleaseArtwork(db::Session& session, const SearchReleaseArtworkParams& searchParams, const db::Release::pointer& release) { - const db::Image::pointer image{ computePreferredReleaseImage(searchContext, release) }; - if (image) - return ReleaseArtwork{ image->getId() }; + db::Artwork::pointer artwork{ computePreferredReleaseImage(session, searchParams, release) }; + if (artwork) + return artwork; // Fallback on embedded Front image - db::TrackEmbeddedImageId trackEmbeddedImageId; { db::TrackEmbeddedImage::FindParameters params; params.setRelease(release->getId()); @@ -164,11 +144,11 @@ namespace lms::scanner params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc); params.setRange(db::Range{ .offset = 0, .size = 1 }); - db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { trackEmbeddedImageId = image->getId(); }); + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); }); } - if (trackEmbeddedImageId.isValid()) - return ReleaseArtwork{ trackEmbeddedImageId }; + if (artwork) + return artwork; // Fallback on embedded media image { @@ -178,66 +158,28 @@ namespace lms::scanner params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc); params.setRange(db::Range{ .offset = 0, .size = 1 }); - db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { trackEmbeddedImageId = image->getId(); }); + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); }); } - if (trackEmbeddedImageId.isValid()) - return ReleaseArtwork{ trackEmbeddedImageId }; - - return ReleaseArtwork{}; + return artwork; } - bool fetchNextReleaseArtworksToUpdate(SearchReleaseArtworkContext& searchContext, ReleaseArtworkAssociationContainer& ReleaseArtworkAssociations) + void updateReleasePreferredArtwork(db::Session& session, const ReleaseArtworkAssociation& releaseArtworkAssociation) { - const db::ReleaseId releaseId{ searchContext.lastRetrievedReleaseId }; - - { - constexpr std::size_t readBatchSize{ 100 }; - - auto transaction{ searchContext.session.createReadTransaction() }; - - db::Release::find(searchContext.session, searchContext.lastRetrievedReleaseId, readBatchSize, [&](const db::Release::pointer& release) { - const ReleaseArtwork preferredArtwork{ computePreferredReleaseArtwork(searchContext, release) }; - const db::Artwork::pointer currentPreferredArtwork{ release->getPreferredArtwork() }; - - if (!isSameArtwork(preferredArtwork, currentPreferredArtwork)) - ReleaseArtworkAssociations.push_back(ReleaseArtworkAssociation{ release, preferredArtwork }); - searchContext.processedReleaseCount++; - }); - } - - return releaseId != searchContext.lastRetrievedReleaseId; + db::Release::updatePreferredArtwork(session, releaseArtworkAssociation.releaseId, releaseArtworkAssociation.preferredArtworkId); } - void updateReleaseArtwork(db::Session& session, const ReleaseArtworkAssociation& ReleaseArtworkAssociation) - { - db::Release::pointer release{ ReleaseArtworkAssociation.release }; - - db::Artwork::pointer artwork; - if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if(&ReleaseArtworkAssociation.preferredArtwork) }) - artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId); - else if (const db::ImageId * imageId{ std::get_if(&ReleaseArtworkAssociation.preferredArtwork) }) - artwork = utils::getOrCreateArtworkFromImage(session, *imageId); - - release.modify()->setPreferredArtwork(artwork); - - if (artwork) - LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork in release '" << release->getName() << "' with image in " << artwork->getAbsoluteFilePath()); - else - LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from release '" << release->getName() << "'"); - } - - void updateReleaseArtworks(db::Session& session, ReleaseArtworkAssociationContainer& imageAssociations) + void updateReleasePreferredArtworks(db::Session& session, ReleaseArtworkAssociationContainer& imageAssociations, bool forceFullBatch) { constexpr std::size_t writeBatchSize{ 50 }; - while (!imageAssociations.empty()) + while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty()) { auto transaction{ session.createWriteTransaction() }; for (std::size_t i{}; !imageAssociations.empty() && i < writeBatchSize; ++i) { - updateReleaseArtwork(session, imageAssociations.front()); + updateReleasePreferredArtwork(session, imageAssociations.front()); imageAssociations.pop_front(); } } @@ -256,6 +198,62 @@ namespace lms::scanner return res; } + bool fetchNextReleaseIdRange(db::Session& session, db::ReleaseId& lastRetrievedId, db::IdRange& idRange) + { + constexpr std::size_t readBatchSize{ 100 }; + + auto transaction{ session.createReadTransaction() }; + + idRange = db::Release::findNextRange(session, lastRetrievedId, readBatchSize); + lastRetrievedId = idRange.last; + + return idRange.isValid(); + } + + class ComputeReleaseArtworkAssociationsJob : public core::IJob + { + public: + ComputeReleaseArtworkAssociationsJob(db::Db& db, const SearchReleaseArtworkParams& searchParams, db::IdRange artistIdRange) + : _db{ db } + , _searchParams{ searchParams } + , _artistIdRange{ artistIdRange } + { + } + + std::span getAssociations() const { return _associations; } + std::size_t getProcessedReleaseCount() const { return _processedReleaseCount; } + + private: + core::LiteralString getName() const override { return "Associate Track Artworks"; } + void run() override + { + auto& session{ _db.getTLSSession() }; + auto transaction{ session.createReadTransaction() }; + + db::Release::find(session, _artistIdRange, [this, &session](const db::Release::pointer& release) { + const db::Artwork::pointer preferredArtwork{ computePreferredReleaseArtwork(session, _searchParams, release) }; + + if (release->getPreferredArtwork() != preferredArtwork) + { + _associations.push_back(ReleaseArtworkAssociation{ release->getId(), preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{} }); + + if (preferredArtwork) + LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork for release '" << release->getName() << "' with image in " << preferredArtwork->getAbsoluteFilePath()); + else + LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from release '" << release->getName() << "'"); + } + + _processedReleaseCount++; + }); + } + + db::Db& _db; + const SearchReleaseArtworkParams& _searchParams; + db::IdRange _artistIdRange; + std::vector _associations; + std::size_t _processedReleaseCount{}; + }; + } // namespace ScanStepAssociateReleaseImages::ScanStepAssociateReleaseImages(InitParams& initParams) @@ -278,21 +276,39 @@ namespace lms::scanner context.currentStepStats.totalElems = db::Release::getCount(session); } - SearchReleaseArtworkContext searchContext{ - .session = session, - .lastRetrievedReleaseId = {}, + const SearchReleaseArtworkParams searchParams{ .releaseImageFileNames = _releaseImageFileNames, }; - ReleaseArtworkAssociationContainer ReleaseArtworkAssociations; - while (fetchNextReleaseArtworksToUpdate(searchContext, ReleaseArtworkAssociations)) - { + ReleaseArtworkAssociationContainer artistArtworkAssociations; + auto processJobsDone = [&](std::span> jobs) { if (_abortScan) return; - updateReleaseArtworks(session, ReleaseArtworkAssociations); - context.currentStepStats.processedElems = searchContext.processedReleaseCount; + for (const auto& job : jobs) + { + const auto& associationJob{ static_cast(*job) }; + const auto& artistAssociations{ associationJob.getAssociations() }; + + artistArtworkAssociations.insert(std::end(artistArtworkAssociations), std::cbegin(artistAssociations), std::cend(artistAssociations)); + + context.currentStepStats.processedElems += associationJob.getProcessedReleaseCount(); + } + + updateReleasePreferredArtworks(session, artistArtworkAssociations, true); _progressCallback(context.currentStepStats); - } + }; + + JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F }; + + db::ReleaseId lastRetrievedReleaseId{}; + db::IdRange artistIdRange; + while (fetchNextReleaseIdRange(session, lastRetrievedReleaseId, artistIdRange)) + queue.push(std::make_unique(_db, searchParams, artistIdRange)); + + queue.finish(); + + // process all remaining associations + updateReleasePreferredArtworks(session, artistArtworkAssociations, false); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp index 035c505d..7514d85f 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp @@ -21,20 +21,23 @@ #include #include -#include +#include +#include "core/IJob.hpp" +#include "core/IJobScheduler.hpp" #include "core/ILogger.hpp" +#include "core/LiteralString.hpp" #include "database/Artwork.hpp" #include "database/Db.hpp" #include "database/Directory.hpp" +#include "database/IdRange.hpp" #include "database/Image.hpp" #include "database/Release.hpp" #include "database/Session.hpp" #include "database/Track.hpp" #include "database/TrackEmbeddedImage.hpp" -#include "database/TrackEmbeddedImageId.hpp" -#include "ArtworkUtils.hpp" +#include "JobQueue.hpp" #include "ScanContext.hpp" namespace lms::scanner @@ -42,37 +45,11 @@ namespace lms::scanner namespace { // May come from an embedded image in a track, or from what has been previously resolved for the release - using TrackArtwork = std::variant; - bool isSameArtwork(TrackArtwork preferredArtwork, const db::ObjectPtr& artwork) - { - if (std::holds_alternative(preferredArtwork)) - return !artwork; - - if (const db::TrackEmbeddedImageId* trackEmbeddedImageId = std::get_if(&preferredArtwork)) - { - assert(trackEmbeddedImageId->isValid()); - return artwork && *trackEmbeddedImageId == artwork->getTrackEmbeddedImageId(); - } - - if (const db::ArtworkId* artworkId = std::get_if(&preferredArtwork)) - { - assert(artworkId->isValid()); - return artwork && *artworkId == artwork->getId(); - } - - return false; - } - - bool isValid(const TrackArtwork& res) - { - return !std::holds_alternative(res); - } - struct TrackArtworksAssociation { - db::Track::pointer track; - TrackArtwork preferredArtwork; - TrackArtwork preferredMediaArtwork; + db::TrackId trackId; + std::optional preferredArtworkId; + std::optional preferredMediaArtworkId; }; using TrackArtworksAssociationContainer = std::deque; @@ -83,11 +60,10 @@ namespace lms::scanner std::size_t processedTrackCount{}; }; - TrackArtwork computePreferredTrackArtwork(SearchTrackArtworkContext& searchContext, const db::Track::pointer& track, TrackArtwork preferredMediaArtwork) + db::Artwork::pointer computePreferredTrackArtwork(db::Session& session, const db::Track::pointer& track, const db::Artwork::pointer& preferredMediaArtwork) { - TrackArtwork res{ preferredMediaArtwork }; - - if (isValid(res)) + db::Artwork::pointer res{ preferredMediaArtwork }; + if (res) return res; // Fallback on front cover for this track @@ -97,10 +73,10 @@ namespace lms::scanner params.setImageTypes({ db::ImageType::FrontCover }); params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc); params.setRange(db::Range{ .offset = 0, .size = 1 }); - db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); }); + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); }); } - if (isValid(res)) + if (res) return res; // Fallback on the artwork already resolved for the release @@ -108,28 +84,26 @@ namespace lms::scanner if (!releaseId.isValid()) return res; - if (const db::Release::pointer release{ db::Release::find(searchContext.session, releaseId) }) - { - if (const db::ArtworkId releaseArtworkId{ release->getPreferredArtworkId() }; releaseArtworkId.isValid()) - res = releaseArtworkId; - } + if (const db::Release::pointer release{ db::Release::find(session, releaseId) }) + res = release->getPreferredArtwork(); return res; } - TrackArtwork computePreferredTrackMediaArtwork(SearchTrackArtworkContext& searchContext, const db::Track::pointer& track) + db::Artwork::pointer computePreferredTrackMediaArtwork(db::Session& session, const db::Track::pointer& track) { - TrackArtwork res; + db::Artwork::pointer res; + { db::TrackEmbeddedImage::FindParameters params; params.setTrack(track->getId()); params.setImageTypes({ db::ImageType::Media }); params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc); params.setRange(db::Range{ .offset = 0, .size = 1 }); - db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); }); + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); }); } - if (isValid(res)) + if (res) return res; // fallback on another track of the same disc @@ -145,95 +119,28 @@ namespace lms::scanner params.setImageTypes({ db::ImageType::Media }); params.setSortMethod(db::TrackEmbeddedImageSortMethod::TrackNumberThenSizeDesc); params.setRange(db::Range{ .offset = 0, .size = 1 }); - db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); }); + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); }); } return res; } - bool fetchNextTrackArtworksToUpdate(SearchTrackArtworkContext& searchContext, TrackArtworksAssociationContainer& TrackArtworksAssociations) + void updateTrackPreferredArtworks(db::Session& session, const TrackArtworksAssociation& trackArtworksAssociation) { - const db::TrackId trackId{ searchContext.lastRetrievedTrackId }; + assert(trackArtworksAssociation.preferredArtworkId || trackArtworksAssociation.preferredMediaArtworkId); - { - constexpr std::size_t readBatchSize{ 100 }; + if (trackArtworksAssociation.preferredArtworkId) + db::Track::updatePreferredArtwork(session, trackArtworksAssociation.trackId, *trackArtworksAssociation.preferredArtworkId); - auto transaction{ searchContext.session.createReadTransaction() }; - - db::Track::find(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize, [&](const db::Track::pointer& track) { - const TrackArtwork preferredMediaArtwork{ computePreferredTrackMediaArtwork(searchContext, track) }; - const TrackArtwork preferredArtwork{ computePreferredTrackArtwork(searchContext, track, preferredMediaArtwork) }; - - const db::Artwork::pointer currentPreferredArtwork{ track->getPreferredArtwork() }; - const db::Artwork::pointer currentPreferredMediaArtwork{ track->getPreferredMediaArtwork() }; - - if (!isSameArtwork(preferredArtwork, currentPreferredArtwork) - || !isSameArtwork(preferredMediaArtwork, currentPreferredMediaArtwork)) - { - TrackArtworksAssociations.push_back(TrackArtworksAssociation{ .track = track, .preferredArtwork = preferredArtwork, .preferredMediaArtwork = preferredMediaArtwork }); - } - - searchContext.processedTrackCount++; - }); - } - - return trackId != searchContext.lastRetrievedTrackId; + if (trackArtworksAssociation.preferredMediaArtworkId) + db::Track::updatePreferredMediaArtwork(session, trackArtworksAssociation.trackId, *trackArtworksAssociation.preferredMediaArtworkId); } - void updateTrackPreferredArtwork(db::Session& session, db::Track::pointer& track, TrackArtwork preferredArtwork) - { - db::Artwork::pointer artwork; - if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if(&preferredArtwork) }) - artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId); - else if (const db::ArtworkId * artworkId{ std::get_if(&preferredArtwork) }) - artwork = db::Artwork::find(session, *artworkId); - - // Using track.modify() is quite CPU intensive as the track class has too many fields - db::Track::updatePreferredArtwork(session, track->getId(), artwork ? artwork->getId() : db::ArtworkId{}); - if (artwork) - LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork in track " << track->getAbsoluteFilePath() << " with image in " << artwork->getAbsoluteFilePath()); - else - LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from track " << track->getAbsoluteFilePath()); - } - - void updateTrackPreferredMediaArtwork(db::Session& session, db::Track::pointer& track, TrackArtwork preferredArtwork) - { - db::Artwork::pointer artwork; - if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if(&preferredArtwork) }) - artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId); - else if (const db::ArtworkId * artworkId{ std::get_if(&preferredArtwork) }) - artwork = db::Artwork::find(session, *artworkId); - - // Using track.modify() is quite CPU intensive as the track class has too many fields - db::Track::updatePreferredMediaArtwork(session, track->getId(), artwork ? artwork->getId() : db::ArtworkId{}); - if (artwork) - LMS_LOG(DBUPDATER, DEBUG, "Updated preferred media artwork in track " << track->getAbsoluteFilePath() << " with image in " << artwork->getAbsoluteFilePath()); - else - LMS_LOG(DBUPDATER, DEBUG, "Removed preferred media artwork from track " << track->getAbsoluteFilePath()); - } - - void updateTrackPreferredArtworks(db::Session& session, const TrackArtworksAssociation& TrackArtworksAssociation) - { - db::Track::pointer track{ TrackArtworksAssociation.track }; - - { - const db::Artwork::pointer currentPreferredArtwork{ track->getPreferredArtwork() }; - if (!isSameArtwork(TrackArtworksAssociation.preferredArtwork, currentPreferredArtwork)) - updateTrackPreferredArtwork(session, track, TrackArtworksAssociation.preferredArtwork); - } - - { - const db::Artwork::pointer currentPreferredMediaArtwork{ track->getPreferredMediaArtwork() }; - if (!isSameArtwork(TrackArtworksAssociation.preferredMediaArtwork, currentPreferredMediaArtwork)) - updateTrackPreferredMediaArtwork(session, track, TrackArtworksAssociation.preferredMediaArtwork); - } - } - - void updateTrackPreferredArtworks(db::Session& session, TrackArtworksAssociationContainer& imageAssociations) + void updateTrackPreferredArtworks(db::Session& session, TrackArtworksAssociationContainer& imageAssociations, bool forceFullBatch) { constexpr std::size_t writeBatchSize{ 50 }; - while (!imageAssociations.empty()) + while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty()) { auto transaction{ session.createWriteTransaction() }; @@ -244,6 +151,76 @@ namespace lms::scanner } } } + + bool fetchNextTrackIdRange(SearchTrackArtworkContext& searchContext, db::IdRange& trackIdRange) + { + constexpr std::size_t readBatchSize{ 100 }; + + auto transaction{ searchContext.session.createReadTransaction() }; + + trackIdRange = db::Track::findNextRange(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize); + searchContext.lastRetrievedTrackId = trackIdRange.last; + + return trackIdRange.isValid(); + } + + class ComputeTrackArtworkAssociationsJob : public core::IJob + { + public: + ComputeTrackArtworkAssociationsJob(db::Db& db, db::IdRange trackIdRange) + : _db{ db } + , _trackIdRange{ trackIdRange } + { + } + + std::span getTrackAssociations() const { return _trackAssociations; } + std::size_t getProcessedTrackCount() const { return _processedTrackCount; } + + private: + core::LiteralString getName() const override { return "Associate Track Artworks"; } + void run() override + { + auto& session{ _db.getTLSSession() }; + auto transaction{ session.createReadTransaction() }; + + db::Track::find(session, _trackIdRange, [&](const db::Track::pointer& track) { + const db::Artwork::pointer preferredMediaArtwork{ computePreferredTrackMediaArtwork(session, track) }; + const db::Artwork::pointer preferredArtwork{ computePreferredTrackArtwork(session, track, preferredMediaArtwork) }; + + TrackArtworksAssociation artworksAssociation{ .trackId = track->getId(), .preferredArtworkId = std::nullopt, .preferredMediaArtworkId = std::nullopt }; + + if (track->getPreferredArtwork() != preferredArtwork) + { + artworksAssociation.preferredArtworkId = preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{}; + + if (preferredArtwork) + LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork in track " << track->getAbsoluteFilePath() << " with image in " << preferredArtwork->getAbsoluteFilePath()); + else + LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from track " << track->getAbsoluteFilePath()); + } + + if (track->getPreferredMediaArtwork() != preferredMediaArtwork) + { + artworksAssociation.preferredMediaArtworkId = preferredMediaArtwork ? preferredMediaArtwork->getId() : db::ArtworkId{}; + + if (preferredMediaArtwork) + LMS_LOG(DBUPDATER, DEBUG, "Updating preferred media artwork in track " << track->getAbsoluteFilePath() << " with image in " << preferredMediaArtwork->getAbsoluteFilePath()); + else + LMS_LOG(DBUPDATER, DEBUG, "Removing preferred media artwork from track " << track->getAbsoluteFilePath()); + } + + if (artworksAssociation.preferredArtworkId || artworksAssociation.preferredMediaArtworkId) + _trackAssociations.push_back(artworksAssociation); + + _processedTrackCount++; + }); + } + + db::Db& _db; + db::IdRange _trackIdRange; + std::vector _trackAssociations; + std::size_t _processedTrackCount{}; + }; } // namespace ScanStepAssociateTrackImages::ScanStepAssociateTrackImages(InitParams& initParams) @@ -270,15 +247,34 @@ namespace lms::scanner .lastRetrievedTrackId = {}, }; - TrackArtworksAssociationContainer TrackArtworksAssociations; - while (fetchNextTrackArtworksToUpdate(searchContext, TrackArtworksAssociations)) - { + TrackArtworksAssociationContainer trackArtworksAssociations; + auto processTracks = [&](std::span> jobs) { if (_abortScan) return; - updateTrackPreferredArtworks(session, TrackArtworksAssociations); - context.currentStepStats.processedElems = searchContext.processedTrackCount; + for (const auto& job : jobs) + { + const auto& associationJob{ static_cast(*job) }; + const auto& trackAssociations{ associationJob.getTrackAssociations() }; + + trackArtworksAssociations.insert(std::end(trackArtworksAssociations), std::cbegin(trackAssociations), std::cend(trackAssociations)); + + context.currentStepStats.processedElems += associationJob.getProcessedTrackCount(); + } + + updateTrackPreferredArtworks(session, trackArtworksAssociations, true); _progressCallback(context.currentStepStats); - } + }; + + JobQueue queue{ getJobScheduler(), 20, processTracks, 1, 0.85F }; + + db::IdRange trackIdRange; + while (fetchNextTrackIdRange(searchContext, trackIdRange)) + queue.push(std::make_unique(_db, trackIdRange)); + + queue.finish(); + + // process all remaining associations + updateTrackPreferredArtworks(session, trackArtworksAssociations, false); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp index 5ed9873e..27f13d50 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp @@ -31,6 +31,7 @@ #include "scanners/IFileScanOperation.hpp" #include "scanners/IFileScanner.hpp" +#include "JobQueue.hpp" #include "ScanContext.hpp" namespace lms::scanner @@ -114,11 +115,16 @@ namespace lms::scanner void ScanStepScanFiles::process(ScanContext& context, const MediaLibraryInfo& mediaLibrary) { - core::IJobScheduler& jobScheduler{ getJobScheduler() }; - assert(jobScheduler.getJobsDoneCount() == 0); - - const std::size_t scanQueueMaxScanRequestCount{ 100 * jobScheduler.getThreadCount() }; + const std::size_t scanQueueMaxScanRequestCount{ 50 * getJobScheduler().getThreadCount() }; constexpr std::size_t processFileResultsBatchSize{ 10 }; + constexpr float drainRatio{ 0.85 }; + + auto processDoneJobs = [&](std::span> jobsDone) { + if (!_abortScan) + processFileScanResults(context, jobsDone); + }; + + JobQueue queue{ getJobScheduler(), scanQueueMaxScanRequestCount, processDoneJobs, processFileResultsBatchSize, drainRatio }; std::vector> jobsDone; std::vector> scanOperations; @@ -148,7 +154,7 @@ namespace lms::scanner if (context.scanOptions.fullScan || scanner->needsScan(fileToScan)) { auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) }; - jobScheduler.scheduleJob(std::make_unique(std::move(scanOperation))); + queue.push(std::make_unique(std::move(scanOperation))); } } @@ -156,30 +162,9 @@ namespace lms::scanner _progressCallback(context.currentStepStats); } - while (jobScheduler.getJobsDoneCount() > (scanQueueMaxScanRequestCount / 2)) - { - jobScheduler.popJobsDone(jobsDone, processFileResultsBatchSize); - processFileScanResults(context, jobsDone); - jobsDone.clear(); - } - - jobScheduler.waitUntilJobCountAtMost(scanQueueMaxScanRequestCount); - return true; }, &excludeDirFileName); - - jobScheduler.wait(); - - while (jobScheduler.popJobsDone(jobsDone, processFileResultsBatchSize) > 0) - { - if (!_abortScan) - processFileScanResults(context, jobsDone); - - jobsDone.clear(); - } - - assert(jobScheduler.getJobsDoneCount() == 0); } void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span> scanJobs) @@ -191,9 +176,6 @@ namespace lms::scanner for (auto& scanJob : scanJobs) { - if (_abortScan) - return; - IFileScanOperation& scanOperation{ static_cast(*scanJob).getScanOperation() }; LMS_LOG(DBUPDATER, DEBUG, scanOperation.getName() << ": processing result for " << scanOperation.getFilePath());