From 1b73ae1e8c48766c0bc9694b43c1b3e6fc7069ff Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 5 Jul 2025 19:28:18 +0200 Subject: [PATCH] Scanner: parallelized processing within the playlist/track association step --- src/libs/database/impl/Artist.cpp | 2 +- src/libs/database/impl/PlayListFile.cpp | 21 ++ src/libs/database/impl/Release.cpp | 2 +- src/libs/database/impl/Track.cpp | 2 +- src/libs/database/include/database/Artist.hpp | 2 +- .../include/database/PlayListFile.hpp | 3 + .../database/include/database/Release.hpp | 2 +- src/libs/database/include/database/Track.hpp | 2 +- src/libs/database/test/Artist.cpp | 18 +- src/libs/database/test/Release.cpp | 18 +- src/libs/database/test/Track.cpp | 18 +- .../steps/ScanStepAssociateArtistImages.cpp | 4 +- .../steps/ScanStepAssociatePlayListTracks.cpp | 207 +++++++++++------- .../steps/ScanStepAssociateReleaseImages.cpp | 4 +- .../steps/ScanStepAssociateTrackImages.cpp | 33 +-- 15 files changed, 194 insertions(+), 144 deletions(-) diff --git a/src/libs/database/impl/Artist.cpp b/src/libs/database/impl/Artist.cpp index d3b106fe..04e44676 100644 --- a/src/libs/database/impl/Artist.cpp +++ b/src/libs/database/impl/Artist.cpp @@ -291,7 +291,7 @@ namespace lms::db utils::forEachQueryRangeResult(query, params.range, func); } - IdRange Artist::findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count) + IdRange Artist::findNextIdRange(Session& session, ArtistId lastRetrievedId, std::size_t count) { auto query{ session.getDboSession()->query>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT a.id FROM artist a WHERE a.id > ? ORDER BY a.id LIMIT ?) sub") }; query.bind(lastRetrievedId); diff --git a/src/libs/database/impl/PlayListFile.cpp b/src/libs/database/impl/PlayListFile.cpp index 49226d58..1f408f4c 100644 --- a/src/libs/database/impl/PlayListFile.cpp +++ b/src/libs/database/impl/PlayListFile.cpp @@ -83,6 +83,27 @@ namespace lms::db }); } + void PlayListFile::find(Session& session, const IdRange& idRange, const std::function& func) + { + assert(idRange.isValid()); + + auto query{ session.getDboSession()->query>("SELECT pl_f from playlist_file pl_f").orderBy("pl_f.id").where("pl_f.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) }; + + utils::forEachQueryResult(query, [&](const PlayListFile::pointer& release) { + func(release); + }); + } + + IdRange PlayListFile::findNextIdRange(Session& session, PlayListFileId lastRetrievedId, std::size_t count) + { + auto query{ session.getDboSession()->query>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT pl_f.id FROM playlist_file pl_f WHERE pl_f.id > ? ORDER BY pl_f.id LIMIT ?) sub") }; + query.bind(lastRetrievedId); + query.bind(static_cast(count)); + + auto res{ utils::fetchQuerySingleResult(query) }; + return IdRange{ .first = std::get<0>(res), .last = std::get<1>(res) }; + } + PlayListFile::pointer PlayListFile::find(Session& session, PlayListFileId id) { session.checkReadTransaction(); diff --git a/src/libs/database/impl/Release.cpp b/src/libs/database/impl/Release.cpp index d78d6f26..9b740d21 100644 --- a/src/libs/database/impl/Release.cpp +++ b/src/libs/database/impl/Release.cpp @@ -515,7 +515,7 @@ namespace lms::db }); } - IdRange Release::findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count) + IdRange Release::findNextIdRange(Session& session, ReleaseId lastRetrievedId, std::size_t count) { auto query{ session.getDboSession()->query>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT r.id FROM release r WHERE r.id > ? ORDER BY r.id LIMIT ?) sub") }; query.bind(lastRetrievedId); diff --git a/src/libs/database/impl/Track.cpp b/src/libs/database/impl/Track.cpp index 9b6384fd..c6d6cadb 100644 --- a/src/libs/database/impl/Track.cpp +++ b/src/libs/database/impl/Track.cpp @@ -303,7 +303,7 @@ namespace lms::db }); } - IdRange Track::findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count) + IdRange Track::findNextIdRange(Session& session, TrackId lastRetrievedId, std::size_t count) { auto query{ session.getDboSession()->query>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT t.id FROM track t WHERE t.id > ? ORDER BY t.id LIMIT ?) sub") }; query.bind(lastRetrievedId); diff --git a/src/libs/database/include/database/Artist.hpp b/src/libs/database/include/database/Artist.hpp index dd38ed0c..051b7025 100644 --- a/src/libs/database/include/database/Artist.hpp +++ b/src/libs/database/include/database/Artist.hpp @@ -129,7 +129,7 @@ namespace lms::db static void find(Session& session, const IdRange& idRange, const std::function& func); static RangeResults find(Session& session, const FindParameters& params); static void find(Session& session, const FindParameters& params, std::function func); - static IdRange findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count); + static IdRange findNextIdRange(Session& session, ArtistId lastRetrievedId, std::size_t count); static RangeResults findIds(Session& session, const FindParameters& params); static RangeResults findOrphanIds(Session& session, std::optional range = std::nullopt); // No track related static bool exists(Session& session, ArtistId id); diff --git a/src/libs/database/include/database/PlayListFile.hpp b/src/libs/database/include/database/PlayListFile.hpp index 568d4dbb..bc53f7ce 100644 --- a/src/libs/database/include/database/PlayListFile.hpp +++ b/src/libs/database/include/database/PlayListFile.hpp @@ -28,6 +28,7 @@ #include #include "database/DirectoryId.hpp" +#include "database/IdRange.hpp" #include "database/Object.hpp" LMS_DECLARE_IDTYPE(PlayListFileId) @@ -50,6 +51,8 @@ namespace lms::db static pointer find(Session& session, const std::filesystem::path& path); static void find(Session& session, PlayListFileId& lastRetrievedId, std::size_t count, const std::function& func); static void findAbsoluteFilePath(Session& session, PlayListFileId& lastRetrievedId, std::size_t count, const std::function& func); + static void find(Session& session, const IdRange& idRange, const std::function& func); + static IdRange findNextIdRange(Session& session, PlayListFileId lastRetrievedId, std::size_t count); // getters const std::filesystem::path& getAbsoluteFilePath() const { return _absoluteFilePath; } diff --git a/src/libs/database/include/database/Release.hpp b/src/libs/database/include/database/Release.hpp index 7316b179..44a10eae 100644 --- a/src/libs/database/include/database/Release.hpp +++ b/src/libs/database/include/database/Release.hpp @@ -251,7 +251,7 @@ namespace lms::db static pointer find(Session& session, ReleaseId id); static void find(Session& session, ReleaseId& lastRetrievedRelease, std::size_t count, const std::function& func, MediaLibraryId library = {}); static void find(Session& session, const IdRange& idRange, const std::function& func); - static IdRange findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count); + static IdRange findNextIdRange(Session& session, ReleaseId lastRetrievedId, std::size_t count); static RangeResults find(Session& session, const FindParameters& parameters); static void find(Session& session, const FindParameters& parameters, const std::function& func); static RangeResults findIds(Session& session, const FindParameters& parameters); diff --git a/src/libs/database/include/database/Track.hpp b/src/libs/database/include/database/Track.hpp index e967596e..3ab58cd5 100644 --- a/src/libs/database/include/database/Track.hpp +++ b/src/libs/database/include/database/Track.hpp @@ -194,7 +194,7 @@ namespace lms::db static pointer find(Session& session, TrackId id); static void find(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function& func, MediaLibraryId library = {}); static void find(Session& session, const IdRange& idRange, const std::function& func); - static IdRange findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count); + static IdRange findNextIdRange(Session& session, TrackId lastRetrievedId, std::size_t count); static void findAbsoluteFilePath(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function& func); static bool exists(Session& session, TrackId id); diff --git a/src/libs/database/test/Artist.cpp b/src/libs/database/test/Artist.cpp index 2e53657d..5b52344a 100644 --- a/src/libs/database/test/Artist.cpp +++ b/src/libs/database/test/Artist.cpp @@ -608,17 +608,17 @@ namespace lms::db::tests } } - TEST_F(DatabaseFixture, Artist_findNextRange) + TEST_F(DatabaseFixture, Artist_findNextIdRange) { { auto transaction{ session.createReadTransaction() }; - auto range{ Artist::findNextRange(session, ArtistId{}, 0) }; + auto range{ Artist::findNextIdRange(session, ArtistId{}, 0) }; EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, ArtistId{}); EXPECT_EQ(range.last, ArtistId{}); - range = Artist::findNextRange(session, ArtistId{}, 100); + range = Artist::findNextIdRange(session, ArtistId{}, 100); EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, ArtistId{}); EXPECT_EQ(range.last, ArtistId{}); @@ -628,22 +628,22 @@ namespace lms::db::tests { auto transaction{ session.createReadTransaction() }; - auto range{ Artist::findNextRange(session, ArtistId{}, 0) }; + auto range{ Artist::findNextIdRange(session, ArtistId{}, 0) }; EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, ArtistId{}); EXPECT_EQ(range.last, ArtistId{}); - range = Artist::findNextRange(session, ArtistId{}, 1); + range = Artist::findNextIdRange(session, ArtistId{}, 1); EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, artist1.getId()); EXPECT_EQ(range.last, artist1.getId()); - range = Artist::findNextRange(session, range.last, 1); + range = Artist::findNextIdRange(session, range.last, 1); EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, ArtistId{}); EXPECT_EQ(range.last, ArtistId{}); - range = Artist::findNextRange(session, ArtistId{}, 100); + range = Artist::findNextIdRange(session, ArtistId{}, 100); EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, artist1.getId()); EXPECT_EQ(range.last, artist1.getId()); @@ -655,12 +655,12 @@ namespace lms::db::tests { auto transaction{ session.createReadTransaction() }; - auto range{ Artist::findNextRange(session, ArtistId{}, 2) }; + auto range{ Artist::findNextIdRange(session, ArtistId{}, 2) }; EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, artist1.getId()); EXPECT_EQ(range.last, artist2.getId()); - range = Artist::findNextRange(session, artist2.getId(), 2); + range = Artist::findNextIdRange(session, artist2.getId(), 2); EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, artist3.getId()); EXPECT_EQ(range.last, artist3.getId()); diff --git a/src/libs/database/test/Release.cpp b/src/libs/database/test/Release.cpp index fc7cbc44..4d134964 100644 --- a/src/libs/database/test/Release.cpp +++ b/src/libs/database/test/Release.cpp @@ -173,17 +173,17 @@ namespace lms::db::tests } } - TEST_F(DatabaseFixture, Release_findNextRange) + TEST_F(DatabaseFixture, Release_findNextIdRange) { { auto transaction{ session.createReadTransaction() }; - auto range{ Release::findNextRange(session, ReleaseId{}, 0) }; + auto range{ Release::findNextIdRange(session, ReleaseId{}, 0) }; EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, ReleaseId{}); EXPECT_EQ(range.last, ReleaseId{}); - range = Release::findNextRange(session, ReleaseId{}, 100); + range = Release::findNextIdRange(session, ReleaseId{}, 100); EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, ReleaseId{}); EXPECT_EQ(range.last, ReleaseId{}); @@ -193,22 +193,22 @@ namespace lms::db::tests { auto transaction{ session.createReadTransaction() }; - auto range{ Release::findNextRange(session, ReleaseId{}, 0) }; + auto range{ Release::findNextIdRange(session, ReleaseId{}, 0) }; EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, ReleaseId{}); EXPECT_EQ(range.last, ReleaseId{}); - range = Release::findNextRange(session, ReleaseId{}, 1); + range = Release::findNextIdRange(session, ReleaseId{}, 1); EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, release1.getId()); EXPECT_EQ(range.last, release1.getId()); - range = Release::findNextRange(session, range.last, 1); + range = Release::findNextIdRange(session, range.last, 1); EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, ReleaseId{}); EXPECT_EQ(range.last, ReleaseId{}); - range = Release::findNextRange(session, ReleaseId{}, 100); + range = Release::findNextIdRange(session, ReleaseId{}, 100); EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, release1.getId()); EXPECT_EQ(range.last, release1.getId()); @@ -220,12 +220,12 @@ namespace lms::db::tests { auto transaction{ session.createReadTransaction() }; - auto range{ Release::findNextRange(session, ReleaseId{}, 2) }; + auto range{ Release::findNextIdRange(session, ReleaseId{}, 2) }; EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, release1.getId()); EXPECT_EQ(range.last, release2.getId()); - range = Release::findNextRange(session, release2.getId(), 2); + range = Release::findNextIdRange(session, release2.getId(), 2); EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, release3.getId()); EXPECT_EQ(range.last, release3.getId()); diff --git a/src/libs/database/test/Track.cpp b/src/libs/database/test/Track.cpp index e88b6ce2..5fb48041 100644 --- a/src/libs/database/test/Track.cpp +++ b/src/libs/database/test/Track.cpp @@ -151,17 +151,17 @@ namespace lms::db::tests } } - TEST_F(DatabaseFixture, Track_findNextRange) + TEST_F(DatabaseFixture, Track_findNextIdRange) { { auto transaction{ session.createReadTransaction() }; - auto range{ Track::findNextRange(session, TrackId{}, 0) }; + auto range{ Track::findNextIdRange(session, TrackId{}, 0) }; EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, TrackId{}); EXPECT_EQ(range.last, TrackId{}); - range = Track::findNextRange(session, TrackId{}, 100); + range = Track::findNextIdRange(session, TrackId{}, 100); EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, TrackId{}); EXPECT_EQ(range.last, TrackId{}); @@ -171,22 +171,22 @@ namespace lms::db::tests { auto transaction{ session.createReadTransaction() }; - auto range{ Track::findNextRange(session, TrackId{}, 0) }; + auto range{ Track::findNextIdRange(session, TrackId{}, 0) }; EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, TrackId{}); EXPECT_EQ(range.last, TrackId{}); - range = Track::findNextRange(session, TrackId{}, 1); + range = Track::findNextIdRange(session, TrackId{}, 1); EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, track1.getId()); EXPECT_EQ(range.last, track1.getId()); - range = Track::findNextRange(session, range.last, 1); + range = Track::findNextIdRange(session, range.last, 1); EXPECT_FALSE(range.isValid()); EXPECT_EQ(range.first, TrackId{}); EXPECT_EQ(range.last, TrackId{}); - range = Track::findNextRange(session, TrackId{}, 100); + range = Track::findNextIdRange(session, TrackId{}, 100); EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, track1.getId()); EXPECT_EQ(range.last, track1.getId()); @@ -198,12 +198,12 @@ namespace lms::db::tests { auto transaction{ session.createReadTransaction() }; - auto range{ Track::findNextRange(session, TrackId{}, 2) }; + auto range{ Track::findNextIdRange(session, TrackId{}, 2) }; EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, track1.getId()); EXPECT_EQ(range.last, track2.getId()); - range = Track::findNextRange(session, track2.getId(), 2); + range = Track::findNextIdRange(session, track2.getId(), 2); EXPECT_TRUE(range.isValid()); EXPECT_EQ(range.first, track3.getId()); EXPECT_EQ(range.last, track3.getId()); diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp index ed9d4b30..a25b83c0 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp @@ -261,7 +261,7 @@ namespace lms::scanner auto transaction{ session.createReadTransaction() }; - idRange = db::Artist::findNextRange(session, lastRetrievedId, readBatchSize); + idRange = db::Artist::findNextIdRange(session, lastRetrievedId, readBatchSize); lastRetrievedId = idRange.last; return idRange.isValid(); @@ -281,7 +281,7 @@ namespace lms::scanner std::size_t getProcessedArtistCount() const { return _processedArtistCount; } private: - core::LiteralString getName() const override { return "Associate Track Artworks"; } + core::LiteralString getName() const override { return "Associate Artist Artworks"; } void run() override { auto& session{ _db.getTLSSession() }; diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp index 24308574..5d009edb 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp @@ -21,7 +21,9 @@ #include #include +#include +#include "core/IJob.hpp" #include "core/ILogger.hpp" #include "database/Db.hpp" #include "database/Directory.hpp" @@ -32,6 +34,7 @@ #include "database/TrackList.hpp" #include "services/scanner/ScanErrors.hpp" +#include "JobQueue.hpp" #include "ScanContext.hpp" #include "ScannerSettings.hpp" @@ -53,15 +56,6 @@ namespace lms::scanner }; using PlayListFileAssociationContainer = std::deque; - struct SearchPlayListFileContext - { - db::Session& session; - db::PlayListFileId lastRetrievedPlayListFileId; - std::size_t processedPlayListFileCount{}; - const ScannerSettings& settings; - std::vector> errors; - }; - db::Track::pointer getMatchingTrack(db::Session& session, const std::filesystem::path& filePath, const db::Directory::pointer& playListDirectory) { db::Track::pointer matchingTrack; @@ -110,64 +104,7 @@ namespace lms::scanner return needUpdate; } - bool fetchNextPlayListFilesToUpdate(SearchPlayListFileContext& searchContext, PlayListFileAssociationContainer& playListFileAssociations) - { - const db::PlayListFileId playListFileIdId{ searchContext.lastRetrievedPlayListFileId }; - - { - constexpr std::size_t readBatchSize{ 20 }; - - auto transaction{ searchContext.session.createReadTransaction() }; - - db::PlayListFile::find(searchContext.session, searchContext.lastRetrievedPlayListFileId, readBatchSize, [&](const db::PlayListFile::pointer& playListFile) { - PlayListFileAssociation playListAssociation; - - playListAssociation.playListFileIdId = playListFile->getId(); - - std::vector> pendingErrors; - - const auto files{ playListFile->getFiles() }; - for (const std::filesystem::path& file : files) - { - // TODO optim: no need to fetch the whole track - const db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) }; - if (track) - playListAssociation.tracks.push_back(TrackInfo{ .trackId = track->getId(), .releaseId = track->getReleaseId() }); - else - pendingErrors.emplace_back(std::make_shared(playListFile->getAbsoluteFilePath(), file)); - } - - if (pendingErrors.size() == files.size()) - { - pendingErrors.clear(); - pendingErrors.emplace_back(std::make_shared(playListFile->getAbsoluteFilePath())); - } - searchContext.errors.insert(std::end(searchContext.errors), std::begin(pendingErrors), std::end(pendingErrors)); - - if (playListAssociation.tracks.empty() - || (searchContext.settings.skipSingleReleasePlayLists && isSingleReleasePlayList(playListAssociation.tracks))) - { - playListAssociation.tracks.clear(); - } - - bool needUpdate{ true }; - if (const db::TrackList::pointer trackList{ playListFile->getTrackList() }) - { - if (!playListAssociation.tracks.empty()) - needUpdate = trackListNeedsUpdate(searchContext.session, playListFile->getName(), playListAssociation.tracks, trackList); - } - - if (needUpdate) - playListFileAssociations.emplace_back(std::move(playListAssociation)); - - searchContext.processedPlayListFileCount++; - }); - } - - return playListFileIdId != searchContext.lastRetrievedPlayListFileId; - } - - void updatePlayListFile(db::Session& session, const PlayListFileAssociation& playListFileAssociation) + void updatePlayList(db::Session& session, const PlayListFileAssociation& playListFileAssociation) { db::PlayListFile::pointer playListFile{ db::PlayListFile::find(session, playListFileAssociation.playListFileIdId) }; assert(playListFile); @@ -198,6 +135,7 @@ namespace lms::scanner trackList.modify()->clear(); for (const TrackInfo trackInfo : playListFileAssociation.tracks) { + // TODO no need to fetch the whole track for that if (db::Track::pointer track{ db::Track::find(session, trackInfo.trackId) }) session.create(track, trackList, playListFile->getLastWriteTime()); } @@ -205,21 +143,110 @@ namespace lms::scanner LMS_LOG(DBUPDATER, DEBUG, std::string_view{ createTrackList ? "Created" : "Updated" } << " associated tracklist for " << playListFile->getAbsoluteFilePath() << " (" << playListFileAssociation.tracks.size() << " tracks)"); } - void updatePlayListFiles(db::Session& session, PlayListFileAssociationContainer& playListFileAssociations) + void updatePlayLists(db::Session& session, PlayListFileAssociationContainer& playListFileAssociations, bool forceFullBatch) { constexpr std::size_t writeBatchSize{ 5 }; - while (!playListFileAssociations.empty()) + while ((forceFullBatch && playListFileAssociations.size() >= writeBatchSize) || !playListFileAssociations.empty()) { auto transaction{ session.createWriteTransaction() }; for (std::size_t i{}; !playListFileAssociations.empty() && i < writeBatchSize; ++i) { - updatePlayListFile(session, playListFileAssociations.front()); + updatePlayList(session, playListFileAssociations.front()); playListFileAssociations.pop_front(); } } } + + bool fetchNextPlayListFileIdRange(db::Session& session, db::PlayListFileId& lastPlayListFileId, db::IdRange& idRange) + { + constexpr std::size_t readBatchSize{ 100 }; + + auto transaction{ session.createReadTransaction() }; + + idRange = db::PlayListFile::findNextIdRange(session, lastPlayListFileId, readBatchSize); + lastPlayListFileId = idRange.last; + + return idRange.isValid(); + } + + class ComputePlayListFileAssociationsJob : public core::IJob + { + public: + ComputePlayListFileAssociationsJob(db::Db& db, const ScannerSettings& settings, db::IdRange playListFileIdRange) + : _db{ db } + , _settings{ settings } + , _playListFileIdRange{ playListFileIdRange } + { + } + + std::span getAssociations() const { return _associations; } + std::size_t getProcessedCount() const { return _processedCount; } + std::span> getErrors() const { return _errors; } + + private: + core::LiteralString getName() const override { return "Associate PlayList Tracks"; } + void run() override + { + auto& session{ _db.getTLSSession() }; + auto transaction{ session.createReadTransaction() }; + + db::PlayListFile::find(session, _playListFileIdRange, [&](const db::PlayListFile::pointer& playListFile) { + PlayListFileAssociation playListAssociation; + + playListAssociation.playListFileIdId = playListFile->getId(); + + std::vector> pendingErrors; + + const auto& files{ playListFile->getFiles() }; + for (const std::filesystem::path& file : files) + { + // TODO optim: no need to fetch the whole track + const db::Track::pointer track{ getMatchingTrack(session, file, playListFile->getDirectory()) }; + if (track) + playListAssociation.tracks.push_back(TrackInfo{ .trackId = track->getId(), .releaseId = track->getReleaseId() }); + else + pendingErrors.emplace_back(std::make_shared(playListFile->getAbsoluteFilePath(), file)); + } + + if (pendingErrors.size() == files.size()) + { + pendingErrors.clear(); + pendingErrors.emplace_back(std::make_shared(playListFile->getAbsoluteFilePath())); + } + _errors.insert(std::end(_errors), std::begin(pendingErrors), std::end(pendingErrors)); + + if (playListAssociation.tracks.empty() + || (_settings.skipSingleReleasePlayLists && isSingleReleasePlayList(playListAssociation.tracks))) + { + playListAssociation.tracks.clear(); + } + + bool needUpdate{ true }; + if (const db::TrackList::pointer trackList{ playListFile->getTrackList() }) + { + if (!playListAssociation.tracks.empty()) + needUpdate = trackListNeedsUpdate(session, playListFile->getName(), playListAssociation.tracks, trackList); + } + else if (playListAssociation.tracks.empty()) + needUpdate = false; + + if (needUpdate) + _associations.emplace_back(std::move(playListAssociation)); + + _processedCount++; + }); + } + + db::Db& _db; + const ScannerSettings& _settings; + db::IdRange _playListFileIdRange; + std::vector _associations; + std::vector> _errors; + std::size_t _processedCount{}; + }; + } // namespace bool ScanStepAssociatePlayListTracks::needProcess(const ScanContext& context) const @@ -242,27 +269,37 @@ namespace lms::scanner context.currentStepStats.totalElems = db::PlayListFile::getCount(session); } - SearchPlayListFileContext searchContext{ - .session = session, - .lastRetrievedPlayListFileId = {}, - .settings = _settings, - .errors = context.stats.errors - }; - - PlayListFileAssociationContainer playListFileAssociations; - while (fetchNextPlayListFilesToUpdate(searchContext, playListFileAssociations)) - { + PlayListFileAssociationContainer playListTrackAssociations; + auto processJobsDone = [&](std::span> jobs) { if (_abortScan) return; - updatePlayListFiles(session, playListFileAssociations); + for (const auto& job : jobs) + { + const auto& associationJob{ static_cast(*job) }; + const auto& associations{ associationJob.getAssociations() }; - context.currentStepStats.processedElems = searchContext.processedPlayListFileCount; - for (const std::shared_ptr& error : searchContext.errors) - addError(context, error); - searchContext.errors.clear(); + playListTrackAssociations.insert(std::end(playListTrackAssociations), std::cbegin(associations), std::cend(associations)); + for (const std::shared_ptr& error : associationJob.getErrors()) + addError(context, error); + context.currentStepStats.processedElems += associationJob.getProcessedCount(); + } + + updatePlayLists(session, playListTrackAssociations, true); _progressCallback(context.currentStepStats); + }; + + { + JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F }; + + db::PlayListFileId lastPlayListFileId; + db::IdRange playListFileIdRange; + while (fetchNextPlayListFileIdRange(session, lastPlayListFileId, playListFileIdRange)) + queue.push(std::make_unique(_db, _settings, playListFileIdRange)); } + + // process all remaining associations + updatePlayLists(session, playListTrackAssociations, false); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp index b6a03229..93fa4512 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp @@ -204,7 +204,7 @@ namespace lms::scanner auto transaction{ session.createReadTransaction() }; - idRange = db::Release::findNextRange(session, lastRetrievedId, readBatchSize); + idRange = db::Release::findNextIdRange(session, lastRetrievedId, readBatchSize); lastRetrievedId = idRange.last; return idRange.isValid(); @@ -224,7 +224,7 @@ namespace lms::scanner std::size_t getProcessedReleaseCount() const { return _processedReleaseCount; } private: - core::LiteralString getName() const override { return "Associate Track Artworks"; } + core::LiteralString getName() const override { return "Associate Release Artworks"; } void run() override { auto& session{ _db.getTLSSession() }; diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp index 7514d85f..9c1475cd 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp @@ -53,13 +53,6 @@ namespace lms::scanner }; using TrackArtworksAssociationContainer = std::deque; - struct SearchTrackArtworkContext - { - db::Session& session; - db::TrackId lastRetrievedTrackId; - std::size_t processedTrackCount{}; - }; - db::Artwork::pointer computePreferredTrackArtwork(db::Session& session, const db::Track::pointer& track, const db::Artwork::pointer& preferredMediaArtwork) { db::Artwork::pointer res{ preferredMediaArtwork }; @@ -152,14 +145,14 @@ namespace lms::scanner } } - bool fetchNextTrackIdRange(SearchTrackArtworkContext& searchContext, db::IdRange& trackIdRange) + bool fetchNextTrackIdRange(db::Session& session, db::TrackId& lastRetrievedTrackId, db::IdRange& trackIdRange) { constexpr std::size_t readBatchSize{ 100 }; - auto transaction{ searchContext.session.createReadTransaction() }; + auto transaction{ session.createReadTransaction() }; - trackIdRange = db::Track::findNextRange(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize); - searchContext.lastRetrievedTrackId = trackIdRange.last; + trackIdRange = db::Track::findNextIdRange(session, lastRetrievedTrackId, readBatchSize); + lastRetrievedTrackId = trackIdRange.last; return trackIdRange.isValid(); } @@ -242,11 +235,6 @@ namespace lms::scanner context.currentStepStats.totalElems = db::Track::getCount(session); } - SearchTrackArtworkContext searchContext{ - .session = session, - .lastRetrievedTrackId = {}, - }; - TrackArtworksAssociationContainer trackArtworksAssociations; auto processTracks = [&](std::span> jobs) { if (_abortScan) @@ -266,13 +254,14 @@ namespace lms::scanner _progressCallback(context.currentStepStats); }; - JobQueue queue{ getJobScheduler(), 20, processTracks, 1, 0.85F }; + { + JobQueue queue{ getJobScheduler(), 20, processTracks, 1, 0.85F }; - db::IdRange trackIdRange; - while (fetchNextTrackIdRange(searchContext, trackIdRange)) - queue.push(std::make_unique(_db, trackIdRange)); - - queue.finish(); + db::TrackId lastRetrievedTrackId; + db::IdRange trackIdRange; + while (fetchNextTrackIdRange(session, lastRetrievedTrackId, trackIdRange)) + queue.push(std::make_unique(_db, trackIdRange)); + } // process all remaining associations updateTrackPreferredArtworks(session, trackArtworksAssociations, false);