From 03425f3c942de32076d3b5ef1a7f9d5106d900c9 Mon Sep 17 00:00:00 2001 From: emeric Date: Thu, 14 Mar 2024 23:21:30 +0100 Subject: [PATCH] Made the last scan step (checking for removed files) faster on big libraries --- src/libs/database/impl/Track.cpp | 26 +++++++++ src/libs/database/include/database/Track.hpp | 3 +- src/libs/database/test/Track.cpp | 56 +++++++++++++++++++ .../impl/ScanStepRemoveOrphanDbFiles.cpp | 53 +++++++----------- 4 files changed, 104 insertions(+), 34 deletions(-) diff --git a/src/libs/database/impl/Track.cpp b/src/libs/database/impl/Track.cpp index 25a5bd80..16d79960 100644 --- a/src/libs/database/impl/Track.cpp +++ b/src/libs/database/impl/Track.cpp @@ -228,6 +228,32 @@ namespace lms::db .resultValue(); } + void Track::find(Session& session, TrackId& lastRetrievedTrack, std::size_t batchSize, bool& moreResults, const std::function& func) + { + session.checkReadTransaction(); + + auto collection{ session.getDboSession().find() + .orderBy("id") + .where("id > ?").bind(lastRetrievedTrack) + .limit(static_cast(batchSize) + 1) + .resultList() }; + + moreResults = false; + + std::size_t count{}; + for (auto itResult{ collection.begin() }; itResult != collection.end(); ++itResult) + { + if (count++ == batchSize) + { + moreResults = true; + break; + } + + func(*itResult); + lastRetrievedTrack = (*itResult)->getId(); + } + } + bool Track::exists(Session& session, TrackId id) { session.checkReadTransaction(); diff --git a/src/libs/database/include/database/Track.hpp b/src/libs/database/include/database/Track.hpp index 68ac088d..c52f9f49 100644 --- a/src/libs/database/include/database/Track.hpp +++ b/src/libs/database/include/database/Track.hpp @@ -110,7 +110,8 @@ namespace lms::db static std::size_t getCount(Session& session); static pointer findByPath(Session& session, const std::filesystem::path& p); static pointer find(Session& session, TrackId id); - static bool exists(Session& session, TrackId id); + static void find(Session& session, TrackId& lastRetrievedTrack, std::size_t batchSize, bool& moreResults, const std::function& func); + static bool exists(Session& session, TrackId id); static std::vector findByRecordingMBID(Session& session, const core::UUID& MBID); static std::vector findByMBID(Session& session, const core::UUID& MBID); static RangeResults findSimilarTrackIds(Session& session, const std::vector& trackIds, std::optional range = std::nullopt); diff --git a/src/libs/database/test/Track.cpp b/src/libs/database/test/Track.cpp index 4ba8883d..0d6683af 100644 --- a/src/libs/database/test/Track.cpp +++ b/src/libs/database/test/Track.cpp @@ -63,6 +63,62 @@ namespace lms::db::tests } } + TEST_F(DatabaseFixture, Track_findByRangedIdBased) + { + ScopedTrack track1{ session, "MyTrackFile1" }; + ScopedTrack track2{ session, "MyTrackFile1" }; + ScopedTrack track3{ session, "MyTrackFile1" }; + + { + auto transaction{ session.createReadTransaction() }; + + bool moreResults; + TrackId lastRetrievedTrackId; + std::vector visitedTracks; + Track::find(session, lastRetrievedTrackId, 10, moreResults, [&](const Track::pointer& track) + { + visitedTracks.push_back(track); + }); + ASSERT_EQ(visitedTracks.size(), 3); + EXPECT_EQ(visitedTracks[0]->getId(), track1.getId()); + EXPECT_EQ(visitedTracks[1]->getId(), track2.getId()); + EXPECT_EQ(visitedTracks[2]->getId(), track3.getId()); + EXPECT_FALSE(moreResults); + EXPECT_EQ(lastRetrievedTrackId, track3.getId()); + } + + { + auto transaction{ session.createReadTransaction() }; + + bool moreResults; + TrackId lastRetrievedTrackId{ track1.getId() }; + std::vector visitedTracks; + Track::find(session, lastRetrievedTrackId, 1, moreResults, [&](const Track::pointer& track) + { + visitedTracks.push_back(track); + }); + ASSERT_EQ(visitedTracks.size(), 1); + EXPECT_EQ(visitedTracks[0]->getId(), track2.getId()); + EXPECT_TRUE(moreResults); + EXPECT_EQ(lastRetrievedTrackId, track2.getId()); + } + + { + auto transaction{ session.createReadTransaction() }; + + bool moreResults; + TrackId lastRetrievedTrackId{ track1.getId() }; + std::vector visitedTracks; + Track::find(session, lastRetrievedTrackId, 0, moreResults, [&](const Track::pointer& track) + { + visitedTracks.push_back(track); + }); + ASSERT_EQ(visitedTracks.size(), 0); + EXPECT_TRUE(moreResults); + EXPECT_EQ(lastRetrievedTrackId, track1.getId()); + } + } + TEST_F(DatabaseFixture, Track_MediaLibrary) { ScopedTrack track{ session, "MyTrackFile" }; diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp index b0f9901e..00ad637a 100644 --- a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp @@ -89,62 +89,49 @@ namespace lms::scanner Session& session{ _db.getTLSSession() }; LMS_LOG(DBUPDATER, DEBUG, "Checking tracks to be removed..."); - std::size_t trackCount{}; - { auto transaction{ session.createReadTransaction() }; - trackCount = Track::getCount(session); + context.currentStepStats.totalElems = Track::getCount(session); } - LMS_LOG(DBUPDATER, DEBUG, trackCount << " tracks to be checked..."); + LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " tracks to be checked..."); - context.currentStepStats.totalElems = trackCount; + // TODO handle only files in context.directory? + std::vector tracksToRemove; - RangeResults trackPaths; - std::vector tracksToRemove; - - // TODO handle only files in context.directory - for (std::size_t i{ trackCount < batchSize ? 0 : trackCount - batchSize }; ; i -= (i > batchSize ? batchSize : i)) + TrackId lastCheckedTrackID; + bool moreResults{ true }; + while (moreResults) { - tracksToRemove.clear(); + if (_abortScan) + break; + tracksToRemove.clear(); { auto transaction{ session.createReadTransaction() }; - trackPaths = Track::findPaths(session, Range{ i, batchSize }); - } + Track::find(session, lastCheckedTrackID, batchSize, moreResults, [&](const Track::pointer& track) + { + if (!checkFile(track->getPath())) + tracksToRemove.push_back(track); - for (const Track::PathResult& trackPath : trackPaths.results) - { - if (_abortScan) - return; - - if (!checkFile(trackPath.path)) - tracksToRemove.push_back(trackPath.trackId); - - context.currentStepStats.processedElems++; + context.currentStepStats.processedElems++; + }); } if (!tracksToRemove.empty()) { auto transaction{ session.createWriteTransaction() }; - for (const TrackId trackId : tracksToRemove) + for (Track::pointer& track : tracksToRemove) { - Track::pointer track{ Track::find(session, trackId) }; - if (track) - { - track.remove(); - context.stats.deletions++; - } + track.remove(); + context.stats.deletions++; } } _progressCallback(context.currentStepStats); - - if (i == 0) - break; } - LMS_LOG(DBUPDATER, DEBUG, trackCount << " tracks checked!"); + LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.processedElems << " tracks checked!"); } void ScanStepRemoveOrphanDbFiles::removeOrphanClusters()