diff --git a/src/libs/services/database/impl/Cluster.cpp b/src/libs/services/database/impl/Cluster.cpp index f93e8de2..cef0c615 100644 --- a/src/libs/services/database/impl/Cluster.cpp +++ b/src/libs/services/database/impl/Cluster.cpp @@ -106,7 +106,7 @@ namespace Database return Utils::execQuery(query, params.range); } - RangeResults Cluster::findOrphans(Session& session, std::optional range) + RangeResults Cluster::findOrphanIds(Session& session, std::optional range) { session.checkSharedLocked(); auto query{ session.getDboSession().query("SELECT DISTINCT c.id FROM cluster c WHERE NOT EXISTS(SELECT 1 FROM track_cluster t_c WHERE t_c.cluster_id = c.id)") }; diff --git a/src/libs/services/database/include/services/database/Cluster.hpp b/src/libs/services/database/include/services/database/Cluster.hpp index 7220d647..c5215c01 100644 --- a/src/libs/services/database/include/services/database/Cluster.hpp +++ b/src/libs/services/database/include/services/database/Cluster.hpp @@ -65,7 +65,7 @@ namespace Database { static RangeResults find(Session& session, const FindParameters& params); static void find(Session& session, const FindParameters& params, std::function _func); static pointer find(Session& session, ClusterId id); - static RangeResults findOrphans(Session& session, std::optional range = std::nullopt); + static RangeResults findOrphanIds(Session& session, std::optional range = std::nullopt); // May be very slow static std::size_t computeTrackCount(Session& session, ClusterId id); diff --git a/src/libs/services/database/test/Cluster.cpp b/src/libs/services/database/test/Cluster.cpp index aeea9be6..cf768e00 100644 --- a/src/libs/services/database/test/Cluster.cpp +++ b/src/libs/services/database/test/Cluster.cpp @@ -55,7 +55,7 @@ TEST_F(DatabaseFixture, Cluster) } { - const auto clusters{ Cluster::findOrphans(session) }; + const auto clusters{ Cluster::findOrphanIds(session) }; ASSERT_EQ(clusters.results.size(), 1); EXPECT_EQ(clusters.results.front(), cluster.getId()); } @@ -91,7 +91,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrack) { auto transaction{ session.createSharedTransaction() }; - EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); + EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty()); auto clusterTypes{ ClusterType::findOrphans(session) }; ASSERT_EQ(clusterTypes.results.size(), 1); EXPECT_EQ(clusterTypes.results.front(), clusterType.getId()); @@ -102,7 +102,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrack) { auto transaction{ session.createSharedTransaction() }; - auto clusters{ Cluster::findOrphans(session) }; + auto clusters{ Cluster::findOrphanIds(session) }; EXPECT_EQ(clusters.results.size(), 2); EXPECT_TRUE(track->getClusters().empty()); EXPECT_TRUE(track->getClusterIds().empty()); @@ -127,7 +127,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrack) { auto transaction{ session.createSharedTransaction() }; - auto clusters{ Cluster::findOrphans(session) }; + auto clusters{ Cluster::findOrphanIds(session) }; ASSERT_EQ(clusters.results.size(), 1); EXPECT_EQ(clusters.results.front(), cluster2.getId()); @@ -225,7 +225,7 @@ TEST_F(DatabaseFixture, Cluster_multiTracks) { auto transaction{ session.createSharedTransaction() }; - EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); + EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty()); EXPECT_EQ(Cluster::computeTrackCount(session, cluster.getId()), tracks.size()); @@ -244,7 +244,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster) { auto transaction{ session.createSharedTransaction() }; - EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); + EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty()); } ScopedClusterType clusterType{ session, "MyClusterType" }; @@ -253,7 +253,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster) { auto transaction{ session.createSharedTransaction() }; - ASSERT_EQ(Cluster::findOrphans(session).results.size(), 2); + ASSERT_EQ(Cluster::findOrphanIds(session).results.size(), 2); EXPECT_TRUE(Release::find(session, Release::FindParameters{}.setClusters({ unusedCluster.getId() })).results.empty()); EXPECT_EQ(Release::find(session, Release::FindParameters{}).results.size(), 1); EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 0); @@ -271,7 +271,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster) auto transaction{ session.createSharedTransaction() }; { - auto clusters{ Cluster::findOrphans(session) }; + auto clusters{ Cluster::findOrphanIds(session) }; ASSERT_EQ(clusters.results.size(), 1); EXPECT_EQ(clusters.results.front(), unusedCluster.getId()); } @@ -330,7 +330,7 @@ TEST_F(DatabaseFixture, SingleTrackSingleArtistMultiClusters) { auto transaction{ session.createSharedTransaction() }; EXPECT_TRUE(ClusterType::findOrphans(session).results.empty()); - EXPECT_EQ(Cluster::findOrphans(session).results.size(), 2); + EXPECT_EQ(Cluster::findOrphanIds(session).results.size(), 2); EXPECT_TRUE(Release::findOrphanIds(session).results.empty()); EXPECT_TRUE(Artist::findOrphanIds(session).results.empty()); } @@ -390,7 +390,7 @@ TEST_F(DatabaseFixture, SingleTrackSingleArtistMultiRolesMultiClusters) { auto transaction{ session.createSharedTransaction() }; - EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); + EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty()); EXPECT_TRUE(Release::findOrphanIds(session).results.empty()); EXPECT_TRUE(Artist::findOrphanIds(session).results.empty()); } @@ -430,7 +430,7 @@ TEST_F(DatabaseFixture, MultiTracksSingleArtistMultiClusters) { auto transaction{ session.createSharedTransaction() }; - EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); + EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty()); EXPECT_TRUE(Artist::findOrphanIds(session).results.empty()); } @@ -541,7 +541,7 @@ TEST_F(DatabaseFixture, SingleTrackSingleReleaseSingleArtistSingleCluster) { auto transaction{ session.createSharedTransaction() }; - EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); + EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty()); EXPECT_TRUE(ClusterType::findOrphans(session).results.empty()); EXPECT_TRUE(Artist::findOrphanIds(session).results.empty()); EXPECT_TRUE(Release::findOrphanIds(session).results.empty()); diff --git a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp index e1893a8a..306167be 100644 --- a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp @@ -26,30 +26,32 @@ namespace Scanner { - void - ScanStepCheckDuplicatedDbFiles::process(ScanContext& context) - { - using namespace Database; + void ScanStepCheckDuplicatedDbFiles::process(ScanContext& context) + { + using namespace Database; - if (_abortScan) - return; + if (_abortScan) + return; - Session& session {_db.getTLSSession()}; - auto transaction {session.createSharedTransaction()}; + Session& session{ _db.getTLSSession() }; + auto transaction{ session.createSharedTransaction() }; - const RangeResults tracks = Track::findIdsTrackMBIDDuplicates(session); - for (const TrackId trackId : tracks.results) - { - const Track::pointer track {Track::find(session, trackId)}; - if (auto trackMBID {track->getTrackMBID()}) - { - LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName(); - context.stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID}); - context.currentStepStats.processedElems++; - _progressCallback(context.currentStepStats); - } - } + const RangeResults tracks = Track::findIdsTrackMBIDDuplicates(session); + for (const TrackId trackId : tracks.results) + { + if (_abortScan) + break; - LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files"; - } + const Track::pointer track{ Track::find(session, trackId) }; + if (auto trackMBID{ track->getTrackMBID() }) + { + LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName(); + context.stats.duplicates.emplace_back(ScanDuplicate{ track->getId(), DuplicateReason::SameTrackMBID }); + context.currentStepStats.processedElems++; + _progressCallback(context.currentStepStats); + } + } + + LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files"; + } } diff --git a/src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp b/src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp index 38399599..ee7f55e7 100644 --- a/src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp +++ b/src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp @@ -57,6 +57,9 @@ namespace Scanner for (const ClusterId clusterId : clusterIds) { + if (_abortScan) + break; + std::size_t trackCount; std::size_t releaseCount; @@ -81,6 +84,6 @@ namespace Scanner return true; }); - LMS_LOG(DBUPDATER, DEBUG) << "Recomputed stats for " << clusterCount << " clusters!"; + LMS_LOG(DBUPDATER, DEBUG) << "Recomputed stats for " << context.currentStepStats.processedElems << " clusters!"; } } diff --git a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp index 02f3dcfa..929ff07e 100644 --- a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp @@ -23,26 +23,25 @@ namespace Scanner { - void - ScanStepDiscoverFiles::process(ScanContext& context) - { - context.stats.filesScanned = 0; - PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path) - { - if (_abortScan) - return false; + void ScanStepDiscoverFiles::process(ScanContext& context) + { + context.stats.filesScanned = 0; + PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path) + { + if (_abortScan) + return false; - if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions)) - { - context.currentStepStats.processedElems++; - _progressCallback(context.currentStepStats); - } + if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions)) + { + context.currentStepStats.processedElems++; + _progressCallback(context.currentStepStats); + } - return true; - }, &excludeDirFileName); + return true; + }, &excludeDirFileName); - context.stats.filesScanned = context.currentStepStats.processedElems; + context.stats.filesScanned = context.currentStepStats.processedElems; - LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'"; - } + LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'"; + } } diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp index 1ae6781d..097c9906 100644 --- a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp @@ -30,6 +30,49 @@ namespace Scanner { + using namespace Database; + + namespace + { + constexpr std::size_t batchSize = 100; + + template + void removeOrphanEntries(Session& session, bool& abortScan) + { + using IdType = typename T::IdType; + + RangeResults entries; + while (!abortScan) + { + { + LMS_LOG(DBUPDATER, DEBUG) << "FIND"; + auto transaction{ session.createSharedTransaction() }; + entries = T::findOrphanIds(session, Range{ 0, batchSize }); + LMS_LOG(DBUPDATER, DEBUG) << "FIND DONE"; + }; + + { + LMS_LOG(DBUPDATER, DEBUG) << "REMOVE"; + + auto transaction{ session.createUniqueTransaction() }; + for (const IdType objectId : entries.results) + { + if (abortScan) + break; + + typename T::pointer entry{ T::find(session, objectId) }; + + LMS_LOG(DBUPDATER, DEBUG) << "Removing '" << entry->getName() << "'"; + entry.remove(); + } + } + + if (!entries.moreResults) + break; + } + } + } + void ScanStepRemoveOrphanDbFiles::process(ScanContext& context) { removeOrphanTracks(context); @@ -45,7 +88,6 @@ namespace Scanner if (_abortScan) return; - static constexpr std::size_t batchSize{ 50 }; Session& session{ _db.getTLSSession() }; LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed..."; @@ -85,7 +127,7 @@ namespace Scanner if (!tracksToRemove.empty()) { - auto transaction{ session.createSharedTransaction() }; + auto transaction{ session.createUniqueTransaction() }; for (const TrackId trackId : tracksToRemove) { @@ -109,57 +151,20 @@ namespace Scanner void ScanStepRemoveOrphanDbFiles::removeOrphanClusters() { - using namespace Database; - LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters..."; - Session& session{ _db.getTLSSession() }; - auto transaction{ session.createUniqueTransaction() }; - - // Now process orphan Cluster (no track) - auto clusterIds{ Cluster::findOrphans(session) }; - for (ClusterId clusterId : clusterIds.results) - { - Cluster::pointer cluster{ Cluster::find(session, clusterId) }; - LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'"; - cluster.remove(); - } + removeOrphanEntries(_db.getTLSSession(), _abortScan); } void ScanStepRemoveOrphanDbFiles::removeOrphanArtists() { - using namespace Database; - LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists..."; - - Session& session{ _db.getTLSSession() }; - auto transaction{ session.createUniqueTransaction() }; - - auto artistIds{ Artist::findOrphanIds(session) }; - for (const ArtistId artistId : artistIds.results) - { - Artist::pointer artist{ Artist::find(session, artistId) }; - LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'"; - artist.remove(); - } + removeOrphanEntries(_db.getTLSSession(), _abortScan); } void ScanStepRemoveOrphanDbFiles::removeOrphanReleases() { - using namespace Database; - LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases..."; - - // TODO, by batch - Session& session{ _db.getTLSSession() }; - auto transaction{ session.createUniqueTransaction() }; - - auto releases{ Release::findOrphanIds(session) }; - for (const ReleaseId releaseId : releases.results) - { - Release::pointer release{ Release::find(session, releaseId) }; - LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'"; - release.remove(); - } + removeOrphanEntries(_db.getTLSSession(), _abortScan); } bool ScanStepRemoveOrphanDbFiles::checkFile(const std::filesystem::path& p) @@ -168,8 +173,7 @@ namespace Scanner { // For each track, make sure the the file still exists // and still belongs to a media directory - if (!std::filesystem::exists(p) - || !std::filesystem::is_regular_file(p)) + if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p)) { LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing"; return false;