owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made scanner abort faster when it has to clean a lot of stuff

Browse Source
This commit is contained in:
emeric
2023-11-14 13:47:08 +01:00
parent 9cfc47fc82
commit b4e515f825
7 changed files with 107 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/database/impl/Cluster.cpp
+1 -1
View File
@@ -106,7 +106,7 @@ namespace Database
return Utils::execQuery<Cluster::pointer>(query, params.range); return Utils::execQuery<Cluster::pointer>(query, params.range);
} }
RangeResults<ClusterId> Cluster::findOrphans(Session& session, std::optional<Range> range) RangeResults<ClusterId> Cluster::findOrphanIds(Session& session, std::optional<Range> range)
{ {
session.checkSharedLocked(); session.checkSharedLocked();
auto query{ session.getDboSession().query<ClusterId>("SELECT DISTINCT c.id FROM cluster c WHERE NOT EXISTS(SELECT 1 FROM track_cluster t_c WHERE t_c.cluster_id = c.id)") }; auto query{ session.getDboSession().query<ClusterId>("SELECT DISTINCT c.id FROM cluster c WHERE NOT EXISTS(SELECT 1 FROM track_cluster t_c WHERE t_c.cluster_id = c.id)") };
src/libs/services/database/include/services/database/Cluster.hpp
+1 -1
View File
@@ -65,7 +65,7 @@ namespace Database {
static RangeResults<pointer> find(Session& session, const FindParameters& params); static RangeResults<pointer> find(Session& session, const FindParameters& params);
static void find(Session& session, const FindParameters& params, std::function<void(const pointer& cluster)> _func); static void find(Session& session, const FindParameters& params, std::function<void(const pointer& cluster)> _func);
static pointer find(Session& session, ClusterId id); static pointer find(Session& session, ClusterId id);
static RangeResults<ClusterId> findOrphans(Session& session, std::optional<Range> range = std::nullopt); static RangeResults<ClusterId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt);
// May be very slow // May be very slow
static std::size_t computeTrackCount(Session& session, ClusterId id); static std::size_t computeTrackCount(Session& session, ClusterId id);
src/libs/services/database/test/Cluster.cpp
+12 -12
View File
@@ -55,7 +55,7 @@ TEST_F(DatabaseFixture, Cluster)
} }
{ {
const auto clusters{ Cluster::findOrphans(session) }; const auto clusters{ Cluster::findOrphanIds(session) };
ASSERT_EQ(clusters.results.size(), 1); ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(clusters.results.front(), cluster.getId()); EXPECT_EQ(clusters.results.front(), cluster.getId());
} }
@@ -91,7 +91,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
auto clusterTypes{ ClusterType::findOrphans(session) }; auto clusterTypes{ ClusterType::findOrphans(session) };
ASSERT_EQ(clusterTypes.results.size(), 1); ASSERT_EQ(clusterTypes.results.size(), 1);
EXPECT_EQ(clusterTypes.results.front(), clusterType.getId()); EXPECT_EQ(clusterTypes.results.front(), clusterType.getId());
@@ -102,7 +102,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
auto clusters{ Cluster::findOrphans(session) }; auto clusters{ Cluster::findOrphanIds(session) };
EXPECT_EQ(clusters.results.size(), 2); EXPECT_EQ(clusters.results.size(), 2);
EXPECT_TRUE(track->getClusters().empty()); EXPECT_TRUE(track->getClusters().empty());
EXPECT_TRUE(track->getClusterIds().empty()); EXPECT_TRUE(track->getClusterIds().empty());
@@ -127,7 +127,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
auto clusters{ Cluster::findOrphans(session) }; auto clusters{ Cluster::findOrphanIds(session) };
ASSERT_EQ(clusters.results.size(), 1); ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(clusters.results.front(), cluster2.getId()); EXPECT_EQ(clusters.results.front(), cluster2.getId());
@@ -225,7 +225,7 @@ TEST_F(DatabaseFixture, Cluster_multiTracks)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
EXPECT_EQ(Cluster::computeTrackCount(session, cluster.getId()), tracks.size()); EXPECT_EQ(Cluster::computeTrackCount(session, cluster.getId()), tracks.size());
@@ -244,7 +244,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
} }
ScopedClusterType clusterType{ session, "MyClusterType" }; ScopedClusterType clusterType{ session, "MyClusterType" };
@@ -253,7 +253,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
ASSERT_EQ(Cluster::findOrphans(session).results.size(), 2); ASSERT_EQ(Cluster::findOrphanIds(session).results.size(), 2);
EXPECT_TRUE(Release::find(session, Release::FindParameters{}.setClusters({ unusedCluster.getId() })).results.empty()); EXPECT_TRUE(Release::find(session, Release::FindParameters{}.setClusters({ unusedCluster.getId() })).results.empty());
EXPECT_EQ(Release::find(session, Release::FindParameters{}).results.size(), 1); EXPECT_EQ(Release::find(session, Release::FindParameters{}).results.size(), 1);
EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 0); EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 0);
@@ -271,7 +271,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
{ {
auto clusters{ Cluster::findOrphans(session) }; auto clusters{ Cluster::findOrphanIds(session) };
ASSERT_EQ(clusters.results.size(), 1); ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(clusters.results.front(), unusedCluster.getId()); EXPECT_EQ(clusters.results.front(), unusedCluster.getId());
} }
@@ -330,7 +330,7 @@ TEST_F(DatabaseFixture, SingleTrackSingleArtistMultiClusters)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(ClusterType::findOrphans(session).results.empty()); EXPECT_TRUE(ClusterType::findOrphans(session).results.empty());
EXPECT_EQ(Cluster::findOrphans(session).results.size(), 2); EXPECT_EQ(Cluster::findOrphanIds(session).results.size(), 2);
EXPECT_TRUE(Release::findOrphanIds(session).results.empty()); EXPECT_TRUE(Release::findOrphanIds(session).results.empty());
EXPECT_TRUE(Artist::findOrphanIds(session).results.empty()); EXPECT_TRUE(Artist::findOrphanIds(session).results.empty());
} }
@@ -390,7 +390,7 @@ TEST_F(DatabaseFixture, SingleTrackSingleArtistMultiRolesMultiClusters)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
EXPECT_TRUE(Release::findOrphanIds(session).results.empty()); EXPECT_TRUE(Release::findOrphanIds(session).results.empty());
EXPECT_TRUE(Artist::findOrphanIds(session).results.empty()); EXPECT_TRUE(Artist::findOrphanIds(session).results.empty());
} }
@@ -430,7 +430,7 @@ TEST_F(DatabaseFixture, MultiTracksSingleArtistMultiClusters)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
EXPECT_TRUE(Artist::findOrphanIds(session).results.empty()); EXPECT_TRUE(Artist::findOrphanIds(session).results.empty());
} }
@@ -541,7 +541,7 @@ TEST_F(DatabaseFixture, SingleTrackSingleReleaseSingleArtistSingleCluster)
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty()); EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
EXPECT_TRUE(ClusterType::findOrphans(session).results.empty()); EXPECT_TRUE(ClusterType::findOrphans(session).results.empty());
EXPECT_TRUE(Artist::findOrphanIds(session).results.empty()); EXPECT_TRUE(Artist::findOrphanIds(session).results.empty());
EXPECT_TRUE(Release::findOrphanIds(session).results.empty()); EXPECT_TRUE(Release::findOrphanIds(session).results.empty());
src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp
+24 -22
View File
@@ -26,30 +26,32 @@
namespace Scanner namespace Scanner
{ {
void void ScanStepCheckDuplicatedDbFiles::process(ScanContext& context)
ScanStepCheckDuplicatedDbFiles::process(ScanContext& context) {
{ using namespace Database;
using namespace Database;
if (_abortScan) if (_abortScan)
return; return;
Session& session {_db.getTLSSession()}; Session& session{ _db.getTLSSession() };
auto transaction {session.createSharedTransaction()}; auto transaction{ session.createSharedTransaction() };
const RangeResults<TrackId> tracks = Track::findIdsTrackMBIDDuplicates(session); const RangeResults<TrackId> tracks = Track::findIdsTrackMBIDDuplicates(session);
for (const TrackId trackId : tracks.results) for (const TrackId trackId : tracks.results)
{ {
const Track::pointer track {Track::find(session, trackId)}; if (_abortScan)
if (auto trackMBID {track->getTrackMBID()}) break;
{
LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
context.stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID});
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
}
LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files"; const Track::pointer track{ Track::find(session, trackId) };
} if (auto trackMBID{ track->getTrackMBID() })
{
LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
context.stats.duplicates.emplace_back(ScanDuplicate{ track->getId(), DuplicateReason::SameTrackMBID });
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
}
LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files";
}
} }
src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp
+4 -1
View File
@@ -57,6 +57,9 @@ namespace Scanner
for (const ClusterId clusterId : clusterIds) for (const ClusterId clusterId : clusterIds)
{ {
if (_abortScan)
break;
std::size_t trackCount; std::size_t trackCount;
std::size_t releaseCount; std::size_t releaseCount;
@@ -81,6 +84,6 @@ namespace Scanner
return true; return true;
}); });
LMS_LOG(DBUPDATER, DEBUG) << "Recomputed stats for " << clusterCount << " clusters!"; LMS_LOG(DBUPDATER, DEBUG) << "Recomputed stats for " << context.currentStepStats.processedElems << " clusters!";
} }
} }
src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp
+17 -18
View File
@@ -23,26 +23,25 @@
namespace Scanner namespace Scanner
{ {
void void ScanStepDiscoverFiles::process(ScanContext& context)
ScanStepDiscoverFiles::process(ScanContext& context) {
{ context.stats.filesScanned = 0;
context.stats.filesScanned = 0; PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path)
PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path) {
{ if (_abortScan)
if (_abortScan) return false;
return false;
if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions)) if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
{ {
context.currentStepStats.processedElems++; context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats); _progressCallback(context.currentStepStats);
} }
return true; return true;
}, &excludeDirFileName); }, &excludeDirFileName);
context.stats.filesScanned = context.currentStepStats.processedElems; context.stats.filesScanned = context.currentStepStats.processedElems;
LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'"; LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'";
} }
} }
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp
+48 -44
View File
@@ -30,6 +30,49 @@
namespace Scanner namespace Scanner
{ {
using namespace Database;
namespace
{
constexpr std::size_t batchSize = 100;
template <typename T>
void removeOrphanEntries(Session& session, bool& abortScan)
{
using IdType = typename T::IdType;
RangeResults<IdType> entries;
while (!abortScan)
{
{
LMS_LOG(DBUPDATER, DEBUG) << "FIND";
auto transaction{ session.createSharedTransaction() };
entries = T::findOrphanIds(session, Range{ 0, batchSize });
LMS_LOG(DBUPDATER, DEBUG) << "FIND DONE";
};
{
LMS_LOG(DBUPDATER, DEBUG) << "REMOVE";
auto transaction{ session.createUniqueTransaction() };
for (const IdType objectId : entries.results)
{
if (abortScan)
break;
typename T::pointer entry{ T::find(session, objectId) };
LMS_LOG(DBUPDATER, DEBUG) << "Removing '" << entry->getName() << "'";
entry.remove();
}
}
if (!entries.moreResults)
break;
}
}
}
void ScanStepRemoveOrphanDbFiles::process(ScanContext& context) void ScanStepRemoveOrphanDbFiles::process(ScanContext& context)
{ {
removeOrphanTracks(context); removeOrphanTracks(context);
@@ -45,7 +88,6 @@ namespace Scanner
if (_abortScan) if (_abortScan)
return; return;
static constexpr std::size_t batchSize{ 50 };
Session& session{ _db.getTLSSession() }; Session& session{ _db.getTLSSession() };
LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed..."; LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
@@ -85,7 +127,7 @@ namespace Scanner
if (!tracksToRemove.empty()) if (!tracksToRemove.empty())
{ {
auto transaction{ session.createSharedTransaction() }; auto transaction{ session.createUniqueTransaction() };
for (const TrackId trackId : tracksToRemove) for (const TrackId trackId : tracksToRemove)
{ {
@@ -109,57 +151,20 @@ namespace Scanner
void ScanStepRemoveOrphanDbFiles::removeOrphanClusters() void ScanStepRemoveOrphanDbFiles::removeOrphanClusters()
{ {
using namespace Database;
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters..."; LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
Session& session{ _db.getTLSSession() }; removeOrphanEntries<Database::Cluster>(_db.getTLSSession(), _abortScan);
auto transaction{ session.createUniqueTransaction() };
// Now process orphan Cluster (no track)
auto clusterIds{ Cluster::findOrphans(session) };
for (ClusterId clusterId : clusterIds.results)
{
Cluster::pointer cluster{ Cluster::find(session, clusterId) };
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'";
cluster.remove();
}
} }
void ScanStepRemoveOrphanDbFiles::removeOrphanArtists() void ScanStepRemoveOrphanDbFiles::removeOrphanArtists()
{ {
using namespace Database;
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists..."; LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists...";
removeOrphanEntries<Database::Artist>(_db.getTLSSession(), _abortScan);
Session& session{ _db.getTLSSession() };
auto transaction{ session.createUniqueTransaction() };
auto artistIds{ Artist::findOrphanIds(session) };
for (const ArtistId artistId : artistIds.results)
{
Artist::pointer artist{ Artist::find(session, artistId) };
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'";
artist.remove();
}
} }
void ScanStepRemoveOrphanDbFiles::removeOrphanReleases() void ScanStepRemoveOrphanDbFiles::removeOrphanReleases()
{ {
using namespace Database;
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases..."; LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases...";
removeOrphanEntries<Database::Release>(_db.getTLSSession(), _abortScan);
// TODO, by batch
Session& session{ _db.getTLSSession() };
auto transaction{ session.createUniqueTransaction() };
auto releases{ Release::findOrphanIds(session) };
for (const ReleaseId releaseId : releases.results)
{
Release::pointer release{ Release::find(session, releaseId) };
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'";
release.remove();
}
} }
bool ScanStepRemoveOrphanDbFiles::checkFile(const std::filesystem::path& p) bool ScanStepRemoveOrphanDbFiles::checkFile(const std::filesystem::path& p)
@@ -168,8 +173,7 @@ namespace Scanner
{ {
// For each track, make sure the the file still exists // For each track, make sure the the file still exists
// and still belongs to a media directory // and still belongs to a media directory
if (!std::filesystem::exists(p) if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
|| !std::filesystem::is_regular_file(p))
{ {
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing"; LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing";
return false; return false;