owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made scanner abort faster when it has to clean a lot of stuff

Browse Source
This commit is contained in:
emeric
2023-11-14 13:47:08 +01:00
parent 9cfc47fc82
commit b4e515f825
7 changed files with 107 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/database/impl/Cluster.cpp
+1 -1
View File
@@ -106,7 +106,7 @@ namespace Database
return Utils::execQuery<Cluster::pointer>(query, params.range);
}
RangeResults<ClusterId> Cluster::findOrphans(Session& session, std::optional<Range> range)
RangeResults<ClusterId> Cluster::findOrphanIds(Session& session, std::optional<Range> range)
{
session.checkSharedLocked();
auto query{ session.getDboSession().query<ClusterId>("SELECT DISTINCT c.id FROM cluster c WHERE NOT EXISTS(SELECT 1 FROM track_cluster t_c WHERE t_c.cluster_id = c.id)") };
src/libs/services/database/include/services/database/Cluster.hpp
+1 -1
View File
@@ -65,7 +65,7 @@ namespace Database {
static RangeResults<pointer> find(Session& session, const FindParameters& params);
static void find(Session& session, const FindParameters& params, std::function<void(const pointer& cluster)> _func);
static pointer find(Session& session, ClusterId id);
static RangeResults<ClusterId> findOrphans(Session& session, std::optional<Range> range = std::nullopt);
static RangeResults<ClusterId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt);
// May be very slow
static std::size_t computeTrackCount(Session& session, ClusterId id);
src/libs/services/database/test/Cluster.cpp
+12 -12
View File
@@ -55,7 +55,7 @@ TEST_F(DatabaseFixture, Cluster)
}
{
const auto clusters{ Cluster::findOrphans(session) };
const auto clusters{ Cluster::findOrphanIds(session) };
ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(clusters.results.front(), cluster.getId());
}
@@ -91,7 +91,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
{
auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty());
EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
auto clusterTypes{ ClusterType::findOrphans(session) };
ASSERT_EQ(clusterTypes.results.size(), 1);
EXPECT_EQ(clusterTypes.results.front(), clusterType.getId());
@@ -102,7 +102,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
{
auto transaction{ session.createSharedTransaction() };
auto clusters{ Cluster::findOrphans(session) };
auto clusters{ Cluster::findOrphanIds(session) };
EXPECT_EQ(clusters.results.size(), 2);
EXPECT_TRUE(track->getClusters().empty());
EXPECT_TRUE(track->getClusterIds().empty());
@@ -127,7 +127,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrack)
{
auto transaction{ session.createSharedTransaction() };
auto clusters{ Cluster::findOrphans(session) };
auto clusters{ Cluster::findOrphanIds(session) };
ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(clusters.results.front(), cluster2.getId());
@@ -225,7 +225,7 @@ TEST_F(DatabaseFixture, Cluster_multiTracks)
{
auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty());
EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
EXPECT_EQ(Cluster::computeTrackCount(session, cluster.getId()), tracks.size());
@@ -244,7 +244,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
{
auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty());
EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
}
ScopedClusterType clusterType{ session, "MyClusterType" };
@@ -253,7 +253,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
{
auto transaction{ session.createSharedTransaction() };
ASSERT_EQ(Cluster::findOrphans(session).results.size(), 2);
ASSERT_EQ(Cluster::findOrphanIds(session).results.size(), 2);
EXPECT_TRUE(Release::find(session, Release::FindParameters{}.setClusters({ unusedCluster.getId() })).results.empty());
EXPECT_EQ(Release::find(session, Release::FindParameters{}).results.size(), 1);
EXPECT_EQ(Cluster::computeReleaseCount(session, cluster.getId()), 0);
@@ -271,7 +271,7 @@ TEST_F(DatabaseFixture, Cluster_singleTrackSingleReleaseSingleCluster)
auto transaction{ session.createSharedTransaction() };
{
auto clusters{ Cluster::findOrphans(session) };
auto clusters{ Cluster::findOrphanIds(session) };
ASSERT_EQ(clusters.results.size(), 1);
EXPECT_EQ(clusters.results.front(), unusedCluster.getId());
}
@@ -330,7 +330,7 @@ TEST_F(DatabaseFixture, SingleTrackSingleArtistMultiClusters)
{
auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(ClusterType::findOrphans(session).results.empty());
EXPECT_EQ(Cluster::findOrphans(session).results.size(), 2);
EXPECT_EQ(Cluster::findOrphanIds(session).results.size(), 2);
EXPECT_TRUE(Release::findOrphanIds(session).results.empty());
EXPECT_TRUE(Artist::findOrphanIds(session).results.empty());
}
@@ -390,7 +390,7 @@ TEST_F(DatabaseFixture, SingleTrackSingleArtistMultiRolesMultiClusters)
{
auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty());
EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
EXPECT_TRUE(Release::findOrphanIds(session).results.empty());
EXPECT_TRUE(Artist::findOrphanIds(session).results.empty());
}
@@ -430,7 +430,7 @@ TEST_F(DatabaseFixture, MultiTracksSingleArtistMultiClusters)
{
auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty());
EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
EXPECT_TRUE(Artist::findOrphanIds(session).results.empty());
}
@@ -541,7 +541,7 @@ TEST_F(DatabaseFixture, SingleTrackSingleReleaseSingleArtistSingleCluster)
{
auto transaction{ session.createSharedTransaction() };
EXPECT_TRUE(Cluster::findOrphans(session).results.empty());
EXPECT_TRUE(Cluster::findOrphanIds(session).results.empty());
EXPECT_TRUE(ClusterType::findOrphans(session).results.empty());
EXPECT_TRUE(Artist::findOrphanIds(session).results.empty());
EXPECT_TRUE(Release::findOrphanIds(session).results.empty());
src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp
+24 -22
View File
@@ -26,30 +26,32 @@
namespace Scanner
{
void
ScanStepCheckDuplicatedDbFiles::process(ScanContext& context)
{
using namespace Database;
void ScanStepCheckDuplicatedDbFiles::process(ScanContext& context)
{
using namespace Database;
if (_abortScan)
return;
if (_abortScan)
return;
Session& session {_db.getTLSSession()};
auto transaction {session.createSharedTransaction()};
Session& session{ _db.getTLSSession() };
auto transaction{ session.createSharedTransaction() };
const RangeResults<TrackId> tracks = Track::findIdsTrackMBIDDuplicates(session);
for (const TrackId trackId : tracks.results)
{
const Track::pointer track {Track::find(session, trackId)};
if (auto trackMBID {track->getTrackMBID()})
{
LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
context.stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID});
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
}
const RangeResults<TrackId> tracks = Track::findIdsTrackMBIDDuplicates(session);
for (const TrackId trackId : tracks.results)
{
if (_abortScan)
break;
LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files";
}
const Track::pointer track{ Track::find(session, trackId) };
if (auto trackMBID{ track->getTrackMBID() })
{
LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
context.stats.duplicates.emplace_back(ScanDuplicate{ track->getId(), DuplicateReason::SameTrackMBID });
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
}
LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files";
}
}
src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp
+4 -1
View File
@@ -57,6 +57,9 @@ namespace Scanner
for (const ClusterId clusterId : clusterIds)
{
if (_abortScan)
break;
std::size_t trackCount;
std::size_t releaseCount;
@@ -81,6 +84,6 @@ namespace Scanner
return true;
});
LMS_LOG(DBUPDATER, DEBUG) << "Recomputed stats for " << clusterCount << " clusters!";
LMS_LOG(DBUPDATER, DEBUG) << "Recomputed stats for " << context.currentStepStats.processedElems << " clusters!";
}
}
src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp
+17 -18
View File
@@ -23,26 +23,25 @@
namespace Scanner
{
void
ScanStepDiscoverFiles::process(ScanContext& context)
{
context.stats.filesScanned = 0;
PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path)
{
if (_abortScan)
return false;
void ScanStepDiscoverFiles::process(ScanContext& context)
{
context.stats.filesScanned = 0;
PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path)
{
if (_abortScan)
return false;
if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
{
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
{
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
return true;
}, &excludeDirFileName);
return true;
}, &excludeDirFileName);
context.stats.filesScanned = context.currentStepStats.processedElems;
context.stats.filesScanned = context.currentStepStats.processedElems;
LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'";
}
LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'";
}
}
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp
+48 -44
View File
@@ -30,6 +30,49 @@
namespace Scanner
{
using namespace Database;
namespace
{
constexpr std::size_t batchSize = 100;
template <typename T>
void removeOrphanEntries(Session& session, bool& abortScan)
{
using IdType = typename T::IdType;
RangeResults<IdType> entries;
while (!abortScan)
{
{
LMS_LOG(DBUPDATER, DEBUG) << "FIND";
auto transaction{ session.createSharedTransaction() };
entries = T::findOrphanIds(session, Range{ 0, batchSize });
LMS_LOG(DBUPDATER, DEBUG) << "FIND DONE";
};
{
LMS_LOG(DBUPDATER, DEBUG) << "REMOVE";
auto transaction{ session.createUniqueTransaction() };
for (const IdType objectId : entries.results)
{
if (abortScan)
break;
typename T::pointer entry{ T::find(session, objectId) };
LMS_LOG(DBUPDATER, DEBUG) << "Removing '" << entry->getName() << "'";
entry.remove();
}
}
if (!entries.moreResults)
break;
}
}
}
void ScanStepRemoveOrphanDbFiles::process(ScanContext& context)
{
removeOrphanTracks(context);
@@ -45,7 +88,6 @@ namespace Scanner
if (_abortScan)
return;
static constexpr std::size_t batchSize{ 50 };
Session& session{ _db.getTLSSession() };
LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
@@ -85,7 +127,7 @@ namespace Scanner
if (!tracksToRemove.empty())
{
auto transaction{ session.createSharedTransaction() };
auto transaction{ session.createUniqueTransaction() };
for (const TrackId trackId : tracksToRemove)
{
@@ -109,57 +151,20 @@ namespace Scanner
void ScanStepRemoveOrphanDbFiles::removeOrphanClusters()
{
using namespace Database;
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
Session& session{ _db.getTLSSession() };
auto transaction{ session.createUniqueTransaction() };
// Now process orphan Cluster (no track)
auto clusterIds{ Cluster::findOrphans(session) };
for (ClusterId clusterId : clusterIds.results)
{
Cluster::pointer cluster{ Cluster::find(session, clusterId) };
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'";
cluster.remove();
}
removeOrphanEntries<Database::Cluster>(_db.getTLSSession(), _abortScan);
}
void ScanStepRemoveOrphanDbFiles::removeOrphanArtists()
{
using namespace Database;
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists...";
Session& session{ _db.getTLSSession() };
auto transaction{ session.createUniqueTransaction() };
auto artistIds{ Artist::findOrphanIds(session) };
for (const ArtistId artistId : artistIds.results)
{
Artist::pointer artist{ Artist::find(session, artistId) };
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'";
artist.remove();
}
removeOrphanEntries<Database::Artist>(_db.getTLSSession(), _abortScan);
}
void ScanStepRemoveOrphanDbFiles::removeOrphanReleases()
{
using namespace Database;
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases...";
// TODO, by batch
Session& session{ _db.getTLSSession() };
auto transaction{ session.createUniqueTransaction() };
auto releases{ Release::findOrphanIds(session) };
for (const ReleaseId releaseId : releases.results)
{
Release::pointer release{ Release::find(session, releaseId) };
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'";
release.remove();
}
removeOrphanEntries<Database::Release>(_db.getTLSSession(), _abortScan);
}
bool ScanStepRemoveOrphanDbFiles::checkFile(const std::filesystem::path& p)
@@ -168,8 +173,7 @@ namespace Scanner
{
// For each track, make sure the the file still exists
// and still belongs to a media directory
if (!std::filesystem::exists(p)
|| !std::filesystem::is_regular_file(p))
if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing";
return false;