owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made the last scan step (checking for removed files) faster on big libraries

Browse Source
This commit is contained in:
emeric
2024-03-14 23:21:30 +01:00
parent 071b86602e
commit 03425f3c94
4 changed files with 104 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/database/impl/Track.cpp
+26
View File
@@ -228,6 +228,32 @@ namespace lms::db
.resultValue();
}
void Track::find(Session& session, TrackId& lastRetrievedTrack, std::size_t batchSize, bool& moreResults, const std::function<void(const Track::pointer&)>& func)
{
session.checkReadTransaction();
auto collection{ session.getDboSession().find<Track>()
.orderBy("id")
.where("id > ?").bind(lastRetrievedTrack)
.limit(static_cast<int>(batchSize) + 1)
.resultList() };
moreResults = false;
std::size_t count{};
for (auto itResult{ collection.begin() }; itResult != collection.end(); ++itResult)
{
if (count++ == batchSize)
{
moreResults = true;
break;
}
func(*itResult);
lastRetrievedTrack = (*itResult)->getId();
}
}
bool Track::exists(Session& session, TrackId id)
{
session.checkReadTransaction();
src/libs/database/include/database/Track.hpp
+2 -1
View File
@@ -110,7 +110,8 @@ namespace lms::db
static std::size_t getCount(Session& session);
static pointer findByPath(Session& session, const std::filesystem::path& p);
static pointer find(Session& session, TrackId id);
static bool exists(Session& session, TrackId id);
static void find(Session& session, TrackId& lastRetrievedTrack, std::size_t batchSize, bool& moreResults, const std::function<void(const Track::pointer&)>& func);
static bool exists(Session& session, TrackId id);
static std::vector<pointer> findByRecordingMBID(Session& session, const core::UUID& MBID);
static std::vector<pointer> findByMBID(Session& session, const core::UUID& MBID);
static RangeResults<TrackId> findSimilarTrackIds(Session& session, const std::vector<TrackId>& trackIds, std::optional<Range> range = std::nullopt);
src/libs/database/test/Track.cpp
+56
View File
@@ -63,6 +63,62 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Track_findByRangedIdBased)
{
ScopedTrack track1{ session, "MyTrackFile1" };
ScopedTrack track2{ session, "MyTrackFile1" };
ScopedTrack track3{ session, "MyTrackFile1" };
{
auto transaction{ session.createReadTransaction() };
bool moreResults;
TrackId lastRetrievedTrackId;
std::vector<Track::pointer> visitedTracks;
Track::find(session, lastRetrievedTrackId, 10, moreResults, [&](const Track::pointer& track)
{
visitedTracks.push_back(track);
});
ASSERT_EQ(visitedTracks.size(), 3);
EXPECT_EQ(visitedTracks[0]->getId(), track1.getId());
EXPECT_EQ(visitedTracks[1]->getId(), track2.getId());
EXPECT_EQ(visitedTracks[2]->getId(), track3.getId());
EXPECT_FALSE(moreResults);
EXPECT_EQ(lastRetrievedTrackId, track3.getId());
}
{
auto transaction{ session.createReadTransaction() };
bool moreResults;
TrackId lastRetrievedTrackId{ track1.getId() };
std::vector<Track::pointer> visitedTracks;
Track::find(session, lastRetrievedTrackId, 1, moreResults, [&](const Track::pointer& track)
{
visitedTracks.push_back(track);
});
ASSERT_EQ(visitedTracks.size(), 1);
EXPECT_EQ(visitedTracks[0]->getId(), track2.getId());
EXPECT_TRUE(moreResults);
EXPECT_EQ(lastRetrievedTrackId, track2.getId());
}
{
auto transaction{ session.createReadTransaction() };
bool moreResults;
TrackId lastRetrievedTrackId{ track1.getId() };
std::vector<Track::pointer> visitedTracks;
Track::find(session, lastRetrievedTrackId, 0, moreResults, [&](const Track::pointer& track)
{
visitedTracks.push_back(track);
});
ASSERT_EQ(visitedTracks.size(), 0);
EXPECT_TRUE(moreResults);
EXPECT_EQ(lastRetrievedTrackId, track1.getId());
}
}
TEST_F(DatabaseFixture, Track_MediaLibrary)
{
ScopedTrack track{ session, "MyTrackFile" };
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp
+20 -33
View File
@@ -89,62 +89,49 @@ namespace lms::scanner
Session& session{ _db.getTLSSession() };
LMS_LOG(DBUPDATER, DEBUG, "Checking tracks to be removed...");
std::size_t trackCount{};
{
auto transaction{ session.createReadTransaction() };
trackCount = Track::getCount(session);
context.currentStepStats.totalElems = Track::getCount(session);
}
LMS_LOG(DBUPDATER, DEBUG, trackCount << " tracks to be checked...");
LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " tracks to be checked...");
context.currentStepStats.totalElems = trackCount;
// TODO handle only files in context.directory?
std::vector<Track::pointer> tracksToRemove;
RangeResults<Track::PathResult> trackPaths;
std::vector<TrackId> tracksToRemove;
// TODO handle only files in context.directory
for (std::size_t i{ trackCount < batchSize ? 0 : trackCount - batchSize }; ; i -= (i > batchSize ? batchSize : i))
TrackId lastCheckedTrackID;
bool moreResults{ true };
while (moreResults)
{
tracksToRemove.clear();
if (_abortScan)
break;
tracksToRemove.clear();
{
auto transaction{ session.createReadTransaction() };
trackPaths = Track::findPaths(session, Range{ i, batchSize });
}
Track::find(session, lastCheckedTrackID, batchSize, moreResults, [&](const Track::pointer& track)
{
if (!checkFile(track->getPath()))
tracksToRemove.push_back(track);
for (const Track::PathResult& trackPath : trackPaths.results)
{
if (_abortScan)
return;
if (!checkFile(trackPath.path))
tracksToRemove.push_back(trackPath.trackId);
context.currentStepStats.processedElems++;
context.currentStepStats.processedElems++;
});
}
if (!tracksToRemove.empty())
{
auto transaction{ session.createWriteTransaction() };
for (const TrackId trackId : tracksToRemove)
for (Track::pointer& track : tracksToRemove)
{
Track::pointer track{ Track::find(session, trackId) };
if (track)
{
track.remove();
context.stats.deletions++;
}
track.remove();
context.stats.deletions++;
}
}
_progressCallback(context.currentStepStats);
if (i == 0)
break;
}
LMS_LOG(DBUPDATER, DEBUG, trackCount << " tracks checked!");
LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.processedElems << " tracks checked!");
}
void ScanStepRemoveOrphanDbFiles::removeOrphanClusters()