owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a scan tracker to speed up queries with high offset in very large database

Browse Source
This commit is contained in:
emeric
2024-03-17 23:36:40 +01:00
parent fef4a90ce0
commit ed1be02aaf
9 changed files with 196 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/database/impl/Track.cpp
+7 -12
View File
@@ -228,27 +228,22 @@ namespace lms::db
.resultValue();
}
void Track::find(Session& session, TrackId& lastRetrievedTrack, std::size_t batchSize, bool& moreResults, const std::function<void(const Track::pointer&)>& func)
void Track::find(Session& session, TrackId& lastRetrievedTrack, std::size_t count, const std::function<void(const Track::pointer&)>& func, MediaLibraryId library)
{
session.checkReadTransaction();
auto collection{ session.getDboSession().find<Track>()
auto query{ session.getDboSession().find<Track>()
.orderBy("id")
.where("id > ?").bind(lastRetrievedTrack)
.limit(static_cast<int>(batchSize) + 1)
.resultList() };
.limit(static_cast<int>(count)) };
moreResults = false;
if (library.isValid())
query.where("media_library_id = ?").bind(library);
auto collection{query.resultList()};
std::size_t count{};
for (auto itResult{ collection.begin() }; itResult != collection.end(); ++itResult)
{
if (count++ == batchSize)
{
moreResults = true;
break;
}
func(*itResult);
lastRetrievedTrack = (*itResult)->getId();
}
src/libs/database/include/database/Track.hpp
+1 -1
View File
@@ -110,7 +110,7 @@ namespace lms::db
static std::size_t getCount(Session& session);
static pointer findByPath(Session& session, const std::filesystem::path& p);
static pointer find(Session& session, TrackId id);
static void find(Session& session, TrackId& lastRetrievedTrack, std::size_t batchSize, bool& moreResults, const std::function<void(const Track::pointer&)>& func);
static void find(Session& session, TrackId& lastRetrievedTrack, std::size_t count, const std::function<void(const Track::pointer&)>& func, MediaLibraryId library = {});
static bool exists(Session& session, TrackId id);
static std::vector<pointer> findByRecordingMBID(Session& session, const core::UUID& MBID);
static std::vector<pointer> findByMBID(Session& session, const core::UUID& MBID);
src/libs/database/test/Track.cpp
+37 -9
View File
@@ -68,14 +68,20 @@ namespace lms::db::tests
ScopedTrack track1{ session, "MyTrackFile1" };
ScopedTrack track2{ session, "MyTrackFile1" };
ScopedTrack track3{ session, "MyTrackFile1" };
ScopedMediaLibrary library{ session };
ScopedMediaLibrary otherLibrary{ session };
{
auto transaction{ session.createWriteTransaction() };
track2.get().modify()->setMediaLibrary(library.get());
}
{
auto transaction{ session.createReadTransaction() };
bool moreResults;
TrackId lastRetrievedTrackId;
std::vector<Track::pointer> visitedTracks;
Track::find(session, lastRetrievedTrackId, 10, moreResults, [&](const Track::pointer& track)
Track::find(session, lastRetrievedTrackId, 10, [&](const Track::pointer& track)
{
visitedTracks.push_back(track);
});
@@ -83,40 +89,62 @@ namespace lms::db::tests
EXPECT_EQ(visitedTracks[0]->getId(), track1.getId());
EXPECT_EQ(visitedTracks[1]->getId(), track2.getId());
EXPECT_EQ(visitedTracks[2]->getId(), track3.getId());
EXPECT_FALSE(moreResults);
EXPECT_EQ(lastRetrievedTrackId, track3.getId());
}
{
auto transaction{ session.createReadTransaction() };
bool moreResults;
TrackId lastRetrievedTrackId{ track1.getId() };
std::vector<Track::pointer> visitedTracks;
Track::find(session, lastRetrievedTrackId, 1, moreResults, [&](const Track::pointer& track)
Track::find(session, lastRetrievedTrackId, 1, [&](const Track::pointer& track)
{
visitedTracks.push_back(track);
});
ASSERT_EQ(visitedTracks.size(), 1);
EXPECT_EQ(visitedTracks[0]->getId(), track2.getId());
EXPECT_TRUE(moreResults);
EXPECT_EQ(lastRetrievedTrackId, track2.getId());
}
{
auto transaction{ session.createReadTransaction() };
bool moreResults;
TrackId lastRetrievedTrackId{ track1.getId() };
std::vector<Track::pointer> visitedTracks;
Track::find(session, lastRetrievedTrackId, 0, moreResults, [&](const Track::pointer& track)
Track::find(session, lastRetrievedTrackId, 0, [&](const Track::pointer& track)
{
visitedTracks.push_back(track);
});
ASSERT_EQ(visitedTracks.size(), 0);
EXPECT_TRUE(moreResults);
EXPECT_EQ(lastRetrievedTrackId, track1.getId());
}
{
auto transaction{ session.createReadTransaction() };
TrackId lastRetrievedTrackId{};
std::vector<Track::pointer> visitedTracks;
Track::find(session, lastRetrievedTrackId, 10, [&](const Track::pointer& track)
{
visitedTracks.push_back(track);
}, otherLibrary.getId());
ASSERT_EQ(visitedTracks.size(), 0);
EXPECT_EQ(lastRetrievedTrackId, TrackId{});
}
{
auto transaction{ session.createReadTransaction() };
TrackId lastRetrievedTrackId{};
std::vector<Track::pointer> visitedTracks;
Track::find(session, lastRetrievedTrackId, 10, [&](const Track::pointer& track)
{
visitedTracks.push_back(track);
}, library.getId());
ASSERT_EQ(visitedTracks.size(), 1);
EXPECT_EQ(visitedTracks[0]->getId(), track2.getId());
EXPECT_EQ(lastRetrievedTrackId, track2.getId());
}
}
TEST_F(DatabaseFixture, Track_MediaLibrary)