owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scanner: parallelized processing within the playlist/track association step

Browse Source
This commit is contained in:
emeric
2025-07-05 19:28:18 +02:00
parent 41504c92e0
commit 1b73ae1e8c
15 changed files with 194 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/database/impl/Artist.cpp
+1 -1
View File
@@ -291,7 +291,7 @@ namespace lms::db
utils::forEachQueryRangeResult(query, params.range, func);
}
IdRange<ArtistId> Artist::findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count)
IdRange<ArtistId> Artist::findNextIdRange(Session& session, ArtistId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<ArtistId, ArtistId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT a.id FROM artist a WHERE a.id > ? ORDER BY a.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
src/libs/database/impl/PlayListFile.cpp
+21
View File
@@ -83,6 +83,27 @@ namespace lms::db
});
}
void PlayListFile::find(Session& session, const IdRange<PlayListFileId>& idRange, const std::function<void(const PlayListFile::pointer&)>& func)
{
assert(idRange.isValid());
auto query{ session.getDboSession()->query<Wt::Dbo::ptr<PlayListFile>>("SELECT pl_f from playlist_file pl_f").orderBy("pl_f.id").where("pl_f.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
utils::forEachQueryResult(query, [&](const PlayListFile::pointer& release) {
func(release);
});
}
IdRange<PlayListFileId> PlayListFile::findNextIdRange(Session& session, PlayListFileId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<PlayListFileId, PlayListFileId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT pl_f.id FROM playlist_file pl_f WHERE pl_f.id > ? ORDER BY pl_f.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
query.bind(static_cast<int>(count));
auto res{ utils::fetchQuerySingleResult(query) };
return IdRange<PlayListFileId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
}
PlayListFile::pointer PlayListFile::find(Session& session, PlayListFileId id)
{
session.checkReadTransaction();
src/libs/database/impl/Release.cpp
+1 -1
View File
@@ -515,7 +515,7 @@ namespace lms::db
});
}
IdRange<ReleaseId> Release::findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count)
IdRange<ReleaseId> Release::findNextIdRange(Session& session, ReleaseId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<ReleaseId, ReleaseId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT r.id FROM release r WHERE r.id > ? ORDER BY r.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
src/libs/database/impl/Track.cpp
+1 -1
View File
@@ -303,7 +303,7 @@ namespace lms::db
});
}
IdRange<TrackId> Track::findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count)
IdRange<TrackId> Track::findNextIdRange(Session& session, TrackId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<TrackId, TrackId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT t.id FROM track t WHERE t.id > ? ORDER BY t.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
src/libs/database/include/database/Artist.hpp
+1 -1
View File
@@ -129,7 +129,7 @@ namespace lms::db
static void find(Session& session, const IdRange<ArtistId>& idRange, const std::function<void(const Artist::pointer&)>& func);
static RangeResults<pointer> find(Session& session, const FindParameters& params);
static void find(Session& session, const FindParameters& params, std::function<void(const pointer&)> func);
static IdRange<ArtistId> findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count);
static IdRange<ArtistId> findNextIdRange(Session& session, ArtistId lastRetrievedId, std::size_t count);
static RangeResults<ArtistId> findIds(Session& session, const FindParameters& params);
static RangeResults<ArtistId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt); // No track related
static bool exists(Session& session, ArtistId id);
src/libs/database/include/database/PlayListFile.hpp
+3
View File
@@ -28,6 +28,7 @@
#include <Wt/WDateTime.h>
#include "database/DirectoryId.hpp"
#include "database/IdRange.hpp"
#include "database/Object.hpp"
LMS_DECLARE_IDTYPE(PlayListFileId)
@@ -50,6 +51,8 @@ namespace lms::db
static pointer find(Session& session, const std::filesystem::path& path);
static void find(Session& session, PlayListFileId& lastRetrievedId, std::size_t count, const std::function<void(const pointer&)>& func);
static void findAbsoluteFilePath(Session& session, PlayListFileId& lastRetrievedId, std::size_t count, const std::function<void(PlayListFileId playListFileId, const std::filesystem::path& absoluteFilePath)>& func);
static void find(Session& session, const IdRange<PlayListFileId>& idRange, const std::function<void(const PlayListFile::pointer&)>& func);
static IdRange<PlayListFileId> findNextIdRange(Session& session, PlayListFileId lastRetrievedId, std::size_t count);
// getters
const std::filesystem::path& getAbsoluteFilePath() const { return _absoluteFilePath; }
src/libs/database/include/database/Release.hpp
+1 -1
View File
@@ -251,7 +251,7 @@ namespace lms::db
static pointer find(Session& session, ReleaseId id);
static void find(Session& session, ReleaseId& lastRetrievedRelease, std::size_t count, const std::function<void(const Release::pointer&)>& func, MediaLibraryId library = {});
static void find(Session& session, const IdRange<ReleaseId>& idRange, const std::function<void(const Release::pointer&)>& func);
static IdRange<ReleaseId> findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count);
static IdRange<ReleaseId> findNextIdRange(Session& session, ReleaseId lastRetrievedId, std::size_t count);
static RangeResults<pointer> find(Session& session, const FindParameters& parameters);
static void find(Session& session, const FindParameters& parameters, const std::function<void(const pointer&)>& func);
static RangeResults<ReleaseId> findIds(Session& session, const FindParameters& parameters);
src/libs/database/include/database/Track.hpp
+1 -1
View File
@@ -194,7 +194,7 @@ namespace lms::db
static pointer find(Session& session, TrackId id);
static void find(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(const Track::pointer&)>& func, MediaLibraryId library = {});
static void find(Session& session, const IdRange<TrackId>& idRange, const std::function<void(const Track::pointer&)>& func);
static IdRange<TrackId> findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count);
static IdRange<TrackId> findNextIdRange(Session& session, TrackId lastRetrievedId, std::size_t count);
static void findAbsoluteFilePath(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(TrackId trackId, const std::filesystem::path& absoluteFilePath)>& func);
static bool exists(Session& session, TrackId id);
src/libs/database/test/Artist.cpp
+9 -9
View File
@@ -608,17 +608,17 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Artist_findNextRange)
TEST_F(DatabaseFixture, Artist_findNextIdRange)
{
{
auto transaction{ session.createReadTransaction() };
auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
auto range{ Artist::findNextIdRange(session, ArtistId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
range = Artist::findNextRange(session, ArtistId{}, 100);
range = Artist::findNextIdRange(session, ArtistId{}, 100);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
@@ -628,22 +628,22 @@ namespace lms::db::tests
{
auto transaction{ session.createReadTransaction() };
auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
auto range{ Artist::findNextIdRange(session, ArtistId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
range = Artist::findNextRange(session, ArtistId{}, 1);
range = Artist::findNextIdRange(session, ArtistId{}, 1);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist1.getId());
EXPECT_EQ(range.last, artist1.getId());
range = Artist::findNextRange(session, range.last, 1);
range = Artist::findNextIdRange(session, range.last, 1);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
range = Artist::findNextRange(session, ArtistId{}, 100);
range = Artist::findNextIdRange(session, ArtistId{}, 100);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist1.getId());
EXPECT_EQ(range.last, artist1.getId());
@@ -655,12 +655,12 @@ namespace lms::db::tests
{
auto transaction{ session.createReadTransaction() };
auto range{ Artist::findNextRange(session, ArtistId{}, 2) };
auto range{ Artist::findNextIdRange(session, ArtistId{}, 2) };
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist1.getId());
EXPECT_EQ(range.last, artist2.getId());
range = Artist::findNextRange(session, artist2.getId(), 2);
range = Artist::findNextIdRange(session, artist2.getId(), 2);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist3.getId());
EXPECT_EQ(range.last, artist3.getId());
src/libs/database/test/Release.cpp
+9 -9
View File
@@ -173,17 +173,17 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Release_findNextRange)
TEST_F(DatabaseFixture, Release_findNextIdRange)
{
{
auto transaction{ session.createReadTransaction() };
auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
auto range{ Release::findNextIdRange(session, ReleaseId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
range = Release::findNextRange(session, ReleaseId{}, 100);
range = Release::findNextIdRange(session, ReleaseId{}, 100);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
@@ -193,22 +193,22 @@ namespace lms::db::tests
{
auto transaction{ session.createReadTransaction() };
auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
auto range{ Release::findNextIdRange(session, ReleaseId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
range = Release::findNextRange(session, ReleaseId{}, 1);
range = Release::findNextIdRange(session, ReleaseId{}, 1);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release1.getId());
EXPECT_EQ(range.last, release1.getId());
range = Release::findNextRange(session, range.last, 1);
range = Release::findNextIdRange(session, range.last, 1);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
range = Release::findNextRange(session, ReleaseId{}, 100);
range = Release::findNextIdRange(session, ReleaseId{}, 100);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release1.getId());
EXPECT_EQ(range.last, release1.getId());
@@ -220,12 +220,12 @@ namespace lms::db::tests
{
auto transaction{ session.createReadTransaction() };
auto range{ Release::findNextRange(session, ReleaseId{}, 2) };
auto range{ Release::findNextIdRange(session, ReleaseId{}, 2) };
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release1.getId());
EXPECT_EQ(range.last, release2.getId());
range = Release::findNextRange(session, release2.getId(), 2);
range = Release::findNextIdRange(session, release2.getId(), 2);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release3.getId());
EXPECT_EQ(range.last, release3.getId());
src/libs/database/test/Track.cpp
+9 -9
View File
@@ -151,17 +151,17 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Track_findNextRange)
TEST_F(DatabaseFixture, Track_findNextIdRange)
{
{
auto transaction{ session.createReadTransaction() };
auto range{ Track::findNextRange(session, TrackId{}, 0) };
auto range{ Track::findNextIdRange(session, TrackId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
range = Track::findNextRange(session, TrackId{}, 100);
range = Track::findNextIdRange(session, TrackId{}, 100);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
@@ -171,22 +171,22 @@ namespace lms::db::tests
{
auto transaction{ session.createReadTransaction() };
auto range{ Track::findNextRange(session, TrackId{}, 0) };
auto range{ Track::findNextIdRange(session, TrackId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
range = Track::findNextRange(session, TrackId{}, 1);
range = Track::findNextIdRange(session, TrackId{}, 1);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track1.getId());
EXPECT_EQ(range.last, track1.getId());
range = Track::findNextRange(session, range.last, 1);
range = Track::findNextIdRange(session, range.last, 1);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
range = Track::findNextRange(session, TrackId{}, 100);
range = Track::findNextIdRange(session, TrackId{}, 100);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track1.getId());
EXPECT_EQ(range.last, track1.getId());
@@ -198,12 +198,12 @@ namespace lms::db::tests
{
auto transaction{ session.createReadTransaction() };
auto range{ Track::findNextRange(session, TrackId{}, 2) };
auto range{ Track::findNextIdRange(session, TrackId{}, 2) };
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track1.getId());
EXPECT_EQ(range.last, track2.getId());
range = Track::findNextRange(session, track2.getId(), 2);
range = Track::findNextIdRange(session, track2.getId(), 2);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track3.getId());
EXPECT_EQ(range.last, track3.getId());