owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scanner: parallelized processing within artwork association steps

Browse Source
This commit is contained in:
emeric
2025-07-04 23:05:35 +02:00
parent 427f777c44
commit 204a4a8857
23 changed files with 982 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/database/impl/Artist.cpp
+31
View File
@@ -237,6 +237,17 @@ namespace lms::db
});
}
void Artist::find(Session& session, const IdRange<ArtistId>& idRange, const std::function<void(const Artist::pointer&)>& func)
{
assert(idRange.isValid());
auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Artist>>("SELECT a from artist a").orderBy("a.id").where("a.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
utils::forEachQueryResult(query, [&](const Artist::pointer& artist) {
func(artist);
});
}
std::vector<Artist::pointer> Artist::find(Session& session, std::string_view name)
{
session.checkReadTransaction();
@@ -280,6 +291,16 @@ namespace lms::db
utils::forEachQueryRangeResult(query, params.range, func);
}
IdRange<ArtistId> Artist::findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<ArtistId, ArtistId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT a.id FROM artist a WHERE a.id > ? ORDER BY a.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
query.bind(static_cast<int>(count));
auto res{ utils::fetchQuerySingleResult(query) };
return IdRange<ArtistId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
}
RangeResults<ArtistId> Artist::findOrphanIds(Session& session, std::optional<Range> range)
{
session.checkReadTransaction();
@@ -304,6 +325,16 @@ AND NOT EXISTS (
return utils::fetchQuerySingleResult(session.getDboSession()->query<int>("SELECT 1 FROM artist").where("id = ?").bind(id)) == 1;
}
void Artist::updatePreferredArtwork(Session& session, ArtistId artistId, ArtworkId artworkId)
{
session.checkWriteTransaction();
if (artworkId.isValid())
utils::executeCommand(*session.getDboSession(), "UPDATE artist SET preferred_artwork_id = ? WHERE id = ?", artworkId, artistId);
else
utils::executeCommand(*session.getDboSession(), "UPDATE artist SET preferred_artwork_id = NULL WHERE id = ?", artistId);
}
std::optional<core::UUID> Artist::getMBID() const
{
return core::UUID::fromString(_mbid);
src/libs/database/impl/Migration.cpp
+18 -1
View File
@@ -35,7 +35,7 @@ namespace lms::db
{
namespace
{
static constexpr Version LMS_DATABASE_VERSION{ 95 };
static constexpr Version LMS_DATABASE_VERSION{ 96 };
}
VersionInfo::VersionInfo()
@@ -1396,6 +1396,22 @@ FROM artist)");
utils::executeCommand(*session.getDboSession(), "ALTER TABLE track DROP COLUMN file_name");
}
void migrateFromV95(Session& session)
{
// Make sure each image and each embessed image has an artwork object
utils::executeCommand(*session.getDboSession(), R"(INSERT INTO artwork (version, track_embedded_image_id)
SELECT 1 AS version, tei.id
FROM track_embedded_image tei
LEFT JOIN artwork art ON tei.id = art.track_embedded_image_id
WHERE art.track_embedded_image_id IS NULL)");
utils::executeCommand(*session.getDboSession(), R"(INSERT INTO artwork (version, image_id)
SELECT 1 AS version, img.id
FROM image img
LEFT JOIN artwork art ON img.id = art.image_id
WHERE art.image_id IS NULL)");
}
bool doDbMigration(Session& session)
{
constexpr std::string_view outdatedMsg{ "Outdated database, please rebuild it (delete the .db file and restart)" };
@@ -1467,6 +1483,7 @@ FROM artist)");
{ 92, migrateFromV92 },
{ 93, migrateFromV93 },
{ 94, migrateFromV94 },
{ 95, migrateFromV95 },
};
bool migrationPerformed{};
src/libs/database/impl/Release.cpp
+31
View File
@@ -504,6 +504,27 @@ namespace lms::db
});
}
void Release::find(Session& session, const IdRange<ReleaseId>& idRange, const std::function<void(const Release::pointer&)>& func)
{
assert(idRange.isValid());
auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Release>>("SELECT r from release r").orderBy("r.id").where("r.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
utils::forEachQueryResult(query, [&](const Release::pointer& release) {
func(release);
});
}
IdRange<ReleaseId> Release::findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<ReleaseId, ReleaseId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT r.id FROM release r WHERE r.id > ? ORDER BY r.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
query.bind(static_cast<int>(count));
auto res{ utils::fetchQuerySingleResult(query) };
return IdRange<ReleaseId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
}
RangeResults<Release::pointer> Release::find(Session& session, const FindParameters& params)
{
session.checkReadTransaction();
@@ -528,6 +549,16 @@ namespace lms::db
return utils::execRangeQuery<ReleaseId>(query, params.range);
}
void Release::updatePreferredArtwork(Session& session, ReleaseId releaseId, ArtworkId artworkId)
{
session.checkWriteTransaction();
if (artworkId.isValid())
utils::executeCommand(*session.getDboSession(), "UPDATE release SET preferred_artwork_id = ? WHERE id = ?", artworkId, releaseId);
else
utils::executeCommand(*session.getDboSession(), "UPDATE release SET preferred_artwork_id = NULL WHERE id = ?", releaseId);
}
std::size_t Release::getCount(Session& session, const FindParameters& params)
{
session.checkReadTransaction();
src/libs/database/impl/Track.cpp
+21
View File
@@ -292,6 +292,27 @@ namespace lms::db
});
}
void Track::find(Session& session, const IdRange<TrackId>& idRange, const std::function<void(const Track::pointer&)>& func)
{
assert(idRange.isValid());
auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Track>>("SELECT t from track t").orderBy("t.id").where("t.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
utils::forEachQueryResult(query, [&](const Track::pointer& track) {
func(track);
});
}
IdRange<TrackId> Track::findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<TrackId, TrackId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT t.id FROM track t WHERE t.id > ? ORDER BY t.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
query.bind(static_cast<int>(count));
auto res{ utils::fetchQuerySingleResult(query) };
return IdRange<TrackId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
}
bool Track::exists(Session& session, TrackId id)
{
session.checkReadTransaction();
src/libs/database/include/database/Artist.hpp
+6
View File
@@ -34,6 +34,7 @@
#include "database/ArtworkId.hpp"
#include "database/ClusterId.hpp"
#include "database/Filters.hpp"
#include "database/IdRange.hpp"
#include "database/MediaLibraryId.hpp"
#include "database/Object.hpp"
#include "database/ReleaseId.hpp"
@@ -125,12 +126,17 @@ namespace lms::db
static pointer find(Session& session, ArtistId id);
static std::vector<pointer> find(Session& session, std::string_view name); // exact match on name field
static void find(Session& session, ArtistId& lastRetrievedArtist, std::size_t count, const std::function<void(const Artist::pointer&)>& func, MediaLibraryId library = {});
static void find(Session& session, const IdRange<ArtistId>& idRange, const std::function<void(const Artist::pointer&)>& func);
static RangeResults<pointer> find(Session& session, const FindParameters& params);
static void find(Session& session, const FindParameters& params, std::function<void(const pointer&)> func);
static IdRange<ArtistId> findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count);
static RangeResults<ArtistId> findIds(Session& session, const FindParameters& params);
static RangeResults<ArtistId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt); // No track related
static bool exists(Session& session, ArtistId id);
// Updates
static void updatePreferredArtwork(Session& session, ArtistId artistId, ArtworkId artworkId);
// Accessors
const std::string& getName() const { return _name; }
const std::string& getSortName() const { return _sortName; }
src/libs/database/include/database/IdRange.hpp
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2025 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
namespace lms::db
{
template<typename IdType>
struct IdRange
{
public:
IdType first;
IdType last;
bool isValid() const { return first.isValid() && last.isValid(); }
};
} // namespace lms::db
src/libs/database/include/database/Release.hpp
+6
View File
@@ -35,6 +35,7 @@
#include "database/CountryId.hpp"
#include "database/DirectoryId.hpp"
#include "database/Filters.hpp"
#include "database/IdRange.hpp"
#include "database/LabelId.hpp"
#include "database/MediaLibraryId.hpp"
#include "database/Object.hpp"
@@ -249,12 +250,17 @@ namespace lms::db
static pointer find(Session& session, const core::UUID& MBID);
static pointer find(Session& session, ReleaseId id);
static void find(Session& session, ReleaseId& lastRetrievedRelease, std::size_t count, const std::function<void(const Release::pointer&)>& func, MediaLibraryId library = {});
static void find(Session& session, const IdRange<ReleaseId>& idRange, const std::function<void(const Release::pointer&)>& func);
static IdRange<ReleaseId> findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count);
static RangeResults<pointer> find(Session& session, const FindParameters& parameters);
static void find(Session& session, const FindParameters& parameters, const std::function<void(const pointer&)>& func);
static RangeResults<ReleaseId> findIds(Session& session, const FindParameters& parameters);
static std::size_t getCount(Session& session, const FindParameters& parameters);
static RangeResults<ReleaseId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt); // not track related
// Updates
static void updatePreferredArtwork(Session& session, ReleaseId id, ArtworkId artworkId);
// Get the cluster of the tracks that belong to this release
// Each clusters are grouped by cluster type, sorted by the number of occurence (max to min)
// size is the max number of cluster per cluster type
src/libs/database/include/database/Track.hpp
+3
View File
@@ -39,6 +39,7 @@
#include "database/ClusterId.hpp"
#include "database/DirectoryId.hpp"
#include "database/Filters.hpp"
#include "database/IdRange.hpp"
#include "database/MediaLibraryId.hpp"
#include "database/Object.hpp"
#include "database/ReleaseId.hpp"
@@ -192,6 +193,8 @@ namespace lms::db
static pointer findByPath(Session& session, const std::filesystem::path& p);
static pointer find(Session& session, TrackId id);
static void find(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(const Track::pointer&)>& func, MediaLibraryId library = {});
static void find(Session& session, const IdRange<TrackId>& idRange, const std::function<void(const Track::pointer&)>& func);
static IdRange<TrackId> findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count);
static void findAbsoluteFilePath(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(TrackId trackId, const std::filesystem::path& absoluteFilePath)>& func);
static bool exists(Session& session, TrackId id);
src/libs/database/test/Artist.cpp
+117
View File
@@ -608,6 +608,93 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Artist_findNextRange)
{
{
auto transaction{ session.createReadTransaction() };
auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
range = Artist::findNextRange(session, ArtistId{}, 100);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
}
ScopedArtist artist1{ session, "Artist1" };
{
auto transaction{ session.createReadTransaction() };
auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
range = Artist::findNextRange(session, ArtistId{}, 1);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist1.getId());
EXPECT_EQ(range.last, artist1.getId());
range = Artist::findNextRange(session, range.last, 1);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
range = Artist::findNextRange(session, ArtistId{}, 100);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist1.getId());
EXPECT_EQ(range.last, artist1.getId());
}
ScopedArtist artist2{ session, "Artist2" };
ScopedArtist artist3{ session, "Artist3" };
{
auto transaction{ session.createReadTransaction() };
auto range{ Artist::findNextRange(session, ArtistId{}, 2) };
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist1.getId());
EXPECT_EQ(range.last, artist2.getId());
range = Artist::findNextRange(session, artist2.getId(), 2);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist3.getId());
EXPECT_EQ(range.last, artist3.getId());
}
}
TEST_F(DatabaseFixture, Artist_findByRange)
{
ScopedArtist artist1{ session, "Artist1" };
ScopedArtist artist2{ session, "Artist2" };
ScopedArtist artist3{ session, "Artist3" };
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Artist::find(session, IdRange<ArtistId>{ .first = artist1.getId(), .last = artist1.getId() }, [&](const db::Artist::pointer& artist) {
count++;
EXPECT_EQ(artist->getId(), artist1.getId());
});
EXPECT_EQ(count, 1);
}
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Artist::find(session, IdRange<ArtistId>{ .first = artist1.getId(), .last = artist3.getId() }, [&](const db::Artist::pointer&) {
count++;
});
EXPECT_EQ(count, 3);
}
}
TEST_F(DatabaseFixture, Artist_sortMethod)
{
ScopedArtist artistA{ session, "artistA" };
@@ -817,4 +904,34 @@ namespace lms::db::tests
EXPECT_EQ(artists.results[3], artistC.getId());
}
}
TEST_F(DatabaseFixture, Artist_updateArtwork)
{
ScopedArtist artist{ session, "MyArtist" };
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(artist->getPreferredArtwork(), Artwork::pointer{});
}
ScopedImage image{ session, "/image1.jpg" };
ScopedArtwork artwork{ session, image.lockAndGet() };
{
auto transaction{ session.createWriteTransaction() };
Artist::updatePreferredArtwork(session, artist->getId(), artwork.getId());
}
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(artist->getPreferredArtwork()->getId(), artwork.getId());
}
{
auto transaction{ session.createWriteTransaction() };
Artist::updatePreferredArtwork(session, artist.getId(), ArtworkId{});
}
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(artist->getPreferredArtwork(), Artwork::pointer{});
}
}
} // namespace lms::db::tests
src/libs/database/test/Release.cpp
+118
View File
@@ -173,6 +173,93 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Release_findNextRange)
{
{
auto transaction{ session.createReadTransaction() };
auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
range = Release::findNextRange(session, ReleaseId{}, 100);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
}
ScopedRelease release1{ session, "Artist1" };
{
auto transaction{ session.createReadTransaction() };
auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
range = Release::findNextRange(session, ReleaseId{}, 1);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release1.getId());
EXPECT_EQ(range.last, release1.getId());
range = Release::findNextRange(session, range.last, 1);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
range = Release::findNextRange(session, ReleaseId{}, 100);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release1.getId());
EXPECT_EQ(range.last, release1.getId());
}
ScopedRelease release2{ session, "Artist2" };
ScopedRelease release3{ session, "Artist3" };
{
auto transaction{ session.createReadTransaction() };
auto range{ Release::findNextRange(session, ReleaseId{}, 2) };
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release1.getId());
EXPECT_EQ(range.last, release2.getId());
range = Release::findNextRange(session, release2.getId(), 2);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release3.getId());
EXPECT_EQ(range.last, release3.getId());
}
}
TEST_F(DatabaseFixture, Release_findByRange)
{
ScopedRelease release1{ session, "Artist1" };
ScopedRelease release2{ session, "Artist2" };
ScopedRelease release3{ session, "Artist3" };
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Release::find(session, IdRange<ReleaseId>{ .first = release1.getId(), .last = release1.getId() }, [&](const db::Release::pointer& release) {
count++;
EXPECT_EQ(release->getId(), release1.getId());
});
EXPECT_EQ(count, 1);
}
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Release::find(session, IdRange<ReleaseId>{ .first = release1.getId(), .last = release3.getId() }, [&](const db::Release::pointer&) {
count++;
});
EXPECT_EQ(count, 3);
}
}
TEST_F(DatabaseFixture, Release_singleTrack)
{
ScopedRelease release{ session, "MyRelease" };
@@ -1431,4 +1518,35 @@ namespace lms::db::tests
EXPECT_EQ(releases.results[1]->getId(), release1->getId());
}
}
TEST_F(DatabaseFixture, Release_updateArtwork)
{
ScopedRelease release{ session, "MyRelease" };
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(release->getPreferredArtwork(), Artwork::pointer{});
}
ScopedImage image{ session, "/image1.jpg" };
ScopedArtwork artwork{ session, image.lockAndGet() };
{
auto transaction{ session.createWriteTransaction() };
Release::updatePreferredArtwork(session, release->getId(), artwork.getId());
}
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(release->getPreferredArtwork()->getId(), artwork.getId());
}
{
auto transaction{ session.createWriteTransaction() };
Release::updatePreferredArtwork(session, release.getId(), ArtworkId{});
}
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(release->getPreferredArtwork(), Artwork::pointer{});
}
}
} // namespace lms::db::tests
src/libs/database/test/Track.cpp
+87
View File
@@ -151,6 +151,93 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Track_findNextRange)
{
{
auto transaction{ session.createReadTransaction() };
auto range{ Track::findNextRange(session, TrackId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
range = Track::findNextRange(session, TrackId{}, 100);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
}
ScopedTrack track1{ session };
{
auto transaction{ session.createReadTransaction() };
auto range{ Track::findNextRange(session, TrackId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
range = Track::findNextRange(session, TrackId{}, 1);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track1.getId());
EXPECT_EQ(range.last, track1.getId());
range = Track::findNextRange(session, range.last, 1);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
range = Track::findNextRange(session, TrackId{}, 100);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track1.getId());
EXPECT_EQ(range.last, track1.getId());
}
ScopedTrack track2{ session };
ScopedTrack track3{ session };
{
auto transaction{ session.createReadTransaction() };
auto range{ Track::findNextRange(session, TrackId{}, 2) };
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track1.getId());
EXPECT_EQ(range.last, track2.getId());
range = Track::findNextRange(session, track2.getId(), 2);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track3.getId());
EXPECT_EQ(range.last, track3.getId());
}
}
TEST_F(DatabaseFixture, Track_findByRange)
{
ScopedTrack track1{ session };
ScopedTrack track2{ session };
ScopedTrack track3{ session };
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Track::find(session, IdRange<TrackId>{ .first = track1.getId(), .last = track1.getId() }, [&](const db::Track::pointer& track) {
count++;
EXPECT_EQ(track->getId(), track1.getId());
});
EXPECT_EQ(count, 1);
}
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Track::find(session, IdRange<TrackId>{ .first = track1.getId(), .last = track3.getId() }, [&](const db::Track::pointer&) {
count++;
});
EXPECT_EQ(count, 3);
}
}
TEST_F(DatabaseFixture, Track_findAbsoluteFilePath)
{
ScopedTrack track{ session };