owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scanner: parallelized processing within artwork association steps

Browse Source
This commit is contained in:
emeric
2025-07-04 23:05:35 +02:00
parent 427f777c44
commit 204a4a8857
23 changed files with 982 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/database/impl/Artist.cpp
+31
View File
@@ -237,6 +237,17 @@ namespace lms::db
});
}
void Artist::find(Session& session, const IdRange<ArtistId>& idRange, const std::function<void(const Artist::pointer&)>& func)
{
assert(idRange.isValid());
auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Artist>>("SELECT a from artist a").orderBy("a.id").where("a.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
utils::forEachQueryResult(query, [&](const Artist::pointer& artist) {
func(artist);
});
}
std::vector<Artist::pointer> Artist::find(Session& session, std::string_view name)
{
session.checkReadTransaction();
@@ -280,6 +291,16 @@ namespace lms::db
utils::forEachQueryRangeResult(query, params.range, func);
}
IdRange<ArtistId> Artist::findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<ArtistId, ArtistId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT a.id FROM artist a WHERE a.id > ? ORDER BY a.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
query.bind(static_cast<int>(count));
auto res{ utils::fetchQuerySingleResult(query) };
return IdRange<ArtistId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
}
RangeResults<ArtistId> Artist::findOrphanIds(Session& session, std::optional<Range> range)
{
session.checkReadTransaction();
@@ -304,6 +325,16 @@ AND NOT EXISTS (
return utils::fetchQuerySingleResult(session.getDboSession()->query<int>("SELECT 1 FROM artist").where("id = ?").bind(id)) == 1;
}
void Artist::updatePreferredArtwork(Session& session, ArtistId artistId, ArtworkId artworkId)
{
session.checkWriteTransaction();
if (artworkId.isValid())
utils::executeCommand(*session.getDboSession(), "UPDATE artist SET preferred_artwork_id = ? WHERE id = ?", artworkId, artistId);
else
utils::executeCommand(*session.getDboSession(), "UPDATE artist SET preferred_artwork_id = NULL WHERE id = ?", artistId);
}
std::optional<core::UUID> Artist::getMBID() const
{
return core::UUID::fromString(_mbid);
src/libs/database/impl/Migration.cpp
+18 -1
View File
@@ -35,7 +35,7 @@ namespace lms::db
{
namespace
{
static constexpr Version LMS_DATABASE_VERSION{ 95 };
static constexpr Version LMS_DATABASE_VERSION{ 96 };
}
VersionInfo::VersionInfo()
@@ -1396,6 +1396,22 @@ FROM artist)");
utils::executeCommand(*session.getDboSession(), "ALTER TABLE track DROP COLUMN file_name");
}
void migrateFromV95(Session& session)
{
// Make sure each image and each embessed image has an artwork object
utils::executeCommand(*session.getDboSession(), R"(INSERT INTO artwork (version, track_embedded_image_id)
SELECT 1 AS version, tei.id
FROM track_embedded_image tei
LEFT JOIN artwork art ON tei.id = art.track_embedded_image_id
WHERE art.track_embedded_image_id IS NULL)");
utils::executeCommand(*session.getDboSession(), R"(INSERT INTO artwork (version, image_id)
SELECT 1 AS version, img.id
FROM image img
LEFT JOIN artwork art ON img.id = art.image_id
WHERE art.image_id IS NULL)");
}
bool doDbMigration(Session& session)
{
constexpr std::string_view outdatedMsg{ "Outdated database, please rebuild it (delete the .db file and restart)" };
@@ -1467,6 +1483,7 @@ FROM artist)");
{ 92, migrateFromV92 },
{ 93, migrateFromV93 },
{ 94, migrateFromV94 },
{ 95, migrateFromV95 },
};
bool migrationPerformed{};
src/libs/database/impl/Release.cpp
+31
View File
@@ -504,6 +504,27 @@ namespace lms::db
});
}
void Release::find(Session& session, const IdRange<ReleaseId>& idRange, const std::function<void(const Release::pointer&)>& func)
{
assert(idRange.isValid());
auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Release>>("SELECT r from release r").orderBy("r.id").where("r.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
utils::forEachQueryResult(query, [&](const Release::pointer& release) {
func(release);
});
}
IdRange<ReleaseId> Release::findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<ReleaseId, ReleaseId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT r.id FROM release r WHERE r.id > ? ORDER BY r.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
query.bind(static_cast<int>(count));
auto res{ utils::fetchQuerySingleResult(query) };
return IdRange<ReleaseId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
}
RangeResults<Release::pointer> Release::find(Session& session, const FindParameters& params)
{
session.checkReadTransaction();
@@ -528,6 +549,16 @@ namespace lms::db
return utils::execRangeQuery<ReleaseId>(query, params.range);
}
void Release::updatePreferredArtwork(Session& session, ReleaseId releaseId, ArtworkId artworkId)
{
session.checkWriteTransaction();
if (artworkId.isValid())
utils::executeCommand(*session.getDboSession(), "UPDATE release SET preferred_artwork_id = ? WHERE id = ?", artworkId, releaseId);
else
utils::executeCommand(*session.getDboSession(), "UPDATE release SET preferred_artwork_id = NULL WHERE id = ?", releaseId);
}
std::size_t Release::getCount(Session& session, const FindParameters& params)
{
session.checkReadTransaction();
src/libs/database/impl/Track.cpp
+21
View File
@@ -292,6 +292,27 @@ namespace lms::db
});
}
void Track::find(Session& session, const IdRange<TrackId>& idRange, const std::function<void(const Track::pointer&)>& func)
{
assert(idRange.isValid());
auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Track>>("SELECT t from track t").orderBy("t.id").where("t.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
utils::forEachQueryResult(query, [&](const Track::pointer& track) {
func(track);
});
}
IdRange<TrackId> Track::findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count)
{
auto query{ session.getDboSession()->query<std::tuple<TrackId, TrackId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT t.id FROM track t WHERE t.id > ? ORDER BY t.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
query.bind(static_cast<int>(count));
auto res{ utils::fetchQuerySingleResult(query) };
return IdRange<TrackId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
}
bool Track::exists(Session& session, TrackId id)
{
session.checkReadTransaction();
src/libs/database/include/database/Artist.hpp
+6
View File
@@ -34,6 +34,7 @@
#include "database/ArtworkId.hpp"
#include "database/ClusterId.hpp"
#include "database/Filters.hpp"
#include "database/IdRange.hpp"
#include "database/MediaLibraryId.hpp"
#include "database/Object.hpp"
#include "database/ReleaseId.hpp"
@@ -125,12 +126,17 @@ namespace lms::db
static pointer find(Session& session, ArtistId id);
static std::vector<pointer> find(Session& session, std::string_view name); // exact match on name field
static void find(Session& session, ArtistId& lastRetrievedArtist, std::size_t count, const std::function<void(const Artist::pointer&)>& func, MediaLibraryId library = {});
static void find(Session& session, const IdRange<ArtistId>& idRange, const std::function<void(const Artist::pointer&)>& func);
static RangeResults<pointer> find(Session& session, const FindParameters& params);
static void find(Session& session, const FindParameters& params, std::function<void(const pointer&)> func);
static IdRange<ArtistId> findNextRange(Session& session, ArtistId lastRetrievedId, std::size_t count);
static RangeResults<ArtistId> findIds(Session& session, const FindParameters& params);
static RangeResults<ArtistId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt); // No track related
static bool exists(Session& session, ArtistId id);
// Updates
static void updatePreferredArtwork(Session& session, ArtistId artistId, ArtworkId artworkId);
// Accessors
const std::string& getName() const { return _name; }
const std::string& getSortName() const { return _sortName; }
src/libs/services/scanner/impl/steps/ArtworkUtils.hpp → src/libs/database/include/database/IdRange.hpp
+10 -16
View File
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2024 Emeric Poupon
* Copyright (C) 2025 Emeric Poupon
*
* This file is part of LMS.
*
@@ -19,21 +19,15 @@
#pragma once
#include <filesystem>
#include "database/ArtworkId.hpp"
#include "database/ImageId.hpp"
#include "database/Object.hpp"
#include "database/TrackEmbeddedImageId.hpp"
namespace lms::db
{
class Artwork;
class Session;
} // namespace lms::db
template<typename IdType>
struct IdRange
{
public:
IdType first;
IdType last;
namespace lms::scanner::utils
{
db::ObjectPtr<db::Artwork> getOrCreateArtworkFromTrackEmbeddedImage(db::Session& session, db::TrackEmbeddedImageId trackEmbeddedImageId);
db::ObjectPtr<db::Artwork> getOrCreateArtworkFromImage(db::Session& session, db::ImageId imageId);
} // namespace lms::scanner::utils
bool isValid() const { return first.isValid() && last.isValid(); }
};
} // namespace lms::db
src/libs/database/include/database/Release.hpp
+6
View File
@@ -35,6 +35,7 @@
#include "database/CountryId.hpp"
#include "database/DirectoryId.hpp"
#include "database/Filters.hpp"
#include "database/IdRange.hpp"
#include "database/LabelId.hpp"
#include "database/MediaLibraryId.hpp"
#include "database/Object.hpp"
@@ -249,12 +250,17 @@ namespace lms::db
static pointer find(Session& session, const core::UUID& MBID);
static pointer find(Session& session, ReleaseId id);
static void find(Session& session, ReleaseId& lastRetrievedRelease, std::size_t count, const std::function<void(const Release::pointer&)>& func, MediaLibraryId library = {});
static void find(Session& session, const IdRange<ReleaseId>& idRange, const std::function<void(const Release::pointer&)>& func);
static IdRange<ReleaseId> findNextRange(Session& session, ReleaseId lastRetrievedId, std::size_t count);
static RangeResults<pointer> find(Session& session, const FindParameters& parameters);
static void find(Session& session, const FindParameters& parameters, const std::function<void(const pointer&)>& func);
static RangeResults<ReleaseId> findIds(Session& session, const FindParameters& parameters);
static std::size_t getCount(Session& session, const FindParameters& parameters);
static RangeResults<ReleaseId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt); // not track related
// Updates
static void updatePreferredArtwork(Session& session, ReleaseId id, ArtworkId artworkId);
// Get the cluster of the tracks that belong to this release
// Each clusters are grouped by cluster type, sorted by the number of occurence (max to min)
// size is the max number of cluster per cluster type
src/libs/database/include/database/Track.hpp
+3
View File
@@ -39,6 +39,7 @@
#include "database/ClusterId.hpp"
#include "database/DirectoryId.hpp"
#include "database/Filters.hpp"
#include "database/IdRange.hpp"
#include "database/MediaLibraryId.hpp"
#include "database/Object.hpp"
#include "database/ReleaseId.hpp"
@@ -192,6 +193,8 @@ namespace lms::db
static pointer findByPath(Session& session, const std::filesystem::path& p);
static pointer find(Session& session, TrackId id);
static void find(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(const Track::pointer&)>& func, MediaLibraryId library = {});
static void find(Session& session, const IdRange<TrackId>& idRange, const std::function<void(const Track::pointer&)>& func);
static IdRange<TrackId> findNextRange(Session& session, TrackId lastRetrievedId, std::size_t count);
static void findAbsoluteFilePath(Session& session, TrackId& lastRetrievedId, std::size_t count, const std::function<void(TrackId trackId, const std::filesystem::path& absoluteFilePath)>& func);
static bool exists(Session& session, TrackId id);
src/libs/database/test/Artist.cpp
+117
View File
@@ -608,6 +608,93 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Artist_findNextRange)
{
{
auto transaction{ session.createReadTransaction() };
auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
range = Artist::findNextRange(session, ArtistId{}, 100);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
}
ScopedArtist artist1{ session, "Artist1" };
{
auto transaction{ session.createReadTransaction() };
auto range{ Artist::findNextRange(session, ArtistId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
range = Artist::findNextRange(session, ArtistId{}, 1);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist1.getId());
EXPECT_EQ(range.last, artist1.getId());
range = Artist::findNextRange(session, range.last, 1);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ArtistId{});
EXPECT_EQ(range.last, ArtistId{});
range = Artist::findNextRange(session, ArtistId{}, 100);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist1.getId());
EXPECT_EQ(range.last, artist1.getId());
}
ScopedArtist artist2{ session, "Artist2" };
ScopedArtist artist3{ session, "Artist3" };
{
auto transaction{ session.createReadTransaction() };
auto range{ Artist::findNextRange(session, ArtistId{}, 2) };
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist1.getId());
EXPECT_EQ(range.last, artist2.getId());
range = Artist::findNextRange(session, artist2.getId(), 2);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, artist3.getId());
EXPECT_EQ(range.last, artist3.getId());
}
}
TEST_F(DatabaseFixture, Artist_findByRange)
{
ScopedArtist artist1{ session, "Artist1" };
ScopedArtist artist2{ session, "Artist2" };
ScopedArtist artist3{ session, "Artist3" };
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Artist::find(session, IdRange<ArtistId>{ .first = artist1.getId(), .last = artist1.getId() }, [&](const db::Artist::pointer& artist) {
count++;
EXPECT_EQ(artist->getId(), artist1.getId());
});
EXPECT_EQ(count, 1);
}
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Artist::find(session, IdRange<ArtistId>{ .first = artist1.getId(), .last = artist3.getId() }, [&](const db::Artist::pointer&) {
count++;
});
EXPECT_EQ(count, 3);
}
}
TEST_F(DatabaseFixture, Artist_sortMethod)
{
ScopedArtist artistA{ session, "artistA" };
@@ -817,4 +904,34 @@ namespace lms::db::tests
EXPECT_EQ(artists.results[3], artistC.getId());
}
}
TEST_F(DatabaseFixture, Artist_updateArtwork)
{
ScopedArtist artist{ session, "MyArtist" };
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(artist->getPreferredArtwork(), Artwork::pointer{});
}
ScopedImage image{ session, "/image1.jpg" };
ScopedArtwork artwork{ session, image.lockAndGet() };
{
auto transaction{ session.createWriteTransaction() };
Artist::updatePreferredArtwork(session, artist->getId(), artwork.getId());
}
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(artist->getPreferredArtwork()->getId(), artwork.getId());
}
{
auto transaction{ session.createWriteTransaction() };
Artist::updatePreferredArtwork(session, artist.getId(), ArtworkId{});
}
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(artist->getPreferredArtwork(), Artwork::pointer{});
}
}
} // namespace lms::db::tests
src/libs/database/test/Release.cpp
+118
View File
@@ -173,6 +173,93 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Release_findNextRange)
{
{
auto transaction{ session.createReadTransaction() };
auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
range = Release::findNextRange(session, ReleaseId{}, 100);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
}
ScopedRelease release1{ session, "Artist1" };
{
auto transaction{ session.createReadTransaction() };
auto range{ Release::findNextRange(session, ReleaseId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
range = Release::findNextRange(session, ReleaseId{}, 1);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release1.getId());
EXPECT_EQ(range.last, release1.getId());
range = Release::findNextRange(session, range.last, 1);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, ReleaseId{});
EXPECT_EQ(range.last, ReleaseId{});
range = Release::findNextRange(session, ReleaseId{}, 100);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release1.getId());
EXPECT_EQ(range.last, release1.getId());
}
ScopedRelease release2{ session, "Artist2" };
ScopedRelease release3{ session, "Artist3" };
{
auto transaction{ session.createReadTransaction() };
auto range{ Release::findNextRange(session, ReleaseId{}, 2) };
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release1.getId());
EXPECT_EQ(range.last, release2.getId());
range = Release::findNextRange(session, release2.getId(), 2);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, release3.getId());
EXPECT_EQ(range.last, release3.getId());
}
}
TEST_F(DatabaseFixture, Release_findByRange)
{
ScopedRelease release1{ session, "Artist1" };
ScopedRelease release2{ session, "Artist2" };
ScopedRelease release3{ session, "Artist3" };
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Release::find(session, IdRange<ReleaseId>{ .first = release1.getId(), .last = release1.getId() }, [&](const db::Release::pointer& release) {
count++;
EXPECT_EQ(release->getId(), release1.getId());
});
EXPECT_EQ(count, 1);
}
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Release::find(session, IdRange<ReleaseId>{ .first = release1.getId(), .last = release3.getId() }, [&](const db::Release::pointer&) {
count++;
});
EXPECT_EQ(count, 3);
}
}
TEST_F(DatabaseFixture, Release_singleTrack)
{
ScopedRelease release{ session, "MyRelease" };
@@ -1431,4 +1518,35 @@ namespace lms::db::tests
EXPECT_EQ(releases.results[1]->getId(), release1->getId());
}
}
TEST_F(DatabaseFixture, Release_updateArtwork)
{
ScopedRelease release{ session, "MyRelease" };
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(release->getPreferredArtwork(), Artwork::pointer{});
}
ScopedImage image{ session, "/image1.jpg" };
ScopedArtwork artwork{ session, image.lockAndGet() };
{
auto transaction{ session.createWriteTransaction() };
Release::updatePreferredArtwork(session, release->getId(), artwork.getId());
}
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(release->getPreferredArtwork()->getId(), artwork.getId());
}
{
auto transaction{ session.createWriteTransaction() };
Release::updatePreferredArtwork(session, release.getId(), ArtworkId{});
}
{
auto transaction{ session.createReadTransaction() };
EXPECT_EQ(release->getPreferredArtwork(), Artwork::pointer{});
}
}
} // namespace lms::db::tests
src/libs/database/test/Track.cpp
+87
View File
@@ -151,6 +151,93 @@ namespace lms::db::tests
}
}
TEST_F(DatabaseFixture, Track_findNextRange)
{
{
auto transaction{ session.createReadTransaction() };
auto range{ Track::findNextRange(session, TrackId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
range = Track::findNextRange(session, TrackId{}, 100);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
}
ScopedTrack track1{ session };
{
auto transaction{ session.createReadTransaction() };
auto range{ Track::findNextRange(session, TrackId{}, 0) };
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
range = Track::findNextRange(session, TrackId{}, 1);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track1.getId());
EXPECT_EQ(range.last, track1.getId());
range = Track::findNextRange(session, range.last, 1);
EXPECT_FALSE(range.isValid());
EXPECT_EQ(range.first, TrackId{});
EXPECT_EQ(range.last, TrackId{});
range = Track::findNextRange(session, TrackId{}, 100);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track1.getId());
EXPECT_EQ(range.last, track1.getId());
}
ScopedTrack track2{ session };
ScopedTrack track3{ session };
{
auto transaction{ session.createReadTransaction() };
auto range{ Track::findNextRange(session, TrackId{}, 2) };
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track1.getId());
EXPECT_EQ(range.last, track2.getId());
range = Track::findNextRange(session, track2.getId(), 2);
EXPECT_TRUE(range.isValid());
EXPECT_EQ(range.first, track3.getId());
EXPECT_EQ(range.last, track3.getId());
}
}
TEST_F(DatabaseFixture, Track_findByRange)
{
ScopedTrack track1{ session };
ScopedTrack track2{ session };
ScopedTrack track3{ session };
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Track::find(session, IdRange<TrackId>{ .first = track1.getId(), .last = track1.getId() }, [&](const db::Track::pointer& track) {
count++;
EXPECT_EQ(track->getId(), track1.getId());
});
EXPECT_EQ(count, 1);
}
{
auto transaction{ session.createReadTransaction() };
std::size_t count{};
Track::find(session, IdRange<TrackId>{ .first = track1.getId(), .last = track3.getId() }, [&](const db::Track::pointer&) {
count++;
});
EXPECT_EQ(count, 3);
}
}
TEST_F(DatabaseFixture, Track_findAbsoluteFilePath)
{
ScopedTrack track{ session };
src/libs/services/scanner/CMakeLists.txt
+1 -1
View File
@@ -8,7 +8,7 @@ add_library(lmsscanner STATIC
impl/scanners/LyricsFileScanner.cpp
impl/scanners/PlayListFileScanner.cpp
impl/scanners/Utils.cpp
impl/steps/ArtworkUtils.cpp
impl/steps/JobQueue.cpp
impl/steps/ScanErrorLogger.cpp
impl/steps/ScanStepArtistReconciliation.cpp
impl/steps/ScanStepAssociateArtistImages.cpp
src/libs/services/scanner/impl/ScannerService.cpp
+4 -4
View File
@@ -448,7 +448,7 @@ namespace lms::scanner
if (!_lastScanSettings)
_lastScanSettings = readScannerSettings(_db.getTLSSession(), lastScanSettingsName);
auto cbFunc{ [this](const ScanStepStats& stats) {
auto progressFunc{ [this](const ScanStepStats& stats) {
notifyInProgressIfNeeded(stats);
} };
@@ -466,7 +466,7 @@ namespace lms::scanner
.jobScheduler = *_jobScheduler,
.settings = _settings,
.lastScanSettings = _lastScanSettings.has_value() ? &(_lastScanSettings.value()) : nullptr,
.progressCallback = cbFunc,
.progressCallback = progressFunc,
.abortScan = _abortScan,
.db = _db,
.fileScanners = fileScanners,
@@ -500,14 +500,14 @@ namespace lms::scanner
_currentScanStepStats = stepStats;
}
const std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
const auto now{ std::chrono::steady_clock::now() };
_events.scanInProgress.emit(stepStats);
_lastScanInProgressEmit = now;
}
void ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
{
std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
const auto now{ std::chrono::steady_clock::now() };
if (now - _lastScanInProgressEmit >= std::chrono::seconds{ 1 })
notifyInProgress(stepStats);
src/libs/services/scanner/impl/ScannerService.hpp
+1 -1
View File
@@ -96,7 +96,7 @@ namespace lms::scanner
Wt::WIOService _ioService;
boost::asio::system_timer _scheduleTimer{ _ioService };
Events _events;
std::chrono::system_clock::time_point _lastScanInProgressEmit;
std::chrono::steady_clock::time_point _lastScanInProgressEmit;
mutable std::shared_mutex _statusMutex;
State _curState{ State::NotScheduled };
src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp
+3
View File
@@ -25,6 +25,7 @@
#include "core/Path.hpp"
#include "core/XxHash3.hpp"
#include "database/Artist.hpp"
#include "database/Artwork.hpp"
#include "database/Cluster.hpp"
#include "database/Db.hpp"
#include "database/Directory.hpp"
@@ -332,6 +333,8 @@ namespace lms::scanner
image.modify()->setWidth(imageInfo.properties.width);
image.modify()->setHeight(imageInfo.properties.height);
image.modify()->setMimeType(imageInfo.mimeType);
session.create<db::Artwork>(image);
}
return image;
src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp
+4
View File
@@ -22,6 +22,7 @@
#include <optional>
#include "core/ILogger.hpp"
#include "database/Artwork.hpp"
#include "database/Db.hpp"
#include "database/Image.hpp"
#include "database/MediaLibrary.hpp"
@@ -84,7 +85,10 @@ namespace lms::scanner
const bool added{ !image };
if (!image)
{
image = dbSession.create<db::Image>(getFilePath());
dbSession.create<db::Artwork>(image);
}
image.modify()->setLastWriteTime(getLastWriteTime());
image.modify()->setFileSize(getFileSize());
src/libs/services/scanner/impl/steps/ArtworkUtils.cpp
-59
View File
@@ -1,59 +0,0 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ArtworkUtils.hpp"
#include "database/Artwork.hpp"
#include "database/Image.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/TrackEmbeddedImage.hpp"
namespace lms::scanner::utils
{
db::ObjectPtr<db::Artwork> getOrCreateArtworkFromTrackEmbeddedImage(db::Session& session, db::TrackEmbeddedImageId trackEmbeddedImageId)
{
assert(trackEmbeddedImageId.isValid());
session.checkWriteTransaction();
db::ObjectPtr<db::Artwork> artwork{ db::Artwork::find(session, trackEmbeddedImageId) };
if (!artwork)
{
db::TrackEmbeddedImage::pointer trackEmbeddedImage{ db::TrackEmbeddedImage::find(session, trackEmbeddedImageId) };
assert(trackEmbeddedImage);
artwork = session.create<db::Artwork>(trackEmbeddedImage);
}
return artwork;
}
db::ObjectPtr<db::Artwork> getOrCreateArtworkFromImage(db::Session& session, db::ImageId imageId)
{
assert(imageId.isValid());
session.checkWriteTransaction();
db::ObjectPtr<db::Artwork> artwork{ db::Artwork::find(session, imageId) };
if (!artwork)
{
db::Image::pointer image{ db::Image::find(session, imageId) };
assert(image);
artwork = session.create<db::Artwork>(image);
}
return artwork;
}
} // namespace lms::scanner::utils
src/libs/services/scanner/impl/steps/JobQueue.cpp
+70
View File
@@ -0,0 +1,70 @@
/*
* Copyright (C) 2025 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "JobQueue.hpp"
#include <cassert>
#include "core/IJob.hpp"
#include "core/IJobScheduler.hpp"
namespace lms::scanner
{
JobQueue::JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float _drainThreshold)
: _scheduler{ scheduler }
, _maxQueueSize{ maxQueueSize }
, _processJobsDoneFunc{ std::move(processJobsDoneFunc) }
, _batchSize{ batchSize }
, _drainThreshold{ _drainThreshold }
{
assert(_scheduler.getJobsDoneCount() == 0);
}
JobQueue::~JobQueue()
{
finish();
}
void JobQueue::push(std::unique_ptr<core::IJob> job)
{
_scheduler.scheduleJob(std::move(job));
drainIfNeeded();
}
void JobQueue::finish()
{
_scheduler.wait();
while (_scheduler.popJobsDone(_jobsDone, _batchSize) > 0)
{
_processJobsDoneFunc(std::span{ _jobsDone });
_jobsDone.clear();
}
}
void JobQueue::drainIfNeeded()
{
while (_scheduler.getJobsDoneCount() > static_cast<std::size_t>(_maxQueueSize * _drainThreshold))
{
_scheduler.popJobsDone(_jobsDone, _batchSize);
_processJobsDoneFunc(std::span{ _jobsDone });
_jobsDone.clear();
}
_scheduler.waitUntilJobCountAtMost(_maxQueueSize);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/JobQueue.hpp
+57
View File
@@ -0,0 +1,57 @@
/*
* Copyright (C) 2025 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include <functional>
#include <memory>
#include <span>
#include <vector>
namespace lms::core
{
class IJob;
class IJobScheduler;
} // namespace lms::core
namespace lms::scanner
{
class JobQueue
{
public:
using ProcessFunction = std::function<void(std::span<std::unique_ptr<core::IJob>>)>;
JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float drainThreshold);
~JobQueue();
JobQueue(const JobQueue&) = delete;
JobQueue& operator=(const JobQueue&) = delete;
// push can wait and invoke the supplied ProcessFunction
void push(std::unique_ptr<core::IJob> job);
void finish();
private:
void drainIfNeeded();
core::IJobScheduler& _scheduler;
const std::size_t _maxQueueSize;
ProcessFunction _processJobsDoneFunc;
const std::size_t _batchSize;
const float _drainThreshold;
std::vector<std::unique_ptr<core::IJob>> _jobsDone;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp
+135 -111
View File
@@ -24,9 +24,9 @@
#include <deque>
#include <set>
#include <span>
#include <variant>
#include "core/IConfig.hpp"
#include "core/IJob.hpp"
#include "core/ILogger.hpp"
#include "core/Path.hpp"
#include "core/String.hpp"
@@ -41,7 +41,7 @@
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "ArtworkUtils.hpp"
#include "JobQueue.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
@@ -49,42 +49,24 @@ namespace lms::scanner
{
namespace
{
using ArtistArtwork = std::variant<std::monostate, db::ImageId, db::ArtworkId>; // TODO handle embedded images in tracks?
bool isSameArtwork(ArtistArtwork preferredArtwork, const db::ObjectPtr<db::Artwork>& artwork)
{
if (std::holds_alternative<std::monostate>(preferredArtwork))
return !artwork;
if (const db::ImageId* imageId = std::get_if<db::ImageId>(&preferredArtwork))
return artwork && *imageId == artwork->getImageId();
if (const db::ArtworkId* artworkId = std::get_if<db::ArtworkId>(&preferredArtwork))
return artwork && *artworkId == artwork->getId();
return false;
}
struct ArtistArtworkAssociation
{
db::Artist::pointer artist;
ArtistArtwork preferredArtwork;
db::ArtistId artistId;
db::ArtworkId preferredArtworkId;
};
using ArtistArtworkAssociationContainer = std::deque<ArtistArtworkAssociation>;
struct SearchArtistImageContext
struct SearchArtistArtworkParams
{
db::Session& session;
db::ArtistId lastRetrievedArtistId;
std::size_t processedArtistCount{};
std::span<const std::string> artistFileNames;
const ScannerSettings& settings;
};
db::Image::pointer findImageInDirectory(SearchArtistImageContext& searchContext, const std::filesystem::path& directoryPath, std::span<const std::string> fileStemsToSearch)
db::Image::pointer findImageInDirectory(db::Session& session, const std::filesystem::path& directoryPath, std::span<const std::string> fileStemsToSearch)
{
db::Image::pointer image;
const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) };
const db::Directory::pointer directory{ db::Directory::find(session, directoryPath) };
if (directory) // may not exist for artists that are split on different media libraries
{
for (std::string_view fileStem : fileStemsToSearch)
@@ -93,7 +75,7 @@ namespace lms::scanner
params.setDirectory(directory->getId());
params.setFileStem(fileStem);
db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) {
db::Image::find(session, params, [&](const db::Image::pointer foundImg) {
if (!image)
image = foundImg;
});
@@ -106,38 +88,38 @@ namespace lms::scanner
return image;
}
db::ImageId getImageFromMbid(SearchArtistImageContext& searchContext, const core::UUID& mbid)
db::Image::pointer getImageFromMbid(db::Session& session, const core::UUID& mbid)
{
db::Image::pointer image;
// Find anywhere, since it is supposed to be unique!
db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid.getAsString()), [&](const db::Image::pointer foundImg) {
db::Image::find(session, db::Image::FindParameters{}.setFileStem(mbid.getAsString()), [&](const db::Image::pointer foundImg) {
if (!image)
image = foundImg;
});
return image ? image->getId() : db::ImageId{};
return image;
}
db::ImageId searchImageInArtistInfoDirectory(SearchArtistImageContext& searchContext, db::ArtistId artistId)
db::Image::pointer searchImageInArtistInfoDirectory(db::Session& session, db::ArtistId artistId)
{
db::Image::pointer image;
std::vector<std::string> fileInfoPaths;
db::ArtistInfo::find(searchContext.session, artistId, [&](const db::ArtistInfo::pointer& artistInfo) {
db::ArtistInfo::find(session, artistId, [&](const db::ArtistInfo::pointer& artistInfo) {
fileInfoPaths.push_back(artistInfo->getAbsoluteFilePath());
if (!image)
image = findImageInDirectory(searchContext, artistInfo->getDirectory()->getAbsolutePath(), std::array<std::string, 2>{ "thumb", "folder" });
image = findImageInDirectory(session, artistInfo->getDirectory()->getAbsolutePath(), std::array<std::string, 2>{ "thumb", "folder" });
});
if (fileInfoPaths.size() > 1)
LMS_LOG(DBUPDATER, DEBUG, "Found " << fileInfoPaths.size() << " artist info files for same artist: " << core::stringUtils::joinStrings(fileInfoPaths, ", "));
return image ? image->getId() : db::ImageId{};
return image;
}
db::ImageId searchImageInDirectories(SearchArtistImageContext& searchContext, db::ArtistId artistId)
db::Image::pointer searchImageInDirectories(db::Session& session, const SearchArtistArtworkParams& searchParams, db::ArtistId artistId)
{
db::Image::pointer image;
@@ -145,7 +127,7 @@ namespace lms::scanner
db::Directory::FindParameters params;
params.setArtist(artistId, { db::TrackArtistLinkType::ReleaseArtist });
db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) {
db::Directory::find(session, params, [&](const db::Directory::pointer& directory) {
releasePaths.insert(directory->getAbsolutePath());
});
@@ -164,9 +146,9 @@ namespace lms::scanner
std::filesystem::path directoryToInspect{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
while (true)
{
image = findImageInDirectory(searchContext, directoryToInspect, searchContext.artistFileNames);
image = findImageInDirectory(session, directoryToInspect, searchParams.artistFileNames);
if (image)
return image->getId();
return image;
std::filesystem::path parentPath{ directoryToInspect.parent_path() };
if (parentPath == directoryToInspect)
@@ -181,106 +163,74 @@ namespace lms::scanner
// /someOtherUserConfiguredArtistFile.jpg
for (const std::filesystem::path& releasePath : releasePaths)
{
image = findImageInDirectory(searchContext, releasePath, searchContext.artistFileNames);
image = findImageInDirectory(session, releasePath, searchParams.artistFileNames);
if (image)
return image->getId();
return image;
}
}
return image ? image->getId() : db::ImageId{};
return image;
}
db::ArtworkId getFirstReleaseArtwork(db::Session& session, const db::Artist::pointer& artist)
db::Artwork::pointer getFirstReleaseArtwork(db::Session& session, const db::Artist::pointer& artist)
{
db::ArtworkId artworkId;
db::Artwork::pointer artwork;
db::Release::FindParameters params;
params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist });
params.setSortMethod(db::ReleaseSortMethod::OriginalDate);
db::Release::find(session, params, [&](const db::Release::pointer& release) {
if (artworkId.isValid())
if (artwork)
return;
artworkId = release->getPreferredArtworkId();
artwork = release->getPreferredArtwork();
});
return artworkId;
return artwork;
}
ArtistArtwork computePreferredArtwork(SearchArtistImageContext& searchContext, const db::Artist::pointer& artist)
db::Artwork::pointer computePreferredArtistArtwork(db::Session& session, const SearchArtistArtworkParams& searchParams, const db::Artist::pointer& artist)
{
db::ImageId imageId;
if (const auto mbid{ artist->getMBID() })
{
imageId = getImageFromMbid(searchContext, *mbid);
if (imageId.isValid())
return ArtistArtwork{ imageId };
const db::Image::pointer image{ getImageFromMbid(session, *mbid) };
if (image)
return db::Artwork::find(session, image->getId());
}
imageId = searchImageInArtistInfoDirectory(searchContext, artist->getId());
if (imageId.isValid())
return ArtistArtwork{ imageId };
imageId = searchImageInDirectories(searchContext, artist->getId());
if (imageId.isValid())
return ArtistArtwork{ imageId };
if (searchContext.settings.artistImageFallbackToRelease)
{
db::ArtworkId artworkId{ getFirstReleaseArtwork(searchContext.session, artist) };
if (artworkId.isValid())
return ArtistArtwork{ artworkId };
}
return ArtistArtwork{};
}
bool fetchNextArtistArtworksToUpdate(SearchArtistImageContext& searchContext, ArtistArtworkAssociationContainer& ArtistArtworkAssociations)
{
const db::ArtistId artistId{ searchContext.lastRetrievedArtistId };
{
constexpr std::size_t readBatchSize{ 100 };
auto transaction{ searchContext.session.createReadTransaction() };
db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) {
ArtistArtwork preferredArtwork{ computePreferredArtwork(searchContext, artist) };
if (!isSameArtwork(preferredArtwork, artist->getPreferredArtwork()))
ArtistArtworkAssociations.push_back(ArtistArtworkAssociation{ artist, preferredArtwork });
searchContext.processedArtistCount++;
});
const db::Image::pointer image{ searchImageInArtistInfoDirectory(session, artist->getId()) };
if (image)
return db::Artwork::find(session, image->getId());
}
return artistId != searchContext.lastRetrievedArtistId;
{
const db::Image::pointer image{ searchImageInDirectories(session, searchParams, artist->getId()) };
if (image)
return db::Artwork::find(session, image->getId());
}
if (searchParams.settings.artistImageFallbackToRelease)
{
const db::Artwork::pointer artwork{ getFirstReleaseArtwork(session, artist) };
if (artwork)
return artwork;
}
return db::Artwork::pointer{};
}
void updateArtistPreferredArtwork(db::Session& session, const ArtistArtworkAssociation& ArtistArtworkAssociation)
void updateArtistPreferredArtwork(db::Session& session, const ArtistArtworkAssociation& artistArtworkAssociation)
{
db::Artist::pointer artist{ ArtistArtworkAssociation.artist };
db::Artwork::pointer artwork;
if (const db::ImageId * imageId{ std::get_if<db::ImageId>(&ArtistArtworkAssociation.preferredArtwork) })
artwork = utils::getOrCreateArtworkFromImage(session, *imageId);
else if (const db::ArtworkId * artworkId{ std::get_if<db::ArtworkId>(&ArtistArtworkAssociation.preferredArtwork) })
artwork = db::Artwork::find(session, *artworkId);
artist.modify()->setPreferredArtwork(artwork);
if (artwork)
LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork for artist '" << artist->getName() << "' with image in " << artwork->getAbsoluteFilePath());
else
LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from artist '" << artist->getName() << "'");
db::Artist::updatePreferredArtwork(session, artistArtworkAssociation.artistId, artistArtworkAssociation.preferredArtworkId);
}
void updateArtistArtworks(db::Session& session, ArtistArtworkAssociationContainer& imageAssociations)
void updateArtistPreferredArtworks(db::Session& session, ArtistArtworkAssociationContainer& imageAssociations, bool forceFullBatch)
{
constexpr std::size_t writeBatchSize{ 50 };
while (!imageAssociations.empty())
while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
{
auto transaction{ session.createWriteTransaction() };
@@ -305,6 +255,62 @@ namespace lms::scanner
return res;
}
bool fetchNextArtistIdRange(db::Session& session, db::ArtistId& lastRetrievedId, db::IdRange<db::ArtistId>& idRange)
{
constexpr std::size_t readBatchSize{ 100 };
auto transaction{ session.createReadTransaction() };
idRange = db::Artist::findNextRange(session, lastRetrievedId, readBatchSize);
lastRetrievedId = idRange.last;
return idRange.isValid();
}
class ComputeArtistArtworkAssociationsJob : public core::IJob
{
public:
ComputeArtistArtworkAssociationsJob(db::Db& db, const SearchArtistArtworkParams& searchParams, db::IdRange<db::ArtistId> artistIdRange)
: _db{ db }
, _searchParams{ searchParams }
, _artistIdRange{ artistIdRange }
{
}
std::span<const ArtistArtworkAssociation> getAssociations() const { return _associations; }
std::size_t getProcessedArtistCount() const { return _processedArtistCount; }
private:
core::LiteralString getName() const override { return "Associate Track Artworks"; }
void run() override
{
auto& session{ _db.getTLSSession() };
auto transaction{ session.createReadTransaction() };
db::Artist::find(session, _artistIdRange, [this, &session](const db::Artist::pointer& artist) {
const db::Artwork::pointer preferredArtwork{ computePreferredArtistArtwork(session, _searchParams, artist) };
if (artist->getPreferredArtwork() != preferredArtwork)
{
_associations.push_back(ArtistArtworkAssociation{ artist->getId(), preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{} });
if (preferredArtwork)
LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork for artist '" << artist->getName() << "' with image in " << preferredArtwork->getAbsoluteFilePath());
else
LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from artist '" << artist->getName() << "'");
}
_processedArtistCount++;
});
}
db::Db& _db;
const SearchArtistArtworkParams& _searchParams;
db::IdRange<db::ArtistId> _artistIdRange;
std::vector<ArtistArtworkAssociation> _associations;
std::size_t _processedArtistCount{};
};
} // namespace
ScanStepAssociateArtistImages::ScanStepAssociateArtistImages(InitParams& initParams)
@@ -333,22 +339,40 @@ namespace lms::scanner
context.currentStepStats.totalElems = db::Artist::getCount(session);
}
SearchArtistImageContext searchContext{
.session = session,
.lastRetrievedArtistId = {},
const SearchArtistArtworkParams searchParams{
.artistFileNames = _artistFileNames,
.settings = _settings,
};
ArtistArtworkAssociationContainer ArtistArtworkAssociations;
while (fetchNextArtistArtworksToUpdate(searchContext, ArtistArtworkAssociations))
{
ArtistArtworkAssociationContainer artistArtworkAssociations;
auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
if (_abortScan)
return;
updateArtistArtworks(session, ArtistArtworkAssociations);
context.currentStepStats.processedElems = searchContext.processedArtistCount;
for (const auto& job : jobs)
{
const auto& associationJob{ static_cast<const ComputeArtistArtworkAssociationsJob&>(*job) };
const auto& artistAssociations{ associationJob.getAssociations() };
artistArtworkAssociations.insert(std::end(artistArtworkAssociations), std::cbegin(artistAssociations), std::cend(artistAssociations));
context.currentStepStats.processedElems += associationJob.getProcessedArtistCount();
}
updateArtistPreferredArtworks(session, artistArtworkAssociations, true);
_progressCallback(context.currentStepStats);
}
};
JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
db::ArtistId lastRetrievedArtistId{};
db::IdRange<db::ArtistId> artistIdRange;
while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange))
queue.push(std::make_unique<ComputeArtistArtworkAssociationsJob>(_db, searchParams, artistIdRange));
queue.finish();
// process all remaining associations
updateArtistPreferredArtworks(session, artistArtworkAssociations, false);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp
+123 -107
View File
@@ -23,9 +23,9 @@
#include <cassert>
#include <deque>
#include <set>
#include <variant>
#include "core/IConfig.hpp"
#include "core/IJob.hpp"
#include "core/ILogger.hpp"
#include "core/Path.hpp"
#include "database/Artwork.hpp"
@@ -36,92 +36,73 @@
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/TrackEmbeddedImage.hpp"
#include "database/TrackEmbeddedImageId.hpp"
#include "ArtworkUtils.hpp"
#include "JobQueue.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
namespace
{
using ReleaseArtwork = std::variant<std::monostate, db::TrackEmbeddedImageId, db::ImageId>;
bool isSameArtwork(ReleaseArtwork preferredArtwork, const db::ObjectPtr<db::Artwork>& artwork)
{
if (std::holds_alternative<std::monostate>(preferredArtwork))
return !artwork;
if (const db::TrackEmbeddedImageId* trackEmbeddedImageId = std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork))
return artwork && *trackEmbeddedImageId == artwork->getTrackEmbeddedImageId();
if (const db::ImageId* imageId = std::get_if<db::ImageId>(&preferredArtwork))
return artwork && *imageId == artwork->getImageId();
return false;
}
struct ReleaseArtworkAssociation
{
db::Release::pointer release;
ReleaseArtwork preferredArtwork;
db::ReleaseId releaseId;
db::ArtworkId preferredArtworkId;
};
using ReleaseArtworkAssociationContainer = std::deque<ReleaseArtworkAssociation>;
struct SearchReleaseArtworkContext
struct SearchReleaseArtworkParams
{
db::Session& session;
db::ReleaseId lastRetrievedReleaseId;
std::size_t processedReleaseCount{};
const std::vector<std::string>& releaseImageFileNames;
};
db::Image::pointer findImageInDirectory(SearchReleaseArtworkContext& searchContext, const std::filesystem::path& directoryPath)
db::Artwork::pointer findImageInDirectory(db::Session& session, const SearchReleaseArtworkParams& searchParams, const std::filesystem::path& directoryPath)
{
db::Image::pointer image;
db::Artwork::pointer artwork;
const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) };
const db::Directory::pointer directory{ db::Directory::find(session, directoryPath) };
if (directory) // may not exist for releases that are split on different media libraries
{
for (std::string_view fileStem : searchContext.releaseImageFileNames)
for (std::string_view fileStem : searchParams.releaseImageFileNames)
{
db::Image::FindParameters params;
params.setDirectory(directory->getId());
params.setFileStem(fileStem);
db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) {
if (!image)
image = foundImg;
db::Image::find(session, params, [&](const db::Image::pointer& image) {
if (!artwork)
artwork = db::Artwork::find(session, image->getId());
});
if (image)
if (artwork)
break;
}
}
return image;
return artwork;
}
db::Image::pointer computePreferredReleaseImage(SearchReleaseArtworkContext& searchContext, const db::Release::pointer& release)
db::Artwork::pointer computePreferredReleaseImage(db::Session& session, const SearchReleaseArtworkParams& searchParams, const db::Release::pointer& release)
{
db::Image::pointer image;
db::Artwork::pointer artwork;
const auto mbid{ release->getMBID() };
if (mbid)
{
// Find anywhere, since it is suppoed to be unique!
db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer foundImg) {
if (!image)
image = foundImg;
db::Image::find(session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer& image) {
if (!artwork)
artwork = db::Artwork::find(session, image->getId());
});
}
if (!image)
if (!artwork)
{
std::set<std::filesystem::path> releasePaths;
db::Directory::FindParameters params;
params.setRelease(release->getId());
db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) {
db::Directory::find(session, params, [&](const db::Directory::pointer& directory) {
releasePaths.insert(directory->getAbsolutePath());
});
@@ -132,31 +113,30 @@ namespace lms::scanner
if (releasePaths.size() > 1)
{
const std::filesystem::path releasePath{ core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) };
image = findImageInDirectory(searchContext, releasePath);
artwork = findImageInDirectory(session, searchParams, releasePath);
}
if (!image)
if (!artwork)
{
for (const std::filesystem::path& releasePath : releasePaths)
{
image = findImageInDirectory(searchContext, releasePath);
if (image)
artwork = findImageInDirectory(session, searchParams, releasePath);
if (artwork)
break;
}
}
}
return image;
return artwork;
}
ReleaseArtwork computePreferredReleaseArtwork(SearchReleaseArtworkContext& searchContext, const db::Release::pointer& release)
db::Artwork::pointer computePreferredReleaseArtwork(db::Session& session, const SearchReleaseArtworkParams& searchParams, const db::Release::pointer& release)
{
const db::Image::pointer image{ computePreferredReleaseImage(searchContext, release) };
if (image)
return ReleaseArtwork{ image->getId() };
db::Artwork::pointer artwork{ computePreferredReleaseImage(session, searchParams, release) };
if (artwork)
return artwork;
// Fallback on embedded Front image
db::TrackEmbeddedImageId trackEmbeddedImageId;
{
db::TrackEmbeddedImage::FindParameters params;
params.setRelease(release->getId());
@@ -164,11 +144,11 @@ namespace lms::scanner
params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc);
params.setRange(db::Range{ .offset = 0, .size = 1 });
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { trackEmbeddedImageId = image->getId(); });
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); });
}
if (trackEmbeddedImageId.isValid())
return ReleaseArtwork{ trackEmbeddedImageId };
if (artwork)
return artwork;
// Fallback on embedded media image
{
@@ -178,66 +158,28 @@ namespace lms::scanner
params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc);
params.setRange(db::Range{ .offset = 0, .size = 1 });
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { trackEmbeddedImageId = image->getId(); });
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); });
}
if (trackEmbeddedImageId.isValid())
return ReleaseArtwork{ trackEmbeddedImageId };
return ReleaseArtwork{};
return artwork;
}
bool fetchNextReleaseArtworksToUpdate(SearchReleaseArtworkContext& searchContext, ReleaseArtworkAssociationContainer& ReleaseArtworkAssociations)
void updateReleasePreferredArtwork(db::Session& session, const ReleaseArtworkAssociation& releaseArtworkAssociation)
{
const db::ReleaseId releaseId{ searchContext.lastRetrievedReleaseId };
{
constexpr std::size_t readBatchSize{ 100 };
auto transaction{ searchContext.session.createReadTransaction() };
db::Release::find(searchContext.session, searchContext.lastRetrievedReleaseId, readBatchSize, [&](const db::Release::pointer& release) {
const ReleaseArtwork preferredArtwork{ computePreferredReleaseArtwork(searchContext, release) };
const db::Artwork::pointer currentPreferredArtwork{ release->getPreferredArtwork() };
if (!isSameArtwork(preferredArtwork, currentPreferredArtwork))
ReleaseArtworkAssociations.push_back(ReleaseArtworkAssociation{ release, preferredArtwork });
searchContext.processedReleaseCount++;
});
}
return releaseId != searchContext.lastRetrievedReleaseId;
db::Release::updatePreferredArtwork(session, releaseArtworkAssociation.releaseId, releaseArtworkAssociation.preferredArtworkId);
}
void updateReleaseArtwork(db::Session& session, const ReleaseArtworkAssociation& ReleaseArtworkAssociation)
{
db::Release::pointer release{ ReleaseArtworkAssociation.release };
db::Artwork::pointer artwork;
if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if<db::TrackEmbeddedImageId>(&ReleaseArtworkAssociation.preferredArtwork) })
artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId);
else if (const db::ImageId * imageId{ std::get_if<db::ImageId>(&ReleaseArtworkAssociation.preferredArtwork) })
artwork = utils::getOrCreateArtworkFromImage(session, *imageId);
release.modify()->setPreferredArtwork(artwork);
if (artwork)
LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork in release '" << release->getName() << "' with image in " << artwork->getAbsoluteFilePath());
else
LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from release '" << release->getName() << "'");
}
void updateReleaseArtworks(db::Session& session, ReleaseArtworkAssociationContainer& imageAssociations)
void updateReleasePreferredArtworks(db::Session& session, ReleaseArtworkAssociationContainer& imageAssociations, bool forceFullBatch)
{
constexpr std::size_t writeBatchSize{ 50 };
while (!imageAssociations.empty())
while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
{
auto transaction{ session.createWriteTransaction() };
for (std::size_t i{}; !imageAssociations.empty() && i < writeBatchSize; ++i)
{
updateReleaseArtwork(session, imageAssociations.front());
updateReleasePreferredArtwork(session, imageAssociations.front());
imageAssociations.pop_front();
}
}
@@ -256,6 +198,62 @@ namespace lms::scanner
return res;
}
bool fetchNextReleaseIdRange(db::Session& session, db::ReleaseId& lastRetrievedId, db::IdRange<db::ReleaseId>& idRange)
{
constexpr std::size_t readBatchSize{ 100 };
auto transaction{ session.createReadTransaction() };
idRange = db::Release::findNextRange(session, lastRetrievedId, readBatchSize);
lastRetrievedId = idRange.last;
return idRange.isValid();
}
class ComputeReleaseArtworkAssociationsJob : public core::IJob
{
public:
ComputeReleaseArtworkAssociationsJob(db::Db& db, const SearchReleaseArtworkParams& searchParams, db::IdRange<db::ReleaseId> artistIdRange)
: _db{ db }
, _searchParams{ searchParams }
, _artistIdRange{ artistIdRange }
{
}
std::span<const ReleaseArtworkAssociation> getAssociations() const { return _associations; }
std::size_t getProcessedReleaseCount() const { return _processedReleaseCount; }
private:
core::LiteralString getName() const override { return "Associate Track Artworks"; }
void run() override
{
auto& session{ _db.getTLSSession() };
auto transaction{ session.createReadTransaction() };
db::Release::find(session, _artistIdRange, [this, &session](const db::Release::pointer& release) {
const db::Artwork::pointer preferredArtwork{ computePreferredReleaseArtwork(session, _searchParams, release) };
if (release->getPreferredArtwork() != preferredArtwork)
{
_associations.push_back(ReleaseArtworkAssociation{ release->getId(), preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{} });
if (preferredArtwork)
LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork for release '" << release->getName() << "' with image in " << preferredArtwork->getAbsoluteFilePath());
else
LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from release '" << release->getName() << "'");
}
_processedReleaseCount++;
});
}
db::Db& _db;
const SearchReleaseArtworkParams& _searchParams;
db::IdRange<db::ReleaseId> _artistIdRange;
std::vector<ReleaseArtworkAssociation> _associations;
std::size_t _processedReleaseCount{};
};
} // namespace
ScanStepAssociateReleaseImages::ScanStepAssociateReleaseImages(InitParams& initParams)
@@ -278,21 +276,39 @@ namespace lms::scanner
context.currentStepStats.totalElems = db::Release::getCount(session);
}
SearchReleaseArtworkContext searchContext{
.session = session,
.lastRetrievedReleaseId = {},
const SearchReleaseArtworkParams searchParams{
.releaseImageFileNames = _releaseImageFileNames,
};
ReleaseArtworkAssociationContainer ReleaseArtworkAssociations;
while (fetchNextReleaseArtworksToUpdate(searchContext, ReleaseArtworkAssociations))
{
ReleaseArtworkAssociationContainer artistArtworkAssociations;
auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
if (_abortScan)
return;
updateReleaseArtworks(session, ReleaseArtworkAssociations);
context.currentStepStats.processedElems = searchContext.processedReleaseCount;
for (const auto& job : jobs)
{
const auto& associationJob{ static_cast<const ComputeReleaseArtworkAssociationsJob&>(*job) };
const auto& artistAssociations{ associationJob.getAssociations() };
artistArtworkAssociations.insert(std::end(artistArtworkAssociations), std::cbegin(artistAssociations), std::cend(artistAssociations));
context.currentStepStats.processedElems += associationJob.getProcessedReleaseCount();
}
updateReleasePreferredArtworks(session, artistArtworkAssociations, true);
_progressCallback(context.currentStepStats);
}
};
JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
db::ReleaseId lastRetrievedReleaseId{};
db::IdRange<db::ReleaseId> artistIdRange;
while (fetchNextReleaseIdRange(session, lastRetrievedReleaseId, artistIdRange))
queue.push(std::make_unique<ComputeReleaseArtworkAssociationsJob>(_db, searchParams, artistIdRange));
queue.finish();
// process all remaining associations
updateReleasePreferredArtworks(session, artistArtworkAssociations, false);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp
+125 -129
View File
@@ -21,20 +21,23 @@
#include <cassert>
#include <deque>
#include <variant>
#include <optional>
#include "core/IJob.hpp"
#include "core/IJobScheduler.hpp"
#include "core/ILogger.hpp"
#include "core/LiteralString.hpp"
#include "database/Artwork.hpp"
#include "database/Db.hpp"
#include "database/Directory.hpp"
#include "database/IdRange.hpp"
#include "database/Image.hpp"
#include "database/Release.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/TrackEmbeddedImage.hpp"
#include "database/TrackEmbeddedImageId.hpp"
#include "ArtworkUtils.hpp"
#include "JobQueue.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
@@ -42,37 +45,11 @@ namespace lms::scanner
namespace
{
// May come from an embedded image in a track, or from what has been previously resolved for the release
using TrackArtwork = std::variant<std::monostate, db::TrackEmbeddedImageId, db::ArtworkId>;
bool isSameArtwork(TrackArtwork preferredArtwork, const db::ObjectPtr<db::Artwork>& artwork)
{
if (std::holds_alternative<std::monostate>(preferredArtwork))
return !artwork;
if (const db::TrackEmbeddedImageId* trackEmbeddedImageId = std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork))
{
assert(trackEmbeddedImageId->isValid());
return artwork && *trackEmbeddedImageId == artwork->getTrackEmbeddedImageId();
}
if (const db::ArtworkId* artworkId = std::get_if<db::ArtworkId>(&preferredArtwork))
{
assert(artworkId->isValid());
return artwork && *artworkId == artwork->getId();
}
return false;
}
bool isValid(const TrackArtwork& res)
{
return !std::holds_alternative<std::monostate>(res);
}
struct TrackArtworksAssociation
{
db::Track::pointer track;
TrackArtwork preferredArtwork;
TrackArtwork preferredMediaArtwork;
db::TrackId trackId;
std::optional<db::ArtworkId> preferredArtworkId;
std::optional<db::ArtworkId> preferredMediaArtworkId;
};
using TrackArtworksAssociationContainer = std::deque<TrackArtworksAssociation>;
@@ -83,11 +60,10 @@ namespace lms::scanner
std::size_t processedTrackCount{};
};
TrackArtwork computePreferredTrackArtwork(SearchTrackArtworkContext& searchContext, const db::Track::pointer& track, TrackArtwork preferredMediaArtwork)
db::Artwork::pointer computePreferredTrackArtwork(db::Session& session, const db::Track::pointer& track, const db::Artwork::pointer& preferredMediaArtwork)
{
TrackArtwork res{ preferredMediaArtwork };
if (isValid(res))
db::Artwork::pointer res{ preferredMediaArtwork };
if (res)
return res;
// Fallback on front cover for this track
@@ -97,10 +73,10 @@ namespace lms::scanner
params.setImageTypes({ db::ImageType::FrontCover });
params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc);
params.setRange(db::Range{ .offset = 0, .size = 1 });
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); });
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); });
}
if (isValid(res))
if (res)
return res;
// Fallback on the artwork already resolved for the release
@@ -108,28 +84,26 @@ namespace lms::scanner
if (!releaseId.isValid())
return res;
if (const db::Release::pointer release{ db::Release::find(searchContext.session, releaseId) })
{
if (const db::ArtworkId releaseArtworkId{ release->getPreferredArtworkId() }; releaseArtworkId.isValid())
res = releaseArtworkId;
}
if (const db::Release::pointer release{ db::Release::find(session, releaseId) })
res = release->getPreferredArtwork();
return res;
}
TrackArtwork computePreferredTrackMediaArtwork(SearchTrackArtworkContext& searchContext, const db::Track::pointer& track)
db::Artwork::pointer computePreferredTrackMediaArtwork(db::Session& session, const db::Track::pointer& track)
{
TrackArtwork res;
db::Artwork::pointer res;
{
db::TrackEmbeddedImage::FindParameters params;
params.setTrack(track->getId());
params.setImageTypes({ db::ImageType::Media });
params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc);
params.setRange(db::Range{ .offset = 0, .size = 1 });
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); });
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); });
}
if (isValid(res))
if (res)
return res;
// fallback on another track of the same disc
@@ -145,95 +119,28 @@ namespace lms::scanner
params.setImageTypes({ db::ImageType::Media });
params.setSortMethod(db::TrackEmbeddedImageSortMethod::TrackNumberThenSizeDesc);
params.setRange(db::Range{ .offset = 0, .size = 1 });
db::TrackEmbeddedImage::find(searchContext.session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = image->getId(); });
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); });
}
return res;
}
bool fetchNextTrackArtworksToUpdate(SearchTrackArtworkContext& searchContext, TrackArtworksAssociationContainer& TrackArtworksAssociations)
void updateTrackPreferredArtworks(db::Session& session, const TrackArtworksAssociation& trackArtworksAssociation)
{
const db::TrackId trackId{ searchContext.lastRetrievedTrackId };
assert(trackArtworksAssociation.preferredArtworkId || trackArtworksAssociation.preferredMediaArtworkId);
{
constexpr std::size_t readBatchSize{ 100 };
if (trackArtworksAssociation.preferredArtworkId)
db::Track::updatePreferredArtwork(session, trackArtworksAssociation.trackId, *trackArtworksAssociation.preferredArtworkId);
auto transaction{ searchContext.session.createReadTransaction() };
db::Track::find(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize, [&](const db::Track::pointer& track) {
const TrackArtwork preferredMediaArtwork{ computePreferredTrackMediaArtwork(searchContext, track) };
const TrackArtwork preferredArtwork{ computePreferredTrackArtwork(searchContext, track, preferredMediaArtwork) };
const db::Artwork::pointer currentPreferredArtwork{ track->getPreferredArtwork() };
const db::Artwork::pointer currentPreferredMediaArtwork{ track->getPreferredMediaArtwork() };
if (!isSameArtwork(preferredArtwork, currentPreferredArtwork)
|| !isSameArtwork(preferredMediaArtwork, currentPreferredMediaArtwork))
{
TrackArtworksAssociations.push_back(TrackArtworksAssociation{ .track = track, .preferredArtwork = preferredArtwork, .preferredMediaArtwork = preferredMediaArtwork });
}
searchContext.processedTrackCount++;
});
}
return trackId != searchContext.lastRetrievedTrackId;
if (trackArtworksAssociation.preferredMediaArtworkId)
db::Track::updatePreferredMediaArtwork(session, trackArtworksAssociation.trackId, *trackArtworksAssociation.preferredMediaArtworkId);
}
void updateTrackPreferredArtwork(db::Session& session, db::Track::pointer& track, TrackArtwork preferredArtwork)
{
db::Artwork::pointer artwork;
if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork) })
artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId);
else if (const db::ArtworkId * artworkId{ std::get_if<db::ArtworkId>(&preferredArtwork) })
artwork = db::Artwork::find(session, *artworkId);
// Using track.modify() is quite CPU intensive as the track class has too many fields
db::Track::updatePreferredArtwork(session, track->getId(), artwork ? artwork->getId() : db::ArtworkId{});
if (artwork)
LMS_LOG(DBUPDATER, DEBUG, "Updated preferred artwork in track " << track->getAbsoluteFilePath() << " with image in " << artwork->getAbsoluteFilePath());
else
LMS_LOG(DBUPDATER, DEBUG, "Removed preferred artwork from track " << track->getAbsoluteFilePath());
}
void updateTrackPreferredMediaArtwork(db::Session& session, db::Track::pointer& track, TrackArtwork preferredArtwork)
{
db::Artwork::pointer artwork;
if (const db::TrackEmbeddedImageId * trackEmbeddedImageId{ std::get_if<db::TrackEmbeddedImageId>(&preferredArtwork) })
artwork = utils::getOrCreateArtworkFromTrackEmbeddedImage(session, *trackEmbeddedImageId);
else if (const db::ArtworkId * artworkId{ std::get_if<db::ArtworkId>(&preferredArtwork) })
artwork = db::Artwork::find(session, *artworkId);
// Using track.modify() is quite CPU intensive as the track class has too many fields
db::Track::updatePreferredMediaArtwork(session, track->getId(), artwork ? artwork->getId() : db::ArtworkId{});
if (artwork)
LMS_LOG(DBUPDATER, DEBUG, "Updated preferred media artwork in track " << track->getAbsoluteFilePath() << " with image in " << artwork->getAbsoluteFilePath());
else
LMS_LOG(DBUPDATER, DEBUG, "Removed preferred media artwork from track " << track->getAbsoluteFilePath());
}
void updateTrackPreferredArtworks(db::Session& session, const TrackArtworksAssociation& TrackArtworksAssociation)
{
db::Track::pointer track{ TrackArtworksAssociation.track };
{
const db::Artwork::pointer currentPreferredArtwork{ track->getPreferredArtwork() };
if (!isSameArtwork(TrackArtworksAssociation.preferredArtwork, currentPreferredArtwork))
updateTrackPreferredArtwork(session, track, TrackArtworksAssociation.preferredArtwork);
}
{
const db::Artwork::pointer currentPreferredMediaArtwork{ track->getPreferredMediaArtwork() };
if (!isSameArtwork(TrackArtworksAssociation.preferredMediaArtwork, currentPreferredMediaArtwork))
updateTrackPreferredMediaArtwork(session, track, TrackArtworksAssociation.preferredMediaArtwork);
}
}
void updateTrackPreferredArtworks(db::Session& session, TrackArtworksAssociationContainer& imageAssociations)
void updateTrackPreferredArtworks(db::Session& session, TrackArtworksAssociationContainer& imageAssociations, bool forceFullBatch)
{
constexpr std::size_t writeBatchSize{ 50 };
while (!imageAssociations.empty())
while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
{
auto transaction{ session.createWriteTransaction() };
@@ -244,6 +151,76 @@ namespace lms::scanner
}
}
}
bool fetchNextTrackIdRange(SearchTrackArtworkContext& searchContext, db::IdRange<db::TrackId>& trackIdRange)
{
constexpr std::size_t readBatchSize{ 100 };
auto transaction{ searchContext.session.createReadTransaction() };
trackIdRange = db::Track::findNextRange(searchContext.session, searchContext.lastRetrievedTrackId, readBatchSize);
searchContext.lastRetrievedTrackId = trackIdRange.last;
return trackIdRange.isValid();
}
class ComputeTrackArtworkAssociationsJob : public core::IJob
{
public:
ComputeTrackArtworkAssociationsJob(db::Db& db, db::IdRange<db::TrackId> trackIdRange)
: _db{ db }
, _trackIdRange{ trackIdRange }
{
}
std::span<const TrackArtworksAssociation> getTrackAssociations() const { return _trackAssociations; }
std::size_t getProcessedTrackCount() const { return _processedTrackCount; }
private:
core::LiteralString getName() const override { return "Associate Track Artworks"; }
void run() override
{
auto& session{ _db.getTLSSession() };
auto transaction{ session.createReadTransaction() };
db::Track::find(session, _trackIdRange, [&](const db::Track::pointer& track) {
const db::Artwork::pointer preferredMediaArtwork{ computePreferredTrackMediaArtwork(session, track) };
const db::Artwork::pointer preferredArtwork{ computePreferredTrackArtwork(session, track, preferredMediaArtwork) };
TrackArtworksAssociation artworksAssociation{ .trackId = track->getId(), .preferredArtworkId = std::nullopt, .preferredMediaArtworkId = std::nullopt };
if (track->getPreferredArtwork() != preferredArtwork)
{
artworksAssociation.preferredArtworkId = preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{};
if (preferredArtwork)
LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork in track " << track->getAbsoluteFilePath() << " with image in " << preferredArtwork->getAbsoluteFilePath());
else
LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from track " << track->getAbsoluteFilePath());
}
if (track->getPreferredMediaArtwork() != preferredMediaArtwork)
{
artworksAssociation.preferredMediaArtworkId = preferredMediaArtwork ? preferredMediaArtwork->getId() : db::ArtworkId{};
if (preferredMediaArtwork)
LMS_LOG(DBUPDATER, DEBUG, "Updating preferred media artwork in track " << track->getAbsoluteFilePath() << " with image in " << preferredMediaArtwork->getAbsoluteFilePath());
else
LMS_LOG(DBUPDATER, DEBUG, "Removing preferred media artwork from track " << track->getAbsoluteFilePath());
}
if (artworksAssociation.preferredArtworkId || artworksAssociation.preferredMediaArtworkId)
_trackAssociations.push_back(artworksAssociation);
_processedTrackCount++;
});
}
db::Db& _db;
db::IdRange<db::TrackId> _trackIdRange;
std::vector<TrackArtworksAssociation> _trackAssociations;
std::size_t _processedTrackCount{};
};
} // namespace
ScanStepAssociateTrackImages::ScanStepAssociateTrackImages(InitParams& initParams)
@@ -270,15 +247,34 @@ namespace lms::scanner
.lastRetrievedTrackId = {},
};
TrackArtworksAssociationContainer TrackArtworksAssociations;
while (fetchNextTrackArtworksToUpdate(searchContext, TrackArtworksAssociations))
{
TrackArtworksAssociationContainer trackArtworksAssociations;
auto processTracks = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
if (_abortScan)
return;
updateTrackPreferredArtworks(session, TrackArtworksAssociations);
context.currentStepStats.processedElems = searchContext.processedTrackCount;
for (const auto& job : jobs)
{
const auto& associationJob{ static_cast<const ComputeTrackArtworkAssociationsJob&>(*job) };
const auto& trackAssociations{ associationJob.getTrackAssociations() };
trackArtworksAssociations.insert(std::end(trackArtworksAssociations), std::cbegin(trackAssociations), std::cend(trackAssociations));
context.currentStepStats.processedElems += associationJob.getProcessedTrackCount();
}
updateTrackPreferredArtworks(session, trackArtworksAssociations, true);
_progressCallback(context.currentStepStats);
}
};
JobQueue queue{ getJobScheduler(), 20, processTracks, 1, 0.85F };
db::IdRange<db::TrackId> trackIdRange;
while (fetchNextTrackIdRange(searchContext, trackIdRange))
queue.push(std::make_unique<ComputeTrackArtworkAssociationsJob>(_db, trackIdRange));
queue.finish();
// process all remaining associations
updateTrackPreferredArtworks(session, trackArtworksAssociations, false);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp
+11 -29
View File
@@ -31,6 +31,7 @@
#include "scanners/IFileScanOperation.hpp"
#include "scanners/IFileScanner.hpp"
#include "JobQueue.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
@@ -114,11 +115,16 @@ namespace lms::scanner
void ScanStepScanFiles::process(ScanContext& context, const MediaLibraryInfo& mediaLibrary)
{
core::IJobScheduler& jobScheduler{ getJobScheduler() };
assert(jobScheduler.getJobsDoneCount() == 0);
const std::size_t scanQueueMaxScanRequestCount{ 100 * jobScheduler.getThreadCount() };
const std::size_t scanQueueMaxScanRequestCount{ 50 * getJobScheduler().getThreadCount() };
constexpr std::size_t processFileResultsBatchSize{ 10 };
constexpr float drainRatio{ 0.85 };
auto processDoneJobs = [&](std::span<std::unique_ptr<core::IJob>> jobsDone) {
if (!_abortScan)
processFileScanResults(context, jobsDone);
};
JobQueue queue{ getJobScheduler(), scanQueueMaxScanRequestCount, processDoneJobs, processFileResultsBatchSize, drainRatio };
std::vector<std::unique_ptr<core::IJob>> jobsDone;
std::vector<std::unique_ptr<IFileScanOperation>> scanOperations;
@@ -148,7 +154,7 @@ namespace lms::scanner
if (context.scanOptions.fullScan || scanner->needsScan(fileToScan))
{
auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) };
jobScheduler.scheduleJob(std::make_unique<FileScanJob>(std::move(scanOperation)));
queue.push(std::make_unique<FileScanJob>(std::move(scanOperation)));
}
}
@@ -156,30 +162,9 @@ namespace lms::scanner
_progressCallback(context.currentStepStats);
}
while (jobScheduler.getJobsDoneCount() > (scanQueueMaxScanRequestCount / 2))
{
jobScheduler.popJobsDone(jobsDone, processFileResultsBatchSize);
processFileScanResults(context, jobsDone);
jobsDone.clear();
}
jobScheduler.waitUntilJobCountAtMost(scanQueueMaxScanRequestCount);
return true;
},
&excludeDirFileName);
jobScheduler.wait();
while (jobScheduler.popJobsDone(jobsDone, processFileResultsBatchSize) > 0)
{
if (!_abortScan)
processFileScanResults(context, jobsDone);
jobsDone.clear();
}
assert(jobScheduler.getJobsDoneCount() == 0);
}
void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<std::unique_ptr<core::IJob>> scanJobs)
@@ -191,9 +176,6 @@ namespace lms::scanner
for (auto& scanJob : scanJobs)
{
if (_abortScan)
return;
IFileScanOperation& scanOperation{ static_cast<FileScanJob&>(*scanJob).getScanOperation() };
LMS_LOG(DBUPDATER, DEBUG, scanOperation.getName() << ": processing result for " << scanOperation.getFilePath());