Associating an artwork to each medium, ref #699

This commit is contained in:
emeric
2025-07-30 20:12:01 +02:00
parent 0ceaf4cfaa
commit 6bf08b94ae
12 changed files with 485 additions and 22 deletions
+6 -2
View File
@@ -43,7 +43,8 @@ namespace lms::db
query.where("d.name LIKE ? ESCAPE '" ESCAPE_CHAR_STR "'").bind("%" + utils::escapeLikeKeyword(keyword) + "%");
if (params.artist.isValid()
|| params.release.isValid())
|| params.release.isValid()
|| params.medium.isValid())
{
query.join("track t ON t.directory_id = d.id");
query.groupBy("d.id");
@@ -55,6 +56,9 @@ namespace lms::db
if (params.parentDirectory.isValid())
query.where("d.parent_directory_id = ?").bind(params.parentDirectory);
if (params.medium.isValid())
query.where("t.medium_id = ?").bind(params.medium);
if (params.release.isValid())
query.where("t.release_id = ?").bind(params.release);
@@ -91,7 +95,7 @@ namespace lms::db
case DirectorySortMethod::None:
break;
case DirectorySortMethod::Name:
query.orderBy("name COLLATE NOCASE");
query.orderBy("d.name COLLATE NOCASE");
break;
}
+42
View File
@@ -83,4 +83,46 @@ namespace lms::db
return utils::fetchQuerySingleResult(query);
}
void Medium::find(Session& session, const IdRange<MediumId>& idRange, const std::function<void(const Medium::pointer&)>& func)
{
assert(idRange.isValid());
auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Medium>>("SELECT m from medium m").orderBy("m.id").where("m.id BETWEEN ? AND ?").bind(idRange.first).bind(idRange.last) };
utils::forEachQueryResult(query, [&](const Medium::pointer& medium) {
func(medium);
});
}
IdRange<MediumId> Medium::findNextIdRange(Session& session, MediumId lastRetrievedId, std::size_t count)
{
session.checkReadTransaction();
auto query{ session.getDboSession()->query<std::tuple<MediumId, MediumId>>("SELECT MIN(sub.id) AS first_id, MAX(sub.id) AS last_id FROM (SELECT m.id FROM medium m WHERE m.id > ? ORDER BY m.id LIMIT ?) sub") };
query.bind(lastRetrievedId);
query.bind(static_cast<int>(count));
auto res{ utils::fetchQuerySingleResult(query) };
return IdRange<MediumId>{ .first = std::get<0>(res), .last = std::get<1>(res) };
}
RangeResults<MediumId> Medium::findOrphanIds(Session& session, std::optional<Range> range)
{
session.checkReadTransaction();
// select the mediums that have no track
auto query{ session.getDboSession()->query<MediumId>("select m.id from medium m LEFT OUTER JOIN track t ON m.id = t.medium_id WHERE t.id IS NULL") };
return utils::execRangeQuery<MediumId>(query, range);
}
void Medium::updatePreferredArtwork(Session& session, MediumId mediumId, ArtworkId artworkId)
{
session.checkWriteTransaction();
if (artworkId.isValid())
utils::executeCommand(*session.getDboSession(), "UPDATE medium SET preferred_artwork_id = ? WHERE id = ?", artworkId, mediumId);
else
utils::executeCommand(*session.getDboSession(), "UPDATE medium SET preferred_artwork_id = NULL WHERE id = ?", mediumId);
}
} // namespace lms::db
@@ -34,6 +34,7 @@
#include "database/objects/ArtistId.hpp"
#include "database/objects/DirectoryId.hpp"
#include "database/objects/MediaLibraryId.hpp"
#include "database/objects/MediumId.hpp"
#include "database/objects/ReleaseId.hpp"
namespace lms::db
@@ -50,8 +51,9 @@ namespace lms::db
{
std::optional<Range> range;
std::vector<std::string_view> keywords; // if non empty, name must match all of these keywords
ArtistId artist; // only directory that involve this artist
ReleaseId release; // only releases that involve this artist
ArtistId artist; // only directoies that involve this artist
MediumId medium; // only directories that involve this medium
ReleaseId release; // only directories that involve this release
core::EnumSet<TrackArtistLinkType> trackArtistLinkTypes; // and for these link types
DirectoryId parentDirectory; // If set, directories that have this parent
bool withNoTrack{}; // If set, directories that do not contain any track
@@ -74,6 +76,11 @@ namespace lms::db
trackArtistLinkTypes = _trackArtistLinkTypes;
return *this;
}
FindParameters& setMedium(MediumId _medium)
{
medium = _medium;
return *this;
}
FindParameters& setRelease(ReleaseId _release)
{
release = _release;
@@ -27,6 +27,7 @@
#include <Wt/Dbo/Field.h>
#include <Wt/Dbo/collection.h>
#include "database/IdRange.hpp"
#include "database/Object.hpp"
#include "database/Types.hpp"
#include "database/objects/ArtworkId.hpp"
@@ -74,7 +75,12 @@ namespace lms::db
static std::size_t getCount(Session& session);
static pointer find(Session& session, MediumId id);
static pointer find(Session& session, ReleaseId id, std::optional<std::size_t> position);
static void find(Session& session, const FindParameters& params, std::function<void(const pointer&)> func);
static void find(Session& session, const IdRange<MediumId>& idRange, const std::function<void(const Medium::pointer&)>& func);
static IdRange<MediumId> findNextIdRange(Session& session, MediumId lastRetrievedId, std::size_t count);
static RangeResults<MediumId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt);
// Updates
static void updatePreferredArtwork(Session& session, MediumId mediumId, ArtworkId artworkId);
// getters
std::string_view getName() const { return _name; }
+55
View File
@@ -20,10 +20,12 @@
#include "Common.hpp"
#include "database/objects/Directory.hpp"
#include "database/objects/Medium.hpp"
namespace lms::db::tests
{
using ScopedDirectory = ScopedEntity<db::Directory>;
using ScopedMedium = ScopedEntity<db::Medium>;
TEST_F(DatabaseFixture, Directory)
{
@@ -285,4 +287,57 @@ namespace lms::db::tests
EXPECT_EQ(res.size(), 0);
}
}
TEST_F(DatabaseFixture, Directory_findByMedium)
{
ScopedDirectory dir1{ session, "/root" };
ScopedDirectory dir2{ session, "/root" };
ScopedTrack track1{ session };
ScopedTrack track2{ session };
ScopedRelease release1{ session, "Release1" };
ScopedMedium medium1{ session, release1.lockAndGet() };
ScopedRelease release2{ session, "Release2" };
ScopedMedium medium2{ session, release1.lockAndGet() };
{
auto transaction{ session.createReadTransaction() };
Directory::FindParameters params;
params.setMedium(medium1.getId());
bool visited{};
Directory::find(session, params, [&](const Directory::pointer&) {
visited = true;
});
EXPECT_FALSE(visited);
}
{
auto transaction{ session.createWriteTransaction() };
track1.get().modify()->setMedium(medium1.get());
track1.get().modify()->setRelease(release1.get());
track1.get().modify()->setDirectory(dir1.get());
track2.get().modify()->setMedium(medium2.get());
track2.get().modify()->setRelease(release2.get());
track2.get().modify()->setDirectory(dir2.get());
}
{
auto transaction{ session.createReadTransaction() };
Directory::FindParameters params;
params.setMedium(medium1.getId());
std::vector<DirectoryId> visitedDirectories;
Directory::find(session, params, [&](const Directory::pointer& dir) {
visitedDirectories.push_back(dir->getId());
});
ASSERT_EQ(visitedDirectories.size(), 1);
EXPECT_EQ(visitedDirectories[0], dir1.getId());
}
}
} // namespace lms::db::tests
+1
View File
@@ -13,6 +13,7 @@ add_library(lmsscanner STATIC
impl/steps/ScanStepArtistReconciliation.cpp
impl/steps/ScanStepAssociateArtistImages.cpp
impl/steps/ScanStepAssociateExternalLyrics.cpp
impl/steps/ScanStepAssociateMediumImages.cpp
impl/steps/ScanStepAssociatePlayListTracks.cpp
impl/steps/ScanStepAssociateReleaseImages.cpp
impl/steps/ScanStepAssociateTrackImages.cpp
@@ -40,6 +40,7 @@
#include "steps/ScanStepArtistReconciliation.hpp"
#include "steps/ScanStepAssociateArtistImages.hpp"
#include "steps/ScanStepAssociateExternalLyrics.hpp"
#include "steps/ScanStepAssociateMediumImages.hpp"
#include "steps/ScanStepAssociatePlayListTracks.hpp"
#include "steps/ScanStepAssociateReleaseImages.hpp"
#include "steps/ScanStepAssociateTrackImages.hpp"
@@ -506,7 +507,8 @@ namespace lms::scanner
_scanSteps.emplace_back(std::make_unique<ScanStepUpdateLibraryFields>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepAssociateReleaseImages>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepAssociateArtistImages>(params)); // must come after ScanStepAssociateReleaseImages
_scanSteps.emplace_back(std::make_unique<ScanStepAssociateTrackImages>(params)); // must come after ScanStepAssociateReleaseImages
_scanSteps.emplace_back(std::make_unique<ScanStepAssociateMediumImages>(params)); // must come after ScanStepAssociateReleaseImages
_scanSteps.emplace_back(std::make_unique<ScanStepAssociateTrackImages>(params)); // must come after ScanStepAssociateMediumImages and ScanStepAssociateReleaseImages
_scanSteps.emplace_back(std::make_unique<ScanStepAssociateExternalLyrics>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepRemoveOrphanedDbEntries>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepCompact>(params));
@@ -0,0 +1,304 @@
/*
* Copyright (C) 2025 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepAssociateMediumImages.hpp"
#include <array>
#include <cassert>
#include <deque>
#include <set>
#include <span>
#include "core/IConfig.hpp"
#include "core/IJob.hpp"
#include "core/ILogger.hpp"
#include "database/IDb.hpp"
#include "database/Session.hpp"
#include "database/Types.hpp"
#include "database/objects/Artist.hpp"
#include "database/objects/ArtistInfo.hpp"
#include "database/objects/Artwork.hpp"
#include "database/objects/ArtworkId.hpp"
#include "database/objects/Directory.hpp"
#include "database/objects/Image.hpp"
#include "database/objects/Medium.hpp"
#include "database/objects/Release.hpp"
#include "database/objects/Track.hpp"
#include "JobQueue.hpp"
#include "ScanContext.hpp"
#include "database/objects/TrackEmbeddedImage.hpp"
namespace lms::scanner
{
namespace
{
struct MediumArtworkAssociation
{
db::MediumId mediumId;
db::ArtworkId preferredArtworkId;
};
using MediumArtworkAssociationContainer = std::deque<MediumArtworkAssociation>;
struct SearchMediumArtworkParams
{
std::span<const std::string_view> mediumFileNames;
};
db::Image::pointer findImageInDirectory(db::Session& session, const db::Directory::pointer& directory, std::span<const std::string_view> fileStemsToSearch)
{
db::Image::pointer image;
for (std::string_view fileStem : fileStemsToSearch)
{
db::Image::FindParameters params;
params.setDirectory(directory->getId());
params.setFileStem(fileStem);
db::Image::find(session, params, [&](const db::Image::pointer foundImg) {
if (!image)
image = foundImg;
});
if (image)
break;
}
return image;
}
db::Image::pointer searchImageInDirectories(db::Session& session, const SearchMediumArtworkParams& searchParams, const db::Medium::pointer& medium)
{
db::Image::pointer image;
std::set<std::filesystem::path> mediumPaths;
db::Directory::FindParameters params;
params.setMedium(medium->getId());
// Expect layout like this:
// Release/Tracks
// /NameOfTheDisc.jpg
// /someOtherUserConfiguredMediumFile.jpg
//
// Or:
// Release/CD X/Tracks'
// /NameOfDisc.jpg.jpg
// /someOtherUserConfiguredMediumFile.jpg
//
// We don't expect mediums to be split across multiple directories, so we can just search for the first directory that matches the medium.
db::Directory::find(session, params, [&](const db::Directory::pointer& directory) {
if (image)
return;
if (const std::string_view mediumName{ medium->getName() }; !mediumName.empty())
image = findImageInDirectory(session, directory, std::span{ &mediumName, 1 });
if (!image)
image = findImageInDirectory(session, directory, searchParams.mediumFileNames);
});
return image;
}
db::TrackEmbeddedImage::pointer getArtworkFromTracks(db::Session& session, const db::Medium::pointer& medium)
{
db::TrackEmbeddedImage::pointer image;
db::TrackEmbeddedImage::FindParameters params;
params.setMedium(medium->getId());
params.setImageType(db::ImageType::Media);
params.setSortMethod(db::TrackEmbeddedImageSortMethod::TrackNumberThenSizeDesc);
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& foundImage) {
if (image)
return;
image = foundImage;
});
return image;
}
db::Artwork::pointer computePreferredMediumArtwork(db::Session& session, const SearchMediumArtworkParams& searchParams, const db::Medium::pointer& medium)
{
if (const db::Image::pointer image{ searchImageInDirectories(session, searchParams, medium) })
return db::Artwork::find(session, image->getId());
if (const db::TrackEmbeddedImage::pointer image{ getArtworkFromTracks(session, medium) })
return db::Artwork::find(session, image->getId());
return db::Artwork::pointer{};
}
void updateMediumPreferredArtwork(db::Session& session, const MediumArtworkAssociation& mediumArtworkAssociation)
{
db::Medium::updatePreferredArtwork(session, mediumArtworkAssociation.mediumId, mediumArtworkAssociation.preferredArtworkId);
}
void updateMediumPreferredArtworks(db::Session& session, MediumArtworkAssociationContainer& imageAssociations, bool forceFullBatch)
{
constexpr std::size_t writeBatchSize{ 50 };
while ((forceFullBatch && imageAssociations.size() >= writeBatchSize) || !imageAssociations.empty())
{
auto transaction{ session.createWriteTransaction() };
for (std::size_t i{}; !imageAssociations.empty() && i < writeBatchSize; ++i)
{
updateMediumPreferredArtwork(session, imageAssociations.front());
imageAssociations.pop_front();
}
}
}
std::vector<std::string> constructArtistFileNames()
{
std::vector<std::string> res;
core::Service<core::IConfig>::get()->visitStrings("medium-image-file-names",
[&res](std::string_view fileName) {
res.emplace_back(fileName);
},
{ "discsubtitle" });
return res;
}
bool fetchNextMediumIdRange(db::Session& session, db::MediumId& lastRetrievedId, db::IdRange<db::MediumId>& idRange)
{
constexpr std::size_t readBatchSize{ 100 };
auto transaction{ session.createReadTransaction() };
idRange = db::Medium::findNextIdRange(session, lastRetrievedId, readBatchSize);
lastRetrievedId = idRange.last;
return idRange.isValid();
}
class ComputeMediumArtworkAssociationsJob : public core::IJob
{
public:
ComputeMediumArtworkAssociationsJob(db::IDb& db, const SearchMediumArtworkParams& searchParams, db::IdRange<db::MediumId> mediumIdRange)
: _db{ db }
, _searchParams{ searchParams }
, _mediumIdRange{ mediumIdRange }
{
}
~ComputeMediumArtworkAssociationsJob() override = default;
ComputeMediumArtworkAssociationsJob(const ComputeMediumArtworkAssociationsJob&) = delete;
ComputeMediumArtworkAssociationsJob& operator=(const ComputeMediumArtworkAssociationsJob&) = delete;
std::span<const MediumArtworkAssociation> getAssociations() const { return _associations; }
std::size_t getProcessedMediumCount() const { return _processedMediumCount; }
private:
core::LiteralString getName() const override { return "Associate Medium Artworks"; }
void run() override
{
auto& session{ _db.getTLSSession() };
auto transaction{ session.createReadTransaction() };
db::Medium::find(session, _mediumIdRange, [this, &session](const db::Medium::pointer& medium) {
const db::Artwork::pointer preferredArtwork{ computePreferredMediumArtwork(session, _searchParams, medium) };
if (medium->getPreferredArtwork() != preferredArtwork)
{
_associations.push_back(MediumArtworkAssociation{ medium->getId(), preferredArtwork ? preferredArtwork->getId() : db::ArtworkId{} });
if (preferredArtwork)
LMS_LOG(DBUPDATER, DEBUG, "Updating preferred artwork for medium '" << medium->getName() << "(from '" << medium->getRelease()->getName() << "') with image in " << preferredArtwork->getAbsoluteFilePath());
else
LMS_LOG(DBUPDATER, DEBUG, "Removing preferred artwork from medium '" << medium->getName() << "(from '" << medium->getRelease()->getName() << "')");
}
_processedMediumCount++;
});
}
db::IDb& _db;
const SearchMediumArtworkParams& _searchParams;
db::IdRange<db::MediumId> _mediumIdRange;
std::vector<MediumArtworkAssociation> _associations;
std::size_t _processedMediumCount{};
};
} // namespace
ScanStepAssociateMediumImages::ScanStepAssociateMediumImages(InitParams& initParams)
: ScanStepBase{ initParams }
, _mediumFileNames{ constructArtistFileNames() }
{
}
bool ScanStepAssociateMediumImages::needProcess(const ScanContext& context) const
{
return context.stats.getChangesCount() > 0;
}
void ScanStepAssociateMediumImages::process(ScanContext& context)
{
auto& session{ _db.getTLSSession() };
{
auto transaction{ session.createReadTransaction() };
context.currentStepStats.totalElems = db::Artist::getCount(session);
}
std::vector<std::string_view> mediumFileNames;
mediumFileNames.reserve(_mediumFileNames.size());
for (const std::string& fileName : _mediumFileNames)
mediumFileNames.push_back(fileName);
const SearchMediumArtworkParams searchParams{
.mediumFileNames = mediumFileNames,
};
MediumArtworkAssociationContainer mediumArtworkAssociations;
auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
if (_abortScan)
return;
for (const auto& job : jobs)
{
const auto& associationJob{ static_cast<const ComputeMediumArtworkAssociationsJob&>(*job) };
const auto& artistAssociations{ associationJob.getAssociations() };
mediumArtworkAssociations.insert(std::end(mediumArtworkAssociations), std::cbegin(artistAssociations), std::cend(artistAssociations));
context.currentStepStats.processedElems += associationJob.getProcessedMediumCount();
}
updateMediumPreferredArtworks(session, mediumArtworkAssociations, true);
_progressCallback(context.currentStepStats);
};
{
JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
db::MediumId lastRetrievedMediumId{};
db::IdRange<db::MediumId> mediumIdRange;
while (fetchNextMediumIdRange(session, lastRetrievedMediumId, mediumIdRange))
queue.push(std::make_unique<ComputeMediumArtworkAssociationsJob>(_db, searchParams, mediumIdRange));
}
// process all remaining associations
updateMediumPreferredArtworks(session, mediumArtworkAssociations, false);
}
} // namespace lms::scanner
@@ -0,0 +1,45 @@
/*
* Copyright (C) 2025 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <string>
#include <vector>
#include "ScanStepBase.hpp"
namespace lms::scanner
{
class ScanStepAssociateMediumImages : public ScanStepBase
{
public:
ScanStepAssociateMediumImages(InitParams& initParams);
~ScanStepAssociateMediumImages() override = default;
ScanStepAssociateMediumImages(const ScanStepAssociateMediumImages&) = delete;
ScanStepAssociateMediumImages& operator=(const ScanStepAssociateMediumImages&) = delete;
private:
ScanStep getStep() const override { return ScanStep::AssociateArtistImages; }
core::LiteralString getStepName() const override { return "Associate medium images"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
const std::vector<std::string> _mediumFileNames;
};
} // namespace lms::scanner
@@ -33,6 +33,7 @@
#include "database/objects/Artwork.hpp"
#include "database/objects/Directory.hpp"
#include "database/objects/Image.hpp"
#include "database/objects/Medium.hpp"
#include "database/objects/Release.hpp"
#include "database/objects/Track.hpp"
#include "database/objects/TrackEmbeddedImage.hpp"
@@ -105,22 +106,9 @@ namespace lms::scanner
if (res)
return res;
// fallback on another track of the same disc
const db::MediumId mediumId{ track->getMediumId() };
if (!mediumId.isValid())
return res;
{
db::TrackEmbeddedImage::FindParameters params;
params.setMedium(track->getMediumId());
params.setImageType(db::ImageType::Media);
params.setSortMethod(db::TrackEmbeddedImageSortMethod::TrackNumberThenSizeDesc);
db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) {
if (!res)
res = db::Artwork::find(session, image->getId());
});
}
// fallback on the medium's preferred artwork
if (const auto medium{ track->getMedium() })
res = medium->getPreferredArtwork();
return res;
}
@@ -25,6 +25,7 @@
#include "database/objects/Artist.hpp"
#include "database/objects/Cluster.hpp"
#include "database/objects/Directory.hpp"
#include "database/objects/Medium.hpp"
#include "database/objects/Release.hpp"
#include "database/objects/Track.hpp"
#include "database/objects/TrackEmbeddedImage.hpp"
@@ -45,6 +46,7 @@ namespace lms::scanner
removeOrphanedClusterTypes(context);
removeOrphanedArtists(context);
removeOrphanedReleases(context);
removeOrphanedMediums(context); // after release so that most entries are removed using the medium foreign key
removeOrphanedReleaseTypes(context);
removeOrphanedLabels(context);
removeOrphanedCountries(context);
@@ -70,6 +72,12 @@ namespace lms::scanner
removeOrphanedEntries<db::Artist>(context);
}
void ScanStepRemoveOrphanedDbEntries::removeOrphanedMediums(ScanContext& context)
{
LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned mediums...");
removeOrphanedEntries<db::Medium>(context);
}
void ScanStepRemoveOrphanedDbEntries::removeOrphanedReleases(ScanContext& context)
{
LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned releases...");
@@ -37,6 +37,7 @@ namespace lms::scanner
void removeOrphanedClusters(ScanContext& context);
void removeOrphanedClusterTypes(ScanContext& context);
void removeOrphanedArtists(ScanContext& context);
void removeOrphanedMediums(ScanContext& context);
void removeOrphanedReleases(ScanContext& context);
void removeOrphanedReleaseTypes(ScanContext& context);
void removeOrphanedLabels(ScanContext& context);