From 71dc99ad46d8f62ff253d5454eb4020350d11cef Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 29 Jun 2024 16:04:16 +0200 Subject: [PATCH 1/6] Added a scan step to scan artist images, ref #435 --- approot/messages.xml | 3 +- approot/messages_fr.xml | 3 +- approot/messages_it.xml | 3 +- approot/messages_pl.xml | 3 +- approot/messages_zh.xml | 3 +- src/libs/database/CMakeLists.txt | 1 + src/libs/database/impl/Artist.cpp | 6 + src/libs/database/impl/Image.cpp | 56 +++ src/libs/database/impl/MediaLibrary.cpp | 2 +- src/libs/database/impl/Migration.cpp | 469 +++++++++--------- src/libs/database/impl/Session.cpp | 4 + src/libs/database/include/database/Artist.hpp | 4 + src/libs/database/include/database/Image.hpp | 86 ++++ .../database/include/database/ImageId.hpp | 24 + src/libs/database/include/database/Object.hpp | 2 +- src/libs/database/test/CMakeLists.txt | 1 + src/libs/database/test/Common.cpp | 2 + src/libs/database/test/Image.cpp | 66 +++ .../image/impl/graphicsmagick/RawImage.cpp | 10 + .../image/impl/graphicsmagick/RawImage.hpp | 3 + src/libs/image/impl/stb/RawImage.hpp | 5 +- src/libs/image/include/image/IRawImage.hpp | 4 + src/libs/metadata/impl/Parser.cpp | 2 +- src/libs/services/cover/impl/CoverService.cpp | 93 +--- src/libs/services/cover/impl/CoverService.hpp | 1 - src/libs/services/scanner/CMakeLists.txt | 6 +- .../scanner/impl/ScanStepScanArtistImages.cpp | 347 +++++++++++++ .../scanner/impl/ScanStepScanArtistImages.hpp | 41 ++ ...anFiles.cpp => ScanStepScanAudioFiles.cpp} | 22 +- ...anFiles.hpp => ScanStepScanAudioFiles.hpp} | 8 +- .../services/scanner/impl/ScannerService.cpp | 9 +- .../include/services/scanner/ScannerStats.hpp | 4 +- src/libs/subsonic/impl/SubsonicId.cpp | 16 + src/libs/subsonic/impl/SubsonicId.hpp | 5 + .../impl/entrypoints/MediaLibraryScanning.cpp | 2 +- src/libs/subsonic/impl/responses/Artist.cpp | 4 +- src/lms/ui/admin/ScannerController.cpp | 11 +- 37 files changed, 983 insertions(+), 348 deletions(-) create mode 100644 src/libs/database/impl/Image.cpp create mode 100644 src/libs/database/include/database/Image.hpp create mode 100644 src/libs/database/include/database/ImageId.hpp create mode 100644 src/libs/database/test/Image.cpp create mode 100644 src/libs/services/scanner/impl/ScanStepScanArtistImages.cpp create mode 100644 src/libs/services/scanner/impl/ScanStepScanArtistImages.hpp rename src/libs/services/scanner/impl/{ScanStepScanFiles.cpp => ScanStepScanAudioFiles.cpp} (95%) rename src/libs/services/scanner/impl/{ScanStepScanFiles.hpp => ScanStepScanAudioFiles.hpp} (95%) diff --git a/approot/messages.xml b/approot/messages.xml index 4f1d5872..306b08aa 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -116,7 +116,8 @@ Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)... Optimizing database... {1}/{2} entries ({3}%)... Reloading similarity engine: {1}%... -Scanning files: {1}/{2} files ({3}%)... +Scanning artist images: {1}/{2} artists ({3}%)... +Scanning audio files: {1}/{2} files ({3}%)... Step status diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 6a61b253..4ed82095 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -116,7 +116,8 @@ Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)... Optimisation de la base de données... {1}/{2} entrées ({3}%)... Rechargement du moteur de recommandation : {1}%... -Scan des fichiers : {1}/{2} fichiers ({3}%)... +Scan des images des artistes: {1}/{2} artists ({3}%)... +Scan des fichiers audio : {1}/{2} fichiers ({3}%)... Statut de l'étape diff --git a/approot/messages_it.xml b/approot/messages_it.xml index 631454f9..2e04db1a 100644 --- a/approot/messages_it.xml +++ b/approot/messages_it.xml @@ -116,7 +116,8 @@ Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)... Ottimizzazione del database... {1}/{2} voci ({3}%)... Ricarica motore di tracce simili: {1}%... -Scansione files: {1}/{2} files ({3}%)... +Scansione delle immagini degli artisti: {1}/{2} artisti ({3}%)... +Scansione dei file audio: {1}/{2} files ({3}%)... Stato passo diff --git a/approot/messages_pl.xml b/approot/messages_pl.xml index b7e0a884..d1bf52da 100644 --- a/approot/messages_pl.xml +++ b/approot/messages_pl.xml @@ -133,7 +133,8 @@ Pobieranie danych o ścieżce z AcousticBrainz: {1}/{2} ścieżek ({3}%)... Optymalizowanie bazy danych... {1}/{2} wpisów ({3}%)... Przeładowywanie silnika podobieństw: {1}%... -Skanowanie plików: {1}/{2} plików ({3}%)... +Skanowanie obrazów artystów: {1}/{2} artystów ({3}%)... +Skanowanie plików: {1}/{2} plików ({3}%)... Obecny krok diff --git a/approot/messages_zh.xml b/approot/messages_zh.xml index 745cfe82..027bddca 100644 --- a/approot/messages_zh.xml +++ b/approot/messages_zh.xml @@ -116,7 +116,8 @@ 从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)... 重载相似引擎中 {1}%... -扫描文件中: {1}/{2} 个文件 ({3}%)... + +扫描文件中: {1}/{2} 个文件 ({3}%)... 当前步骤状态 diff --git a/src/libs/database/CMakeLists.txt b/src/libs/database/CMakeLists.txt index b5e5a194..4c55f425 100644 --- a/src/libs/database/CMakeLists.txt +++ b/src/libs/database/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(lmsdatabase SHARED impl/AuthToken.cpp impl/Cluster.cpp impl/Db.cpp + impl/Image.cpp impl/Listen.cpp impl/MediaLibrary.cpp impl/Migration.cpp diff --git a/src/libs/database/impl/Artist.cpp b/src/libs/database/impl/Artist.cpp index ae93df40..065fbd1e 100644 --- a/src/libs/database/impl/Artist.cpp +++ b/src/libs/database/impl/Artist.cpp @@ -22,6 +22,7 @@ #include "core/ILogger.hpp" #include "database/Cluster.hpp" +#include "database/Image.hpp" #include "database/Release.hpp" #include "database/Session.hpp" #include "database/Track.hpp" @@ -273,6 +274,11 @@ namespace lms::db utils::forEachQueryRangeResult(query, params.range, func); } + ObjectPtr Artist::getImage() const + { + return ObjectPtr{ _image.lock() }; + } + RangeResults Artist::findSimilarArtistIds(core::EnumSet artistLinkTypes, std::optional range) const { assert(session()); diff --git a/src/libs/database/impl/Image.cpp b/src/libs/database/impl/Image.cpp new file mode 100644 index 00000000..2a0f5e3e --- /dev/null +++ b/src/libs/database/impl/Image.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "database/Image.hpp" + +#include + +#include "database/Artist.hpp" +#include "database/Session.hpp" + +#include "IdTypeTraits.hpp" +#include "PathTraits.hpp" +#include "Utils.hpp" + +namespace lms::db +{ + Image::Image(const std::filesystem::path& p) + : _path{ p } + { + } + + Image::pointer Image::create(Session& session, const std::filesystem::path& p) + { + return session.getDboSession()->add(std::unique_ptr{ new Image{ p } }); + } + + std::size_t Image::getCount(Session& session) + { + session.checkReadTransaction(); + + return utils::fetchQuerySingleResult(session.getDboSession()->query("SELECT COUNT(*) FROM image")); + } + + Image::pointer Image::find(Session& session, ImageId id) + { + session.checkReadTransaction(); + + return utils::fetchQuerySingleResult(session.getDboSession()->find().where("id = ?").bind(id)); + } +} // namespace lms::db diff --git a/src/libs/database/impl/MediaLibrary.cpp b/src/libs/database/impl/MediaLibrary.cpp index 74a7a2e1..59655bc1 100644 --- a/src/libs/database/impl/MediaLibrary.cpp +++ b/src/libs/database/impl/MediaLibrary.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013-2016 Emeric Poupon + * Copyright (C) 2024 Emeric Poupon * * This file is part of LMS. * diff --git a/src/libs/database/impl/Migration.cpp b/src/libs/database/impl/Migration.cpp index 4dacba8b..f18c41cd 100644 --- a/src/libs/database/impl/Migration.cpp +++ b/src/libs/database/impl/Migration.cpp @@ -35,7 +35,7 @@ namespace lms::db { namespace { - static constexpr Version LMS_DATABASE_VERSION{ 59 }; + static constexpr Version LMS_DATABASE_VERSION{ 60 }; } VersionInfo::VersionInfo() @@ -86,11 +86,13 @@ namespace lms::db::Migration Db& _db; }; - static void migrateFromV33(Session& session) + namespace { - // remove name from track_artist_link - // Drop Auth mode - session.getDboSession()->execute(R"( + void migrateFromV33(Session& session) + { + // remove name from track_artist_link + // Drop Auth mode + session.getDboSession()->execute(R"( CREATE TABLE IF NOT EXISTS "track_artist_link_backup" ( "id" integer primary key autoincrement, "version" integer not null, @@ -101,49 +103,49 @@ CREATE TABLE IF NOT EXISTS "track_artist_link_backup" ( constraint "fk_track_artist_link_artist" foreign key ("artist_id") references "artist" ("id") on delete cascade deferrable initially deferred ); ))"); - session.getDboSession()->execute("INSERT INTO track_artist_link_backup SELECT id, version, type, track_id, artist_id FROM track_artist_link"); - session.getDboSession()->execute("DROP TABLE track_artist_link"); - session.getDboSession()->execute("ALTER TABLE track_artist_link_backup RENAME TO track_artist_link"); - } + session.getDboSession()->execute("INSERT INTO track_artist_link_backup SELECT id, version, type, track_id, artist_id FROM track_artist_link"); + session.getDboSession()->execute("DROP TABLE track_artist_link"); + session.getDboSession()->execute("ALTER TABLE track_artist_link_backup RENAME TO track_artist_link"); + } - static void migrateFromV34(Session& session) - { - // Add scrobbling state - // By default, everything needs to be sent - session.getDboSession()->execute("ALTER TABLE starred_artist ADD scrobbling_state INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*ScrobblingState::PendingAdd*/ 0)) + ")"); - session.getDboSession()->execute("ALTER TABLE starred_release ADD scrobbling_state INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*ScrobblingState::PendingAdd*/ 0)) + ")"); - session.getDboSession()->execute("ALTER TABLE starred_track ADD scrobbling_state INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*ScrobblingState::PendingAdd*/ 0)) + ")"); - } + void migrateFromV34(Session& session) + { + // Add scrobbling state + // By default, everything needs to be sent + session.getDboSession()->execute("ALTER TABLE starred_artist ADD scrobbling_state INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*ScrobblingState::PendingAdd*/ 0)) + ")"); + session.getDboSession()->execute("ALTER TABLE starred_release ADD scrobbling_state INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*ScrobblingState::PendingAdd*/ 0)) + ")"); + session.getDboSession()->execute("ALTER TABLE starred_track ADD scrobbling_state INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*ScrobblingState::PendingAdd*/ 0)) + ")"); + } - static void migrateFromV35(Session& session) - { - // Add creattion/last modif date time for tracklists - session.getDboSession()->execute("ALTER TABLE tracklist ADD creation_date_time TEXT"); - session.getDboSession()->execute("ALTER TABLE tracklist ADD last_modified_date_time TEXT"); - } + void migrateFromV35(Session& session) + { + // Add creattion/last modif date time for tracklists + session.getDboSession()->execute("ALTER TABLE tracklist ADD creation_date_time TEXT"); + session.getDboSession()->execute("ALTER TABLE tracklist ADD last_modified_date_time TEXT"); + } - static void migrateFromV36(Session& session) - { - // Increased precision for track durations (now in milliseconds instead of secodns) - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + void migrateFromV36(Session& session) + { + // Increased precision for track durations (now in milliseconds instead of secodns) + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - static void migrateFromV37(Session& session) - { - // Support Performer tags (via subtypes) - session.getDboSession()->execute("ALTER TABLE track_artist_link ADD subtype TEXT"); + void migrateFromV37(Session& session) + { + // Support Performer tags (via subtypes) + session.getDboSession()->execute("ALTER TABLE track_artist_link ADD subtype TEXT"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - static void migrateFromV38(Session& session) - { - // migrate release-specific tags from Track to Release - session.getDboSession()->execute("ALTER TABLE release ADD total_disc INTEGER"); + void migrateFromV38(Session& session) + { + // migrate release-specific tags from Track to Release + session.getDboSession()->execute("ALTER TABLE release ADD total_disc INTEGER"); - session.getDboSession()->execute(R"( + session.getDboSession()->execute(R"( CREATE TABLE IF NOT EXISTS "track_backup" ( "id" integer primary key autoincrement, "version" integer not null, @@ -170,178 +172,178 @@ CREATE TABLE IF NOT EXISTS "track_backup" ( constraint "fk_track_release" foreign key ("release_id") references "release" ("id") on delete cascade deferrable initially deferred ); ))"); - session.getDboSession()->execute("INSERT INTO track_backup SELECT id, version, scan_version, track_number, disc_number, total_track, disc_subtitle, name, duration, date, original_date, file_path, file_last_write, file_added, has_cover, mbid, recording_mbid, copyright, copyright_url, track_replay_gain, release_replay_gain, release_id FROM track"); - session.getDboSession()->execute("DROP TABLE track"); - session.getDboSession()->execute("ALTER TABLE track_backup RENAME TO track"); + session.getDboSession()->execute("INSERT INTO track_backup SELECT id, version, scan_version, track_number, disc_number, total_track, disc_subtitle, name, duration, date, original_date, file_path, file_last_write, file_added, has_cover, mbid, recording_mbid, copyright, copyright_url, track_replay_gain, release_replay_gain, release_id FROM track"); + session.getDboSession()->execute("DROP TABLE track"); + session.getDboSession()->execute("ALTER TABLE track_backup RENAME TO track"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - static void migrateFromV39(Session& session) - { - // add release type - session.getDboSession()->execute("ALTER TABLE release ADD primary_type INTEGER"); - session.getDboSession()->execute("ALTER TABLE release ADD secondary_types INTEGER"); + void migrateFromV39(Session& session) + { + // add release type + session.getDboSession()->execute("ALTER TABLE release ADD primary_type INTEGER"); + session.getDboSession()->execute("ALTER TABLE release ADD secondary_types INTEGER"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - static void migrateFromV40(Session& session) - { - // add artist_display_name in Release and Track - session.getDboSession()->execute("ALTER TABLE release ADD artist_display_name TEXT NOT NULL DEFAULT ''"); - session.getDboSession()->execute("ALTER TABLE track ADD artist_display_name TEXT NOT NULL DEFAULT ''"); + void migrateFromV40(Session& session) + { + // add artist_display_name in Release and Track + session.getDboSession()->execute("ALTER TABLE release ADD artist_display_name TEXT NOT NULL DEFAULT ''"); + session.getDboSession()->execute("ALTER TABLE track ADD artist_display_name TEXT NOT NULL DEFAULT ''"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - static void migrateFromV41(Session& session) - { - // add artist_display_name in Release and Track - session.getDboSession()->execute("ALTER TABLE user RENAME COLUMN subsonic_transcode_format TO subsonic_default_transcode_format"); - session.getDboSession()->execute("ALTER TABLE user RENAME COLUMN subsonic_transcode_bitrate TO subsonic_default_transcode_bitrate"); - session.getDboSession()->execute("ALTER TABLE user DROP COLUMN subsonic_transcode_enable"); - } + void migrateFromV41(Session& session) + { + // add artist_display_name in Release and Track + session.getDboSession()->execute("ALTER TABLE user RENAME COLUMN subsonic_transcode_format TO subsonic_default_transcode_format"); + session.getDboSession()->execute("ALTER TABLE user RENAME COLUMN subsonic_transcode_bitrate TO subsonic_default_transcode_bitrate"); + session.getDboSession()->execute("ALTER TABLE user DROP COLUMN subsonic_transcode_enable"); + } - static void migrateFromV42(Session& session) - { - session.getDboSession()->execute("DROP INDEX IF EXISTS listen_scrobbler_idx"); - session.getDboSession()->execute("DROP INDEX IF EXISTS listen_user_scrobbler_idx"); - session.getDboSession()->execute("DROP INDEX IF EXISTS listen_user_track_scrobbler_date_time_idx"); - session.getDboSession()->execute("DROP INDEX IF EXISTS starred_artist_user_scrobbler_idx"); - session.getDboSession()->execute("DROP INDEX IF EXISTS starred_artist_artist_user_scrobbler_idx"); - session.getDboSession()->execute("DROP INDEX IF EXISTS starred_release_user_scrobbler_idx"); - session.getDboSession()->execute("DROP INDEX IF EXISTS starred_release_release_user_scrobbler_idx"); - session.getDboSession()->execute("DROP INDEX IF EXISTS starred_track_user_scrobbler_idx"); - session.getDboSession()->execute("DROP INDEX IF EXISTS starred_track_track_user_scrobbler_idx"); + void migrateFromV42(Session& session) + { + session.getDboSession()->execute("DROP INDEX IF EXISTS listen_scrobbler_idx"); + session.getDboSession()->execute("DROP INDEX IF EXISTS listen_user_scrobbler_idx"); + session.getDboSession()->execute("DROP INDEX IF EXISTS listen_user_track_scrobbler_date_time_idx"); + session.getDboSession()->execute("DROP INDEX IF EXISTS starred_artist_user_scrobbler_idx"); + session.getDboSession()->execute("DROP INDEX IF EXISTS starred_artist_artist_user_scrobbler_idx"); + session.getDboSession()->execute("DROP INDEX IF EXISTS starred_release_user_scrobbler_idx"); + session.getDboSession()->execute("DROP INDEX IF EXISTS starred_release_release_user_scrobbler_idx"); + session.getDboSession()->execute("DROP INDEX IF EXISTS starred_track_user_scrobbler_idx"); + session.getDboSession()->execute("DROP INDEX IF EXISTS starred_track_track_user_scrobbler_idx"); - // New feedback service that now handles the star/unstar stuff (that was previously handled by the scrobbling service) - session.getDboSession()->execute("ALTER TABLE user RENAME COLUMN scrobbler TO scrobbling_backend"); - session.getDboSession()->execute("ALTER TABLE user ADD feedback_backend INTEGER"); - session.getDboSession()->execute("ALTER TABLE listen RENAME COLUMN scrobbler TO backend"); - session.getDboSession()->execute("ALTER TABLE listen RENAME COLUMN scrobbling_state TO sync_state"); - session.getDboSession()->execute("ALTER TABLE starred_artist RENAME COLUMN scrobbler TO backend"); - session.getDboSession()->execute("ALTER TABLE starred_artist RENAME COLUMN scrobbling_state TO sync_state"); - session.getDboSession()->execute("ALTER TABLE starred_release RENAME COLUMN scrobbler TO backend"); - session.getDboSession()->execute("ALTER TABLE starred_release RENAME COLUMN scrobbling_state TO sync_state"); - session.getDboSession()->execute("ALTER TABLE starred_track RENAME COLUMN scrobbler TO backend"); - session.getDboSession()->execute("ALTER TABLE starred_track RENAME COLUMN scrobbling_state TO sync_state"); + // New feedback service that now handles the star/unstar stuff (that was previously handled by the scrobbling service) + session.getDboSession()->execute("ALTER TABLE user RENAME COLUMN scrobbler TO scrobbling_backend"); + session.getDboSession()->execute("ALTER TABLE user ADD feedback_backend INTEGER"); + session.getDboSession()->execute("ALTER TABLE listen RENAME COLUMN scrobbler TO backend"); + session.getDboSession()->execute("ALTER TABLE listen RENAME COLUMN scrobbling_state TO sync_state"); + session.getDboSession()->execute("ALTER TABLE starred_artist RENAME COLUMN scrobbler TO backend"); + session.getDboSession()->execute("ALTER TABLE starred_artist RENAME COLUMN scrobbling_state TO sync_state"); + session.getDboSession()->execute("ALTER TABLE starred_release RENAME COLUMN scrobbler TO backend"); + session.getDboSession()->execute("ALTER TABLE starred_release RENAME COLUMN scrobbling_state TO sync_state"); + session.getDboSession()->execute("ALTER TABLE starred_track RENAME COLUMN scrobbler TO backend"); + session.getDboSession()->execute("ALTER TABLE starred_track RENAME COLUMN scrobbling_state TO sync_state"); - session.getDboSession()->execute("UPDATE user SET feedback_backend = scrobbling_backend"); - } + session.getDboSession()->execute("UPDATE user SET feedback_backend = scrobbling_backend"); + } - static void migrateFromV43(Session& session) - { - // add counts in genre table - session.getDboSession()->execute("ALTER TABLE cluster ADD track_count INTEGER"); - session.getDboSession()->execute("ALTER TABLE cluster ADD release_count INTEGER"); + void migrateFromV43(Session& session) + { + // add counts in genre table + session.getDboSession()->execute("ALTER TABLE cluster ADD track_count INTEGER"); + session.getDboSession()->execute("ALTER TABLE cluster ADD release_count INTEGER"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - static void migrateFromV44(Session& session) - { - // add bitrate - session.getDboSession()->execute("ALTER TABLE track ADD bitrate INTEGER NOT NULL DEFAULT 0"); + void migrateFromV44(Session& session) + { + // add bitrate + session.getDboSession()->execute("ALTER TABLE track ADD bitrate INTEGER NOT NULL DEFAULT 0"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - void migrateFromV45(Session& session) - { - // add subsonic_enable_transcoding_by_default, default is disabled - session.getDboSession()->execute("ALTER TABLE user ADD subsonic_enable_transcoding_by_default INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*User::defaultSubsonicEnableTranscodingByDefault*/ 0)) + ")"); - } + void migrateFromV45(Session& session) + { + // add subsonic_enable_transcoding_by_default, default is disabled + session.getDboSession()->execute("ALTER TABLE user ADD subsonic_enable_transcoding_by_default INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*User::defaultSubsonicEnableTranscodingByDefault*/ 0)) + ")"); + } - void migrateFromV46(Session& session) - { - // add extra tags to parse - session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "cluster_type_backup" ( + void migrateFromV46(Session& session) + { + // add extra tags to parse + session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "cluster_type_backup" ( "id" integer primary key autoincrement, "version" integer not null, "name" text not null );)"); - session.getDboSession()->execute("INSERT INTO cluster_type_backup SELECT id, version, name FROM cluster_type"); - session.getDboSession()->execute("DROP TABLE cluster_type"); - session.getDboSession()->execute("ALTER TABLE cluster_type_backup RENAME TO cluster_type"); + session.getDboSession()->execute("INSERT INTO cluster_type_backup SELECT id, version, name FROM cluster_type"); + session.getDboSession()->execute("DROP TABLE cluster_type"); + session.getDboSession()->execute("ALTER TABLE cluster_type_backup RENAME TO cluster_type"); - session.getDboSession()->execute("ALTER TABLE scan_settings ADD COLUMN extra_tags_to_scan TEXT"); + session.getDboSession()->execute("ALTER TABLE scan_settings ADD COLUMN extra_tags_to_scan TEXT"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - void migrateFromV47(Session& session) - { - // release type, new way - session.getDboSession()->execute("ALTER TABLE release DROP primary_type"); - session.getDboSession()->execute("ALTER TABLE release DROP secondary_types"); + void migrateFromV47(Session& session) + { + // release type, new way + session.getDboSession()->execute("ALTER TABLE release DROP primary_type"); + session.getDboSession()->execute("ALTER TABLE release DROP secondary_types"); - session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "release_type" ( + session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "release_type" ( "id" integer primary key autoincrement, "version" integer not null, "name" text not null))"); - session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "release_release_type" ( + session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "release_release_type" ( "release_type_id" bigint, "release_id" bigint, primary key ("release_type_id", "release_id"), constraint "fk_release_release_type_key1" foreign key ("release_type_id") references "release_type" ("id") on delete cascade deferrable initially deferred, constraint "fk_release_release_type_key2" foreign key ("release_id") references "release" ("id") on delete cascade deferrable initially deferred ))"); - session.getDboSession()->execute(R"(CREATE INDEX "release_release_type_release_type" on "release_release_type" ("release_type_id"))"); - session.getDboSession()->execute(R"(CREATE INDEX "release_release_type_release" on "release_release_type" ("release_id"))"); + session.getDboSession()->execute(R"(CREATE INDEX "release_release_type_release_type" on "release_release_type" ("release_type_id"))"); + session.getDboSession()->execute(R"(CREATE INDEX "release_release_type_release" on "release_release_type" ("release_id"))"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - void migrateFromV48(Session& session) - { - // Regression for the extra tags not being parsed - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + void migrateFromV48(Session& session) + { + // Regression for the extra tags not being parsed + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - void migrateFromV49(Session& session) - { - // Add year / originalYear fields, as date / originalDate are not enough (we don't want a wrong date but year or nothing) - session.getDboSession()->execute("ALTER TABLE track ADD year INTEGER"); - session.getDboSession()->execute("ALTER TABLE track ADD original_year INTEGER"); + void migrateFromV49(Session& session) + { + // Add year / originalYear fields, as date / originalDate are not enough (we don't want a wrong date but year or nothing) + session.getDboSession()->execute("ALTER TABLE track ADD year INTEGER"); + session.getDboSession()->execute("ALTER TABLE track ADD original_year INTEGER"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - void migrateFromV50(Session& session) - { - // MediaLibrary support - session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "media_library" ( + void migrateFromV50(Session& session) + { + // MediaLibrary support + session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "media_library" ( "id" integer primary key autoincrement, "version" integer not null, "path" text not null, "name" text not null ))"); - const int scanSettingsId{ session.getDboSession()->query("SELECT id FROM scan_settings") }; + const int scanSettingsId{ session.getDboSession()->query("SELECT id FROM scan_settings") }; - // Convert the existing media_directory in the scan_settings table to a media_library with id '1' - session.getDboSession()->execute(R"(INSERT INTO "media_library" ("id", "version", "path", "name") + // Convert the existing media_directory in the scan_settings table to a media_library with id '1' + session.getDboSession()->execute(R"(INSERT INTO "media_library" ("id", "version", "path", "name") SELECT 1, 0, s_s.media_directory, "Main" FROM scan_settings s_s WHERE id = ?)") - .bind(scanSettingsId); + .bind(scanSettingsId); - // Remove the outdated column in scan_settings - session.getDboSession()->execute("ALTER TABLE scan_settings DROP media_directory"); + // Remove the outdated column in scan_settings + session.getDboSession()->execute("ALTER TABLE scan_settings DROP media_directory"); - // Add the media_library column in tracks, with id '1' - session.getDboSession()->execute(R"( + // Add the media_library column in tracks, with id '1' + session.getDboSession()->execute(R"( CREATE TABLE IF NOT EXISTS "track_backup" ( "id" integer primary key autoincrement, "version" integer not null, @@ -374,8 +376,8 @@ CREATE TABLE IF NOT EXISTS "track_backup" ( constraint "fk_track_media_library" foreign key ("media_library_id") references "media_library" ("id") on delete set null deferrable initially deferred ))"); - // Migrate data, with the new media_library_id field set to 1 - session.getDboSession()->execute(R"(INSERT INTO track_backup + // Migrate data, with the new media_library_id field set to 1 + session.getDboSession()->execute(R"(INSERT INTO track_backup SELECT id, version, @@ -405,76 +407,96 @@ SELECT release_id, 1 FROM track)"); - session.getDboSession()->execute("DROP TABLE track"); - session.getDboSession()->execute("ALTER TABLE track_backup RENAME TO track"); - } + session.getDboSession()->execute("DROP TABLE track"); + session.getDboSession()->execute("ALTER TABLE track_backup RENAME TO track"); + } - void migrateFromV51(Session& session) - { - // Add custom artist tag delimiters, no need to rescan since it has no effect when empty - session.getDboSession()->execute("ALTER TABLE scan_settings ADD artist_tag_delimiters TEXT NOT NULL DEFAULT ''"); - session.getDboSession()->execute("ALTER TABLE scan_settings ADD default_tag_delimiters TEXT NOT NULL DEFAULT ''"); - } + void migrateFromV51(Session& session) + { + // Add custom artist tag delimiters, no need to rescan since it has no effect when empty + session.getDboSession()->execute("ALTER TABLE scan_settings ADD artist_tag_delimiters TEXT NOT NULL DEFAULT ''"); + session.getDboSession()->execute("ALTER TABLE scan_settings ADD default_tag_delimiters TEXT NOT NULL DEFAULT ''"); + } - void migrateFromV52(Session& session) - { - // Add sort name for releases - session.getDboSession()->execute("ALTER TABLE release ADD sort_name TEXT NOT NULL DEFAULT ''"); + void migrateFromV52(Session& session) + { + // Add sort name for releases + session.getDboSession()->execute("ALTER TABLE release ADD sort_name TEXT NOT NULL DEFAULT ''"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - void migrateFromV53(Session& session) - { - // Add release group mbid - session.getDboSession()->execute("ALTER TABLE release ADD group_mbid TEXT NOT NULL DEFAULT ''"); + void migrateFromV53(Session& session) + { + // Add release group mbid + session.getDboSession()->execute("ALTER TABLE release ADD group_mbid TEXT NOT NULL DEFAULT ''"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - void migrateFromV54(Session& session) - { - // Add file size + relative file path - session.getDboSession()->execute("ALTER TABLE track RENAME COLUMN file_path TO absolute_file_path"); - session.getDboSession()->execute("ALTER TABLE track ADD file_size BIGINT NOT NULL DEFAULT(0)"); - session.getDboSession()->execute("ALTER TABLE track ADD relative_file_path TEXT NOT NULL DEFAULT ''"); + void migrateFromV54(Session& session) + { + // Add file size + relative file path + session.getDboSession()->execute("ALTER TABLE track RENAME COLUMN file_path TO absolute_file_path"); + session.getDboSession()->execute("ALTER TABLE track ADD file_size BIGINT NOT NULL DEFAULT(0)"); + session.getDboSession()->execute("ALTER TABLE track ADD relative_file_path TEXT NOT NULL DEFAULT ''"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - void migrateFromV55(Session& session) - { - // Add bitsPerSample, channelCount and sampleRate - session.getDboSession()->execute("ALTER TABLE track ADD bits_per_sample INTEGER NOT NULL DEFAULT(0)"); - session.getDboSession()->execute("ALTER TABLE track ADD channel_count INTEGER NOT NULL DEFAULT(0)"); - session.getDboSession()->execute("ALTER TABLE track ADD sample_rate INTEGER NOT NULL DEFAULT(0)"); + void migrateFromV55(Session& session) + { + // Add bitsPerSample, channelCount and sampleRate + session.getDboSession()->execute("ALTER TABLE track ADD bits_per_sample INTEGER NOT NULL DEFAULT(0)"); + session.getDboSession()->execute("ALTER TABLE track ADD channel_count INTEGER NOT NULL DEFAULT(0)"); + session.getDboSession()->execute("ALTER TABLE track ADD sample_rate INTEGER NOT NULL DEFAULT(0)"); - // Just increment the scan version of the settings to make the next scheduled scan rescan everything - session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); - } + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } - void migrateFromV56(Session& session) - { - // Make sure we remove all the previoulsy created index, the createIndexesIfNeeded will recreate them all - std::vector indexeNames{ utils::fetchQueryResults(session.getDboSession()->query(R"(SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE '%_idx')")) }; - for (const auto& indexName : indexeNames) - session.getDboSession()->execute("DROP INDEX " + indexName); - } + void migrateFromV56(Session& session) + { + // Make sure we remove all the previoulsy created index, the createIndexesIfNeeded will recreate them all + std::vector indexeNames{ utils::fetchQueryResults(session.getDboSession()->query(R"(SELECT name FROM sqlite_master WHERE type = 'index' AND name LIKE '%_idx')")) }; + for (const auto& indexName : indexeNames) + session.getDboSession()->execute("DROP INDEX " + indexName); + } - void migrateFromV57(Session& session) - { - // useless index, may have been already removed in the previous step - session.getDboSession()->execute("DROP INDEX IF EXISTS cluster_name_idx"); - } + void migrateFromV57(Session& session) + { + // useless index, may have been already removed in the previous step + session.getDboSession()->execute("DROP INDEX IF EXISTS cluster_name_idx"); + } - void migrateFromV58(Session& session) - { - // DSF support - session.getDboSession()->execute("UPDATE scan_settings SET audio_file_extensions = audio_file_extensions || ' .dsf'"); - } + void migrateFromV58(Session& session) + { + // DSF support + session.getDboSession()->execute("UPDATE scan_settings SET audio_file_extensions = audio_file_extensions || ' .dsf'"); + } + + void migrateFromV59(Session& session) + { + // Dedicated image table + session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "image" ( + "id" integer primary key autoincrement, + "version" integer not null, + "path" text not null, + "file_last_write" text, + "file_size" integer not null, + "width" integer not null, + "height" integer not null, + "artist_id" bigint, + constraint "fk_image_artist" foreign key ("artist_id") references "artist" ("id") on delete cascade deferrable initially deferred +))"); + + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } + } // namespace bool doDbMigration(Session& session) { @@ -511,6 +533,7 @@ SELECT { 56, migrateFromV56 }, { 57, migrateFromV57 }, { 58, migrateFromV58 }, + { 59, migrateFromV59 }, }; bool migrationPerformed{}; diff --git a/src/libs/database/impl/Session.cpp b/src/libs/database/impl/Session.cpp index d0c5c90d..9364e030 100644 --- a/src/libs/database/impl/Session.cpp +++ b/src/libs/database/impl/Session.cpp @@ -26,6 +26,7 @@ #include "database/AuthToken.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" +#include "database/Image.hpp" #include "database/Listen.hpp" #include "database/MediaLibrary.hpp" #include "database/Release.hpp" @@ -92,6 +93,7 @@ namespace lms::db _session.mapClass("auth_token"); _session.mapClass("cluster"); _session.mapClass("cluster_type"); + _session.mapClass("image"); _session.mapClass("listen"); _session.mapClass("media_library"); _session.mapClass("release"); @@ -177,6 +179,8 @@ namespace lms::db _session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)"); _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS image_artist_idx ON image(artist_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)"); _session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)"); _session.execute("CREATE INDEX IF NOT EXISTS listen_user_backend_idx ON listen(user_id,backend)"); diff --git a/src/libs/database/include/database/Artist.hpp b/src/libs/database/include/database/Artist.hpp index 2081f9d9..2b914463 100644 --- a/src/libs/database/include/database/Artist.hpp +++ b/src/libs/database/include/database/Artist.hpp @@ -44,6 +44,7 @@ namespace lms::db class Cluster; class ClusterType; + class Image; class Release; class Session; class StarredArtist; @@ -139,6 +140,7 @@ namespace lms::db const std::string& getName() const { return _name; } const std::string& getSortName() const { return _sortName; } std::optional getMBID() const { return core::UUID::fromString(_MBID); } + ObjectPtr getImage() const; // No artistLinkTypes means get them all RangeResults findSimilarArtistIds(core::EnumSet artistLinkTypes = {}, std::optional range = std::nullopt) const; @@ -159,6 +161,7 @@ namespace lms::db Wt::Dbo::field(a, _sortName, "sort_name"); Wt::Dbo::field(a, _MBID, "mbid"); + Wt::Dbo::hasOne(a, _image, "artist"); Wt::Dbo::hasMany(a, _trackArtistLinks, Wt::Dbo::ManyToOne, "artist"); Wt::Dbo::hasMany(a, _starredArtists, Wt::Dbo::ManyToMany, "user_starred_artists", "", Wt::Dbo::OnDeleteCascade); } @@ -175,6 +178,7 @@ namespace lms::db std::string _sortName; std::string _MBID; // Musicbrainz Identifier + Wt::Dbo::weak_ptr _image; Wt::Dbo::collection> _trackArtistLinks; // Tracks involving this artist Wt::Dbo::collection> _starredArtists; // starred entries for this artist }; diff --git a/src/libs/database/include/database/Image.hpp b/src/libs/database/include/database/Image.hpp new file mode 100644 index 00000000..d164392b --- /dev/null +++ b/src/libs/database/include/database/Image.hpp @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include +#include + +#include "database/ArtistId.hpp" +#include "database/ImageId.hpp" +#include "database/Object.hpp" + +namespace lms::db +{ + class Artist; + class Session; + + class Image final : public Object + { + public: + Image() = default; + + // find + static std::size_t getCount(Session& session); + static pointer find(Session& session, ImageId id); + + // getters + const std::filesystem::path& getPath() const { return _path; } + const Wt::WDateTime& getLastWriteTime() const { return _fileLastWrite; } + std::size_t getFileSize() const { return _fileSize; } + std::size_t getWidth() const { return _width; } + std::size_t getHeight() const { return _height; } + + // setters + void setPath(const std::filesystem::path& p) { _path = p; } + void setLastWriteTime(Wt::WDateTime time) { _fileLastWrite = time; } + void setFileSize(std::size_t fileSize) { _fileSize = fileSize; } + void setWidth(std::size_t width) { _width = width; } + void setHeight(std::size_t height) { _height = height; } + void setArtist(const ObjectPtr& artist) { _artist = getDboPtr(artist); } + + template + void persist(Action& a) + { + Wt::Dbo::field(a, _path, "path"); + Wt::Dbo::field(a, _fileLastWrite, "file_last_write"); + Wt::Dbo::field(a, _fileSize, "file_size"); + + Wt::Dbo::field(a, _width, "width"); + Wt::Dbo::field(a, _height, "height"); + + Wt::Dbo::belongsTo(a, _artist, "artist", Wt::Dbo::OnDeleteCascade); + } + + private: + friend class Session; + Image(const std::filesystem::path& p); + static pointer create(Session& session, const std::filesystem::path& p); + + std::filesystem::path _path; + Wt::WDateTime _fileLastWrite; + int _fileSize{}; + int _width{}; + int _height{}; + + Wt::Dbo::ptr _artist; + }; +} // namespace lms::db diff --git a/src/libs/database/include/database/ImageId.hpp b/src/libs/database/include/database/ImageId.hpp new file mode 100644 index 00000000..5b4424f0 --- /dev/null +++ b/src/libs/database/include/database/ImageId.hpp @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "database/IdType.hpp" + +LMS_DECLARE_IDTYPE(ImageId) diff --git a/src/libs/database/include/database/Object.hpp b/src/libs/database/include/database/Object.hpp index 187b0c3f..b7acee4f 100644 --- a/src/libs/database/include/database/Object.hpp +++ b/src/libs/database/include/database/Object.hpp @@ -95,7 +95,7 @@ namespace lms::db // Can get raw dbo ptr only from Objects template - static Wt::Dbo::ptr getDboPtr(ObjectPtr ptr) + static Wt::Dbo::ptr getDboPtr(const ObjectPtr& ptr) { return ptr._obj; } diff --git a/src/libs/database/test/CMakeLists.txt b/src/libs/database/test/CMakeLists.txt index 451581c8..6f161970 100644 --- a/src/libs/database/test/CMakeLists.txt +++ b/src/libs/database/test/CMakeLists.txt @@ -4,6 +4,7 @@ add_executable(test-database Cluster.cpp Common.cpp DatabaseTest.cpp + Image.cpp Listen.cpp Migration.cpp Release.cpp diff --git a/src/libs/database/test/Common.cpp b/src/libs/database/test/Common.cpp index b61399b7..f9b277e0 100644 --- a/src/libs/database/test/Common.cpp +++ b/src/libs/database/test/Common.cpp @@ -22,6 +22,7 @@ #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" +#include "database/Image.hpp" #include "database/Listen.hpp" #include "database/MediaLibrary.hpp" #include "database/Release.hpp" @@ -80,6 +81,7 @@ namespace lms::db::tests EXPECT_EQ(Cluster::getCount(session), 0); EXPECT_EQ(ClusterType::getCount(session), 0); EXPECT_EQ(Listen::getCount(session), 0); + EXPECT_EQ(Image::getCount(session), 0); EXPECT_EQ(MediaLibrary::getCount(session), 0); EXPECT_EQ(Release::getCount(session), 0); EXPECT_EQ(StarredArtist::getCount(session), 0); diff --git a/src/libs/database/test/Image.cpp b/src/libs/database/test/Image.cpp new file mode 100644 index 00000000..99a7c55c --- /dev/null +++ b/src/libs/database/test/Image.cpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "Common.hpp" + +#include "database/Image.hpp" + +namespace lms::db::tests +{ + using ScopedImage = ScopedEntity; + + TEST_F(DatabaseFixture, Image) + { + ScopedImage image{ session, "/path/to/image" }; + + { + auto transaction{ session.createReadTransaction() }; + EXPECT_EQ(Image::getCount(session), 1); + + Image::pointer img{ Image::find(session, image.getId()) }; + ASSERT_NE(img, Image::pointer{}); + EXPECT_EQ(img->getPath(), "/path/to/image"); + EXPECT_EQ(img->getWidth(), 0); + EXPECT_EQ(img->getHeight(), 0); + EXPECT_EQ(img->getFileSize(), 0); + } + + { + auto transaction{ session.createWriteTransaction() }; + + Image::pointer img{ Image::find(session, image.getId()) }; + ASSERT_NE(img, Image::pointer{}); + img.modify()->setPath("/path/to/another/image"); + img.modify()->setWidth(640); + img.modify()->setHeight(480); + img.modify()->setFileSize(1024 * 1024); + } + + { + auto transaction{ session.createReadTransaction() }; + + Image::pointer img{ Image::find(session, image.getId()) }; + ASSERT_NE(img, Image::pointer{}); + EXPECT_EQ(img->getPath(), "/path/to/another/image"); + EXPECT_EQ(img->getWidth(), 640); + EXPECT_EQ(img->getHeight(), 480); + EXPECT_EQ(img->getFileSize(), 1024 * 1024); + } + } +} // namespace lms::db::tests \ No newline at end of file diff --git a/src/libs/image/impl/graphicsmagick/RawImage.cpp b/src/libs/image/impl/graphicsmagick/RawImage.cpp index 408c8e90..5979ae61 100644 --- a/src/libs/image/impl/graphicsmagick/RawImage.cpp +++ b/src/libs/image/impl/graphicsmagick/RawImage.cpp @@ -104,6 +104,16 @@ namespace lms::image::GraphicsMagick } } + ImageSize RawImage::getWidth() const + { + return _image.size().width(); + } + + ImageSize RawImage::getHeight() const + { + return _image.size().height(); + } + void RawImage::resize(ImageSize width) { try diff --git a/src/libs/image/impl/graphicsmagick/RawImage.hpp b/src/libs/image/impl/graphicsmagick/RawImage.hpp index b217f9d2..13fc7b59 100644 --- a/src/libs/image/impl/graphicsmagick/RawImage.hpp +++ b/src/libs/image/impl/graphicsmagick/RawImage.hpp @@ -35,6 +35,9 @@ namespace lms::image::GraphicsMagick RawImage(const std::byte* encodedData, std::size_t encodedDataSize); RawImage(const std::filesystem::path& path); + ImageSize getWidth() const override; + ImageSize getHeight() const override; + void resize(ImageSize width) override; std::unique_ptr encodeToJPEG(unsigned quality) const override; diff --git a/src/libs/image/impl/stb/RawImage.hpp b/src/libs/image/impl/stb/RawImage.hpp index 65fd822c..ceaccd8c 100644 --- a/src/libs/image/impl/stb/RawImage.hpp +++ b/src/libs/image/impl/stb/RawImage.hpp @@ -33,11 +33,12 @@ namespace lms::image::STB RawImage(const std::byte* encodedData, std::size_t encodedDataSize); RawImage(const std::filesystem::path& path); + ImageSize getWidth() const override; + ImageSize getHeight() const override; + void resize(ImageSize width) override; std::unique_ptr encodeToJPEG(unsigned quality) const override; - ImageSize getWidth() const; - ImageSize getHeight() const; const std::byte* getData() const; private: diff --git a/src/libs/image/include/image/IRawImage.hpp b/src/libs/image/include/image/IRawImage.hpp index 19b074a8..75ed8c88 100644 --- a/src/libs/image/include/image/IRawImage.hpp +++ b/src/libs/image/include/image/IRawImage.hpp @@ -27,6 +27,10 @@ namespace lms::image { public: virtual ~IRawImage() = default; + + virtual ImageSize getWidth() const = 0; + virtual ImageSize getHeight() const = 0; + virtual void resize(ImageSize width) = 0; virtual std::unique_ptr encodeToJPEG(unsigned quality) const = 0; }; diff --git a/src/libs/metadata/impl/Parser.cpp b/src/libs/metadata/impl/Parser.cpp index ffff85c8..d99ffec5 100644 --- a/src/libs/metadata/impl/Parser.cpp +++ b/src/libs/metadata/impl/Parser.cpp @@ -301,7 +301,7 @@ namespace lms::metadata // But to please most users, if we find a custom delimiter in the Artist tag, we construct the artist diplay string with a "nicer" join if (!_artistTagDelimiters.empty() && track.artists.size() > 1 - && getTagValuesAs(tagReader, { TagType::Artist }, _artistTagDelimiters).size() > 1) + && getTagValuesAs(tagReader, TagType::Artist, _artistTagDelimiters).size() > 1) { std::vector artistNames; std::transform(std::cbegin(track.artists), std::cend(track.artists), std::back_inserter(artistNames), [](const Artist& artist) -> std::string_view { return artist.name; }); diff --git a/src/libs/services/cover/impl/CoverService.cpp b/src/libs/services/cover/impl/CoverService.cpp index acc8e848..80571a85 100644 --- a/src/libs/services/cover/impl/CoverService.cpp +++ b/src/libs/services/cover/impl/CoverService.cpp @@ -30,6 +30,7 @@ #include "core/Utils.hpp" #include "database/Artist.hpp" #include "database/Db.hpp" +#include "database/Image.hpp" #include "database/Release.hpp" #include "database/Session.hpp" #include "database/Track.hpp" @@ -86,19 +87,6 @@ namespace lms::cover return res; } - std::vector constructArtistFileNames() - { - std::vector res; - - core::Service::get()->visitStrings("artist-image-file-names", - [&res](std::string_view fileName) { - res.emplace_back(fileName); - }, - { "artist" }); - - return res; - } - bool isFileSupported(const std::filesystem::path& file, const std::vector& extensions) { return (std::find(std::cbegin(extensions), std::cend(extensions), file.extension()) != std::cend(extensions)); @@ -118,7 +106,6 @@ namespace lms::cover , _cache{ core::Service::get()->getULong("cover-max-cache-size", 30) * 1000 * 1000 } , _maxFileSize{ core::Service::get()->getULong("cover-max-file-size", 10) * 1000 * 1000 } , _preferredFileNames{ constructPreferredFileNames() } - , _artistFileNames{ constructArtistFileNames() } { setJpegQuality(core::Service::get()->getULong("cover-jpeg-quality", 75)); @@ -389,87 +376,15 @@ namespace lms::cover if (artistImage) return artistImage; - std::string artistName; - std::string artistMBID; - - std::set releasePaths; - std::set multiArtistReleasePaths; - { Session& session{ _db.getTLSSession() }; auto transaction{ session.createReadTransaction() }; - const Artist::pointer artist{ Artist::find(session, artistId) }; - if (!artist) - return artistImage; - - artistName = artist->getName(); - if (auto mbid{ artist->getMBID() }) - artistMBID = mbid->getAsString(); - - Track::FindParameters params; - params.setArtist(artistId, { TrackArtistLinkType::ReleaseArtist }); - - Track::find(session, params, [&](const Track::pointer& track) { - Artist::FindParameters artistFindParams; - artistFindParams.setTrack(track->getId()); - artistFindParams.setLinkType(TrackArtistLinkType::ReleaseArtist); - - const auto releaseArtists{ Artist::findIds(session, artistFindParams) }; - if (releaseArtists.results.size() == 1) - releasePaths.insert(track->getAbsoluteFilePath().parent_path()); - else - multiArtistReleasePaths.insert(track->getAbsoluteFilePath().parent_path()); - }); - } - - std::vector artistFileNames; - if (!artistMBID.empty()) - artistFileNames.push_back(artistMBID); - artistFileNames.push_back(artistName); - - std::vector artistFileNamesWithGenericNames{ artistFileNames }; - artistFileNamesWithGenericNames.insert(artistFileNamesWithGenericNames.end(), std::cbegin(_artistFileNames), std::cend(_artistFileNames)); - - // Expect layout like this: - // ReleaseArtist/Release/Tracks' - // /artist-mbid.jpg - // /artist-name.jpg - // /artist.jpg - if (!releasePaths.empty()) - { - const std::filesystem::path artistPath{ releasePaths.size() == 1 ? releasePaths.begin()->parent_path() : core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) }; - artistImage = getFromDirectory(artistPath, width, artistFileNamesWithGenericNames, false); - } - - // Expect layout like this: - // ReleaseArtist/Release/Tracks' - // /artist-mbid.jpg - // /artist-name.jpg - // /artist.jpg - if (!artistImage) - { - for (const std::filesystem::path& releasePath : releasePaths) + if (const Artist::pointer artist{ db::Artist::find(session, artistId) }) { - artistImage = getFromDirectory(releasePath, width, artistFileNamesWithGenericNames, false); - if (artistImage) - break; - } - } - - // Expect layout like this: - // Only search for the artist's name in the release path, as we can't map a generic name to several artists - // ReleaseArtist/Release/Tracks' - // /artist-name.jpg - // /artist-mbid.jpg - if (!artistImage) - { - for (const std::filesystem::path& releasePath : multiArtistReleasePaths) - { - artistImage = getFromDirectory(releasePath, width, artistFileNames, false); - if (artistImage) - break; + if (const db::Image::pointer image{ artist->getImage() }) + artistImage = getFromCoverFile(image->getPath(), width); } } diff --git a/src/libs/services/cover/impl/CoverService.hpp b/src/libs/services/cover/impl/CoverService.hpp index 0eb0bf94..a5899bc4 100644 --- a/src/libs/services/cover/impl/CoverService.hpp +++ b/src/libs/services/cover/impl/CoverService.hpp @@ -76,7 +76,6 @@ namespace lms::cover static inline const std::vector _fileExtensions{ ".jpg", ".jpeg", ".png", ".bmp" }; // TODO parametrize const std::size_t _maxFileSize; const std::vector _preferredFileNames; - const std::vector _artistFileNames; unsigned _jpegQuality; }; diff --git a/src/libs/services/scanner/CMakeLists.txt b/src/libs/services/scanner/CMakeLists.txt index dbe27c9c..28b02f43 100644 --- a/src/libs/services/scanner/CMakeLists.txt +++ b/src/libs/services/scanner/CMakeLists.txt @@ -8,7 +8,8 @@ add_library(lmsscanner SHARED impl/ScanStepDiscoverFiles.cpp impl/ScanStepOptimize.cpp impl/ScanStepRemoveOrphanDbFiles.cpp - impl/ScanStepScanFiles.cpp + impl/ScanStepScanArtistImages.cpp + impl/ScanStepScanAudioFiles.cpp ) target_include_directories(lmsscanner INTERFACE @@ -20,10 +21,11 @@ target_include_directories(lmsscanner PRIVATE ) target_link_libraries(lmsscanner PRIVATE + lmscore lmsdatabase + lmsimage lmsmetadata lmsrecommendation - lmscore ) target_link_libraries(lmsscanner PUBLIC diff --git a/src/libs/services/scanner/impl/ScanStepScanArtistImages.cpp b/src/libs/services/scanner/impl/ScanStepScanArtistImages.cpp new file mode 100644 index 00000000..9696231a --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepScanArtistImages.cpp @@ -0,0 +1,347 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepScanArtistImages.hpp" + +#include +#include +#include +#include + +#include "core/IConfig.hpp" +#include "core/ILogger.hpp" +#include "core/Path.hpp" +#include "database/Artist.hpp" +#include "database/Db.hpp" +#include "database/Image.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "image/Exception.hpp" +#include "image/Image.hpp" + +namespace lms::scanner +{ + namespace + { + constexpr std::size_t readBatchSize{ 10 }; + constexpr std::size_t writeBatchSize{ 5 }; + + struct ImageInfo + { + operator bool() const { return !imagePath.empty(); } + void clear() + { + imagePath.clear(); + lastWriteTime = {}; + fileSize = {}; + height = {}; + width = {}; + } + + std::filesystem::path imagePath; + Wt::WDateTime lastWriteTime; + std::size_t fileSize{}; + std::size_t height{}; + std::size_t width{}; + }; + + bool tryDecodeImage(const std::filesystem::path& imagePath, ImageInfo& imageInfo) + { + assert(!imageInfo); + + try + { + std::unique_ptr rawImage{ image::decodeImage(imagePath) }; + imageInfo.imagePath = imagePath; + imageInfo.fileSize = std::filesystem::file_size(imagePath); + imageInfo.width = rawImage->getWidth(); + imageInfo.height = rawImage->getHeight(); + imageInfo.lastWriteTime = core::pathUtils::getLastWriteTime(imagePath); + } + catch (const image::Exception& e) + { + LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file '" << imagePath.string() << "': " << e.what()); + return false; + } + + return true; + } + + struct ArtistImageInfo + { + db::ArtistId artistId; + ImageInfo imageInfo; + }; + + using ArtistImageInfoContainer = std::deque; + + bool isFileSupported(const std::filesystem::path& file) + { + static const std::array fileExtensions{ ".jpg", ".jpeg", ".png", ".bmp" }; // TODO parametrize + + return (std::find(std::cbegin(fileExtensions), std::cend(fileExtensions), file.extension()) != std::cend(fileExtensions)); + } + + std::multimap getImagePaths(const std::filesystem::path& directoryPath, const std::vector& fileNames) + { + std::multimap res; + std::error_code ec; + + std::filesystem::directory_iterator itPath(directoryPath, ec); + const std::filesystem::directory_iterator itEnd; + while (!ec && itPath != itEnd) + { + const std::filesystem::path& path{ *itPath }; + const std::string stem{ path.stem().string() }; + if (isFileSupported(path) + && std::any_of(std::cbegin(fileNames), std::cend(fileNames), [&](const std::string& fileName) { return core::stringUtils::stringCaseInsensitiveEqual(stem, fileName); })) + { + res.emplace(stem, path); + } + + itPath.increment(ec); + } + + return res; + } + + bool findImageInDirectory(const std::filesystem::path& directory, const std::vector& fileNames, ImageInfo& imageInfo) + { + assert(!imageInfo); + + const std::multimap coverPaths{ getImagePaths(directory, fileNames) }; + + for (const std::string_view fileName : fileNames) + { + const auto range{ coverPaths.equal_range(std::string{ fileName }) }; + for (auto it{ range.first }; it != range.second; ++it) + { + if (tryDecodeImage(it->second, imageInfo)) + return true; + } + } + + return false; + } + + void fetchArtistImageInfo(db::Session& session, const std::vector& genericArtistFileNames, const db::Artist::pointer& artist, ImageInfo& imageInfo) + { + const std::string artistMBID{ [&] { + std::string artistMBID; + if (auto mbid{ artist->getMBID() }) + artistMBID = mbid->getAsString(); + return artistMBID; + }() }; + + std::set releasePaths; + std::set multiArtistReleasePaths; + + db::Track::FindParameters params; + params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist }); + + db::Track::find(session, params, [&](const db::Track::pointer& track) { + db::Artist::FindParameters artistFindParams; + artistFindParams.setTrack(track->getId()); + artistFindParams.setLinkType(db::TrackArtistLinkType::ReleaseArtist); + + const auto releaseArtists{ db::Artist::findIds(session, artistFindParams) }; + if (releaseArtists.results.size() == 1) + releasePaths.insert(track->getAbsoluteFilePath().parent_path()); + else + multiArtistReleasePaths.insert(track->getAbsoluteFilePath().parent_path()); + }); + + std::vector artistFileNames; + if (!artistMBID.empty()) + artistFileNames.push_back(artistMBID); + artistFileNames.push_back(artist->getName()); + + std::vector artistFileNamesWithGenericNames{ artistFileNames }; + artistFileNamesWithGenericNames.insert(artistFileNamesWithGenericNames.end(), std::cbegin(genericArtistFileNames), std::cend(genericArtistFileNames)); + + // Expect layout like this: + // ReleaseArtist/Release/Tracks' + // /artist-mbid.jpg + // /artist-name.jpg + // /artist.jpg + if (!releasePaths.empty()) + { + const std::filesystem::path artistPath{ releasePaths.size() == 1 ? releasePaths.begin()->parent_path() : core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) }; + if (findImageInDirectory(artistPath, artistFileNamesWithGenericNames, imageInfo)) + return; + } + + // Expect layout like this: + // ReleaseArtist/Release/Tracks' + // /artist-mbid.jpg + // /artist-name.jpg + // /artist.jpg + for (const std::filesystem::path& releasePath : releasePaths) + { + // TODO: what if an artist has released an album that bears their name? + if (findImageInDirectory(releasePath, artistFileNamesWithGenericNames, imageInfo)) + return; + } + + // Expect layout like this: + // Only search for the artist's name in the release path, as we can't map a generic name to several artists + // ReleaseArtist/Release/Tracks' + // /artist-name.jpg + // /artist-mbid.jpg + for (const std::filesystem::path& releasePath : multiArtistReleasePaths) + { + if (findImageInDirectory(releasePath, artistFileNames, imageInfo)) + return; + } + } + + bool artistImageNeedsUpdate(const db::Image::pointer& image, const ImageInfo& imageInfo) + { + if (!imageInfo && !image) // no image as before + return false; + else if (!imageInfo && image) // no longer has image + return true; + else if (imageInfo && !image) // image has been added + return true; + + assert(imageInfo); + // artist image still here, consider it is the same only if the last modified time is the same + return imageInfo.lastWriteTime != image->getLastWriteTime(); + } + + struct SearchImageContext + { + db::Session& session; + db::ArtistId lastRetrievedArtistId; + const std::vector& artistFileNames; + bool fullScan; + }; + + bool fetchNextArtistImagesToUpdate(SearchImageContext& searchContext, ArtistImageInfoContainer& artistImageInfoList) + { + const db::ArtistId artistId{ searchContext.lastRetrievedArtistId }; + ImageInfo imageInfo; + + { + auto transaction{ searchContext.session.createReadTransaction() }; + + db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) { + imageInfo.clear(); + + fetchArtistImageInfo(searchContext.session, searchContext.artistFileNames, artist, imageInfo); + if (imageInfo) + LMS_LOG(DBUPDATER, DEBUG, "Found artist image for artist '" << artist->getName() << "' at '" << imageInfo.imagePath << "'"); + + if (searchContext.fullScan || artistImageNeedsUpdate(artist->getImage(), imageInfo)) + artistImageInfoList.push_back(ArtistImageInfo{ artist->getId(), imageInfo }); + }); + } + + return artistId != searchContext.lastRetrievedArtistId; + } + + void updateArtistImage(db::Session& session, const ArtistImageInfo& artistImageInfo) + { + db::Artist::pointer artist{ db::Artist::find(session, artistImageInfo.artistId) }; + assert(artist); + + db::Image::pointer image{ artist->getImage() }; + const ImageInfo& imageInfo{ artistImageInfo.imageInfo }; + + if (!imageInfo) + { + if (image) + image.remove(); + return; + } + + if (!image) + { + image = session.create(imageInfo.imagePath); + image.modify()->setArtist(artist); + } + else + image.modify()->setPath(imageInfo.imagePath); + + image.modify()->setLastWriteTime(imageInfo.lastWriteTime); + image.modify()->setFileSize(imageInfo.fileSize); + image.modify()->setHeight(imageInfo.height); + image.modify()->setWidth(imageInfo.width); + } + + void updateArtistImages(db::Session& session, ArtistImageInfoContainer& imageInfoList) + { + if (imageInfoList.empty()) + return; + + auto transaction{ session.createWriteTransaction() }; + + for (std::size_t i{}; !imageInfoList.empty() && i < writeBatchSize; ++i) + { + updateArtistImage(session, imageInfoList.front()); + imageInfoList.pop_front(); + } + } + + std::vector constructArtistFileNames() + { + std::vector res; + + core::Service::get()->visitStrings("artist-image-file-names", + [&res](std::string_view fileName) { + res.emplace_back(fileName); + }, + { "artist" }); + + return res; + } + + } // namespace + + ScanStepScanArtistImages::ScanStepScanArtistImages(InitParams& initParams) + : ScanStepBase{ initParams } + , _artistFileNames{ constructArtistFileNames() } + { + } + + void ScanStepScanArtistImages::process(ScanContext& context) + { + auto& session{ _db.getTLSSession() }; + + { + auto transaction{ session.createReadTransaction() }; + context.currentStepStats.totalElems = db::Artist::getCount(session); + } + + SearchImageContext searchContext{ + .session = session, + .lastRetrievedArtistId = {}, + .artistFileNames = _artistFileNames, + .fullScan = context.scanOptions.fullScan + }; + + ArtistImageInfoContainer imageInfoList; + while (fetchNextArtistImagesToUpdate(searchContext, imageInfoList)) + { + updateArtistImages(session, imageInfoList); + context.currentStepStats.processedElems += readBatchSize; + _progressCallback(context.currentStepStats); + } + } +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepScanArtistImages.hpp b/src/libs/services/scanner/impl/ScanStepScanArtistImages.hpp new file mode 100644 index 00000000..0765dda1 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepScanArtistImages.hpp @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include "ScanStepBase.hpp" + +namespace lms::scanner +{ + class ScanStepScanArtistImages : public ScanStepBase + { + public: + ScanStepScanArtistImages(InitParams& initParams); + + private: + ScanStep getStep() const override { return ScanStep::ScanArtistImages; } + core::LiteralString getStepName() const override { return "Scan artist images"; } + void process(ScanContext& context) override; + + const std::vector _artistFileNames; + }; +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/ScanStepScanAudioFiles.cpp similarity index 95% rename from src/libs/services/scanner/impl/ScanStepScanFiles.cpp rename to src/libs/services/scanner/impl/ScanStepScanAudioFiles.cpp index e296b38b..6896d2db 100644 --- a/src/libs/services/scanner/impl/ScanStepScanFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepScanAudioFiles.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "ScanStepScanFiles.hpp" +#include "ScanStepScanAudioFiles.hpp" #include "core/Exception.hpp" #include "core/IConfig.hpp" @@ -301,14 +301,14 @@ namespace lms::scanner } } // namespace - ScanStepScanFiles::MetadataScanQueue::MetadataScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort) + ScanStepScanAudioFiles::MetadataScanQueue::MetadataScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort) : _metadataParser{ parser } , _scanContextRunner{ _scanContext, threadCount, "ScannerMetadata" } , _abort{ abort } { } - void ScanStepScanFiles::MetadataScanQueue::pushScanRequest(const std::filesystem::path& path) + void ScanStepScanAudioFiles::MetadataScanQueue::pushScanRequest(const std::filesystem::path& path) { { std::scoped_lock lock{ _mutex }; @@ -348,13 +348,13 @@ namespace lms::scanner }); } - std::size_t ScanStepScanFiles::MetadataScanQueue::getResultsCount() const + std::size_t ScanStepScanAudioFiles::MetadataScanQueue::getResultsCount() const { std::scoped_lock lock{ _mutex }; return _scanResults.size(); } - size_t ScanStepScanFiles::MetadataScanQueue::popResults(std::vector& results, std::size_t maxCount) + size_t ScanStepScanAudioFiles::MetadataScanQueue::popResults(std::vector& results, std::size_t maxCount) { results.clear(); results.reserve(maxCount); @@ -372,7 +372,7 @@ namespace lms::scanner return results.size(); } - void ScanStepScanFiles::MetadataScanQueue::wait(std::size_t maxScanRequestCount) + void ScanStepScanAudioFiles::MetadataScanQueue::wait(std::size_t maxScanRequestCount) { LMS_SCOPED_TRACE_OVERVIEW("Scanner", "WaitParseResults"); @@ -380,7 +380,7 @@ namespace lms::scanner _condVar.wait(lock, [=, this] { return _ongoingScanCount <= maxScanRequestCount; }); } - ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams) + ScanStepScanAudioFiles::ScanStepScanAudioFiles(InitParams& initParams) : ScanStepBase{ initParams } , _metadataParser{ metadata::createParser(metadata::ParserBackend::TagLib, getParserReadStyle()) } // For now, always use TagLib , _metadataScanQueue{ *_metadataParser, getScanMetaDataThreadCount(), _abortScan } @@ -388,7 +388,7 @@ namespace lms::scanner LMS_LOG(DBUPDATER, INFO, "Using " << _metadataScanQueue.getThreadCount() << " thread(s) for scanning file metadata"); } - void ScanStepScanFiles::process(ScanContext& context) + void ScanStepScanAudioFiles::process(ScanContext& context) { const std::size_t scanQueueMaxScanRequestCount{ 100 * _metadataScanQueue.getThreadCount() }; const std::size_t processMetaDataBatchSize{ 5 }; @@ -446,7 +446,7 @@ namespace lms::scanner } } - bool ScanStepScanFiles::checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo) + bool ScanStepScanAudioFiles::checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo) { ScanStats& stats{ context.stats }; @@ -498,7 +498,7 @@ namespace lms::scanner return true; // need to scan } - void ScanStepScanFiles::processMetaDataScanResults(ScanContext& context, std::span scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo) + void ScanStepScanAudioFiles::processMetaDataScanResults(ScanContext& context, std::span scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo) { LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ProcessScanResults"); @@ -525,7 +525,7 @@ namespace lms::scanner } } - void ScanStepScanFiles::processFileMetaData(ScanContext& context, const std::filesystem::path& file, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo) + void ScanStepScanAudioFiles::processFileMetaData(ScanContext& context, const std::filesystem::path& file, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo) { ScanStats& stats{ context.stats }; diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp b/src/libs/services/scanner/impl/ScanStepScanAudioFiles.hpp similarity index 95% rename from src/libs/services/scanner/impl/ScanStepScanFiles.hpp rename to src/libs/services/scanner/impl/ScanStepScanAudioFiles.hpp index 7af546da..238418a3 100644 --- a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepScanAudioFiles.hpp @@ -34,14 +34,14 @@ namespace lms::scanner { - class ScanStepScanFiles : public ScanStepBase + class ScanStepScanAudioFiles : public ScanStepBase { public: - ScanStepScanFiles(InitParams& initParams); + ScanStepScanAudioFiles(InitParams& initParams); private: - ScanStep getStep() const override { return ScanStep::ScanFiles; } - core::LiteralString getStepName() const override { return "Scan files"; } + ScanStep getStep() const override { return ScanStep::ScanAudioFiles; } + core::LiteralString getStepName() const override { return "Scan audio files"; } void process(ScanContext& context) override; bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo); diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index 852d28d2..26f44579 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -36,7 +36,8 @@ #include "ScanStepDiscoverFiles.hpp" #include "ScanStepOptimize.hpp" #include "ScanStepRemoveOrphanDbFiles.hpp" -#include "ScanStepScanFiles.hpp" +#include "ScanStepScanArtistImages.hpp" +#include "ScanStepScanAudioFiles.hpp" namespace lms::scanner { @@ -335,11 +336,13 @@ namespace lms::scanner _abortScan, _db }; - + + // Order is important _scanSteps.clear(); _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); diff --git a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp index b056faf8..fdc77c14 100644 --- a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp +++ b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp @@ -57,6 +57,7 @@ namespace lms::scanner DuplicateReason reason; }; + // Alphabetical order enum class ScanStep { CheckForMissingFiles, @@ -67,7 +68,8 @@ namespace lms::scanner FetchTrackFeatures, Optimize, ReloadSimilarityEngine, - ScanFiles, + ScanArtistImages, + ScanAudioFiles, }; static inline constexpr unsigned ScanProgressStepCount{ 9 }; diff --git a/src/libs/subsonic/impl/SubsonicId.cpp b/src/libs/subsonic/impl/SubsonicId.cpp index 9ec69508..508e1c93 100644 --- a/src/libs/subsonic/impl/SubsonicId.cpp +++ b/src/libs/subsonic/impl/SubsonicId.cpp @@ -76,6 +76,22 @@ namespace lms::core::stringUtils return std::nullopt; } + template<> + std::optional readAs(std::string_view str) + { + std::vector values{ core::stringUtils::splitString(str, '-') }; + if (values.size() != 2) + return std::nullopt; + + if (values[0] != "im") + return std::nullopt; + + if (const auto value{ core::stringUtils::readAs(values[1]) }) + return db::ImageId{ *value }; + + return std::nullopt; + } + template<> std::optional readAs(std::string_view str) { diff --git a/src/libs/subsonic/impl/SubsonicId.hpp b/src/libs/subsonic/impl/SubsonicId.hpp index d7a88efa..a08713df 100644 --- a/src/libs/subsonic/impl/SubsonicId.hpp +++ b/src/libs/subsonic/impl/SubsonicId.hpp @@ -21,6 +21,7 @@ #include "core/String.hpp" #include "database/ArtistId.hpp" +#include "database/ImageId.hpp" #include "database/MediaLibraryId.hpp" #include "database/ReleaseId.hpp" #include "database/TrackId.hpp" @@ -33,6 +34,7 @@ namespace lms::api::subsonic }; std::string idToString(db::ArtistId id); + std::string idToString(db::ImageId id); std::string idToString(db::MediaLibraryId id); std::string idToString(db::ReleaseId id); std::string idToString(db::TrackId id); @@ -49,6 +51,9 @@ namespace lms::core::stringUtils template<> std::optional readAs(std::string_view str); + template<> + std::optional readAs(std::string_view str); + template<> std::optional readAs(std::string_view str); diff --git a/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp b/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp index 3a899b2b..f2447616 100644 --- a/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp +++ b/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp @@ -39,7 +39,7 @@ namespace lms::api::subsonic::Scan { std::size_t count{}; - if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanFiles) + if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanAudioFiles) count = scanStatus.currentScanStepStats->processedElems; statusResponse.setAttribute("count", count); diff --git a/src/libs/subsonic/impl/responses/Artist.cpp b/src/libs/subsonic/impl/responses/Artist.cpp index ca382b29..734b02cd 100644 --- a/src/libs/subsonic/impl/responses/Artist.cpp +++ b/src/libs/subsonic/impl/responses/Artist.cpp @@ -23,6 +23,7 @@ #include "core/Service.hpp" #include "core/String.hpp" #include "database/Artist.hpp" +#include "database/Image.hpp" #include "database/Release.hpp" #include "database/TrackArtistLink.hpp" #include "database/User.hpp" @@ -92,7 +93,8 @@ namespace lms::api::subsonic artistNode.setAttribute("id", idToString(artist->getId())); artistNode.setAttribute("name", artist->getName()); - artistNode.setAttribute("coverArt", idToString(artist->getId())); + if (const db::Image::pointer artistImage{ artist->getImage() }) + artistNode.setAttribute("coverArt", idToString(artist->getId())); if (id3) { diff --git a/src/lms/ui/admin/ScannerController.cpp b/src/lms/ui/admin/ScannerController.cpp index 33b4aa50..c4162ef2 100644 --- a/src/lms/ui/admin/ScannerController.cpp +++ b/src/lms/ui/admin/ScannerController.cpp @@ -310,8 +310,15 @@ namespace lms::ui .arg(stepStats.progress())); break; - case scanner::ScanStep::ScanFiles: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files") + case scanner::ScanStep::ScanArtistImages: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-artist-images") + .arg(stepStats.processedElems) + .arg(stepStats.totalElems) + .arg(stepStats.progress())); + break; + + case scanner::ScanStep::ScanAudioFiles: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-audio-files") .arg(stepStats.processedElems) .arg(stepStats.totalElems) .arg(stepStats.progress())); From 45c8b828655d9ba408d655981b2e02c0e02893cf Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 29 Jun 2024 16:14:28 +0200 Subject: [PATCH 2/6] Format the code --- src/libs/database/include/database/Image.hpp | 2 +- src/libs/services/scanner/impl/ScannerService.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libs/database/include/database/Image.hpp b/src/libs/database/include/database/Image.hpp index d164392b..218bc963 100644 --- a/src/libs/database/include/database/Image.hpp +++ b/src/libs/database/include/database/Image.hpp @@ -21,8 +21,8 @@ #include -#include #include +#include #include "database/ArtistId.hpp" #include "database/ImageId.hpp" diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index 26f44579..04d7b291 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -336,7 +336,7 @@ namespace lms::scanner _abortScan, _db }; - + // Order is important _scanSteps.clear(); _scanSteps.push_back(std::make_unique(params)); From d1324c1c3a1bad7406b41dfd752f746bfe7dd184 Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 6 Jul 2024 13:53:41 +0200 Subject: [PATCH 3/6] Put all directories and images in database, use the info to associate an image to each artist --- approot/messages.xml | 11 +- approot/messages_fr.xml | 11 +- approot/messages_it.xml | 11 +- approot/messages_pl.xml | 11 +- approot/messages_zh.xml | 9 +- conf/lms.conf | 2 +- src/libs/database/CMakeLists.txt | 1 + src/libs/database/impl/Artist.cpp | 12 +- src/libs/database/impl/Cluster.cpp | 1 + src/libs/database/impl/Directory.cpp | 162 ++++++++ src/libs/database/impl/Image.cpp | 65 +++- src/libs/database/impl/Listen.cpp | 2 +- src/libs/database/impl/Migration.cpp | 136 ++++++- src/libs/database/impl/Release.cpp | 8 +- src/libs/database/impl/Session.cpp | 9 + src/libs/database/impl/Track.cpp | 11 +- src/libs/database/impl/TrackFeatures.cpp | 1 + src/libs/database/impl/TrackList.cpp | 4 +- src/libs/database/impl/User.cpp | 6 +- src/libs/database/include/database/Artist.hpp | 1 + .../database/include/database/Directory.hpp | 97 +++++ .../database/include/database/DirectoryId.hpp | 24 ++ src/libs/database/include/database/Image.hpp | 45 ++- src/libs/database/include/database/Track.hpp | 5 + src/libs/database/test/CMakeLists.txt | 1 + src/libs/database/test/Directory.cpp | 155 ++++++++ src/libs/database/test/Image.cpp | 41 ++- src/libs/database/test/Migration.cpp | 25 ++ src/libs/image/CMakeLists.txt | 2 + src/libs/image/impl/graphicsmagick/Image.cpp | 61 +++ .../image/impl/graphicsmagick/RawImage.cpp | 34 +- src/libs/image/impl/stb/Image.cpp | 50 +++ src/libs/image/impl/stb/RawImage.cpp | 19 - src/libs/image/include/image/Image.hpp | 2 + src/libs/services/cover/impl/CoverService.cpp | 4 +- src/libs/services/scanner/CMakeLists.txt | 10 +- .../services/scanner/impl/FileScanQueue.cpp | 149 ++++++++ .../services/scanner/impl/FileScanQueue.hpp | 83 +++++ .../impl/ScanStepAssociateArtistImages.cpp | 232 ++++++++++++ ....hpp => ScanStepAssociateArtistImages.hpp} | 8 +- ...pp => ScanStepCheckForDuplicatedFiles.cpp} | 4 +- ...pp => ScanStepCheckForDuplicatedFiles.hpp} | 4 +- .../impl/ScanStepCheckForRemovedFiles.cpp | 139 +++++++ ...s.hpp => ScanStepCheckForRemovedFiles.hpp} | 16 +- .../scanner/impl/ScanStepDiscoverFiles.cpp | 8 +- .../impl/ScanStepRemoveOrphanDbFiles.cpp | 199 ---------- .../impl/ScanStepRemoveOrphanedDbEntries.cpp | 125 +++++++ .../impl/ScanStepRemoveOrphanedDbEntries.hpp | 44 +++ .../scanner/impl/ScanStepScanArtistImages.cpp | 347 ------------------ .../scanner/impl/ScanStepScanAudioFiles.hpp | 88 ----- ...anAudioFiles.cpp => ScanStepScanFiles.cpp} | 304 ++++++++------- .../scanner/impl/ScanStepScanFiles.hpp | 57 +++ .../services/scanner/impl/ScannerService.cpp | 32 +- .../services/scanner/impl/ScannerSettings.hpp | 3 +- .../include/services/scanner/ScannerStats.hpp | 22 +- src/libs/subsonic/impl/SubsonicId.cpp | 16 - src/libs/subsonic/impl/SubsonicId.hpp | 5 - .../impl/entrypoints/MediaLibraryScanning.cpp | 2 +- src/lms/ui/admin/ScannerController.cpp | 47 +-- 59 files changed, 2023 insertions(+), 960 deletions(-) create mode 100644 src/libs/database/impl/Directory.cpp create mode 100644 src/libs/database/include/database/Directory.hpp create mode 100644 src/libs/database/include/database/DirectoryId.hpp create mode 100644 src/libs/database/test/Directory.cpp create mode 100644 src/libs/image/impl/graphicsmagick/Image.cpp create mode 100644 src/libs/image/impl/stb/Image.cpp create mode 100644 src/libs/services/scanner/impl/FileScanQueue.cpp create mode 100644 src/libs/services/scanner/impl/FileScanQueue.hpp create mode 100644 src/libs/services/scanner/impl/ScanStepAssociateArtistImages.cpp rename src/libs/services/scanner/impl/{ScanStepScanArtistImages.hpp => ScanStepAssociateArtistImages.hpp} (81%) rename src/libs/services/scanner/impl/{ScanStepCheckDuplicatedDbFiles.cpp => ScanStepCheckForDuplicatedFiles.cpp} (94%) rename src/libs/services/scanner/impl/{ScanStepCheckDuplicatedDbFiles.hpp => ScanStepCheckForDuplicatedFiles.hpp} (92%) create mode 100644 src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.cpp rename src/libs/services/scanner/impl/{ScanStepRemoveOrphanDbFiles.hpp => ScanStepCheckForRemovedFiles.hpp} (73%) delete mode 100644 src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp create mode 100644 src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp create mode 100644 src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.hpp delete mode 100644 src/libs/services/scanner/impl/ScanStepScanArtistImages.cpp delete mode 100644 src/libs/services/scanner/impl/ScanStepScanAudioFiles.hpp rename src/libs/services/scanner/impl/{ScanStepScanAudioFiles.cpp => ScanStepScanFiles.cpp} (76%) create mode 100644 src/libs/services/scanner/impl/ScanStepScanFiles.hpp diff --git a/approot/messages.xml b/approot/messages.xml index 306b08aa..77257036 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -87,7 +87,8 @@ Cannot get track duration -Cannot parse file +Cannot parse audio file +Cannot parse image file Cannot read file Compact the database. Caution: this may take a while and will block the whole application during the compact step! {1} duplicate files: @@ -108,16 +109,16 @@ Not scheduled Scheduled on {1} Scanning: step {1}/{2} +Associating artist images: {1}%... Checking for duplicate files... {1} files -Checking files... {1}% +Checking for removed files... {1}% Compacting database... Computing stats... {1}% Discovering files: {1} files Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)... -Optimizing database... {1}/{2} entries ({3}%)... +Optimizing database... {1}%... Reloading similarity engine: {1}%... -Scanning artist images: {1}/{2} artists ({3}%)... -Scanning audio files: {1}/{2} files ({3}%)... +Scanning files: {1}/{2} ({3}%)... Step status diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 4ed82095..a8e1fa29 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -87,7 +87,8 @@ Impossible de récupérer la durée de la piste -Impossible d'analyser le fichier +Impossible d'analyser le fichier audio +Impossible d'analyser le fichier image Impossible de lire le fichier Compacter la base de données. Attention : cette opération peut prendre du temps et va vérouiller l'application pendant toute l'étape de compactage! {1} fichiers dupliqués : @@ -108,16 +109,16 @@ Non planifié Planifié le {1} En cours de scan : étape {1}/{2} +Association des images des artistes: {1}%... Vérification des fichiers dupliqués... {1} fichiers -Vérification des fichiers... {1}% +Vérification des fichiers... {1}% Compactage de la base de données... Calcul des statistiques... {1}% Découverte des fichiers : {1} fichiers Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)... -Optimisation de la base de données... {1}/{2} entrées ({3}%)... +Optimisation de la base de données... {1}%... Rechargement du moteur de recommandation : {1}%... -Scan des images des artistes: {1}/{2} artists ({3}%)... -Scan des fichiers audio : {1}/{2} fichiers ({3}%)... +Scan des fichiers : {1}/{2} ({3}%)... Statut de l'étape diff --git a/approot/messages_it.xml b/approot/messages_it.xml index 2e04db1a..ec421e0c 100644 --- a/approot/messages_it.xml +++ b/approot/messages_it.xml @@ -87,7 +87,8 @@ Non sono stato in grado di determinare la durata della traccia -Non in grado di analizzare il file +Impossibile analizzare il file audio +Impossibile analizzare il file immagine Non in grado di leggere il file Compatta il database. Attenzione: ciò potrebbe richiedere del tempo e bloccherà l'intera applicazione durante il passaggio di compattazione! {1} file duplicati: @@ -108,16 +109,16 @@ Non pianificato Pianificato il {1} Scansione: passo {1}/{2} +Associando immagini degli artisti: {1}%... Controllo duplicati... {1} files -Controllo file... {1}% +Controllo file... {1}% Compattazione del database... Calcolo statistiche... {1}% File trovati: {1} files Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)... -Ottimizzazione del database... {1}/{2} voci ({3}%)... +Ottimizzazione del database... {1}%... Ricarica motore di tracce simili: {1}%... -Scansione delle immagini degli artisti: {1}/{2} artisti ({3}%)... -Scansione dei file audio: {1}/{2} files ({3}%)... +Scansione dei file: {1}/{2} ({3}%)... Stato passo diff --git a/approot/messages_pl.xml b/approot/messages_pl.xml index d1bf52da..5a1c27df 100644 --- a/approot/messages_pl.xml +++ b/approot/messages_pl.xml @@ -88,7 +88,8 @@ Nie udało się ustalić długości ścieżki -Nie udało się przeparsować pliku +Nie można przeanalizować pliku audio +Nie można przeanalizować pliku obrazu Nie udało się odczytać pliku Sprasuj bazę danych. Uwaga: może to trochę zająć, a cała aplikacja będzie w tym czasie zablokowana! @@ -117,12 +118,13 @@ Nie zaplanowano Zaplanowano na {1} Skanowanie: krok {1}/{2} +Kojarzenie obrazów artystów: {1}%... Sprawdzanie duplikatów... {1} plik Sprawdzanie duplikatów... {1} pliki Sprawdzanie duplikatów... {1} plików -Sprawdzanie plików... {1}% +Sprawdzanie plików... {1}% Prasowanie bazy danych... Obliczanie statystyk... {1}% @@ -131,10 +133,9 @@ Odkrywanie plików: {1} plików Pobieranie danych o ścieżce z AcousticBrainz: {1}/{2} ścieżek ({3}%)... -Optymalizowanie bazy danych... {1}/{2} wpisów ({3}%)... +Optymalizowanie bazy danych... {1}%... Przeładowywanie silnika podobieństw: {1}%... -Skanowanie obrazów artystów: {1}/{2} artystów ({3}%)... -Skanowanie plików: {1}/{2} plików ({3}%)... +Skanowanie plików: {1}/{2} ({3}%)... Obecny krok diff --git a/approot/messages_zh.xml b/approot/messages_zh.xml index 027bddca..7586bc01 100644 --- a/approot/messages_zh.xml +++ b/approot/messages_zh.xml @@ -87,7 +87,8 @@ 无法获得音轨时间 -无法解析文件 +无法解析文件 + 无法读取文件 {1} 个重复文件： @@ -109,15 +110,15 @@ 计划于 {1} 扫描中: 阶段 {1}/{2} -检查文件中... {1}% +检查文件中... {1}% + 检索文件中: {1} 文件 从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)... 重载相似引擎中 {1}%... - -扫描文件中: {1}/{2} 个文件 ({3}%)... +扫描文件中: {1}/{2} 个文件 ({3}%)... 当前步骤状态 diff --git a/conf/lms.conf b/conf/lms.conf index 58d89bbb..4fcca840 100644 --- a/conf/lms.conf +++ b/conf/lms.conf @@ -87,7 +87,7 @@ cover-jpeg-quality = 75; cover-preferred-file-names = ("cover", "front"); # File names for artist images (order is important) -# Files whose name is the artist's MBID, then the artist's name, are searched before the names in this list +# Note: files whose name is the artist's MBID are always searched before the names in this list. You can place the MBID files anywhere in your libraries. artist-image-file-names = ("artist"); # Playqueue max entry count diff --git a/src/libs/database/CMakeLists.txt b/src/libs/database/CMakeLists.txt index 4c55f425..2d3c2a16 100644 --- a/src/libs/database/CMakeLists.txt +++ b/src/libs/database/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(lmsdatabase SHARED impl/AuthToken.cpp impl/Cluster.cpp impl/Db.cpp + impl/Directory.cpp impl/Image.cpp impl/Listen.cpp impl/MediaLibrary.cpp diff --git a/src/libs/database/impl/Artist.cpp b/src/libs/database/impl/Artist.cpp index 065fbd1e..94125bd3 100644 --- a/src/libs/database/impl/Artist.cpp +++ b/src/libs/database/impl/Artist.cpp @@ -22,6 +22,7 @@ #include "core/ILogger.hpp" #include "database/Cluster.hpp" +#include "database/Directory.hpp" #include "database/Image.hpp" #include "database/Release.hpp" #include "database/Session.hpp" @@ -222,19 +223,19 @@ namespace lms::db { session.checkReadTransaction(); - return utils::fetchQueryResults(session.getDboSession()->find().where("name = ?").bind(std::string{ name, 0, _maxNameLength }).orderBy("LENGTH(mbid) DESC")); // put mbid entries first + return utils::fetchQueryResults(session.getDboSession()->query>("SELECT a FROM artist a").where("a.name = ?").bind(std::string{ name, 0, _maxNameLength }).orderBy("LENGTH(a.mbid) DESC")); // put mbid entries first } Artist::pointer Artist::find(Session& session, const core::UUID& mbid) { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("mbid = ?").bind(std::string{ mbid.getAsString() })); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT a FROM artist a").where("a.mbid = ?").bind(std::string{ mbid.getAsString() })); } Artist::pointer Artist::find(Session& session, ArtistId id) { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("id = ?").bind(id)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT a FROM artist a").where("a.id = ?").bind(id)); } bool Artist::exists(Session& session, ArtistId id) @@ -364,4 +365,9 @@ namespace lms::db _sortName = std::string(sortName, 0, _maxNameLength); } + void Artist::setImage(ObjectPtr image) + { + _image = getDboPtr(image); + } + } // namespace lms::db diff --git a/src/libs/database/impl/Cluster.cpp b/src/libs/database/impl/Cluster.cpp index c84a9149..af2cabb6 100644 --- a/src/libs/database/impl/Cluster.cpp +++ b/src/libs/database/impl/Cluster.cpp @@ -20,6 +20,7 @@ #include "database/Cluster.hpp" #include "database/Artist.hpp" +#include "database/Directory.hpp" #include "database/MediaLibrary.hpp" #include "database/Release.hpp" #include "database/ScanSettings.hpp" diff --git a/src/libs/database/impl/Directory.cpp b/src/libs/database/impl/Directory.cpp new file mode 100644 index 00000000..8b9ba07a --- /dev/null +++ b/src/libs/database/impl/Directory.cpp @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "database/Directory.hpp" + +#include "database/Session.hpp" + +#include "IdTypeTraits.hpp" +#include "PathTraits.hpp" +#include "Utils.hpp" + +namespace lms::db +{ + namespace + { + Wt::Dbo::Query> createQuery(Session& session, const Directory::FindParameters& params) + { + auto query{ session.getDboSession()->query>("SELECT d FROM directory d") }; + + if (params.artist.isValid()) + { + query.join("track t ON t.directory_id = d.id") + .join("artist a ON a.id = t_a_l.artist_id") + .join("track_artist_link t_a_l ON t_a_l.track_id = t.id") + .where("a.id = ?") + .bind(params.artist); + + if (!params.trackArtistLinkTypes.empty()) + { + std::ostringstream oss; + + bool first{ true }; + for (TrackArtistLinkType linkType : params.trackArtistLinkTypes) + { + if (!first) + oss << " OR "; + oss << "t_a_l.type = ?"; + query.bind(linkType); + + first = false; + } + query.where(oss.str()); + } + + query.groupBy("d.id"); + } + + return query; + } + } // namespace + + Directory::Directory(const std::filesystem::path& p) + { + setAbsolutePath(p); + } + + Directory::pointer Directory::create(Session& session, const std::filesystem::path& p) + { + return session.getDboSession()->add(std::unique_ptr{ new Directory{ p } }); + } + + std::size_t Directory::getCount(Session& session) + { + session.checkReadTransaction(); + + return utils::fetchQuerySingleResult(session.getDboSession()->query("SELECT COUNT(*) FROM directory")); + } + + Directory::pointer Directory::find(Session& session, DirectoryId id) + { + session.checkReadTransaction(); + + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT d from directory d").where("d.id = ?").bind(id)); + } + + Directory::pointer Directory::find(Session& session, const std::filesystem::path& path) + { + session.checkReadTransaction(); + + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT d from directory d").where("d.absolute_path = ?").bind(path)); + } + + void Directory::find(Session& session, DirectoryId& lastRetrievedDirectory, std::size_t count, const std::function& func) + { + session.checkReadTransaction(); + + auto query{ session.getDboSession()->query>("SELECT d from directory d").orderBy("d.id").where("d.id > ?").bind(lastRetrievedDirectory).limit(static_cast(count)) }; + + utils::forEachQueryResult(query, [&](const Directory::pointer& image) { + func(image); + lastRetrievedDirectory = image->getId(); + }); + } + + void Directory::find(Session& session, const FindParameters& params, const std::function& func) + { + auto query{ createQuery(session, params) }; + utils::forEachQueryResult(query, [&func](const Directory::pointer& dir) { + func(dir); + }); + } + + RangeResults Directory::findOrphanIds(Session& session, std::optional range) + { + session.checkReadTransaction(); + + auto query{ session.getDboSession()->query("SELECT d.id FROM directory d") }; + query.leftJoin("directory d_child ON d.id = d_child.parent_directory_id"); + query.leftJoin("track t ON d.id = t.directory_id"); + query.leftJoin("image i ON d.id = i.directory_id"); + query.where("d_child.id IS NULL"); + query.where("t.directory_id IS NULL"); + query.where("i.directory_id IS NULL"); + + return utils::execRangeQuery(query, range); + } + + void Directory::setAbsolutePath(const std::filesystem::path& p) + { + assert(p.is_absolute()); + + if (!p.has_filename() && p.has_parent_path()) + { + _absolutePath = p.parent_path(); + _name = _absolutePath.filename(); + } + else + { + _absolutePath = p; + _name = p.filename(); + } + } + + void Directory::setParent(ObjectPtr parent) + { +#ifndef NDEBUG + if (parent) + { + assert(_absolutePath.has_parent_path()); + assert(parent->getAbsolutePath() == _absolutePath.parent_path()); + } +#endif + + _parent = getDboPtr(parent); + } +} // namespace lms::db diff --git a/src/libs/database/impl/Image.cpp b/src/libs/database/impl/Image.cpp index 2a0f5e3e..62db5af7 100644 --- a/src/libs/database/impl/Image.cpp +++ b/src/libs/database/impl/Image.cpp @@ -22,6 +22,7 @@ #include #include "database/Artist.hpp" +#include "database/Directory.hpp" #include "database/Session.hpp" #include "IdTypeTraits.hpp" @@ -30,9 +31,24 @@ namespace lms::db { - Image::Image(const std::filesystem::path& p) - : _path{ p } + namespace { + Wt::Dbo::Query> createQuery(Session& session, const Image::FindParameters& params) + { + auto query{ session.getDboSession()->query>("SELECT i FROM image i") }; + + if (params.directory.isValid()) + query.where("i.directory_id = ?").bind(params.directory); + if (!params.fileStem.empty()) + query.where("i.stem = ?").bind(params.fileStem); + + return query; + } + } // namespace + + Image::Image(const std::filesystem::path& p) + { + setAbsoluteFilePath(p); } Image::pointer Image::create(Session& session, const std::filesystem::path& p) @@ -51,6 +67,49 @@ namespace lms::db { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("id = ?").bind(id)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT i from image i").where("i.id = ?").bind(id)); } + + Image::pointer Image::find(Session& session, const std::filesystem::path& path) + { + session.checkReadTransaction(); + + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT i from image i").where("i.absolute_file_path = ?").bind(path)); + } + + void Image::find(Session& session, ImageId& lastRetrievedImage, std::size_t count, const std::function& func) + { + session.checkReadTransaction(); + + auto query{ session.getDboSession()->query>("SELECT i from image i").orderBy("i.id").where("i.id > ?").bind(lastRetrievedImage).limit(static_cast(count)) }; + + utils::forEachQueryResult(query, [&](const Image::pointer& image) { + func(image); + lastRetrievedImage = image->getId(); + }); + } + + RangeResults Image::find(Session& session, const FindParameters& params) + { + session.checkReadTransaction(); + + auto query{ createQuery(session, params) }; + return utils::execRangeQuery(query, params.range); + } + + void Image::find(Session& session, const FindParameters& params, const std::function& func) + { + auto query{ createQuery(session, params) }; + utils::forEachQueryResult(query, [&](const Image::pointer& image) { + func(image); + }); + } + + void Image::setAbsoluteFilePath(const std::filesystem::path& p) + { + assert(p.is_absolute()); + _fileAbsolutePath = p; + _fileStem = p.stem().string(); + } + } // namespace lms::db diff --git a/src/libs/database/impl/Listen.cpp b/src/libs/database/impl/Listen.cpp index bc862b5b..9c5b6c17 100644 --- a/src/libs/database/impl/Listen.cpp +++ b/src/libs/database/impl/Listen.cpp @@ -210,7 +210,7 @@ namespace lms::db Listen::pointer Listen::find(Session& session, ListenId id) { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("id = ?").bind(id)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT l from listen l").where("l.id = ?").bind(id)); } RangeResults Listen::find(Session& session, const FindParameters& parameters) diff --git a/src/libs/database/impl/Migration.cpp b/src/libs/database/impl/Migration.cpp index f18c41cd..e28d1233 100644 --- a/src/libs/database/impl/Migration.cpp +++ b/src/libs/database/impl/Migration.cpp @@ -35,7 +35,7 @@ namespace lms::db { namespace { - static constexpr Version LMS_DATABASE_VERSION{ 60 }; + static constexpr Version LMS_DATABASE_VERSION{ 61 }; } VersionInfo::VersionInfo() @@ -485,6 +485,7 @@ SELECT "id" integer primary key autoincrement, "version" integer not null, "path" text not null, + "stem" text not null, "file_last_write" text, "file_size" integer not null, "width" integer not null, @@ -496,6 +497,138 @@ SELECT // Just increment the scan version of the settings to make the next scheduled scan rescan everything session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); } + + void migrateFromV60(Session& session) + { + // Dedicated directory table + session.getDboSession()->execute(R"(CREATE TABLE IF NOT EXISTS "directory" ( + "id" integer primary key autoincrement, + "version" integer not null, + "absolute_path" text not null, + "name" text not null, + "parent_directory_id" bigint, + constraint "fk_directory_directory" foreign key ("parent_directory_id") references "directory" ("id") on delete cascade deferrable initially deferred +))"); + + // Add a ref in track, need to recreate a new table + session.getDboSession()->execute(R"( +CREATE TABLE IF NOT EXISTS "track_backup" ( + "id" integer primary key autoincrement, + "version" integer not null, + "scan_version" integer not null, + "track_number" integer, + "disc_number" integer, + "total_track" integer, + "disc_subtitle" text not null, + "name" text not null, + "duration" integer, + "bitrate" integer not null, + "bits_per_sample" integer not null, + "channel_count" integer not null, + "sample_rate" integer not null, + "date" text, + "year" integer, + "original_date" text, + "original_year" integer, + "absolute_file_path" text not null, + "relative_file_path" text not null, + "file_size" bigint not null, + "file_last_write" text, + "file_added" text, + "has_cover" boolean not null, + "mbid" text not null, + "recording_mbid" text not null, + "copyright" text not null, + "copyright_url" text not null, + "track_replay_gain" real, + "release_replay_gain" real, + "artist_display_name" text not null, + "release_id" bigint, + "media_library_id" bigint, + "directory_id" bigint, + constraint "fk_track_release" foreign key ("release_id") references "release" ("id") on delete cascade deferrable initially deferred, + constraint "fk_track_media_library" foreign key ("media_library_id") references "media_library" ("id") on delete set null deferrable initially deferred, + constraint "fk_track_directory" foreign key ("directory_id") references "directory" ("id") on delete cascade deferrable initially deferred +))"); + // Migrate data, with the new directory_id field set to null + session.getDboSession()->execute(R"(INSERT INTO track_backup +SELECT + id, + version, + scan_version, + track_number, + disc_number, + total_track, + disc_subtitle, + name, + duration, + bitrate, + bits_per_sample, + channel_count, + sample_rate, + date, + year, + original_date, + original_year, + absolute_file_path, + relative_file_path, + file_size, + file_last_write, + file_added, + has_cover, + mbid, + recording_mbid, + copyright, + copyright_url, + track_replay_gain, + release_replay_gain, + artist_display_name, + release_id, + media_library_id, + NULL + FROM track)"); + session.getDboSession()->execute("DROP TABLE track"); + session.getDboSession()->execute("ALTER TABLE track_backup RENAME TO track"); + + // Add a ref in image + rename path to absolute_file_path, need to recreate a new table + session.getDboSession()->execute(R"( + CREATE TABLE IF NOT EXISTS "image_backup" ( + "id" integer primary key autoincrement, + "version" integer not null, + "absolute_file_path" text not null, + "stem" text not null, + "file_last_write" text, + "file_size" integer not null, + "width" integer not null, + "height" integer not null, + "artist_id" bigint, + "directory_id" bigint, + constraint "fk_image_artist" foreign key ("artist_id") references "artist" ("id") on delete cascade deferrable initially deferred, + constraint "fk_image_directory" foreign key ("directory_id") references "directory" ("id") on delete cascade deferrable initially deferred +))"); + + // Migrate data, with the new directory_id field set to null + session.getDboSession()->execute(R"(INSERT INTO image_backup +SELECT + id, + version, + path, + stem, + file_last_write, + file_size, + width, + height, + artist_id, + NULL + FROM image + )"); + session.getDboSession()->execute("DROP TABLE image"); + session.getDboSession()->execute("ALTER TABLE image_backup RENAME TO image"); + + // Just increment the scan version of the settings to make the next scheduled scan rescan everything + session.getDboSession()->execute("UPDATE scan_settings SET scan_version = scan_version + 1"); + } + } // namespace bool doDbMigration(Session& session) @@ -534,6 +667,7 @@ SELECT { 57, migrateFromV57 }, { 58, migrateFromV58 }, { 59, migrateFromV59 }, + { 60, migrateFromV60 }, }; bool migrationPerformed{}; diff --git a/src/libs/database/impl/Release.cpp b/src/libs/database/impl/Release.cpp index bb3fea1e..b7322ff4 100644 --- a/src/libs/database/impl/Release.cpp +++ b/src/libs/database/impl/Release.cpp @@ -226,14 +226,14 @@ namespace lms::db { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("id = ?").bind(id)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT r_t from release_type r_t").where("r_t.id = ?").bind(id)); } ReleaseType::pointer ReleaseType::find(Session& session, std::string_view name) { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("name = ?").bind(name)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT r_t from release_type r_t").where("r_t.name = ?").bind(name)); } Release::Release(const std::string& name, const std::optional& MBID) @@ -258,14 +258,14 @@ namespace lms::db { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("mbid = ?").bind(mbid.getAsString())); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT r from release r").where("r.mbid = ?").bind(mbid.getAsString())); } Release::pointer Release::find(Session& session, ReleaseId id) { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("id = ?").bind(id)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT r from release r").where("r.id = ?").bind(id)); } bool Release::exists(Session& session, ReleaseId id) diff --git a/src/libs/database/impl/Session.cpp b/src/libs/database/impl/Session.cpp index 9364e030..03c9a2f7 100644 --- a/src/libs/database/impl/Session.cpp +++ b/src/libs/database/impl/Session.cpp @@ -26,6 +26,7 @@ #include "database/AuthToken.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" +#include "database/Directory.hpp" #include "database/Image.hpp" #include "database/Listen.hpp" #include "database/MediaLibrary.hpp" @@ -93,6 +94,7 @@ namespace lms::db _session.mapClass("auth_token"); _session.mapClass("cluster"); _session.mapClass("cluster_type"); + _session.mapClass("directory"); _session.mapClass("image"); _session.mapClass("listen"); _session.mapClass("media_library"); @@ -179,7 +181,14 @@ namespace lms::db _session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)"); _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS directory_id_idx ON directory(id)"); + _session.execute("CREATE INDEX IF NOT EXISTS directory_path_idx ON directory(absolute_path)"); + _session.execute("CREATE INDEX IF NOT EXISTS image_artist_idx ON image(artist_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS image_directory_idx ON image(directory_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS image_id_idx ON image(id)"); + _session.execute("CREATE INDEX IF NOT EXISTS image_path_idx ON image(absolute_file_path)"); + _session.execute("CREATE INDEX IF NOT EXISTS image_stem_idx ON image(stem)"); _session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)"); _session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)"); diff --git a/src/libs/database/impl/Track.cpp b/src/libs/database/impl/Track.cpp index 1e11fd34..80c10c18 100644 --- a/src/libs/database/impl/Track.cpp +++ b/src/libs/database/impl/Track.cpp @@ -24,6 +24,7 @@ #include "core/ILogger.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" +#include "database/Directory.hpp" #include "database/MediaLibrary.hpp" #include "database/Release.hpp" #include "database/Session.hpp" @@ -219,21 +220,21 @@ namespace lms::db { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("absolute_file_path = ?").bind(p.string())); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT t from track t").where("t.absolute_file_path = ?").bind(p.string())); } Track::pointer Track::find(Session& session, TrackId id) { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("id = ?").bind(id)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT t from track t").where("t.id = ?").bind(id)); } void Track::find(Session& session, TrackId& lastRetrievedTrack, std::size_t count, const std::function& func, MediaLibraryId library) { session.checkReadTransaction(); - auto query{ session.getDboSession()->find().orderBy("id").where("id > ?").bind(lastRetrievedTrack).limit(static_cast(count)) }; + auto query{ session.getDboSession()->query>("SELECT t from track t").orderBy("t.id").where("t.id > ?").bind(lastRetrievedTrack).limit(static_cast(count)) }; if (library.isValid()) query.where("media_library_id = ?").bind(library); @@ -255,14 +256,14 @@ namespace lms::db { session.checkReadTransaction(); - return utils::fetchQueryResults(session.getDboSession()->find().where("mbid = ?").bind(mbid.getAsString())); + return utils::fetchQueryResults(session.getDboSession()->query>("SELECT t from track t").where("t.mbid = ?").bind(mbid.getAsString())); } std::vector Track::findByRecordingMBID(Session& session, const core::UUID& mbid) { session.checkReadTransaction(); - return utils::fetchQueryResults(session.getDboSession()->find().where("recording_mbid = ?").bind(mbid.getAsString())); + return utils::fetchQueryResults(session.getDboSession()->query>("SELECT t from track t").where("t.recording_mbid = ?").bind(mbid.getAsString())); } RangeResults Track::findIdsTrackMBIDDuplicates(Session& session, std::optional range) diff --git a/src/libs/database/impl/TrackFeatures.cpp b/src/libs/database/impl/TrackFeatures.cpp index f2987081..53e68c7f 100644 --- a/src/libs/database/impl/TrackFeatures.cpp +++ b/src/libs/database/impl/TrackFeatures.cpp @@ -23,6 +23,7 @@ #include #include "core/ILogger.hpp" +#include "database/Directory.hpp" #include "database/Session.hpp" #include "database/Track.hpp" diff --git a/src/libs/database/impl/TrackList.cpp b/src/libs/database/impl/TrackList.cpp index 454964d7..1e1018e9 100644 --- a/src/libs/database/impl/TrackList.cpp +++ b/src/libs/database/impl/TrackList.cpp @@ -136,7 +136,7 @@ namespace lms::db session.checkReadTransaction(); assert(userId.isValid()); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("name = ?").bind(name).where("type = ?").bind(type).where("user_id = ?").bind(userId)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("select t_l from tracklist t_l").where("t_l.name = ?").bind(name).where("t_l.type = ?").bind(type).where("t_l.user_id = ?").bind(userId)); } RangeResults TrackList::find(Session& session, const FindParameters& params) @@ -157,7 +157,7 @@ namespace lms::db { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("id = ?").bind(id)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("select t_l from tracklist t_l").where("t_l.id = ?").bind(id)); } bool TrackList::isEmpty() const diff --git a/src/libs/database/impl/User.cpp b/src/libs/database/impl/User.cpp index 944a24f7..22dd0bbb 100644 --- a/src/libs/database/impl/User.cpp +++ b/src/libs/database/impl/User.cpp @@ -78,17 +78,17 @@ namespace lms::db { session.checkReadTransaction(); - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("type = ?").bind(UserType::DEMO)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT u from user u").where("u.type = ?").bind(UserType::DEMO)); } User::pointer User::find(Session& session, UserId id) { - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("id = ?").bind(id)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT u from user u").where("u.id = ?").bind(id)); } User::pointer User::find(Session& session, std::string_view name) { - return utils::fetchQuerySingleResult(session.getDboSession()->find().where("login_name = ?").bind(name)); + return utils::fetchQuerySingleResult(session.getDboSession()->query>("SELECT u from user u").where("u.login_name = ?").bind(name)); } void User::setSubsonicDefaultTranscodingOutputBitrate(Bitrate bitrate) diff --git a/src/libs/database/include/database/Artist.hpp b/src/libs/database/include/database/Artist.hpp index 2b914463..4f1dbbd4 100644 --- a/src/libs/database/include/database/Artist.hpp +++ b/src/libs/database/include/database/Artist.hpp @@ -153,6 +153,7 @@ namespace lms::db void setName(std::string_view name) { _name = name; } void setMBID(const std::optional& mbid) { _MBID = mbid ? mbid->getAsString() : ""; } void setSortName(const std::string& sortName); + void setImage(ObjectPtr image); template void persist(Action& a) diff --git a/src/libs/database/include/database/Directory.hpp b/src/libs/database/include/database/Directory.hpp new file mode 100644 index 00000000..51564cf6 --- /dev/null +++ b/src/libs/database/include/database/Directory.hpp @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include + +#include "core/EnumSet.hpp" +#include "database/ArtistId.hpp" +#include "database/DirectoryId.hpp" +#include "database/Object.hpp" +#include "database/Types.hpp" + +namespace lms::db +{ + class Session; + + class Directory final : public Object + { + public: + Directory() = default; + + struct FindParameters + { + std::optional range; + ArtistId artist; // only tracks that involve this artist + core::EnumSet trackArtistLinkTypes; // and for these link types + + FindParameters& setRange(std::optional _range) + { + range = _range; + return *this; + } + FindParameters& setArtist(ArtistId _artist, core::EnumSet _trackArtistLinkTypes = {}) + { + artist = _artist; + trackArtistLinkTypes = _trackArtistLinkTypes; + return *this; + } + }; + + // find + static std::size_t getCount(Session& session); + static pointer find(Session& session, DirectoryId id); + static pointer find(Session& session, const std::filesystem::path& path); + static void find(Session& session, DirectoryId& lastRetrievedDirectory, std::size_t count, const std::function& func); + static void find(Session& session, const FindParameters& parameters, const std::function& func); + static RangeResults findOrphanIds(Session& session, std::optional range = std::nullopt); + + // getters + const std::filesystem::path& getAbsolutePath() const { return _absolutePath; } + std::string_view getName() const { return _name; } + ObjectPtr getParent() const { return _parent; } + + // setters + void setAbsolutePath(const std::filesystem::path& p); + void setParent(ObjectPtr parent); + + template + void persist(Action& a) + { + Wt::Dbo::field(a, _absolutePath, "absolute_path"); + Wt::Dbo::field(a, _name, "name"); + + Wt::Dbo::belongsTo(a, _parent, "parent_directory", Wt::Dbo::OnDeleteCascade); + } + + private: + friend class Session; + Directory(const std::filesystem::path& p); + static pointer create(Session& session, const std::filesystem::path& p); + + std::filesystem::path _absolutePath; + std::string _name; + + Wt::Dbo::ptr _parent; + }; +} // namespace lms::db diff --git a/src/libs/database/include/database/DirectoryId.hpp b/src/libs/database/include/database/DirectoryId.hpp new file mode 100644 index 00000000..1e557e60 --- /dev/null +++ b/src/libs/database/include/database/DirectoryId.hpp @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "database/IdType.hpp" + +LMS_DECLARE_IDTYPE(DirectoryId) diff --git a/src/libs/database/include/database/Image.hpp b/src/libs/database/include/database/Image.hpp index 218bc963..d2081835 100644 --- a/src/libs/database/include/database/Image.hpp +++ b/src/libs/database/include/database/Image.hpp @@ -20,17 +20,21 @@ #pragma once #include +#include #include #include #include "database/ArtistId.hpp" +#include "database/DirectoryId.hpp" #include "database/ImageId.hpp" #include "database/Object.hpp" +#include "database/Types.hpp" namespace lms::db { class Artist; + class Directory; class Session; class Image final : public Object @@ -38,29 +42,59 @@ namespace lms::db public: Image() = default; + struct FindParameters + { + std::optional range; + std::string fileStem; // if set, images with this file stem + DirectoryId directory; // if set, images in this directory + + FindParameters& setRange(std::optional _range) + { + range = _range; + return *this; + } + FindParameters& setFileStem(std::string_view _fileStem) + { + fileStem = _fileStem; + return *this; + } + FindParameters& setDirectory(DirectoryId _directory) + { + directory = _directory; + return *this; + } + }; + // find static std::size_t getCount(Session& session); static pointer find(Session& session, ImageId id); + static pointer find(Session& session, const std::filesystem::path& file); + static RangeResults find(Session& session, const FindParameters& params); + static void find(Session& session, const FindParameters& parameters, const std::function& func); + static void find(Session& session, ImageId& lastRetrievedImage, std::size_t count, const std::function& func); // getters - const std::filesystem::path& getPath() const { return _path; } + const std::filesystem::path& getAbsoluteFilePath() const { return _fileAbsolutePath; } + std::string_view getFileStem() const { return _fileStem; } const Wt::WDateTime& getLastWriteTime() const { return _fileLastWrite; } std::size_t getFileSize() const { return _fileSize; } std::size_t getWidth() const { return _width; } std::size_t getHeight() const { return _height; } // setters - void setPath(const std::filesystem::path& p) { _path = p; } + void setAbsoluteFilePath(const std::filesystem::path& p); void setLastWriteTime(Wt::WDateTime time) { _fileLastWrite = time; } void setFileSize(std::size_t fileSize) { _fileSize = fileSize; } void setWidth(std::size_t width) { _width = width; } void setHeight(std::size_t height) { _height = height; } void setArtist(const ObjectPtr& artist) { _artist = getDboPtr(artist); } + void setDirectory(const ObjectPtr& directory) { _directory = getDboPtr(directory); } template void persist(Action& a) { - Wt::Dbo::field(a, _path, "path"); + Wt::Dbo::field(a, _fileAbsolutePath, "absolute_file_path"); + Wt::Dbo::field(a, _fileStem, "stem"); Wt::Dbo::field(a, _fileLastWrite, "file_last_write"); Wt::Dbo::field(a, _fileSize, "file_size"); @@ -68,6 +102,7 @@ namespace lms::db Wt::Dbo::field(a, _height, "height"); Wt::Dbo::belongsTo(a, _artist, "artist", Wt::Dbo::OnDeleteCascade); + Wt::Dbo::belongsTo(a, _directory, "directory", Wt::Dbo::OnDeleteCascade); } private: @@ -75,12 +110,14 @@ namespace lms::db Image(const std::filesystem::path& p); static pointer create(Session& session, const std::filesystem::path& p); - std::filesystem::path _path; + std::filesystem::path _fileAbsolutePath; + std::string _fileStem; Wt::WDateTime _fileLastWrite; int _fileSize{}; int _width{}; int _height{}; Wt::Dbo::ptr _artist; + Wt::Dbo::ptr _directory; }; } // namespace lms::db diff --git a/src/libs/database/include/database/Track.hpp b/src/libs/database/include/database/Track.hpp index 0df31aee..19829e66 100644 --- a/src/libs/database/include/database/Track.hpp +++ b/src/libs/database/include/database/Track.hpp @@ -50,6 +50,7 @@ namespace lms::db class Artist; class Cluster; class ClusterType; + class Directory; class MediaLibrary; class Release; class Session; @@ -225,6 +226,7 @@ namespace lms::db void setRelease(ObjectPtr release) { _release = getDboPtr(release); } void setClusters(const std::vector>& clusters); void setMediaLibrary(ObjectPtr mediaLibrary) { _mediaLibrary = getDboPtr(mediaLibrary); } + void setDirectory(ObjectPtr directory) { _directory = getDboPtr(directory); } std::size_t getScanVersion() const { return _scanVersion; } std::optional getTrackNumber() const { return _trackNumber; } @@ -263,6 +265,7 @@ namespace lms::db std::vector> getClusters() const; std::vector getClusterIds() const; ObjectPtr getMediaLibrary() const { return _mediaLibrary; } + ObjectPtr getDirectory() const { return _directory; } std::vector>> getClusterGroups(const std::vector& clusterTypes, std::size_t size) const; @@ -299,6 +302,7 @@ namespace lms::db Wt::Dbo::field(a, _artistDisplayName, "artist_display_name"); Wt::Dbo::belongsTo(a, _release, "release", Wt::Dbo::OnDeleteCascade); Wt::Dbo::belongsTo(a, _mediaLibrary, "media_library", Wt::Dbo::OnDeleteSetNull); // don't delete track on media library removal, we want to wait for the next scan to have a chance to migrate files + Wt::Dbo::belongsTo(a, _directory, "directory", Wt::Dbo::OnDeleteCascade); Wt::Dbo::hasMany(a, _trackArtistLinks, Wt::Dbo::ManyToOne, "track"); Wt::Dbo::hasMany(a, _clusters, Wt::Dbo::ManyToMany, "track_cluster", "", Wt::Dbo::OnDeleteCascade); } @@ -342,6 +346,7 @@ namespace lms::db Wt::Dbo::ptr _release; Wt::Dbo::ptr _mediaLibrary; + Wt::Dbo::ptr _directory; Wt::Dbo::collection> _trackArtistLinks; Wt::Dbo::collection> _clusters; }; diff --git a/src/libs/database/test/CMakeLists.txt b/src/libs/database/test/CMakeLists.txt index 6f161970..f6cfe3bf 100644 --- a/src/libs/database/test/CMakeLists.txt +++ b/src/libs/database/test/CMakeLists.txt @@ -4,6 +4,7 @@ add_executable(test-database Cluster.cpp Common.cpp DatabaseTest.cpp + Directory.cpp Image.cpp Listen.cpp Migration.cpp diff --git a/src/libs/database/test/Directory.cpp b/src/libs/database/test/Directory.cpp new file mode 100644 index 00000000..da7152ed --- /dev/null +++ b/src/libs/database/test/Directory.cpp @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "Common.hpp" + +#include "database/Directory.hpp" + +namespace lms::db::tests +{ + using ScopedDirectory = ScopedEntity; + + TEST_F(DatabaseFixture, Directory) + { + ScopedDirectory directory{ session, "/path/to/dir/" }; + + { + auto transaction{ session.createReadTransaction() }; + EXPECT_EQ(Directory::getCount(session), 1); + + Directory::pointer dir{ Directory::find(session, directory.getId()) }; + ASSERT_NE(dir, Directory::pointer{}); + EXPECT_EQ(dir->getAbsolutePath(), "/path/to/dir"); + EXPECT_EQ(dir->getName(), "dir"); + } + + { + auto transaction{ session.createWriteTransaction() }; + + Directory::pointer dir{ Directory::find(session, directory.getId()) }; + ASSERT_NE(dir, Directory::pointer{}); + dir.modify()->setAbsolutePath("/path/to/another/dir2"); + } + + { + auto transaction{ session.createReadTransaction() }; + + Directory::pointer dir{ Directory::find(session, directory.getId()) }; + ASSERT_NE(dir, Directory::pointer{}); + EXPECT_EQ(dir->getAbsolutePath(), "/path/to/another/dir2"); + EXPECT_EQ(dir->getName(), "dir2"); + } + + { + auto transaction{ session.createWriteTransaction() }; + + Directory::pointer dir{ Directory::find(session, directory.getId()) }; + ASSERT_NE(dir, Directory::pointer{}); + dir.modify()->setAbsolutePath("/foo/"); + } + + { + auto transaction{ session.createReadTransaction() }; + + Directory::pointer dir{ Directory::find(session, directory.getId()) }; + ASSERT_NE(dir, Directory::pointer{}); + EXPECT_EQ(dir->getAbsolutePath(), "/foo"); + EXPECT_EQ(dir->getName(), "foo"); + } + + { + auto transaction{ session.createWriteTransaction() }; + + Directory::pointer dir{ Directory::find(session, directory.getId()) }; + ASSERT_NE(dir, Directory::pointer{}); + dir.modify()->setAbsolutePath("/"); + } + + { + auto transaction{ session.createReadTransaction() }; + + Directory::pointer dir{ Directory::find(session, directory.getId()) }; + ASSERT_NE(dir, Directory::pointer{}); + EXPECT_EQ(dir->getAbsolutePath(), "/"); + EXPECT_EQ(dir->getName(), ""); + } + + { + auto transaction{ session.createReadTransaction() }; + + Directory::pointer dir{ Directory::find(session, "/") }; + ASSERT_NE(dir, Directory::pointer{}); + EXPECT_EQ(dir->getId(), directory.getId()); + } + } + + TEST_F(DatabaseFixture, parent) + { + ScopedDirectory parent{ session, "/path/to/dir/" }; + ScopedDirectory child{ session, "/path/to/dir/child" }; + + { + auto transaction{ session.createReadTransaction() }; + + auto dir{ child->getParent() }; + EXPECT_EQ(dir, Directory::pointer{}); + } + + { + auto transaction{ session.createWriteTransaction() }; + + child.get().modify()->setParent(parent.lockAndGet()); + } + + { + auto transaction{ session.createReadTransaction() }; + + auto dir{ child->getParent() }; + ASSERT_NE(dir, Directory::pointer{}); + EXPECT_EQ(dir->getId(), parent.getId()); + } + } + + TEST_F(DatabaseFixture, Directory_orphaned) + { + ScopedDirectory parent{ session, "/path/to/dir/" }; + ScopedDirectory child{ session, "/path/to/dir/child" }; + + { + auto transaction{ session.createReadTransaction() }; + + const auto directories{ Directory::findOrphanIds(session).results }; + EXPECT_EQ(directories.size(), 2); + } + + { + auto transaction{ session.createWriteTransaction() }; + + child.get().modify()->setParent(parent.lockAndGet()); + } + + { + auto transaction{ session.createReadTransaction() }; + + const auto directories{ Directory::findOrphanIds(session).results }; + ASSERT_EQ(directories.size(), 1); + EXPECT_EQ(directories.front(), child.getId()); + } + } +} // namespace lms::db::tests \ No newline at end of file diff --git a/src/libs/database/test/Image.cpp b/src/libs/database/test/Image.cpp index 99a7c55c..b32cdd5f 100644 --- a/src/libs/database/test/Image.cpp +++ b/src/libs/database/test/Image.cpp @@ -19,10 +19,12 @@ #include "Common.hpp" +#include "database/Directory.hpp" #include "database/Image.hpp" namespace lms::db::tests { + using ScopedDirectory = ScopedEntity; using ScopedImage = ScopedEntity; TEST_F(DatabaseFixture, Image) @@ -35,7 +37,8 @@ namespace lms::db::tests Image::pointer img{ Image::find(session, image.getId()) }; ASSERT_NE(img, Image::pointer{}); - EXPECT_EQ(img->getPath(), "/path/to/image"); + EXPECT_EQ(img->getAbsoluteFilePath(), "/path/to/image"); + EXPECT_EQ(img->getFileStem(), "image"); EXPECT_EQ(img->getWidth(), 0); EXPECT_EQ(img->getHeight(), 0); EXPECT_EQ(img->getFileSize(), 0); @@ -46,7 +49,7 @@ namespace lms::db::tests Image::pointer img{ Image::find(session, image.getId()) }; ASSERT_NE(img, Image::pointer{}); - img.modify()->setPath("/path/to/another/image"); + img.modify()->setAbsoluteFilePath("/path/to/another/image2"); img.modify()->setWidth(640); img.modify()->setHeight(480); img.modify()->setFileSize(1024 * 1024); @@ -57,10 +60,42 @@ namespace lms::db::tests Image::pointer img{ Image::find(session, image.getId()) }; ASSERT_NE(img, Image::pointer{}); - EXPECT_EQ(img->getPath(), "/path/to/another/image"); + EXPECT_EQ(img->getAbsoluteFilePath(), "/path/to/another/image2"); + EXPECT_EQ(img->getFileStem(), "image2"); EXPECT_EQ(img->getWidth(), 640); EXPECT_EQ(img->getHeight(), 480); EXPECT_EQ(img->getFileSize(), 1024 * 1024); } + + { + auto transaction{ session.createReadTransaction() }; + + Image::pointer img{ Image::find(session, "/path/to/another/image2") }; + ASSERT_NE(img, Image::pointer{}); + EXPECT_EQ(img->getId(), image->getId()); + } + } + + TEST_F(DatabaseFixture, Image_inDirectory) + { + ScopedImage image{ session, "/path/to/image" }; + ScopedDirectory directory{ session, "/path/to" }; + + { + auto transaction{ session.createReadTransaction() }; + EXPECT_EQ(Image::find(session, Image::FindParameters{}.setDirectory(directory.getId())).results.size(), 0); + } + + { + auto transaction{ session.createWriteTransaction() }; + image.get().modify()->setDirectory(directory.get()); + } + + { + auto transaction{ session.createReadTransaction() }; + const auto results{ Image::find(session, Image::FindParameters{}.setDirectory(directory.getId())).results }; + ASSERT_EQ(results.size(), 1); + EXPECT_EQ(results.front()->getId(), image.getId()); + } } } // namespace lms::db::tests \ No newline at end of file diff --git a/src/libs/database/test/Migration.cpp b/src/libs/database/test/Migration.cpp index e6d53fc4..af119b2a 100644 --- a/src/libs/database/test/Migration.cpp +++ b/src/libs/database/test/Migration.cpp @@ -18,8 +18,14 @@ */ #include "Common.hpp" + #include "core/String.hpp" #include "database/Db.hpp" +#include "database/Directory.hpp" +#include "database/Image.hpp" +#include "database/StarredArtist.hpp" +#include "database/StarredRelease.hpp" +#include "database/StarredTrack.hpp" namespace lms::db::tests { @@ -318,5 +324,24 @@ VALUES // Now perform full migration db.getTLSSession().migrateSchemaIfNeeded(); + + // Now perform some dummy finds to ensure all fields are correctly mapped + { + auto transaction{ session.createReadTransaction() }; + + EXPECT_FALSE(Artist::find(session, ArtistId{})); + EXPECT_FALSE(Cluster::find(session, ClusterId{})); + EXPECT_FALSE(ClusterType::find(session, ClusterTypeId{})); + EXPECT_FALSE(Directory::find(session, DirectoryId{})); + EXPECT_FALSE(Image::find(session, ImageId{})); + EXPECT_FALSE(Listen::find(session, ListenId{})); + EXPECT_FALSE(Release::find(session, ReleaseId{})); + EXPECT_FALSE(StarredArtist::find(session, StarredArtistId{})); + EXPECT_FALSE(StarredRelease::find(session, StarredReleaseId{})); + EXPECT_FALSE(StarredTrack::find(session, StarredTrackId{})); + EXPECT_FALSE(Track::find(session, TrackId{})); + EXPECT_FALSE(TrackList::find(session, TrackListId{})); + EXPECT_FALSE(User::find(session, UserId{})); + } } } // namespace lms::db::tests \ No newline at end of file diff --git a/src/libs/image/CMakeLists.txt b/src/libs/image/CMakeLists.txt index 586b038b..5c041943 100644 --- a/src/libs/image/CMakeLists.txt +++ b/src/libs/image/CMakeLists.txt @@ -25,6 +25,7 @@ if (${LMS_IMAGE_BACKEND} STREQUAL "stb") message(STATUS "Using stb (resize version ${STB_IMAGE_RESIZE_VERSION})") target_sources(lmsimage PRIVATE + impl/stb/Image.cpp impl/stb/JPEGImage.cpp impl/stb/RawImage.cpp ) @@ -36,6 +37,7 @@ elseif (${LMS_IMAGE_BACKEND} STREQUAL "graphicsmagick") message(STATUS "Using graphicsmagick") target_sources(lmsimage PRIVATE + impl/graphicsmagick/Image.cpp impl/graphicsmagick/JPEGImage.cpp impl/graphicsmagick/RawImage.cpp ) diff --git a/src/libs/image/impl/graphicsmagick/Image.cpp b/src/libs/image/impl/graphicsmagick/Image.cpp new file mode 100644 index 00000000..920bed60 --- /dev/null +++ b/src/libs/image/impl/graphicsmagick/Image.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2015 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "image/Image.hpp" + +#include + +#include "core/ILogger.hpp" +#include "core/ITraceLogger.hpp" + +namespace lms::image +{ + std::unique_ptr decodeImage(const std::byte* encodedData, std::size_t encodedDataSize) + { + return std::make_unique(encodedData, encodedDataSize); + } + + std::unique_ptr decodeImage(const std::filesystem::path& path) + { + return std::make_unique(path); + } + + void init(const std::filesystem::path& path) + { + Magick::InitializeMagick(path.string().c_str()); + + if (auto nbThreads{ MagickLib::GetMagickResourceLimit(MagickLib::ThreadsResource) }; nbThreads != 1) + LMS_LOG(COVER, WARNING, "Consider setting env var OMP_NUM_THREADS=1 to save resources"); + + if (!MagickLib::SetMagickResourceLimit(MagickLib::ThreadsResource, 1)) + LMS_LOG(COVER, ERROR, "Cannot set Magick thread resource limit to 1!"); + + if (!MagickLib::SetMagickResourceLimit(MagickLib::DiskResource, 0)) + LMS_LOG(COVER, ERROR, "Cannot set Magick disk resource limit to 0!"); + + LMS_LOG(COVER, INFO, "Magick threads resource limit = " << GetMagickResourceLimit(MagickLib::ThreadsResource)); + LMS_LOG(COVER, INFO, "Magick Disk resource limit = " << GetMagickResourceLimit(MagickLib::DiskResource)); + } + + std::span getSupportedFileExtensions() + { + static const std::array fileExtensions{ ".jpg", ".jpeg", ".png", ".bmp" }; + return fileExtensions; + } +} // namespace lms::image diff --git a/src/libs/image/impl/graphicsmagick/RawImage.cpp b/src/libs/image/impl/graphicsmagick/RawImage.cpp index 5979ae61..d935d125 100644 --- a/src/libs/image/impl/graphicsmagick/RawImage.cpp +++ b/src/libs/image/impl/graphicsmagick/RawImage.cpp @@ -19,6 +19,9 @@ #include "RawImage.hpp" +#include +#include + #include #include "core/ILogger.hpp" @@ -26,39 +29,8 @@ #include "JPEGImage.hpp" -namespace lms::image -{ - std::unique_ptr decodeImage(const std::byte* encodedData, std::size_t encodedDataSize) - { - return std::make_unique(encodedData, encodedDataSize); - } - - std::unique_ptr decodeImage(const std::filesystem::path& path) - { - return std::make_unique(path); - } - - void init(const std::filesystem::path& path) - { - Magick::InitializeMagick(path.string().c_str()); - - if (auto nbThreads{ MagickLib::GetMagickResourceLimit(MagickLib::ThreadsResource) }; nbThreads != 1) - LMS_LOG(COVER, WARNING, "Consider setting env var OMP_NUM_THREADS=1 to save resources"); - - if (!MagickLib::SetMagickResourceLimit(MagickLib::ThreadsResource, 1)) - LMS_LOG(COVER, ERROR, "Cannot set Magick thread resource limit to 1!"); - - if (!MagickLib::SetMagickResourceLimit(MagickLib::DiskResource, 0)) - LMS_LOG(COVER, ERROR, "Cannot set Magick disk resource limit to 0!"); - - LMS_LOG(COVER, INFO, "Magick threads resource limit = " << GetMagickResourceLimit(MagickLib::ThreadsResource)); - LMS_LOG(COVER, INFO, "Magick Disk resource limit = " << GetMagickResourceLimit(MagickLib::DiskResource)); - } -} // namespace lms::image - namespace lms::image::GraphicsMagick { - RawImage::RawImage(const std::byte* encodedData, std::size_t encodedDataSize) { try diff --git a/src/libs/image/impl/stb/Image.cpp b/src/libs/image/impl/stb/Image.cpp new file mode 100644 index 00000000..06b953aa --- /dev/null +++ b/src/libs/image/impl/stb/Image.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "image/Image.hpp" + +#include + +#include "RawImage.hpp" +#include "core/ITraceLogger.hpp" + +namespace lms::image +{ + std::unique_ptr decodeImage(const std::byte* encodedData, std::size_t encodedDataSize) + { + LMS_SCOPED_TRACE_DETAILED("Image", "DecodeBuffer"); + return std::make_unique(encodedData, encodedDataSize); + } + + std::unique_ptr decodeImage(const std::filesystem::path& path) + { + LMS_SCOPED_TRACE_DETAILED("Image", "DecodeFile"); + return std::make_unique(path); + } + + void init(const std::filesystem::path&) + { + } + + std::span getSupportedFileExtensions() + { + static const std::array fileExtensions{ ".jpg", ".jpeg", ".png", ".bmp" }; + return fileExtensions; + } +} // namespace lms::image \ No newline at end of file diff --git a/src/libs/image/impl/stb/RawImage.cpp b/src/libs/image/impl/stb/RawImage.cpp index 569158f4..44a02101 100644 --- a/src/libs/image/impl/stb/RawImage.cpp +++ b/src/libs/image/impl/stb/RawImage.cpp @@ -41,25 +41,6 @@ #include "JPEGImage.hpp" -namespace lms::image -{ - std::unique_ptr decodeImage(const std::byte* encodedData, std::size_t encodedDataSize) - { - LMS_SCOPED_TRACE_DETAILED("Image", "DecodeBuffer"); - return std::make_unique(encodedData, encodedDataSize); - } - - std::unique_ptr decodeImage(const std::filesystem::path& path) - { - LMS_SCOPED_TRACE_DETAILED("Image", "DecodeFile"); - return std::make_unique(path); - } - - void init(const std::filesystem::path&) - { - } -} // namespace lms::image - namespace lms::image::STB { RawImage::RawImage(const std::byte* encodedData, std::size_t encodedDataSize) diff --git a/src/libs/image/include/image/Image.hpp b/src/libs/image/include/image/Image.hpp index 36184663..7bf6592f 100644 --- a/src/libs/image/include/image/Image.hpp +++ b/src/libs/image/include/image/Image.hpp @@ -21,6 +21,7 @@ #include #include +#include #include "image/IEncodedImage.hpp" #include "image/IRawImage.hpp" @@ -28,6 +29,7 @@ namespace lms::image { void init(const std::filesystem::path& path); + std::span getSupportedFileExtensions(); std::unique_ptr decodeImage(const std::byte* encodedData, std::size_t encodedDataSize); std::unique_ptr decodeImage(const std::filesystem::path& path); std::unique_ptr readSvgFile(const std::filesystem::path& path); diff --git a/src/libs/services/cover/impl/CoverService.cpp b/src/libs/services/cover/impl/CoverService.cpp index 80571a85..7c621aed 100644 --- a/src/libs/services/cover/impl/CoverService.cpp +++ b/src/libs/services/cover/impl/CoverService.cpp @@ -381,10 +381,10 @@ namespace lms::cover auto transaction{ session.createReadTransaction() }; - if (const Artist::pointer artist{ db::Artist::find(session, artistId) }) + if (const Artist::pointer artist{ Artist::find(session, artistId) }) { if (const db::Image::pointer image{ artist->getImage() }) - artistImage = getFromCoverFile(image->getPath(), width); + artistImage = getFromCoverFile(image->getAbsoluteFilePath(), width); } } diff --git a/src/libs/services/scanner/CMakeLists.txt b/src/libs/services/scanner/CMakeLists.txt index 28b02f43..41df9cc9 100644 --- a/src/libs/services/scanner/CMakeLists.txt +++ b/src/libs/services/scanner/CMakeLists.txt @@ -1,15 +1,17 @@ add_library(lmsscanner SHARED + impl/FileScanQueue.cpp impl/ScannerService.cpp impl/ScannerStats.cpp - impl/ScanStepCheckDuplicatedDbFiles.cpp + impl/ScanStepAssociateArtistImages.cpp + impl/ScanStepCheckForDuplicatedFiles.cpp + impl/ScanStepCheckForRemovedFiles.cpp impl/ScanStepCompact.cpp impl/ScanStepComputeClusterStats.cpp impl/ScanStepDiscoverFiles.cpp impl/ScanStepOptimize.cpp - impl/ScanStepRemoveOrphanDbFiles.cpp - impl/ScanStepScanArtistImages.cpp - impl/ScanStepScanAudioFiles.cpp + impl/ScanStepRemoveOrphanedDbEntries.cpp + impl/ScanStepScanFiles.cpp ) target_include_directories(lmsscanner INTERFACE diff --git a/src/libs/services/scanner/impl/FileScanQueue.cpp b/src/libs/services/scanner/impl/FileScanQueue.cpp new file mode 100644 index 00000000..6497d24c --- /dev/null +++ b/src/libs/services/scanner/impl/FileScanQueue.cpp @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "FileScanQueue.hpp" + +#include "core/Exception.hpp" +#include "core/IConfig.hpp" +#include "core/ILogger.hpp" +#include "core/ITraceLogger.hpp" +#include "core/Path.hpp" +#include "image/Exception.hpp" +#include "image/Image.hpp" +#include "metadata/Exception.hpp" + +namespace lms::scanner +{ + FileScanQueue::FileScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort) + : _metadataParser{ parser } + , _scanContextRunner{ _scanContext, threadCount, "FileScan" } + , _abort{ abort } + { + } + + void FileScanQueue::pushScanRequest(const std::filesystem::path& path, ScanRequestType type) + { + { + std::scoped_lock lock{ _mutex }; + _ongoingScanCount += 1; + } + + _scanContext.post([=, this] { + if (_abort) + { + std::scoped_lock lock{ _mutex }; + _ongoingScanCount -= 1; + } + else + { + FileScanResult result; + result.path = path; + + switch (type) + { + case ScanRequestType::AudioFile: + result.scanData = scanAudioFile(path); + break; + case ScanRequestType::ImageFile: + result.scanData = scanImageFile(path); + } + + { + std::scoped_lock lock{ _mutex }; + + _scanResults.emplace_back(std::move(result)); + _ongoingScanCount -= 1; + } + } + + _condVar.notify_all(); + }); + } + + AudioFileScanData FileScanQueue::scanAudioFile(const std::filesystem::path& path) + { + LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanAudioFile"); + std::unique_ptr track; + + try + { + track = _metadataParser.parse(path); + } + catch (const metadata::Exception& e) + { + LMS_LOG(DBUPDATER, INFO, "Failed to parse audio file '" << path.string() << "'"); + } + + return track; + } + + ImageFileScanData FileScanQueue::scanImageFile(const std::filesystem::path& path) + { + LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanImageFile"); + + std::optional optInfo; + + try + { + std::unique_ptr rawImage{ image::decodeImage(path) }; + ImageInfo& imageInfo{ optInfo.emplace() }; + imageInfo.width = rawImage->getWidth(); + imageInfo.height = rawImage->getHeight(); + } + catch (const image::Exception& e) + { + LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file '" << path.string() << "': " << e.what()); + } + + return optInfo; + } + + std::size_t FileScanQueue::getResultsCount() const + { + std::scoped_lock lock{ _mutex }; + return _scanResults.size(); + } + + size_t FileScanQueue::popResults(std::vector& results, std::size_t maxCount) + { + results.clear(); + results.reserve(maxCount); + + { + std::scoped_lock lock{ _mutex }; + + while (results.size() < maxCount && !_scanResults.empty()) + { + results.push_back(std::move(_scanResults.front())); + _scanResults.pop_front(); + } + } + + return results.size(); + } + + void FileScanQueue::wait(std::size_t maxScanRequestCount) + { + LMS_SCOPED_TRACE_OVERVIEW("Scanner", "WaitParseResults"); + + std::unique_lock lock{ _mutex }; + _condVar.wait(lock, [=, this] { return _ongoingScanCount <= maxScanRequestCount; }); + } + +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/FileScanQueue.hpp b/src/libs/services/scanner/impl/FileScanQueue.hpp new file mode 100644 index 00000000..c850a30b --- /dev/null +++ b/src/libs/services/scanner/impl/FileScanQueue.hpp @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "core/IOContextRunner.hpp" +#include "metadata/IParser.hpp" + +namespace lms::scanner +{ + struct ImageInfo + { + std::size_t height{}; + std::size_t width{}; + }; + + using AudioFileScanData = std::unique_ptr; + using ImageFileScanData = std::optional; + struct FileScanResult + { + std::filesystem::path path; + std::variant scanData; + }; + + class FileScanQueue + { + public: + FileScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort); + + std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); } + + enum ScanRequestType + { + AudioFile, + ImageFile, + }; + void pushScanRequest(const std::filesystem::path& path, ScanRequestType type); + + std::size_t getResultsCount() const; + size_t popResults(std::vector& results, std::size_t maxCount); + + void wait(std::size_t maxScanRequestCount = 0); // wait until ongoing scan request count <= maxScanRequestCount + + private: + AudioFileScanData scanAudioFile(const std::filesystem::path& path); + ImageFileScanData scanImageFile(const std::filesystem::path& path); + + metadata::IParser& _metadataParser; + boost::asio::io_context _scanContext; + core::IOContextRunner _scanContextRunner; + + mutable std::mutex _mutex; + std::size_t _ongoingScanCount{}; + std::deque _scanResults; + std::condition_variable _condVar; + bool& _abort; + }; + +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepAssociateArtistImages.cpp b/src/libs/services/scanner/impl/ScanStepAssociateArtistImages.cpp new file mode 100644 index 00000000..66d8ba0e --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepAssociateArtistImages.cpp @@ -0,0 +1,232 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepAssociateArtistImages.hpp" + +#include +#include +#include +#include + +#include "core/IConfig.hpp" +#include "core/ILogger.hpp" +#include "core/Path.hpp" +#include "database/Artist.hpp" +#include "database/Db.hpp" +#include "database/Directory.hpp" +#include "database/Image.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" +#include "image/Exception.hpp" +#include "image/Image.hpp" + +namespace lms::scanner +{ + namespace + { + constexpr std::size_t readBatchSize{ 100 }; + constexpr std::size_t writeBatchSize{ 10 }; + + struct ArtistImageAssociation + { + db::ArtistId artistId; + db::ImageId imageId; + }; + using ArtistImageAssociationContainer = std::deque; + + struct SearchImageContext + { + db::Session& session; + db::ArtistId lastRetrievedArtistId; + const std::vector& artistFileNames; + }; + + db::Image::pointer findImageInDirectory(SearchImageContext& searchContext, const std::filesystem::path& directoryPath) + { + db::Image::pointer image; + + const db::Directory::pointer directory{ db::Directory::find(searchContext.session, directoryPath) }; + if (directory) // may not exist for artists that are split on different media libraries + { + for (std::string_view fileStem : searchContext.artistFileNames) + { + db::Image::FindParameters params; + params.setDirectory(directory->getId()); + params.setFileStem(fileStem); + + db::Image::find(searchContext.session, params, [&](const db::Image::pointer foundImg) { + if (!image) + image = foundImg; + }); + + if (image) + break; + } + } + + return image; + } + + db::Image::pointer computeBestArtistImage(SearchImageContext& searchContext, const db::Artist::pointer& artist) + { + db::Image::pointer image; + + const auto mbid{ artist->getMBID() }; + if (mbid) + { + // Find anywhere, since it is suppoed to be unique! + db::Image::find(searchContext.session, db::Image::FindParameters{}.setFileStem(mbid->getAsString()), [&](const db::Image::pointer foundImg) { + if (!image) + image = foundImg; + }); + } + + if (!image) + { + std::set releasePaths; + db::Directory::FindParameters params; + params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist }); + + db::Directory::find(searchContext.session, params, [&](const db::Directory::pointer& directory) { + releasePaths.insert(directory->getAbsolutePath()); + }); + + // Expect layout like this: + // ReleaseArtist/Release/Tracks' + // /artist.jpg + // /someOtherUserConfiguredArtistFile.jpg + if (!releasePaths.empty()) + { + const std::filesystem::path artistPath{ releasePaths.size() == 1 ? releasePaths.begin()->parent_path() : core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) }; + image = findImageInDirectory(searchContext, artistPath); + } + + if (!image) + { + // Expect layout like this: + // ReleaseArtist/Release/Tracks' + // /artist.jpg + // /someOtherUserConfiguredArtistFile.jpg + for (const std::filesystem::path& releasePath : releasePaths) + { + image = findImageInDirectory(searchContext, releasePath); + if (image) + break; + } + } + } + + return image; + } + + bool fetchNextArtistImagesToUpdate(SearchImageContext& searchContext, ArtistImageAssociationContainer& artistImageAssociations) + { + const db::ArtistId artistId{ searchContext.lastRetrievedArtistId }; + + { + auto transaction{ searchContext.session.createReadTransaction() }; + + db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) { + db::Image::pointer image{ computeBestArtistImage(searchContext, artist) }; + + if (image != artist->getImage()) + { + LMS_LOG(DBUPDATER, DEBUG, "Updating artist image for artist '" << artist->getName() << "', using '" << (image ? image->getAbsoluteFilePath().c_str() : "") << "'"); + artistImageAssociations.push_back(ArtistImageAssociation{ artist->getId(), image ? image->getId() : db::ImageId{} }); + } + }); + } + + return artistId != searchContext.lastRetrievedArtistId; + } + + void updateArtistImage(db::Session& session, const ArtistImageAssociation& artistImageAssociation) + { + db::Artist::pointer artist{ db::Artist::find(session, artistImageAssociation.artistId) }; + assert(artist); + + db::Image::pointer image; + if (artistImageAssociation.imageId.isValid()) + image = db::Image::find(session, artistImageAssociation.imageId); + + artist.modify()->setImage(image); + } + + void updateArtistImages(db::Session& session, ArtistImageAssociationContainer& imageAssociations) + { + if (imageAssociations.empty()) + return; + + auto transaction{ session.createWriteTransaction() }; + + for (std::size_t i{}; !imageAssociations.empty() && i < writeBatchSize; ++i) + { + updateArtistImage(session, imageAssociations.front()); + imageAssociations.pop_front(); + } + } + + std::vector constructArtistFileNames() + { + std::vector res; + + core::Service::get()->visitStrings("artist-image-file-names", + [&res](std::string_view fileName) { + res.emplace_back(fileName); + }, + { "artist" }); + + return res; + } + + } // namespace + + ScanStepAssociateArtistImages::ScanStepAssociateArtistImages(InitParams& initParams) + : ScanStepBase{ initParams } + , _artistFileNames{ constructArtistFileNames() } + { + } + + void ScanStepAssociateArtistImages::process(ScanContext& context) + { + if (context.stats.nbChanges() == 0) + return; + + auto& session{ _db.getTLSSession() }; + + { + auto transaction{ session.createReadTransaction() }; + context.currentStepStats.totalElems = db::Artist::getCount(session); + } + + SearchImageContext searchContext{ + .session = session, + .lastRetrievedArtistId = {}, + .artistFileNames = _artistFileNames, + }; + + ArtistImageAssociationContainer artistImageAssociations; + while (fetchNextArtistImagesToUpdate(searchContext, artistImageAssociations)) + { + updateArtistImages(session, artistImageAssociations); + context.currentStepStats.processedElems += readBatchSize; + _progressCallback(context.currentStepStats); + } + } +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepScanArtistImages.hpp b/src/libs/services/scanner/impl/ScanStepAssociateArtistImages.hpp similarity index 81% rename from src/libs/services/scanner/impl/ScanStepScanArtistImages.hpp rename to src/libs/services/scanner/impl/ScanStepAssociateArtistImages.hpp index 0765dda1..4d65399d 100644 --- a/src/libs/services/scanner/impl/ScanStepScanArtistImages.hpp +++ b/src/libs/services/scanner/impl/ScanStepAssociateArtistImages.hpp @@ -26,14 +26,14 @@ namespace lms::scanner { - class ScanStepScanArtistImages : public ScanStepBase + class ScanStepAssociateArtistImages : public ScanStepBase { public: - ScanStepScanArtistImages(InitParams& initParams); + ScanStepAssociateArtistImages(InitParams& initParams); private: - ScanStep getStep() const override { return ScanStep::ScanArtistImages; } - core::LiteralString getStepName() const override { return "Scan artist images"; } + ScanStep getStep() const override { return ScanStep::AssociateArtistImages; } + core::LiteralString getStepName() const override { return "Associate artist images"; } void process(ScanContext& context) override; const std::vector _artistFileNames; diff --git a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp b/src/libs/services/scanner/impl/ScanStepCheckForDuplicatedFiles.cpp similarity index 94% rename from src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp rename to src/libs/services/scanner/impl/ScanStepCheckForDuplicatedFiles.cpp index 224342d7..9b92321b 100644 --- a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepCheckForDuplicatedFiles.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "ScanStepCheckDuplicatedDbFiles.hpp" +#include "ScanStepCheckForDuplicatedFiles.hpp" #include "core/ILogger.hpp" #include "database/Db.hpp" @@ -26,7 +26,7 @@ namespace lms::scanner { - void ScanStepCheckDuplicatedDbFiles::process(ScanContext& context) + void ScanStepCheckForDuplicatedFiles::process(ScanContext& context) { using namespace db; diff --git a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp b/src/libs/services/scanner/impl/ScanStepCheckForDuplicatedFiles.hpp similarity index 92% rename from src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp rename to src/libs/services/scanner/impl/ScanStepCheckForDuplicatedFiles.hpp index e0910796..ca16031f 100644 --- a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepCheckForDuplicatedFiles.hpp @@ -23,14 +23,14 @@ namespace lms::scanner { - class ScanStepCheckDuplicatedDbFiles : public ScanStepBase + class ScanStepCheckForDuplicatedFiles : public ScanStepBase { public: using ScanStepBase::ScanStepBase; private: core::LiteralString getStepName() const override { return "Check for duplicated files"; } - ScanStep getStep() const override { return ScanStep::CheckForDuplicateFiles; } + ScanStep getStep() const override { return ScanStep::CheckForDuplicatedFiles; } void process(ScanContext& context) override; }; } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.cpp b/src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.cpp new file mode 100644 index 00000000..0fb4875b --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.cpp @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepCheckForRemovedFiles.hpp" + +#include "core/ILogger.hpp" +#include "core/Path.hpp" +#include "database/Db.hpp" +#include "database/Image.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" + +namespace lms::scanner +{ + namespace + { + constexpr std::size_t batchSize = 100; + } + + void ScanStepCheckForRemovedFiles::process(ScanContext& context) + { + if (_abortScan) + return; + + db::Session& session{ _db.getTLSSession() }; + + { + auto transaction{ session.createReadTransaction() }; + context.currentStepStats.totalElems = 0; + context.currentStepStats.totalElems += db::Track::getCount(session); + context.currentStepStats.totalElems += db::Image::getCount(session); + } + LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " files to be checked..."); + + checkForRemovedFiles(context, _settings.supportedAudioFileExtensions); + checkForRemovedFiles(context, _settings.supportedImageFileExtensions); + } + + template + void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context, const std::vector& supportedFileExtensions) + { + using namespace db; + + if (_abortScan) + return; + + Session& session{ _db.getTLSSession() }; + + std::vector objectsToRemove; + + typename Object::IdType lastCheckedId; + bool endReached{}; + while (!endReached) + { + if (_abortScan) + break; + + objectsToRemove.clear(); + { + auto transaction{ session.createReadTransaction() }; + + endReached = true; + Object::find(session, lastCheckedId, batchSize, [&](const typename Object::pointer& object) { + endReached = false; + + if (!checkFile(object->getAbsoluteFilePath(), supportedFileExtensions)) + objectsToRemove.push_back(object); + + context.currentStepStats.processedElems++; + }); + } + + if (!objectsToRemove.empty()) + { + auto transaction{ session.createWriteTransaction() }; + + for (typename Object::pointer& object : objectsToRemove) + { + object.remove(); + context.stats.deletions++; + } + } + + _progressCallback(context.currentStepStats); + } + } + + bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p, const std::vector& allowedExtensions) + { + try + { + // For each track, make sure the the file still exists + // and still belongs to a media directory + if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p)) + { + LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': missing"); + return false; + } + + if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries), + [&](const ScannerSettings::MediaLibraryInfo& libraryInfo) { + return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName); + })) + { + LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': out of media directory"); + return false; + } + + if (!core::pathUtils::hasFileAnyExtension(p, allowedExtensions)) + { + LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': file format no longer handled"); + return false; + } + + return true; + } + catch (std::filesystem::filesystem_error& e) + { + LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file '" << p.string() << "': " << e.what()); + return false; + } + } +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp b/src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.hpp similarity index 73% rename from src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp rename to src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.hpp index 8b3750d4..5c79ca72 100644 --- a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.hpp @@ -25,21 +25,19 @@ namespace lms::scanner { - class ScanStepRemoveOrphanDbFiles : public ScanStepBase + class ScanStepCheckForRemovedFiles : public ScanStepBase { public: using ScanStepBase::ScanStepBase; private: - core::LiteralString getStepName() const override { return "Check orphaned entries"; } - ScanStep getStep() const override { return ScanStep::CheckForMissingFiles; } + core::LiteralString getStepName() const override { return "Check for removed files"; } + ScanStep getStep() const override { return ScanStep::CheckForRemovedFiles; } void process(ScanContext& context) override; - void removeOrphanTracks(ScanContext& context); - void removeOrphanClusters(); - void removeOrphanClusterTypes(); - void removeOrphanArtists(); - void removeOrphanReleases(); - bool checkFile(const std::filesystem::path& p); + template + void checkForRemovedFiles(ScanContext& context, const std::vector& supportedFileExtensions); + + bool checkFile(const std::filesystem::path& p, const std::vector& allowedExtensions); }; } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp index 43d0ebb9..b9f9acee 100644 --- a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp @@ -26,7 +26,7 @@ namespace lms::scanner { void ScanStepDiscoverFiles::process(ScanContext& context) { - context.stats.filesScanned = 0; + context.stats.totalFileCount = 0; for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries) { @@ -36,7 +36,7 @@ namespace lms::scanner if (_abortScan) return false; - if (!ec && core::pathUtils::hasFileAnyExtension(path, _settings.supportedExtensions)) + if (!ec && (core::pathUtils::hasFileAnyExtension(path, _settings.supportedAudioFileExtensions) || core::pathUtils::hasFileAnyExtension(path, _settings.supportedImageFileExtensions))) { context.currentStepStats.processedElems++; currentDirectoryProcessElemsCount++; @@ -50,8 +50,8 @@ namespace lms::scanner LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in '" << mediaLibrary.rootDirectory << "'"); } - context.stats.filesScanned = context.currentStepStats.processedElems; + context.stats.totalFileCount = context.currentStepStats.processedElems; - LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.filesScanned << " files in all directories"); + LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.totalFileCount << " files in all directories"); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp deleted file mode 100644 index b86f35ac..00000000 --- a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright (C) 2023 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "ScanStepRemoveOrphanDbFiles.hpp" - -#include "core/ILogger.hpp" -#include "core/Path.hpp" -#include "database/Artist.hpp" -#include "database/Cluster.hpp" -#include "database/Db.hpp" -#include "database/Release.hpp" -#include "database/Session.hpp" -#include "database/Track.hpp" - -namespace lms::scanner -{ - using namespace db; - - namespace - { - constexpr std::size_t batchSize = 100; - - template - void removeOrphanEntries(Session& session, bool& abortScan) - { - using IdType = typename T::IdType; - - RangeResults entries; - while (!abortScan) - { - { - auto transaction{ session.createReadTransaction() }; - - entries = T::findOrphanIds(session, Range{ 0, batchSize }); - }; - - { - auto transaction{ session.createWriteTransaction() }; - - for (const IdType objectId : entries.results) - { - if (abortScan) - break; - - typename T::pointer entry{ T::find(session, objectId) }; - - entry.remove(); - } - } - - if (!entries.moreResults) - break; - } - } - } // namespace - - void ScanStepRemoveOrphanDbFiles::process(ScanContext& context) - { - removeOrphanTracks(context); - removeOrphanClusters(); - removeOrphanClusterTypes(); - removeOrphanArtists(); - removeOrphanReleases(); - } - - void ScanStepRemoveOrphanDbFiles::removeOrphanTracks(ScanContext& context) - { - using namespace db; - - if (_abortScan) - return; - - Session& session{ _db.getTLSSession() }; - - LMS_LOG(DBUPDATER, DEBUG, "Checking tracks to be removed..."); - { - auto transaction{ session.createReadTransaction() }; - context.currentStepStats.totalElems = Track::getCount(session); - } - LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " tracks to be checked..."); - - // TODO handle only files in context.directory? - std::vector tracksToRemove; - - TrackId lastCheckedTrackID; - bool endReached{}; - while (!endReached) - { - if (_abortScan) - break; - - tracksToRemove.clear(); - { - auto transaction{ session.createReadTransaction() }; - - endReached = true; - Track::find(session, lastCheckedTrackID, batchSize, [&](const Track::pointer& track) { - endReached = false; - - if (!checkFile(track->getAbsoluteFilePath())) - tracksToRemove.push_back(track); - - context.currentStepStats.processedElems++; - }); - } - - if (!tracksToRemove.empty()) - { - auto transaction{ session.createWriteTransaction() }; - - for (Track::pointer& track : tracksToRemove) - { - track.remove(); - context.stats.deletions++; - } - } - - _progressCallback(context.currentStepStats); - } - - LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.processedElems << " tracks checked!"); - } - - void ScanStepRemoveOrphanDbFiles::removeOrphanClusters() - { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphan clusters..."); - removeOrphanEntries(_db.getTLSSession(), _abortScan); - } - - void ScanStepRemoveOrphanDbFiles::removeOrphanClusterTypes() - { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphan cluster types..."); - removeOrphanEntries(_db.getTLSSession(), _abortScan); - } - - void ScanStepRemoveOrphanDbFiles::removeOrphanArtists() - { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphan artists..."); - removeOrphanEntries(_db.getTLSSession(), _abortScan); - } - - void ScanStepRemoveOrphanDbFiles::removeOrphanReleases() - { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphan releases..."); - removeOrphanEntries(_db.getTLSSession(), _abortScan); - } - - bool ScanStepRemoveOrphanDbFiles::checkFile(const std::filesystem::path& p) - { - try - { - // For each track, make sure the the file still exists - // and still belongs to a media directory - if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p)) - { - LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': missing"); - return false; - } - - if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries), - [&](const ScannerSettings::MediaLibraryInfo& libraryInfo) { - return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName); - })) - { - LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': out of media directory"); - return false; - } - - if (!core::pathUtils::hasFileAnyExtension(p, _settings.supportedExtensions)) - { - LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': file format no longer handled"); - return false; - } - - return true; - } - catch (std::filesystem::filesystem_error& e) - { - LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file '" << p.string() << "': " << e.what()); - return false; - } - } -} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp new file mode 100644 index 00000000..f3df1910 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepRemoveOrphanedDbEntries.hpp" + +#include "core/ILogger.hpp" +#include "core/Path.hpp" +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Db.hpp" +#include "database/Directory.hpp" +#include "database/Release.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" + +namespace lms::scanner +{ + using namespace db; + + namespace + { + constexpr std::size_t batchSize = 100; + + template + void removeOrphanedEntries(Session& session, bool& abortScan) + { + using IdType = typename T::IdType; + + RangeResults entries; + while (!abortScan) + { + { + auto transaction{ session.createReadTransaction() }; + + entries = T::findOrphanIds(session, Range{ 0, batchSize }); + }; + + { + auto transaction{ session.createWriteTransaction() }; + + for (const IdType objectId : entries.results) + { + if (abortScan) + break; + + typename T::pointer entry{ T::find(session, objectId) }; + + entry.remove(); + } + } + + if (!entries.moreResults) + break; + } + } + } // namespace + + void ScanStepRemoveOrphanedDbEntries::process(ScanContext& context) + { + auto& session{ _db.getTLSSession() }; + + { + auto transaction{ session.createReadTransaction() }; + context.currentStepStats.totalElems = 0; + context.currentStepStats.totalElems += Cluster::getCount(session); + context.currentStepStats.totalElems += ClusterType::getCount(session); + context.currentStepStats.totalElems += Artist::getCount(session); + context.currentStepStats.totalElems += Release::getCount(session); + context.currentStepStats.totalElems += Directory::getCount(session); + } + LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " database entries to be checked..."); + + removeOrphanedClusters(); + removeOrphanedClusterTypes(); + removeOrphanedArtists(); + removeOrphanedReleases(); + removeOrphanedDirectories(); + } + + void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusters() + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned clusters..."); + removeOrphanedEntries(_db.getTLSSession(), _abortScan); + } + + void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusterTypes() + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned cluster types..."); + removeOrphanedEntries(_db.getTLSSession(), _abortScan); + } + + void ScanStepRemoveOrphanedDbEntries::removeOrphanedArtists() + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned artists..."); + removeOrphanedEntries(_db.getTLSSession(), _abortScan); + } + + void ScanStepRemoveOrphanedDbEntries::removeOrphanedReleases() + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned releases..."); + removeOrphanedEntries(_db.getTLSSession(), _abortScan); + } + + void ScanStepRemoveOrphanedDbEntries::removeOrphanedDirectories() + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned releases..."); + removeOrphanedEntries(_db.getTLSSession(), _abortScan); + } +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.hpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.hpp new file mode 100644 index 00000000..007bdd76 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include "ScanStepBase.hpp" + +namespace lms::scanner +{ + class ScanStepRemoveOrphanedDbEntries : public ScanStepBase + { + public: + using ScanStepBase::ScanStepBase; + + private: + core::LiteralString getStepName() const override { return "Remove orphaned DB entries"; } + ScanStep getStep() const override { return ScanStep::RemoveOrphanedDbEntries; } + void process(ScanContext& context) override; + + void removeOrphanedClusters(); + void removeOrphanedClusterTypes(); + void removeOrphanedArtists(); + void removeOrphanedReleases(); + void removeOrphanedDirectories(); + }; +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepScanArtistImages.cpp b/src/libs/services/scanner/impl/ScanStepScanArtistImages.cpp deleted file mode 100644 index 9696231a..00000000 --- a/src/libs/services/scanner/impl/ScanStepScanArtistImages.cpp +++ /dev/null @@ -1,347 +0,0 @@ -/* - * Copyright (C) 2024 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "ScanStepScanArtistImages.hpp" - -#include -#include -#include -#include - -#include "core/IConfig.hpp" -#include "core/ILogger.hpp" -#include "core/Path.hpp" -#include "database/Artist.hpp" -#include "database/Db.hpp" -#include "database/Image.hpp" -#include "database/Session.hpp" -#include "database/Track.hpp" -#include "image/Exception.hpp" -#include "image/Image.hpp" - -namespace lms::scanner -{ - namespace - { - constexpr std::size_t readBatchSize{ 10 }; - constexpr std::size_t writeBatchSize{ 5 }; - - struct ImageInfo - { - operator bool() const { return !imagePath.empty(); } - void clear() - { - imagePath.clear(); - lastWriteTime = {}; - fileSize = {}; - height = {}; - width = {}; - } - - std::filesystem::path imagePath; - Wt::WDateTime lastWriteTime; - std::size_t fileSize{}; - std::size_t height{}; - std::size_t width{}; - }; - - bool tryDecodeImage(const std::filesystem::path& imagePath, ImageInfo& imageInfo) - { - assert(!imageInfo); - - try - { - std::unique_ptr rawImage{ image::decodeImage(imagePath) }; - imageInfo.imagePath = imagePath; - imageInfo.fileSize = std::filesystem::file_size(imagePath); - imageInfo.width = rawImage->getWidth(); - imageInfo.height = rawImage->getHeight(); - imageInfo.lastWriteTime = core::pathUtils::getLastWriteTime(imagePath); - } - catch (const image::Exception& e) - { - LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file '" << imagePath.string() << "': " << e.what()); - return false; - } - - return true; - } - - struct ArtistImageInfo - { - db::ArtistId artistId; - ImageInfo imageInfo; - }; - - using ArtistImageInfoContainer = std::deque; - - bool isFileSupported(const std::filesystem::path& file) - { - static const std::array fileExtensions{ ".jpg", ".jpeg", ".png", ".bmp" }; // TODO parametrize - - return (std::find(std::cbegin(fileExtensions), std::cend(fileExtensions), file.extension()) != std::cend(fileExtensions)); - } - - std::multimap getImagePaths(const std::filesystem::path& directoryPath, const std::vector& fileNames) - { - std::multimap res; - std::error_code ec; - - std::filesystem::directory_iterator itPath(directoryPath, ec); - const std::filesystem::directory_iterator itEnd; - while (!ec && itPath != itEnd) - { - const std::filesystem::path& path{ *itPath }; - const std::string stem{ path.stem().string() }; - if (isFileSupported(path) - && std::any_of(std::cbegin(fileNames), std::cend(fileNames), [&](const std::string& fileName) { return core::stringUtils::stringCaseInsensitiveEqual(stem, fileName); })) - { - res.emplace(stem, path); - } - - itPath.increment(ec); - } - - return res; - } - - bool findImageInDirectory(const std::filesystem::path& directory, const std::vector& fileNames, ImageInfo& imageInfo) - { - assert(!imageInfo); - - const std::multimap coverPaths{ getImagePaths(directory, fileNames) }; - - for (const std::string_view fileName : fileNames) - { - const auto range{ coverPaths.equal_range(std::string{ fileName }) }; - for (auto it{ range.first }; it != range.second; ++it) - { - if (tryDecodeImage(it->second, imageInfo)) - return true; - } - } - - return false; - } - - void fetchArtistImageInfo(db::Session& session, const std::vector& genericArtistFileNames, const db::Artist::pointer& artist, ImageInfo& imageInfo) - { - const std::string artistMBID{ [&] { - std::string artistMBID; - if (auto mbid{ artist->getMBID() }) - artistMBID = mbid->getAsString(); - return artistMBID; - }() }; - - std::set releasePaths; - std::set multiArtistReleasePaths; - - db::Track::FindParameters params; - params.setArtist(artist->getId(), { db::TrackArtistLinkType::ReleaseArtist }); - - db::Track::find(session, params, [&](const db::Track::pointer& track) { - db::Artist::FindParameters artistFindParams; - artistFindParams.setTrack(track->getId()); - artistFindParams.setLinkType(db::TrackArtistLinkType::ReleaseArtist); - - const auto releaseArtists{ db::Artist::findIds(session, artistFindParams) }; - if (releaseArtists.results.size() == 1) - releasePaths.insert(track->getAbsoluteFilePath().parent_path()); - else - multiArtistReleasePaths.insert(track->getAbsoluteFilePath().parent_path()); - }); - - std::vector artistFileNames; - if (!artistMBID.empty()) - artistFileNames.push_back(artistMBID); - artistFileNames.push_back(artist->getName()); - - std::vector artistFileNamesWithGenericNames{ artistFileNames }; - artistFileNamesWithGenericNames.insert(artistFileNamesWithGenericNames.end(), std::cbegin(genericArtistFileNames), std::cend(genericArtistFileNames)); - - // Expect layout like this: - // ReleaseArtist/Release/Tracks' - // /artist-mbid.jpg - // /artist-name.jpg - // /artist.jpg - if (!releasePaths.empty()) - { - const std::filesystem::path artistPath{ releasePaths.size() == 1 ? releasePaths.begin()->parent_path() : core::pathUtils::getLongestCommonPath(std::cbegin(releasePaths), std::cend(releasePaths)) }; - if (findImageInDirectory(artistPath, artistFileNamesWithGenericNames, imageInfo)) - return; - } - - // Expect layout like this: - // ReleaseArtist/Release/Tracks' - // /artist-mbid.jpg - // /artist-name.jpg - // /artist.jpg - for (const std::filesystem::path& releasePath : releasePaths) - { - // TODO: what if an artist has released an album that bears their name? - if (findImageInDirectory(releasePath, artistFileNamesWithGenericNames, imageInfo)) - return; - } - - // Expect layout like this: - // Only search for the artist's name in the release path, as we can't map a generic name to several artists - // ReleaseArtist/Release/Tracks' - // /artist-name.jpg - // /artist-mbid.jpg - for (const std::filesystem::path& releasePath : multiArtistReleasePaths) - { - if (findImageInDirectory(releasePath, artistFileNames, imageInfo)) - return; - } - } - - bool artistImageNeedsUpdate(const db::Image::pointer& image, const ImageInfo& imageInfo) - { - if (!imageInfo && !image) // no image as before - return false; - else if (!imageInfo && image) // no longer has image - return true; - else if (imageInfo && !image) // image has been added - return true; - - assert(imageInfo); - // artist image still here, consider it is the same only if the last modified time is the same - return imageInfo.lastWriteTime != image->getLastWriteTime(); - } - - struct SearchImageContext - { - db::Session& session; - db::ArtistId lastRetrievedArtistId; - const std::vector& artistFileNames; - bool fullScan; - }; - - bool fetchNextArtistImagesToUpdate(SearchImageContext& searchContext, ArtistImageInfoContainer& artistImageInfoList) - { - const db::ArtistId artistId{ searchContext.lastRetrievedArtistId }; - ImageInfo imageInfo; - - { - auto transaction{ searchContext.session.createReadTransaction() }; - - db::Artist::find(searchContext.session, searchContext.lastRetrievedArtistId, readBatchSize, [&](const db::Artist::pointer& artist) { - imageInfo.clear(); - - fetchArtistImageInfo(searchContext.session, searchContext.artistFileNames, artist, imageInfo); - if (imageInfo) - LMS_LOG(DBUPDATER, DEBUG, "Found artist image for artist '" << artist->getName() << "' at '" << imageInfo.imagePath << "'"); - - if (searchContext.fullScan || artistImageNeedsUpdate(artist->getImage(), imageInfo)) - artistImageInfoList.push_back(ArtistImageInfo{ artist->getId(), imageInfo }); - }); - } - - return artistId != searchContext.lastRetrievedArtistId; - } - - void updateArtistImage(db::Session& session, const ArtistImageInfo& artistImageInfo) - { - db::Artist::pointer artist{ db::Artist::find(session, artistImageInfo.artistId) }; - assert(artist); - - db::Image::pointer image{ artist->getImage() }; - const ImageInfo& imageInfo{ artistImageInfo.imageInfo }; - - if (!imageInfo) - { - if (image) - image.remove(); - return; - } - - if (!image) - { - image = session.create(imageInfo.imagePath); - image.modify()->setArtist(artist); - } - else - image.modify()->setPath(imageInfo.imagePath); - - image.modify()->setLastWriteTime(imageInfo.lastWriteTime); - image.modify()->setFileSize(imageInfo.fileSize); - image.modify()->setHeight(imageInfo.height); - image.modify()->setWidth(imageInfo.width); - } - - void updateArtistImages(db::Session& session, ArtistImageInfoContainer& imageInfoList) - { - if (imageInfoList.empty()) - return; - - auto transaction{ session.createWriteTransaction() }; - - for (std::size_t i{}; !imageInfoList.empty() && i < writeBatchSize; ++i) - { - updateArtistImage(session, imageInfoList.front()); - imageInfoList.pop_front(); - } - } - - std::vector constructArtistFileNames() - { - std::vector res; - - core::Service::get()->visitStrings("artist-image-file-names", - [&res](std::string_view fileName) { - res.emplace_back(fileName); - }, - { "artist" }); - - return res; - } - - } // namespace - - ScanStepScanArtistImages::ScanStepScanArtistImages(InitParams& initParams) - : ScanStepBase{ initParams } - , _artistFileNames{ constructArtistFileNames() } - { - } - - void ScanStepScanArtistImages::process(ScanContext& context) - { - auto& session{ _db.getTLSSession() }; - - { - auto transaction{ session.createReadTransaction() }; - context.currentStepStats.totalElems = db::Artist::getCount(session); - } - - SearchImageContext searchContext{ - .session = session, - .lastRetrievedArtistId = {}, - .artistFileNames = _artistFileNames, - .fullScan = context.scanOptions.fullScan - }; - - ArtistImageInfoContainer imageInfoList; - while (fetchNextArtistImagesToUpdate(searchContext, imageInfoList)) - { - updateArtistImages(session, imageInfoList); - context.currentStepStats.processedElems += readBatchSize; - _progressCallback(context.currentStepStats); - } - } -} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepScanAudioFiles.hpp b/src/libs/services/scanner/impl/ScanStepScanAudioFiles.hpp deleted file mode 100644 index 238418a3..00000000 --- a/src/libs/services/scanner/impl/ScanStepScanAudioFiles.hpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (C) 2023 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "core/IOContextRunner.hpp" -#include "metadata/IParser.hpp" - -#include "ScanStepBase.hpp" - -namespace lms::scanner -{ - class ScanStepScanAudioFiles : public ScanStepBase - { - public: - ScanStepScanAudioFiles(InitParams& initParams); - - private: - ScanStep getStep() const override { return ScanStep::ScanAudioFiles; } - core::LiteralString getStepName() const override { return "Scan audio files"; } - void process(ScanContext& context) override; - - bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo); - struct MetaDataScanResult - { - std::filesystem::path path; - std::unique_ptr trackMetaData; - }; - void processMetaDataScanResults(ScanContext& context, std::span scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo); - void processFileMetaData(ScanContext& context, const std::filesystem::path& file, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo); - - std::unique_ptr _metadataParser; - const std::vector _extraTagsToParse; - - class MetadataScanQueue - { - public: - MetadataScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort); - - std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); } - - void pushScanRequest(const std::filesystem::path& path); - - std::size_t getResultsCount() const; - size_t popResults(std::vector& results, std::size_t maxCount); - - void wait(std::size_t maxScanRequestCount = 0); // wait until ongoing scan request count <= maxScanRequestCount - - private: - metadata::IParser& _metadataParser; - boost::asio::io_context _scanContext; - core::IOContextRunner _scanContextRunner; - - mutable std::mutex _mutex; - std::size_t _ongoingScanCount{}; - std::deque _scanResults; - std::condition_variable _condVar; - bool& _abort; - }; - MetadataScanQueue _metadataScanQueue; - - std::deque _metaDataScanResults; - }; -} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepScanAudioFiles.cpp b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp similarity index 76% rename from src/libs/services/scanner/impl/ScanStepScanAudioFiles.cpp rename to src/libs/services/scanner/impl/ScanStepScanFiles.cpp index 6896d2db..df807ad5 100644 --- a/src/libs/services/scanner/impl/ScanStepScanAudioFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "ScanStepScanAudioFiles.hpp" +#include "ScanStepScanFiles.hpp" #include "core/Exception.hpp" #include "core/IConfig.hpp" @@ -27,6 +27,8 @@ #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" +#include "database/Directory.hpp" +#include "database/Image.hpp" #include "database/MediaLibrary.hpp" #include "database/Release.hpp" #include "database/Session.hpp" @@ -102,6 +104,22 @@ namespace lms::scanner return res; } + Directory::pointer getOrCreateDirectory(Session& session, const std::filesystem::path& path, const std::filesystem::path& rootPath) + { + Directory::pointer directory{ Directory::find(session, path) }; + if (!directory) + { + Directory::pointer parentDirectory; + if (path != rootPath) + parentDirectory = getOrCreateDirectory(session, path.parent_path(), rootPath); + + directory = session.create(path); + directory.modify()->setParent(parentDirectory); + } + + return directory; + } + Artist::pointer createArtist(Session& session, const metadata::Artist& artistInfo) { Artist::pointer artist{ session.create(artistInfo.name) }; @@ -301,98 +319,16 @@ namespace lms::scanner } } // namespace - ScanStepScanAudioFiles::MetadataScanQueue::MetadataScanQueue(metadata::IParser& parser, std::size_t threadCount, bool& abort) - : _metadataParser{ parser } - , _scanContextRunner{ _scanContext, threadCount, "ScannerMetadata" } - , _abort{ abort } - { - } - - void ScanStepScanAudioFiles::MetadataScanQueue::pushScanRequest(const std::filesystem::path& path) - { - { - std::scoped_lock lock{ _mutex }; - _ongoingScanCount += 1; - } - - _scanContext.post([=, this] { - LMS_SCOPED_TRACE_OVERVIEW("Scanner", "AudioFileParseJob"); - - std::unique_ptr track; - - if (_abort) - { - std::scoped_lock lock{ _mutex }; - _ongoingScanCount -= 1; - } - else - { - try - { - track = _metadataParser.parse(path); - } - catch (const metadata::Exception& e) - { - LMS_LOG(DBUPDATER, INFO, "Failed to parse '" << path.string() << "'"); - } - - { - std::scoped_lock lock{ _mutex }; - - _scanResults.emplace_back(MetaDataScanResult{ std::move(path), std::move(track) }); - _ongoingScanCount -= 1; - } - } - - _condVar.notify_all(); - }); - } - - std::size_t ScanStepScanAudioFiles::MetadataScanQueue::getResultsCount() const - { - std::scoped_lock lock{ _mutex }; - return _scanResults.size(); - } - - size_t ScanStepScanAudioFiles::MetadataScanQueue::popResults(std::vector& results, std::size_t maxCount) - { - results.clear(); - results.reserve(maxCount); - - { - std::scoped_lock lock{ _mutex }; - - while (results.size() < maxCount && !_scanResults.empty()) - { - results.push_back(std::move(_scanResults.front())); - _scanResults.pop_front(); - } - } - - return results.size(); - } - - void ScanStepScanAudioFiles::MetadataScanQueue::wait(std::size_t maxScanRequestCount) - { - LMS_SCOPED_TRACE_OVERVIEW("Scanner", "WaitParseResults"); - - std::unique_lock lock{ _mutex }; - _condVar.wait(lock, [=, this] { return _ongoingScanCount <= maxScanRequestCount; }); - } - - ScanStepScanAudioFiles::ScanStepScanAudioFiles(InitParams& initParams) + ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams) : ScanStepBase{ initParams } , _metadataParser{ metadata::createParser(metadata::ParserBackend::TagLib, getParserReadStyle()) } // For now, always use TagLib - , _metadataScanQueue{ *_metadataParser, getScanMetaDataThreadCount(), _abortScan } + , _fileScanQueue{ *_metadataParser, getScanMetaDataThreadCount(), _abortScan } { - LMS_LOG(DBUPDATER, INFO, "Using " << _metadataScanQueue.getThreadCount() << " thread(s) for scanning file metadata"); + LMS_LOG(DBUPDATER, INFO, "Using " << _fileScanQueue.getThreadCount() << " thread(s) for scanning file metadata"); } - void ScanStepScanAudioFiles::process(ScanContext& context) + void ScanStepScanFiles::process(ScanContext& context) { - const std::size_t scanQueueMaxScanRequestCount{ 100 * _metadataScanQueue.getThreadCount() }; - const std::size_t processMetaDataBatchSize{ 5 }; - { std::vector tagsToParse{ _extraTagsToParse }; tagsToParse.insert(std::end(tagsToParse), std::cbegin(_settings.extraTags), std::cend(_settings.extraTags)); @@ -401,56 +337,77 @@ namespace lms::scanner _metadataParser->setDefaultTagDelimiters(_settings.defaultTagDelimiters); } - std::vector scanResults; - context.currentStepStats.totalElems = context.stats.filesScanned; + context.currentStepStats.totalElems = context.stats.totalFileCount; for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries) - { - core::pathUtils::exploreFilesRecursive( - mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) { - LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile"); + process(context, mediaLibrary); + } - if (_abortScan) - return false; + void ScanStepScanFiles::process(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary) + { + const std::size_t scanQueueMaxScanRequestCount{ 100 * _fileScanQueue.getThreadCount() }; + const std::size_t processFileResultsBatchSize{ 5 }; - if (ec) + std::vector scanResults; + + core::pathUtils::exploreFilesRecursive( + mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) { + LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile"); + + if (_abortScan) + return false; + + if (ec) + { + LMS_LOG(DBUPDATER, ERROR, "Cannot scan file '" << path.string() << "': " << ec.message()); + context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() }); + } + else + { + bool fileToProcess{}; + if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedAudioFileExtensions)) { - LMS_LOG(DBUPDATER, ERROR, "Cannot process entry '" << path.string() << "': " << ec.message()); - context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() }); + fileToProcess = true; + if (checkAudioFileNeedScan(context, path, mediaLibrary)) + _fileScanQueue.pushScanRequest(path, FileScanQueue::ScanRequestType::AudioFile); } - else if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedExtensions)) + else if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedImageFileExtensions)) { - if (checkFileNeedScan(context, path, mediaLibrary)) - _metadataScanQueue.pushScanRequest(path); + fileToProcess = true; + if (checkImageFileNeedScan(context, path)) + _fileScanQueue.pushScanRequest(path, FileScanQueue::ScanRequestType::ImageFile); + } + if (fileToProcess) + { context.currentStepStats.processedElems++; _progressCallback(context.currentStepStats); } + } - while (_metadataScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2)) - { - _metadataScanQueue.popResults(scanResults, processMetaDataBatchSize); - processMetaDataScanResults(context, scanResults, mediaLibrary); - } + while (_fileScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2)) + { + _fileScanQueue.popResults(scanResults, processFileResultsBatchSize); + processFileScanResults(context, scanResults, mediaLibrary); + } - _metadataScanQueue.wait(scanQueueMaxScanRequestCount); + _fileScanQueue.wait(scanQueueMaxScanRequestCount); - return true; - }, - &excludeDirFileName); + return true; + }, + &excludeDirFileName); - _metadataScanQueue.wait(); + _fileScanQueue.wait(); - while (!_abortScan && _metadataScanQueue.popResults(scanResults, processMetaDataBatchSize) > 0) - processMetaDataScanResults(context, scanResults, mediaLibrary); - } + while (!_abortScan && _fileScanQueue.popResults(scanResults, processFileResultsBatchSize) > 0) + processFileScanResults(context, scanResults, mediaLibrary); } - bool ScanStepScanAudioFiles::checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo) + bool ScanStepScanFiles::checkAudioFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo) { ScanStats& stats{ context.stats }; - Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) }; + const Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) }; // Should rarely fail as we are currently iterating it if (!lastWriteTime.isValid()) { @@ -498,35 +455,77 @@ namespace lms::scanner return true; // need to scan } - void ScanStepScanAudioFiles::processMetaDataScanResults(ScanContext& context, std::span scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo) + bool ScanStepScanFiles::checkImageFileNeedScan(ScanContext& context, const std::filesystem::path& file) + { + ScanStats& stats{ context.stats }; + + const Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) }; + // Should rarely fail as we are currently iterating it + if (!lastWriteTime.isValid()) + { + stats.skips++; + return false; + } + + if (!context.scanOptions.fullScan) + { + db::Session& dbSession{ _db.getTLSSession() }; + auto transaction{ _db.getTLSSession().createReadTransaction() }; + + const db::Image::pointer image{ db::Image::find(dbSession, file) }; + if (image && image->getLastWriteTime() == lastWriteTime) + { + stats.skips++; + return false; + } + } + + return true; // need to scan + } + + void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo) { LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ProcessScanResults"); db::Session& dbSession{ _db.getTLSSession() }; auto transaction{ dbSession.createWriteTransaction() }; - for (const MetaDataScanResult& scanResult : scanResults) + for (const FileScanResult& scanResult : scanResults) { - LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessScanResult"); - if (_abortScan) return; - if (scanResult.trackMetaData) + if (const AudioFileScanData * scanData{ std::get_if(&scanResult.scanData) }) { - context.stats.scans++; - - processFileMetaData(context, scanResult.path, *scanResult.trackMetaData, libraryInfo); + if (metadata::Track * track{ scanData->get() }) + { + context.stats.scans++; + processAudioFileScanData(context, scanResult.path, *track, libraryInfo); + } + else + { + context.stats.errors.emplace_back(scanResult.path, ScanErrorType::CannotReadAudioFile); + } } - else + else if (const ImageFileScanData * scanData{ std::get_if(&scanResult.scanData) }) { - context.stats.errors.emplace_back(scanResult.path, ScanErrorType::CannotParseFile); + if (scanData->has_value()) + { + context.stats.scans++; + processImageFileScanData(context, scanResult.path, scanData->value(), libraryInfo); + } + else + { + context.stats.errors.emplace_back(scanResult.path, ScanErrorType::CannotReadImageFile); + } } } } - void ScanStepScanAudioFiles::processFileMetaData(ScanContext& context, const std::filesystem::path& file, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo) + void ScanStepScanFiles::processAudioFileScanData(ScanContext& context, const std::filesystem::path& file, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo) { + LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessAudioScanData"); + ScanStats& stats{ context.stats }; const std::optional fileInfo{ retrieveFileInfo(file, libraryInfo.rootDirectory) }; @@ -637,6 +636,8 @@ namespace lms::scanner track.modify()->setLastWriteTime(fileInfo->lastWriteTime); track.modify()->setMediaLibrary(MediaLibrary::find(dbSession, libraryInfo.id)); // may be null if settings are updated in // => next scan will correct this + track.modify()->setDirectory(getOrCreateDirectory(dbSession, file.parent_path(), libraryInfo.rootDirectory)); + track.modify()->clearArtistLinks(); // Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files for (const Artist::pointer& artist : getOrCreateArtists(dbSession, trackMetadata.artists, false)) @@ -712,12 +713,57 @@ namespace lms::scanner if (added) { - LMS_LOG(DBUPDATER, DEBUG, "Added '" << file.string() << "'"); + LMS_LOG(DBUPDATER, DEBUG, "Added audio file '" << file.string() << "'"); stats.additions++; } else { - LMS_LOG(DBUPDATER, DEBUG, "Updated '" << file.string() << "'"); + LMS_LOG(DBUPDATER, DEBUG, "Updated audio file '" << file.string() << "'"); + stats.updates++; + } + } + + void ScanStepScanFiles::processImageFileScanData(ScanContext& context, const std::filesystem::path& file, const ImageInfo& imageInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo) + { + LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessImageScanData"); + + ScanStats& stats{ context.stats }; + + const std::optional fileInfo{ retrieveFileInfo(file, libraryInfo.rootDirectory) }; + if (!fileInfo) + { + stats.skips++; + return; + } + + db::Session& dbSession{ _db.getTLSSession() }; + db::Image::pointer image{ db::Image::find(dbSession, file) }; + + bool added; + if (!image) + { + image = dbSession.create(file); + added = true; + } + else + { + added = false; + } + + image.modify()->setLastWriteTime(fileInfo->lastWriteTime); + image.modify()->setFileSize(fileInfo->fileSize); + image.modify()->setHeight(imageInfo.height); + image.modify()->setWidth(imageInfo.width); + image.modify()->setDirectory(getOrCreateDirectory(dbSession, file.parent_path(), libraryInfo.rootDirectory)); + + if (added) + { + LMS_LOG(DBUPDATER, DEBUG, "Added image '" << file.string() << "'"); + stats.additions++; + } + else + { + LMS_LOG(DBUPDATER, DEBUG, "Updated image '" << file.string() << "'"); stats.updates++; } } diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp new file mode 100644 index 00000000..54d5ac66 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +#include "metadata/IParser.hpp" + +#include "FileScanQueue.hpp" +#include "ScanStepBase.hpp" + +namespace lms::scanner +{ + class ScanStepScanFiles : public ScanStepBase + { + public: + ScanStepScanFiles(InitParams& initParams); + + private: + ScanStep getStep() const override { return ScanStep::ScanFiles; } + core::LiteralString getStepName() const override { return "Scan files"; } + void process(ScanContext& context) override; + void process(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary); + + bool checkAudioFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo); + bool checkImageFileNeedScan(ScanContext& context, const std::filesystem::path& file); + + void processFileScanResults(ScanContext& context, std::span scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo); + void processAudioFileScanData(ScanContext& context, const std::filesystem::path& path, const metadata::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo); + void processImageFileScanData(ScanContext& context, const std::filesystem::path& path, const ImageInfo& imageInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo); + + std::unique_ptr _metadataParser; + const std::vector _extraTagsToParse; + + FileScanQueue _fileScanQueue; + }; +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index 04d7b291..321116f6 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -29,15 +29,17 @@ #include "database/MediaLibrary.hpp" #include "database/ScanSettings.hpp" #include "database/TrackFeatures.hpp" +#include "image/Image.hpp" -#include "ScanStepCheckDuplicatedDbFiles.hpp" +#include "ScanStepAssociateArtistImages.hpp" +#include "ScanStepCheckForDuplicatedFiles.hpp" +#include "ScanStepCheckForRemovedFiles.hpp" #include "ScanStepCompact.hpp" #include "ScanStepComputeClusterStats.hpp" #include "ScanStepDiscoverFiles.hpp" #include "ScanStepOptimize.hpp" -#include "ScanStepRemoveOrphanDbFiles.hpp" -#include "ScanStepScanArtistImages.hpp" -#include "ScanStepScanAudioFiles.hpp" +#include "ScanStepRemoveOrphanedDbEntries.hpp" +#include "ScanStepScanFiles.hpp" namespace lms::scanner { @@ -340,13 +342,14 @@ namespace lms::scanner // Order is important _scanSteps.clear(); _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); } ScannerSettings ScannerService::readSettings() @@ -364,9 +367,16 @@ namespace lms::scanner newSettings.updatePeriod = scanSettings->getUpdatePeriod(); { - const auto fileExtensions{ scanSettings->getAudioFileExtensions() }; - newSettings.supportedExtensions.reserve(fileExtensions.size()); - std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions), + const auto audioFileExtensions{ scanSettings->getAudioFileExtensions() }; + newSettings.supportedAudioFileExtensions.reserve(audioFileExtensions.size()); + std::transform(std::cbegin(audioFileExtensions), std::end(audioFileExtensions), std::back_inserter(newSettings.supportedAudioFileExtensions), + [](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; }); + } + + { + const auto imageFileExtensions{ image::getSupportedFileExtensions() }; + newSettings.supportedImageFileExtensions.reserve(imageFileExtensions.size()); + std::transform(std::cbegin(imageFileExtensions), std::end(imageFileExtensions), std::back_inserter(newSettings.supportedImageFileExtensions), [](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; }); } diff --git a/src/libs/services/scanner/impl/ScannerSettings.hpp b/src/libs/services/scanner/impl/ScannerSettings.hpp index 48ba2971..4b228694 100644 --- a/src/libs/services/scanner/impl/ScannerSettings.hpp +++ b/src/libs/services/scanner/impl/ScannerSettings.hpp @@ -35,7 +35,8 @@ namespace lms::scanner std::size_t scanVersion{}; Wt::WTime startTime; db::ScanSettings::UpdatePeriod updatePeriod{ db::ScanSettings::UpdatePeriod::Never }; - std::vector supportedExtensions; + std::vector supportedAudioFileExtensions; + std::vector supportedImageFileExtensions; bool skipDuplicateMBID{}; std::vector extraTags; std::vector artistTagDelimiters; diff --git a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp index fdc77c14..22ce1d1e 100644 --- a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp +++ b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp @@ -30,10 +30,11 @@ namespace lms::scanner { enum class ScanErrorType { - CannotReadFile, // cannot read file - CannotParseFile, // cannot parse file - NoAudioTrack, // no audio track found - BadDuration, // bad duration + CannotReadFile, // cannot read file + CannotReadAudioFile, // cannot parse audio file + CannotReadImageFile, // cannot parse image file + NoAudioTrack, // no audio track found + BadDuration, // bad duration }; enum class DuplicateReason @@ -60,18 +61,19 @@ namespace lms::scanner // Alphabetical order enum class ScanStep { - CheckForMissingFiles, - CheckForDuplicateFiles, + AssociateArtistImages, + CheckForDuplicatedFiles, + CheckForRemovedFiles, ComputeClusterStats, Compact, DiscoverFiles, FetchTrackFeatures, Optimize, ReloadSimilarityEngine, - ScanArtistImages, - ScanAudioFiles, + RemoveOrphanedDbEntries, + ScanFiles, }; - static inline constexpr unsigned ScanProgressStepCount{ 9 }; + static inline constexpr unsigned ScanProgressStepCount{ 11 }; // reduced scan stats struct ScanStepStats @@ -92,7 +94,7 @@ namespace lms::scanner Wt::WDateTime startTime; Wt::WDateTime stopTime; - std::size_t filesScanned{}; // Total number of files scanned (estimated) + std::size_t totalFileCount{}; // Total number of files (estimated) std::size_t skips{}; // no change since last scan std::size_t scans{}; // actually scanned filed diff --git a/src/libs/subsonic/impl/SubsonicId.cpp b/src/libs/subsonic/impl/SubsonicId.cpp index 508e1c93..9ec69508 100644 --- a/src/libs/subsonic/impl/SubsonicId.cpp +++ b/src/libs/subsonic/impl/SubsonicId.cpp @@ -76,22 +76,6 @@ namespace lms::core::stringUtils return std::nullopt; } - template<> - std::optional readAs(std::string_view str) - { - std::vector values{ core::stringUtils::splitString(str, '-') }; - if (values.size() != 2) - return std::nullopt; - - if (values[0] != "im") - return std::nullopt; - - if (const auto value{ core::stringUtils::readAs(values[1]) }) - return db::ImageId{ *value }; - - return std::nullopt; - } - template<> std::optional readAs(std::string_view str) { diff --git a/src/libs/subsonic/impl/SubsonicId.hpp b/src/libs/subsonic/impl/SubsonicId.hpp index a08713df..d7a88efa 100644 --- a/src/libs/subsonic/impl/SubsonicId.hpp +++ b/src/libs/subsonic/impl/SubsonicId.hpp @@ -21,7 +21,6 @@ #include "core/String.hpp" #include "database/ArtistId.hpp" -#include "database/ImageId.hpp" #include "database/MediaLibraryId.hpp" #include "database/ReleaseId.hpp" #include "database/TrackId.hpp" @@ -34,7 +33,6 @@ namespace lms::api::subsonic }; std::string idToString(db::ArtistId id); - std::string idToString(db::ImageId id); std::string idToString(db::MediaLibraryId id); std::string idToString(db::ReleaseId id); std::string idToString(db::TrackId id); @@ -51,9 +49,6 @@ namespace lms::core::stringUtils template<> std::optional readAs(std::string_view str); - template<> - std::optional readAs(std::string_view str); - template<> std::optional readAs(std::string_view str); diff --git a/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp b/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp index f2447616..3a899b2b 100644 --- a/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp +++ b/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp @@ -39,7 +39,7 @@ namespace lms::api::subsonic::Scan { std::size_t count{}; - if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanAudioFiles) + if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanFiles) count = scanStatus.currentScanStepStats->processedElems; statusResponse.setAttribute("count", count); diff --git a/src/lms/ui/admin/ScannerController.cpp b/src/lms/ui/admin/ScannerController.cpp index c4162ef2..b087b79b 100644 --- a/src/lms/ui/admin/ScannerController.cpp +++ b/src/lms/ui/admin/ScannerController.cpp @@ -119,8 +119,10 @@ namespace lms::ui { case scanner::ScanErrorType::CannotReadFile: return Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-file"); - case scanner::ScanErrorType::CannotParseFile: - return Wt::WString::tr("Lms.Admin.ScannerController.cannot-parse-file"); + case scanner::ScanErrorType::CannotReadAudioFile: + return Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-audio-file"); + case scanner::ScanErrorType::CannotReadImageFile: + return Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-image-file"); case scanner::ScanErrorType::NoAudioTrack: return Wt::WString::tr("Lms.Admin.ScannerController.no-audio-track"); case scanner::ScanErrorType::BadDuration: @@ -267,58 +269,59 @@ namespace lms::ui switch (stepStats.currentStep) { - case ScanStep::CheckForDuplicateFiles: + case ScanStep::AssociateArtistImages: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-associating-artist-images") + .arg(stepStats.progress())); + break; + + case ScanStep::CheckForDuplicatedFiles: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files") .arg(stepStats.processedElems)); break; - case scanner::ScanStep::CheckForMissingFiles: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files") + case ScanStep::CheckForRemovedFiles: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-removed-files") .arg(stepStats.progress())); break; - case scanner::ScanStep::Compact: + case ScanStep::Compact: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compact")); break; - case scanner::ScanStep::ComputeClusterStats: + case ScanStep::ComputeClusterStats: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats") .arg(stepStats.progress())); break; - case scanner::ScanStep::DiscoverFiles: + case ScanStep::DiscoverFiles: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files") .arg(stepStats.processedElems)); break; - case scanner::ScanStep::FetchTrackFeatures: + case ScanStep::FetchTrackFeatures: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features") .arg(stepStats.processedElems) .arg(stepStats.totalElems) .arg(stepStats.progress())); break; - case scanner::ScanStep::Optimize: + case ScanStep::Optimize: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-optimize") - .arg(stepStats.processedElems) - .arg(stepStats.totalElems) .arg(stepStats.progress())); break; - case scanner::ScanStep::ReloadSimilarityEngine: + case ScanStep::RemoveOrphanedDbEntries: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-removing-orphaned-entries") + .arg(stepStats.progress())); + break; + + case ScanStep::ReloadSimilarityEngine: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine") .arg(stepStats.progress())); break; - case scanner::ScanStep::ScanArtistImages: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-artist-images") - .arg(stepStats.processedElems) - .arg(stepStats.totalElems) - .arg(stepStats.progress())); - break; - - case scanner::ScanStep::ScanAudioFiles: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-audio-files") + case ScanStep::ScanFiles: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files") .arg(stepStats.processedElems) .arg(stepStats.totalElems) .arg(stepStats.progress())); From 437f9d63edf224bbb24709b57401e90c4de3923d Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 6 Jul 2024 14:12:58 +0200 Subject: [PATCH 4/6] Fixed build when using graphicmagicks backend --- approot/messages_fr.xml | 2 +- src/libs/image/impl/graphicsmagick/Image.cpp | 3 +++ src/libs/image/impl/graphicsmagick/JPEGImage.cpp | 3 +++ src/libs/image/impl/graphicsmagick/RawImage.cpp | 3 +++ src/libs/services/scanner/impl/FileScanQueue.cpp | 2 +- 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index a8e1fa29..4b77b59f 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -111,7 +111,7 @@ En cours de scan : étape {1}/{2} Association des images des artistes: {1}%... Vérification des fichiers dupliqués... {1} fichiers -Vérification des fichiers... {1}% +Vérification des fichiers supprimés... {1}% Compactage de la base de données... Calcul des statistiques... {1}% Découverte des fichiers : {1} fichiers diff --git a/src/libs/image/impl/graphicsmagick/Image.cpp b/src/libs/image/impl/graphicsmagick/Image.cpp index 920bed60..44a59786 100644 --- a/src/libs/image/impl/graphicsmagick/Image.cpp +++ b/src/libs/image/impl/graphicsmagick/Image.cpp @@ -21,6 +21,7 @@ #include +#include "RawImage.hpp" #include "core/ILogger.hpp" #include "core/ITraceLogger.hpp" @@ -28,11 +29,13 @@ namespace lms::image { std::unique_ptr decodeImage(const std::byte* encodedData, std::size_t encodedDataSize) { + LMS_SCOPED_TRACE_DETAILED("Image", "DecodeBuffer"); return std::make_unique(encodedData, encodedDataSize); } std::unique_ptr decodeImage(const std::filesystem::path& path) { + LMS_SCOPED_TRACE_DETAILED("Image", "DecodeFile"); return std::make_unique(path); } diff --git a/src/libs/image/impl/graphicsmagick/JPEGImage.cpp b/src/libs/image/impl/graphicsmagick/JPEGImage.cpp index 36b00d2c..94738b20 100644 --- a/src/libs/image/impl/graphicsmagick/JPEGImage.cpp +++ b/src/libs/image/impl/graphicsmagick/JPEGImage.cpp @@ -20,6 +20,7 @@ #include "JPEGImage.hpp" #include "core/ILogger.hpp" +#include "core/ITraceLogger.hpp" #include "image/Exception.hpp" #include "RawImage.hpp" @@ -28,6 +29,8 @@ namespace lms::image::GraphicsMagick { JPEGImage::JPEGImage(const RawImage& rawImage, unsigned quality) { + LMS_SCOPED_TRACE_DETAILED("Image", "WriteJPEG"); + try { Magick::Image image{ rawImage.getMagickImage() }; diff --git a/src/libs/image/impl/graphicsmagick/RawImage.cpp b/src/libs/image/impl/graphicsmagick/RawImage.cpp index d935d125..f3550478 100644 --- a/src/libs/image/impl/graphicsmagick/RawImage.cpp +++ b/src/libs/image/impl/graphicsmagick/RawImage.cpp @@ -25,6 +25,7 @@ #include #include "core/ILogger.hpp" +#include "core/ITraceLogger.hpp" #include "image/Exception.hpp" #include "JPEGImage.hpp" @@ -90,6 +91,8 @@ namespace lms::image::GraphicsMagick { try { + LMS_SCOPED_TRACE_DETAILED("Image", "Resize"); + _image.resize(Magick::Geometry{ static_cast(width), static_cast(width) }); } catch (Magick::Exception& e) diff --git a/src/libs/services/scanner/impl/FileScanQueue.cpp b/src/libs/services/scanner/impl/FileScanQueue.cpp index 6497d24c..0e083663 100644 --- a/src/libs/services/scanner/impl/FileScanQueue.cpp +++ b/src/libs/services/scanner/impl/FileScanQueue.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Emeric Poupon + * Copyright (C) 2024 Emeric Poupon * * This file is part of LMS. * From a15cccb4bb415655b17309854c70f18088951f7c Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 6 Jul 2024 14:19:14 +0200 Subject: [PATCH 5/6] Fixed last directory entries not being removed --- .../scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp index f3df1910..db4e7428 100644 --- a/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp @@ -51,6 +51,9 @@ namespace lms::scanner entries = T::findOrphanIds(session, Range{ 0, batchSize }); }; + if (entries.results.empty()) + break; + { auto transaction{ session.createWriteTransaction() }; @@ -64,9 +67,6 @@ namespace lms::scanner entry.remove(); } } - - if (!entries.moreResults) - break; } } } // namespace @@ -119,7 +119,7 @@ namespace lms::scanner void ScanStepRemoveOrphanedDbEntries::removeOrphanedDirectories() { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned releases..."); + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned directories..."); removeOrphanedEntries(_db.getTLSSession(), _abortScan); } } // namespace lms::scanner From 89ebe3157d0e073b7f8ae5a033ad2c35d38e5f6a Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 6 Jul 2024 16:26:43 +0200 Subject: [PATCH 6/6] Better user feedback --- approot/messages.xml | 1 + approot/messages_fr.xml | 1 + approot/messages_it.xml | 1 + approot/messages_pl.xml | 1 + approot/messages_zh.xml | 1 + .../impl/ScanStepRemoveOrphanedDbEntries.cpp | 144 ++++++++---------- .../impl/ScanStepRemoveOrphanedDbEntries.hpp | 13 +- src/lms/ui/admin/ScannerController.cpp | 2 +- 8 files changed, 79 insertions(+), 85 deletions(-) diff --git a/approot/messages.xml b/approot/messages.xml index 77257036..d1adae65 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -118,6 +118,7 @@ Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)... Optimizing database... {1}%... Reloading similarity engine: {1}%... +Removing orphaned entries: {1} entries... Scanning files: {1}/{2} ({3}%)... Step status diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 4b77b59f..d07e92ec 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -118,6 +118,7 @@ Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)... Optimisation de la base de données... {1}%... Rechargement du moteur de recommandation : {1}%... +Retrait des entrées orphelines: {1} entrées... Scan des fichiers : {1}/{2} ({3}%)... Statut de l'étape diff --git a/approot/messages_it.xml b/approot/messages_it.xml index ec421e0c..7f8b7fa6 100644 --- a/approot/messages_it.xml +++ b/approot/messages_it.xml @@ -118,6 +118,7 @@ Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)... Ottimizzazione del database... {1}%... Ricarica motore di tracce simili: {1}%... +Rimozione voci orfane: {1} voci... Scansione dei file: {1}/{2} ({3}%)... Stato passo diff --git a/approot/messages_pl.xml b/approot/messages_pl.xml index 5a1c27df..bf83a8af 100644 --- a/approot/messages_pl.xml +++ b/approot/messages_pl.xml @@ -135,6 +135,7 @@ Pobieranie danych o ścieżce z AcousticBrainz: {1}/{2} ścieżek ({3}%)... Optymalizowanie bazy danych... {1}%... Przeładowywanie silnika podobieństw: {1}%... +Usuwanie osieroconych wpisów: {1} wpisów... Skanowanie plików: {1}/{2} ({3}%)... Obecny krok diff --git a/approot/messages_zh.xml b/approot/messages_zh.xml index 7586bc01..c8608334 100644 --- a/approot/messages_zh.xml +++ b/approot/messages_zh.xml @@ -118,6 +118,7 @@ 从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)... 重载相似引擎中 {1}%... + 扫描文件中: {1}/{2} 个文件 ({3}%)... 当前步骤状态 diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp index db4e7428..5ffd546a 100644 --- a/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.cpp @@ -31,95 +31,81 @@ namespace lms::scanner { - using namespace db; + void ScanStepRemoveOrphanedDbEntries::process(ScanContext& context) + { + removeOrphanedClusters(context); + removeOrphanedClusterTypes(context); + removeOrphanedArtists(context); + removeOrphanedReleases(context); + removeOrphanedDirectories(context); + } - namespace + void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusters(ScanContext& context) + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned clusters..."); + removeOrphanedEntries(context); + } + + void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusterTypes(ScanContext& context) + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned cluster types..."); + removeOrphanedEntries(context); + } + + void ScanStepRemoveOrphanedDbEntries::removeOrphanedArtists(ScanContext& context) + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned artists..."); + removeOrphanedEntries(context); + } + + void ScanStepRemoveOrphanedDbEntries::removeOrphanedReleases(ScanContext& context) + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned releases..."); + removeOrphanedEntries(context); + } + + void ScanStepRemoveOrphanedDbEntries::removeOrphanedDirectories(ScanContext& context) + { + LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned directories..."); + removeOrphanedEntries(context); + } + + template + void ScanStepRemoveOrphanedDbEntries::removeOrphanedEntries(ScanStepRemoveOrphanedDbEntries::ScanContext& context) { constexpr std::size_t batchSize = 100; - template - void removeOrphanedEntries(Session& session, bool& abortScan) + using IdType = typename T::IdType; + + db::Session& session{ _db.getTLSSession() }; + + db::RangeResults entries; + while (!_abortScan) { - using IdType = typename T::IdType; - - RangeResults entries; - while (!abortScan) { + auto transaction{ session.createReadTransaction() }; + + entries = T::findOrphanIds(session, db::Range{ 0, batchSize }); + }; + + if (entries.results.empty()) + break; + + { + auto transaction{ session.createWriteTransaction() }; + + for (const IdType objectId : entries.results) { - auto transaction{ session.createReadTransaction() }; + if (_abortScan) + break; - entries = T::findOrphanIds(session, Range{ 0, batchSize }); - }; - - if (entries.results.empty()) - break; - - { - auto transaction{ session.createWriteTransaction() }; - - for (const IdType objectId : entries.results) - { - if (abortScan) - break; - - typename T::pointer entry{ T::find(session, objectId) }; - - entry.remove(); - } + typename T::pointer entry{ T::find(session, objectId) }; + entry.remove(); } } + + context.currentStepStats.processedElems += entries.results.size(); + _progressCallback(context.currentStepStats); } - } // namespace - - void ScanStepRemoveOrphanedDbEntries::process(ScanContext& context) - { - auto& session{ _db.getTLSSession() }; - - { - auto transaction{ session.createReadTransaction() }; - context.currentStepStats.totalElems = 0; - context.currentStepStats.totalElems += Cluster::getCount(session); - context.currentStepStats.totalElems += ClusterType::getCount(session); - context.currentStepStats.totalElems += Artist::getCount(session); - context.currentStepStats.totalElems += Release::getCount(session); - context.currentStepStats.totalElems += Directory::getCount(session); - } - LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " database entries to be checked..."); - - removeOrphanedClusters(); - removeOrphanedClusterTypes(); - removeOrphanedArtists(); - removeOrphanedReleases(); - removeOrphanedDirectories(); - } - - void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusters() - { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned clusters..."); - removeOrphanedEntries(_db.getTLSSession(), _abortScan); - } - - void ScanStepRemoveOrphanedDbEntries::removeOrphanedClusterTypes() - { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned cluster types..."); - removeOrphanedEntries(_db.getTLSSession(), _abortScan); - } - - void ScanStepRemoveOrphanedDbEntries::removeOrphanedArtists() - { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned artists..."); - removeOrphanedEntries(_db.getTLSSession(), _abortScan); - } - - void ScanStepRemoveOrphanedDbEntries::removeOrphanedReleases() - { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned releases..."); - removeOrphanedEntries(_db.getTLSSession(), _abortScan); - } - - void ScanStepRemoveOrphanedDbEntries::removeOrphanedDirectories() - { - LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned directories..."); - removeOrphanedEntries(_db.getTLSSession(), _abortScan); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.hpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.hpp index 007bdd76..3dea1d33 100644 --- a/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.hpp +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanedDbEntries.hpp @@ -35,10 +35,13 @@ namespace lms::scanner ScanStep getStep() const override { return ScanStep::RemoveOrphanedDbEntries; } void process(ScanContext& context) override; - void removeOrphanedClusters(); - void removeOrphanedClusterTypes(); - void removeOrphanedArtists(); - void removeOrphanedReleases(); - void removeOrphanedDirectories(); + void removeOrphanedClusters(ScanContext& context); + void removeOrphanedClusterTypes(ScanContext& context); + void removeOrphanedArtists(ScanContext& context); + void removeOrphanedReleases(ScanContext& context); + void removeOrphanedDirectories(ScanContext& context); + + template + void removeOrphanedEntries(ScanStepRemoveOrphanedDbEntries::ScanContext& context); }; } // namespace lms::scanner diff --git a/src/lms/ui/admin/ScannerController.cpp b/src/lms/ui/admin/ScannerController.cpp index b087b79b..4ccf7fda 100644 --- a/src/lms/ui/admin/ScannerController.cpp +++ b/src/lms/ui/admin/ScannerController.cpp @@ -312,7 +312,7 @@ namespace lms::ui case ScanStep::RemoveOrphanedDbEntries: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-removing-orphaned-entries") - .arg(stepStats.progress())); + .arg(stepStats.processedElems)); break; case ScanStep::ReloadSimilarityEngine: