From 571c0bc5f4077b1e96f06c4e560c84aa5ef4cdc7 Mon Sep 17 00:00:00 2001 From: emeric Date: Mon, 26 May 2025 23:33:52 +0200 Subject: [PATCH] More details for scan error reports #676 --- approot/messages.xml | 6 +- approot/messages_es.xml | 6 +- approot/messages_fr.xml | 6 +- approot/messages_it.xml | 6 +- approot/messages_pl.xml | 6 +- approot/messages_zh.xml | 6 +- src/libs/core/impl/ArchiveZipper.cpp | 16 +- src/libs/core/impl/ChildProcess.cpp | 31 +-- src/libs/core/impl/Path.cpp | 17 +- src/libs/core/impl/String.cpp | 8 - src/libs/core/include/core/Path.hpp | 4 +- src/libs/core/include/core/String.hpp | 2 - src/libs/metadata/CMakeLists.txt | 11 +- src/libs/metadata/impl/ArtistInfo.cpp | 8 +- src/libs/metadata/impl/AudioFileParser.cpp | 108 ++++---- src/libs/metadata/impl/ITagReader.hpp | 1 - src/libs/metadata/impl/Utils.cpp | 2 +- .../{ => avformat}/AvFormatImageReader.cpp | 8 +- .../{ => avformat}/AvFormatImageReader.hpp | 4 +- .../impl/{ => avformat}/AvFormatTagReader.cpp | 8 +- .../impl/{ => avformat}/AvFormatTagReader.hpp | 6 +- src/libs/metadata/impl/avformat/Utils.cpp | 49 ++++ src/libs/metadata/impl/avformat/Utils.hpp | 28 +++ .../metadata/impl/{ => taglib}/TagLibDefs.hpp | 4 + .../impl/{ => taglib}/TagLibImageReader.cpp | 49 ++-- .../impl/{ => taglib}/TagLibImageReader.hpp | 8 +- .../impl/{ => taglib}/TagLibTagReader.cpp | 83 +++---- .../impl/{ => taglib}/TagLibTagReader.hpp | 11 +- src/libs/metadata/impl/taglib/Utils.cpp | 230 ++++++++++++++++++ src/libs/metadata/impl/taglib/Utils.hpp | 39 +++ .../metadata/include/metadata/ArtistInfo.hpp | 2 +- .../metadata/include/metadata/Exception.hpp | 26 +- .../include/metadata/IAudioFileParser.hpp | 1 + src/libs/metadata/test/PlayList.cpp | 1 + src/libs/services/scanner/CMakeLists.txt | 2 + .../services/scanner/impl/ScannerService.cpp | 16 +- .../services/scanner/impl/ScannerStats.cpp | 9 +- .../impl/scanners/ArtistInfoFileScanner.cpp | 129 ++++------ .../impl/scanners/ArtistInfoFileScanner.hpp | 9 +- .../impl/scanners/AudioFileScanOperation.cpp | 126 +++++----- .../impl/scanners/AudioFileScanOperation.hpp | 29 +-- .../impl/scanners/AudioFileScanner.cpp | 65 ++--- .../impl/scanners/AudioFileScanner.hpp | 5 +- .../impl/scanners/FileScanOperationBase.cpp | 34 +++ .../impl/scanners/FileScanOperationBase.hpp | 69 ++++++ .../scanner/impl/scanners/FileToScan.hpp | 7 +- .../impl/scanners/IFileScanOperation.hpp | 23 +- .../scanner/impl/scanners/IFileScanner.hpp | 7 +- .../impl/scanners/ImageFileScanner.cpp | 109 +++------ .../impl/scanners/ImageFileScanner.hpp | 8 +- .../impl/scanners/LyricsFileScanner.cpp | 121 ++++----- .../impl/scanners/LyricsFileScanner.hpp | 8 +- .../impl/scanners/PlayListFileScanner.cpp | 116 ++++----- .../impl/scanners/PlayListFileScanner.hpp | 8 +- .../services/scanner/impl/scanners/Utils.cpp | 56 +---- .../services/scanner/impl/scanners/Utils.hpp | 13 - .../scanner/impl/steps/FileScanQueue.cpp | 2 +- .../services/scanner/impl/steps/IScanStep.hpp | 5 +- .../scanner/impl/steps/ScanErrorLogger.cpp | 93 +++++++ .../scanner/impl/steps/ScanErrorLogger.hpp | 43 ++++ .../steps/ScanStepArtistReconciliation.cpp | 1 + .../steps/ScanStepAssociateArtistImages.cpp | 2 + .../steps/ScanStepAssociateExternalLyrics.cpp | 2 + .../steps/ScanStepAssociatePlayListTracks.cpp | 29 ++- .../steps/ScanStepAssociateReleaseImages.cpp | 2 + .../scanner/impl/steps/ScanStepBase.cpp | 56 +++++ .../scanner/impl/steps/ScanStepBase.hpp | 20 +- .../steps/ScanStepCheckForDuplicatedFiles.cpp | 2 + .../steps/ScanStepCheckForRemovedFiles.cpp | 33 +-- .../steps/ScanStepCheckForRemovedFiles.hpp | 5 +- .../scanner/impl/steps/ScanStepCompact.cpp | 2 + .../steps/ScanStepComputeClusterStats.cpp | 2 + .../impl/steps/ScanStepDiscoverFiles.cpp | 15 +- .../scanner/impl/steps/ScanStepOptimize.cpp | 2 + .../steps/ScanStepRemoveOrphanedDbEntries.cpp | 2 + .../scanner/impl/steps/ScanStepScanFiles.cpp | 86 +++++-- .../scanner/impl/steps/ScanStepScanFiles.hpp | 2 +- .../steps/ScanStepUpdateLibraryFields.cpp | 1 + .../include/services/scanner/ScanErrors.hpp | 205 ++++++++++++++++ .../include/services/scanner/ScannerStats.hpp | 32 +-- src/lms/CMakeLists.txt | 1 + src/lms/ui/LmsApplication.cpp | 2 +- src/lms/ui/admin/ScannerController.cpp | 112 +-------- src/lms/ui/admin/ScannerReportResource.cpp | 168 +++++++++++++ src/lms/ui/admin/ScannerReportResource.hpp | 45 ++++ src/tools/metadata/LmsMetadata.cpp | 2 +- 86 files changed, 1794 insertions(+), 950 deletions(-) rename src/libs/metadata/impl/{ => avformat}/AvFormatImageReader.cpp (92%) rename src/libs/metadata/impl/{ => avformat}/AvFormatImageReader.hpp (94%) rename src/libs/metadata/impl/{ => avformat}/AvFormatTagReader.cpp (98%) rename src/libs/metadata/impl/{ => avformat}/AvFormatTagReader.hpp (93%) create mode 100644 src/libs/metadata/impl/avformat/Utils.cpp create mode 100644 src/libs/metadata/impl/avformat/Utils.hpp rename src/libs/metadata/impl/{ => taglib}/TagLibDefs.hpp (94%) rename src/libs/metadata/impl/{ => taglib}/TagLibImageReader.cpp (95%) rename src/libs/metadata/impl/{ => taglib}/TagLibImageReader.hpp (87%) rename src/libs/metadata/impl/{ => taglib}/TagLibTagReader.cpp (92%) rename src/libs/metadata/impl/{ => taglib}/TagLibTagReader.hpp (90%) create mode 100644 src/libs/metadata/impl/taglib/Utils.cpp create mode 100644 src/libs/metadata/impl/taglib/Utils.hpp create mode 100644 src/libs/services/scanner/impl/scanners/FileScanOperationBase.cpp create mode 100644 src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp create mode 100644 src/libs/services/scanner/impl/steps/ScanErrorLogger.cpp create mode 100644 src/libs/services/scanner/impl/steps/ScanErrorLogger.hpp create mode 100644 src/libs/services/scanner/include/services/scanner/ScanErrors.hpp create mode 100644 src/lms/ui/admin/ScannerReportResource.cpp create mode 100644 src/lms/ui/admin/ScannerReportResource.hpp diff --git a/approot/messages.xml b/approot/messages.xml index 1f775e67..056dddeb 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -100,9 +100,10 @@ Cannot get track duration +Unable to read the image at index {1} Cannot parse artist info file Cannot parse audio file -Cannot read file +Cannot read file ({1}) Cannot parse image file Cannot parse lyrics file Cannot parse playlist file @@ -115,6 +116,9 @@ Last scan Not available {1} files were scanned in {2} at {4} on {3} (UTC) - {5} errors, {6} duplicates +Missing artist name +Unable to find track paths +Unable to find the path "{1}" No audio track Duplicated file hash Duplicated track MBID diff --git a/approot/messages_es.xml b/approot/messages_es.xml index 7f47b917..325890c3 100644 --- a/approot/messages_es.xml +++ b/approot/messages_es.xml @@ -100,9 +100,10 @@ No ha sido posible obtener la duración de la pista +No se puede leer la imagen en el índice {1} No ha sido posible analizar el fichero de información sobre el artista No ha sido posible analizar el fichero de audio -No ha sido posible leer el fichero +No ha sido posible leer el fichero ({1}) No ha sido posible analizar del fichero de imagen No ha sido posible analizar el fichero de letras No ha sido posible analizar el fichero de la lista de distribución @@ -115,6 +116,9 @@ Último escaneo No disponible {1} ficheros analizados en {2} el {3} a las {4} (UTC) - {5} errores, {6} duplicados +Falta el nombre del artista +No se pudieron encontrar las rutas de las pistas +No se pudo encontrar la ruta "{1}" No es una pista de audio Hash del fichero duplicado MBID de la pista duplicado diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index de9c8039..5fea473e 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -100,9 +100,10 @@ Impossible de récupérer la durée de la piste +Impossible de lire l'image à l'indice {1} Impossible d'analyser le fichier d'informations sur l'artiste Impossible d'analyser le fichier audio -Impossible de lire le fichier +Impossible de lire le fichier ({1}) Impossible d'analyser le fichier image Impossible d'analyser le fichier de paroles Impossible d'analyser le fichier de liste de lecture @@ -115,6 +116,9 @@ Dernier scan Non disponible {1} fichiers analysés en {2} à {4} le {3} (UTC) - {5} erreurs, {6} duplicatas +Nom de l'artiste manquant +Impossible de trouver les chemins des pistes +Impossible de trouver le chemin "{1}" Pas de piste audio Hash dupliqué Track MBID dupliqué diff --git a/approot/messages_it.xml b/approot/messages_it.xml index 6545bb5d..91935ade 100644 --- a/approot/messages_it.xml +++ b/approot/messages_it.xml @@ -100,9 +100,10 @@ Non sono stato in grado di determinare la durata della traccia +Impossibile leggere l'immagine all'indice {1} Impossibile analizzare il file delle informazioni sull'artista Impossibile analizzare il file audio -Non in grado di leggere il file +Non in grado di leggere il file ({1}) Impossibile analizzare il file immagine Impossibile analizzare il file dei testi Impossibile analizzare il file della playlist @@ -115,6 +116,9 @@ Ultima scansione Non disponibile {1} file scansionati in {2} alle {3} il {4} (UTC) - {5} errori, {6} duplicati +Nome dell'artista mancante +Impossibile trovare i percorsi delle tracce +Impossibile trovare il percorso "{1}" Nessuna traccia audio Hash doppio Track MBID doppio diff --git a/approot/messages_pl.xml b/approot/messages_pl.xml index cb1af168..78c2d88b 100644 --- a/approot/messages_pl.xml +++ b/approot/messages_pl.xml @@ -101,9 +101,10 @@ Nie udało się ustalić długości ścieżki +Nie można odczytać obrazu pod indeksem {1} Nie można przetworzyć pliku z informacjami o artyście" Nie można przeanalizować pliku audio -Nie udało się odczytać pliku +Nie udało się odczytać pliku ({1}) Nie można przeanalizować pliku obrazu Nie można przetworzyć pliku z tekstem Nie można przeanalizować pliku playlisty @@ -124,6 +125,9 @@ Ostatnie skanowanie Niedostępne Przeskanowano {1} plików w {2} o {4} dnia {3} (UTC) - {5} błędów, {6} duplikatów +Brakuje nazwy artysty +Nie można znaleźć ścieżek utworów +Nie można znaleźć ścieżki „{1}” Brak ścieżki dźwiękowej Zduplikowny hash pliku Zduplikowany MBID pliku diff --git a/approot/messages_zh.xml b/approot/messages_zh.xml index 7b81cbe3..860bb5d2 100644 --- a/approot/messages_zh.xml +++ b/approot/messages_zh.xml @@ -100,9 +100,10 @@ 无法获得音轨时间 + 无法解析文件 -无法读取文件 +无法读取文件 ({1}) @@ -114,6 +115,9 @@ 最后扫描 不可用 已扫描 {1} 个文件，用时 {2}，在 {3} 的 {4} (UTC) - {5} 个错误，{6} 个重复项 + + + 没有音轨 相同文件哈希值 相同 MBID diff --git a/src/libs/core/impl/ArchiveZipper.cpp b/src/libs/core/impl/ArchiveZipper.cpp index 15042e87..9cdfb52a 100644 --- a/src/libs/core/impl/ArchiveZipper.cpp +++ b/src/libs/core/impl/ArchiveZipper.cpp @@ -22,12 +22,12 @@ #include #include #include +#include #include #include #include "core/ILogger.hpp" -#include "core/String.hpp" namespace lms::zip { @@ -43,9 +43,13 @@ namespace lms::zip : Exception{ "File '" + p.string() + "': " + std::string{ message } } { } + }; - FileException(const std::filesystem::path& p, std::string_view message, int err) - : Exception{ "File '" + p.string() + "': " + std::string{ message } + ": " + core::stringUtils::systemErrorToString(err) } + class FileStdException : public FileException + { + public: + FileStdException(const std::filesystem::path& p, std::string_view message, int err) + : FileException{ p, std::string{ message } + "': " + std::error_code{ err, std::generic_category() }.message() } { } }; @@ -240,7 +244,7 @@ namespace lms::zip std::ifstream ifs{ _currentEntry->filePath, std::ios_base::binary }; if (!ifs) - throw FileException{ _currentEntry->filePath, "cannot open file", errno }; + throw FileStdException{ _currentEntry->filePath, "cannot open file", errno }; ifs.seekg(0, std::ios::end); const std::uint64_t fileSize{ static_cast(ifs.tellg()) }; @@ -254,10 +258,10 @@ namespace lms::zip // read from file if (!ifs.seekg(_currentEntryOffset, std::ios::beg)) - throw FileException{ _currentEntry->filePath, "seek failed", errno }; + throw FileStdException{ _currentEntry->filePath, "seek failed", errno }; if (!ifs.read(reinterpret_cast(_readBuffer.data()), bytesToRead)) - throw FileException{ _currentEntry->filePath, "read failed", errno }; + throw FileStdException{ _currentEntry->filePath, "read failed", errno }; const std::uint64_t actualBytesRead{ static_cast(ifs.gcount()) }; diff --git a/src/libs/core/impl/ChildProcess.cpp b/src/libs/core/impl/ChildProcess.cpp index 0f7b2ef9..2dc57207 100644 --- a/src/libs/core/impl/ChildProcess.cpp +++ b/src/libs/core/impl/ChildProcess.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -35,7 +36,6 @@ #include #include "core/ILogger.hpp" -#include "core/String.hpp" namespace lms::core { @@ -44,8 +44,8 @@ namespace lms::core class SystemException : public ChildProcessException { public: - SystemException(int err, const std::string& errMsg) - : ChildProcessException{ errMsg + ": " + stringUtils::systemErrorToString(err) } + SystemException(std::error_code err, const std::string& errMsg) + : ChildProcessException{ errMsg + ": " + err.message() } { } @@ -69,20 +69,20 @@ namespace lms::core // Use 'pipe' instead of 'pipe2', more portable if (pipe(pipefd) < 0) - throw SystemException{ errno, "pipe failed!" }; + throw SystemException{ std::error_code{ errno, std::generic_category() }, "pipe failed!" }; // Manually set the O_NONBLOCK and O_CLOEXEC flags for both ends of the pipe if (fcntl(pipefd[0], F_SETFL, O_NONBLOCK) == -1) - throw SystemException{ errno, "fcntl failed to set O_NONBLOCK!" }; + throw SystemException{ std::error_code{ errno, std::generic_category() }, "fcntl failed to set O_NONBLOCK!" }; if (fcntl(pipefd[1], F_SETFL, O_NONBLOCK) == -1) - throw SystemException{ errno, "fcntl failed to set O_NONBLOCK!" }; + throw SystemException{ std::error_code{ errno, std::generic_category() }, "fcntl failed to set O_NONBLOCK!" }; if (fcntl(pipefd[0], F_SETFD, FD_CLOEXEC) == -1) - throw SystemException{ errno, "fcntl failed to set FD_CLOEXEC!" }; + throw SystemException{ std::error_code{ errno, std::generic_category() }, "fcntl failed to set FD_CLOEXEC!" }; if (fcntl(pipefd[1], F_SETFD, FD_CLOEXEC) == -1) - throw SystemException{ errno, "fcntl failed to set FD_CLOEXEC!" }; + throw SystemException{ std::error_code{ errno, std::generic_category() }, "fcntl failed to set FD_CLOEXEC!" }; #if defined(__linux__) && defined(F_SETPIPE_SZ) for (const int fd : { pipefd[0], pipefd[1] }) @@ -94,7 +94,7 @@ namespace lms::core if (pipeSizeRes == -1) { const int err{ errno }; - LMS_LOG(CHILDPROCESS, DEBUG, "F_GETPIPE_SZ failed: " << stringUtils::systemErrorToString(err)); + LMS_LOG(CHILDPROCESS, DEBUG, "F_GETPIPE_SZ failed: " << (std::error_code{ err, std::generic_category() }.message())); } else { @@ -103,10 +103,10 @@ namespace lms::core #endif if (currentPipeSize < targetPipeSize) { - if (fcntl(fd, F_SETPIPE_SZ, targetPipeSize) == -1) + if (::fcntl(fd, F_SETPIPE_SZ, targetPipeSize) == -1) { const int err{ errno }; - LMS_LOG(CHILDPROCESS, DEBUG, "F_SETPIPE_SZ failed: " << stringUtils::systemErrorToString(err)); + LMS_LOG(CHILDPROCESS, DEBUG, "F_SETPIPE_SZ failed: " << (std::error_code{ err, std::generic_category() }.message())); } } } @@ -114,7 +114,7 @@ namespace lms::core int res{ fork() }; if (res == -1) - throw SystemException{ errno, "fork failed!" }; + throw SystemException{ std::error_code{ errno, std::generic_category() }, "fork failed!" }; if (res == 0) // CHILD { @@ -168,7 +168,10 @@ namespace lms::core // process may already have finished LMS_LOG(CHILDPROCESS, DEBUG, "Killing child process..."); if (::kill(_childPID, SIGKILL) == -1) - LMS_LOG(CHILDPROCESS, DEBUG, "Kill failed: " << stringUtils::systemErrorToString(errno)); + { + const int err{ errno }; + LMS_LOG(CHILDPROCESS, DEBUG, "Kill failed: " << (std::error_code{ err, std::generic_category() }.message())); + } } bool ChildProcess::wait(bool block) @@ -179,7 +182,7 @@ namespace lms::core const pid_t pid{ waitpid(_childPID, &wstatus, block ? 0 : WNOHANG) }; if (pid == -1) - throw SystemException{ errno, "waitpid failed!" }; + throw SystemException{ std::error_code{ errno, std::generic_category() }, "waitpid failed!" }; if (pid == 0) return false; diff --git a/src/libs/core/impl/Path.cpp b/src/libs/core/impl/Path.cpp index d0f983e7..248042c7 100644 --- a/src/libs/core/impl/Path.cpp +++ b/src/libs/core/impl/Path.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -66,16 +67,24 @@ namespace lms::core::pathUtils return std::filesystem::create_directory(dir); } - Wt::WDateTime getLastWriteTime(const std::filesystem::path& file) + Wt::WDateTime getLastWriteTime(const std::filesystem::path& file, std::error_code& ec) { + Wt::WDateTime res; + struct stat sb { }; - if (stat(file.c_str(), &sb) == -1) - throw LmsException("Failed to get stats on file '" + file.string() + "'"); + { + ec = std::error_code{ errno, std::generic_category() }; + } + else + { + ec = std::error_code{}; + res = Wt::WDateTime::fromTime_t(sb.st_mtime); + } - return Wt::WDateTime::fromTime_t(sb.st_mtime); + return res; } bool exploreFilesRecursive(const std::filesystem::path& directory, std::function cb, const std::filesystem::path* excludeDirFileName) diff --git a/src/libs/core/impl/String.cpp b/src/libs/core/impl/String.cpp index 8258348a..d976d26d 100644 --- a/src/libs/core/impl/String.cpp +++ b/src/libs/core/impl/String.cpp @@ -20,7 +20,6 @@ #include "core/String.hpp" #include -#include #include #include #include @@ -509,11 +508,4 @@ namespace lms::core::stringUtils return "[" + std::to_string(mins) + ":" + (secs < 10 ? "0" : "") + std::to_string(secs) + "." + (millis < 100 ? (millis < 10 ? "00" : "0") : "") + std::to_string(millis) + "]"; } - - std::string systemErrorToString(int err) - { - std::array buffer{}; - ::strerror_r(err, buffer.data(), buffer.size()); - return std::string{ buffer.data() }; - } } // namespace lms::core::stringUtils diff --git a/src/libs/core/include/core/Path.hpp b/src/libs/core/include/core/Path.hpp index 500c9af3..1d9688bd 100644 --- a/src/libs/core/include/core/Path.hpp +++ b/src/libs/core/include/core/Path.hpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include @@ -35,7 +35,7 @@ namespace lms::core::pathUtils bool ensureDirectory(const std::filesystem::path& dir); // Get the last write time since Epoch - Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir); + Wt::WDateTime getLastWriteTime(const std::filesystem::path& file, std::error_code& ec); // returns false if aborted by user bool exploreFilesRecursive(const std::filesystem::path& directory, std::function cb, const std::filesystem::path* excludeDirFileName = {}); diff --git a/src/libs/core/include/core/String.hpp b/src/libs/core/include/core/String.hpp index 7c99a7cc..b65df63a 100644 --- a/src/libs/core/include/core/String.hpp +++ b/src/libs/core/include/core/String.hpp @@ -119,6 +119,4 @@ namespace lms::core::stringUtils // to "[minutes:seconds.milliseconds]" std::string formatTimestamp(std::chrono::milliseconds timestamp); - - std::string systemErrorToString(int err); } // namespace lms::core::stringUtils \ No newline at end of file diff --git a/src/libs/metadata/CMakeLists.txt b/src/libs/metadata/CMakeLists.txt index d5dfc9fd..ea401ef1 100644 --- a/src/libs/metadata/CMakeLists.txt +++ b/src/libs/metadata/CMakeLists.txt @@ -11,12 +11,14 @@ endif() add_library(lmsmetadata STATIC impl/ArtistInfo.cpp impl/AudioFileParser.cpp - impl/AvFormatImageReader.cpp - impl/AvFormatTagReader.cpp + impl/avformat/AvFormatImageReader.cpp + impl/avformat/AvFormatTagReader.cpp + impl/avformat/Utils.cpp impl/Lyrics.cpp impl/PlayList.cpp - impl/TagLibImageReader.cpp - impl/TagLibTagReader.cpp + impl/taglib/TagLibImageReader.cpp + impl/taglib/TagLibTagReader.cpp + impl/taglib/Utils.cpp impl/Utils.cpp ) @@ -26,6 +28,7 @@ target_include_directories(lmsmetadata INTERFACE target_include_directories(lmsmetadata PRIVATE include + impl ) target_link_libraries(lmsmetadata PRIVATE diff --git a/src/libs/metadata/impl/ArtistInfo.cpp b/src/libs/metadata/impl/ArtistInfo.cpp index 639bbd42..85b4e96b 100644 --- a/src/libs/metadata/impl/ArtistInfo.cpp +++ b/src/libs/metadata/impl/ArtistInfo.cpp @@ -27,10 +27,10 @@ namespace lms::metadata { - std::span getSupportedInfoFileExtensions() + std::span getSupportedArtistInfoFiles() { - static const std::array fileExtensions{ ".nfo" }; - return fileExtensions; + static const std::array files{ "artist.nfo" }; + return files; } ArtistInfo parseArtistInfo(std::istream& is) @@ -56,7 +56,7 @@ namespace lms::metadata } catch (boost::property_tree::ptree_error& error) { - LMS_LOG(RECOMMENDATION, ERROR, "Cannot read artist xml info: " << error.what()); + LMS_LOG(METADATA, ERROR, "Cannot read artist xml info: " << error.what()); throw ArtistInfoParseException{ error.what() }; } } diff --git a/src/libs/metadata/impl/AudioFileParser.cpp b/src/libs/metadata/impl/AudioFileParser.cpp index a75d7c03..29cfa356 100644 --- a/src/libs/metadata/impl/AudioFileParser.cpp +++ b/src/libs/metadata/impl/AudioFileParser.cpp @@ -28,11 +28,13 @@ #include "core/String.hpp" #include "metadata/Exception.hpp" -#include "AvFormatImageReader.hpp" -#include "AvFormatTagReader.hpp" -#include "TagLibImageReader.hpp" -#include "TagLibTagReader.hpp" #include "Utils.hpp" +#include "avformat/AvFormatImageReader.hpp" +#include "avformat/AvFormatTagReader.hpp" +#include "avformat/Utils.hpp" +#include "taglib/TagLibImageReader.hpp" +#include "taglib/TagLibTagReader.hpp" +#include "taglib/Utils.hpp" namespace lms::metadata { @@ -338,82 +340,56 @@ namespace lms::metadata std::span AudioFileParser::getSupportedExtensions() const { - // TODO: use backend capability to retrieve supported formats - static const std::array fileExtensions{ - ".aac", - ".alac", - ".aif", - ".aiff", - ".ape", - ".dsf", - ".flac", - ".m4a", - ".m4b", - ".mp3", - ".mpc", - ".oga", - ".ogg", - ".opus", - ".shn", - ".wav", - ".wma", - ".wv", - }; - return fileExtensions; + switch (_params.backend) + { + case ParserBackend::TagLib: + return taglib::utils::getSupportedExtensions(); + break; + + case ParserBackend::AvFormat: + return avformat::utils::getSupportedExtensions(); + break; + } + + return {}; } std::unique_ptr AudioFileParser::parseMetaData(const std::filesystem::path& p) const { - try + std::unique_ptr tagReader; + switch (_params.backend) { - std::unique_ptr tagReader; - switch (_params.backend) - { - case ParserBackend::TagLib: - tagReader = std::make_unique(p, _params.readStyle, _params.debug); - break; + case ParserBackend::TagLib: + tagReader = std::make_unique(p, _params.readStyle, _params.debug); + break; - case ParserBackend::AvFormat: - tagReader = std::make_unique(p, _params.debug); - break; - } - if (!tagReader) - throw ParseException{ "Unhandled parser backend" }; + case ParserBackend::AvFormat: + tagReader = std::make_unique(p, _params.debug); + break; + } + if (!tagReader) + throw AudioFileParsingException{ "Unhandled parser backend" }; - return parseMetaData(*tagReader); - } - catch (const Exception& e) - { - LMS_LOG(METADATA, ERROR, "File " << p << ": metadata parsing failed"); - throw ParseException{}; - } + return parseMetaData(*tagReader); } void AudioFileParser::parseImages(const std::filesystem::path& p, ImageVisitor visitor) const { - try + std::unique_ptr imageReader; + switch (_params.backend) { - std::unique_ptr imageReader; - switch (_params.backend) - { - case ParserBackend::TagLib: - imageReader = std::make_unique(p); - break; + case ParserBackend::TagLib: + imageReader = std::make_unique(p); + break; - case ParserBackend::AvFormat: - imageReader = std::make_unique(p); - break; - } - if (!imageReader) - throw ParseException{ "Unhandled parser backend" }; + case ParserBackend::AvFormat: + imageReader = std::make_unique(p); + break; + } + if (!imageReader) + throw AudioFileParsingException{ "Unhandled parser backend" }; - parseImages(*imageReader, std::move(visitor)); - } - catch (const Exception& e) - { - LMS_LOG(METADATA, ERROR, "File " << p << ": image parsing failed"); - throw ParseException{}; - } + parseImages(*imageReader, std::move(visitor)); } std::unique_ptr AudioFileParser::parseMetaData(const ITagReader& tagReader) const diff --git a/src/libs/metadata/impl/ITagReader.hpp b/src/libs/metadata/impl/ITagReader.hpp index 7526815a..0b4bc8db 100644 --- a/src/libs/metadata/impl/ITagReader.hpp +++ b/src/libs/metadata/impl/ITagReader.hpp @@ -19,7 +19,6 @@ #pragma once -#include #include #include "metadata/Types.hpp" diff --git a/src/libs/metadata/impl/Utils.cpp b/src/libs/metadata/impl/Utils.cpp index 1f7158a9..1152b95e 100644 --- a/src/libs/metadata/impl/Utils.cpp +++ b/src/libs/metadata/impl/Utils.cpp @@ -108,7 +108,7 @@ namespace lms::metadata::utils return "accurate"; } - throw core::LmsException{ "Unknown read style" }; + throw Exception{ "Unknown read style" }; } PerformerArtist extractPerformerAndRole(std::string_view entry) diff --git a/src/libs/metadata/impl/AvFormatImageReader.cpp b/src/libs/metadata/impl/avformat/AvFormatImageReader.cpp similarity index 92% rename from src/libs/metadata/impl/AvFormatImageReader.cpp rename to src/libs/metadata/impl/avformat/AvFormatImageReader.cpp index a0c4a97a..7ec95d1c 100644 --- a/src/libs/metadata/impl/AvFormatImageReader.cpp +++ b/src/libs/metadata/impl/avformat/AvFormatImageReader.cpp @@ -18,10 +18,12 @@ */ #include "AvFormatImageReader.hpp" + #include "av/IAudioFile.hpp" #include "av/Types.hpp" +#include "metadata/Exception.hpp" -namespace lms::metadata +namespace lms::metadata::avformat { AvFormatImageReader::AvFormatImageReader(const std::filesystem::path& p) { @@ -31,7 +33,7 @@ namespace lms::metadata } catch (av::Exception& e) { - throw ParseException{}; + throw AudioFileParsingException{ e.what() }; } } @@ -56,4 +58,4 @@ namespace lms::metadata }); } -} // namespace lms::metadata +} // namespace lms::metadata::avformat diff --git a/src/libs/metadata/impl/AvFormatImageReader.hpp b/src/libs/metadata/impl/avformat/AvFormatImageReader.hpp similarity index 94% rename from src/libs/metadata/impl/AvFormatImageReader.hpp rename to src/libs/metadata/impl/avformat/AvFormatImageReader.hpp index 033d1e45..0658b8a7 100644 --- a/src/libs/metadata/impl/AvFormatImageReader.hpp +++ b/src/libs/metadata/impl/avformat/AvFormatImageReader.hpp @@ -26,7 +26,7 @@ namespace lms::av class IAudioFile; } -namespace lms::metadata +namespace lms::metadata::avformat { class AvFormatImageReader : public IImageReader { @@ -42,4 +42,4 @@ namespace lms::metadata std::unique_ptr _audioFile; }; -} // namespace lms::metadata +} // namespace lms::metadata::avformat diff --git a/src/libs/metadata/impl/AvFormatTagReader.cpp b/src/libs/metadata/impl/avformat/AvFormatTagReader.cpp similarity index 98% rename from src/libs/metadata/impl/AvFormatTagReader.cpp rename to src/libs/metadata/impl/avformat/AvFormatTagReader.cpp index aa3feb0f..04f50f62 100644 --- a/src/libs/metadata/impl/AvFormatTagReader.cpp +++ b/src/libs/metadata/impl/avformat/AvFormatTagReader.cpp @@ -25,7 +25,7 @@ #include "core/String.hpp" #include "metadata/Exception.hpp" -namespace lms::metadata +namespace lms::metadata::avformat { namespace { @@ -165,10 +165,12 @@ namespace lms::metadata } catch (av::Exception& e) { - throw ParseException{}; + throw AudioFileParsingException{ e.what() }; } } + AvFormatTagReader::~AvFormatTagReader() = default; + void AvFormatTagReader::visitTagValues(TagType tag, TagValueVisitor visitor) const { auto itTagNames{ avFormatTagMapping.find(tag) }; @@ -223,4 +225,4 @@ namespace lms::metadata visitor("", value); }); } -} // namespace lms::metadata +} // namespace lms::metadata::avformat diff --git a/src/libs/metadata/impl/AvFormatTagReader.hpp b/src/libs/metadata/impl/avformat/AvFormatTagReader.hpp similarity index 93% rename from src/libs/metadata/impl/AvFormatTagReader.hpp rename to src/libs/metadata/impl/avformat/AvFormatTagReader.hpp index a6cd7c4f..44b38c38 100644 --- a/src/libs/metadata/impl/AvFormatTagReader.hpp +++ b/src/libs/metadata/impl/avformat/AvFormatTagReader.hpp @@ -25,13 +25,13 @@ #include "ITagReader.hpp" -namespace lms::metadata +namespace lms::metadata::avformat { class AvFormatTagReader : public ITagReader { public: AvFormatTagReader(const std::filesystem::path& path, bool debug); - ~AvFormatTagReader() override = default; + ~AvFormatTagReader() override; AvFormatTagReader(const AvFormatTagReader&) = delete; AvFormatTagReader& operator=(const AvFormatTagReader&) = delete; @@ -46,4 +46,4 @@ namespace lms::metadata av::IAudioFile::MetadataMap _metaDataMap; av::ContainerInfo _containerInfo; }; -} // namespace lms::metadata +} // namespace lms::metadata::avformat diff --git a/src/libs/metadata/impl/avformat/Utils.cpp b/src/libs/metadata/impl/avformat/Utils.cpp new file mode 100644 index 00000000..83055d2f --- /dev/null +++ b/src/libs/metadata/impl/avformat/Utils.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "Utils.hpp" + +namespace lms::metadata::avformat::utils +{ + std::span getSupportedExtensions() + { + // TODO: use av capability to retrieve supported formats + static const std::array fileExtensions{ + ".aac", + ".alac", + ".aif", + ".aiff", + ".ape", + ".dsf", + ".flac", + ".m4a", + ".m4b", + ".mp3", + ".mpc", + ".oga", + ".ogg", + ".opus", + ".shn", + ".wav", + ".wma", + ".wv", + }; + return fileExtensions; + } +} // namespace lms::metadata::avformat::utils \ No newline at end of file diff --git a/src/libs/metadata/impl/avformat/Utils.hpp b/src/libs/metadata/impl/avformat/Utils.hpp new file mode 100644 index 00000000..b4f655fa --- /dev/null +++ b/src/libs/metadata/impl/avformat/Utils.hpp @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +namespace lms::metadata::avformat::utils +{ + std::span getSupportedExtensions(); +} // namespace lms::metadata::avformat::utils \ No newline at end of file diff --git a/src/libs/metadata/impl/TagLibDefs.hpp b/src/libs/metadata/impl/taglib/TagLibDefs.hpp similarity index 94% rename from src/libs/metadata/impl/TagLibDefs.hpp rename to src/libs/metadata/impl/taglib/TagLibDefs.hpp index 69e97a8a..4f13bf3e 100644 --- a/src/libs/metadata/impl/TagLibDefs.hpp +++ b/src/libs/metadata/impl/taglib/TagLibDefs.hpp @@ -21,6 +21,10 @@ #include +#if (TAGLIB_MAJOR_VERSION > 2) + #define TAGLIB_HAS_DSF 1 +#endif + // TAGLIB_HAS_MP4_ITEM_TYPE if version >= 2.0.1 #if ((TAGLIB_MAJOR_VERSION > 2) || (TAGLIB_MAJOR_VERSION == 2 && TAGLIB_MINOR_VERSION > 0) || (TAGLIB_MAJOR_VERSION == 2 && TAGLIB_PATCH_VERSION >= 1)) #define TAGLIB_HAS_MP4_ITEM_TYPE 1 diff --git a/src/libs/metadata/impl/TagLibImageReader.cpp b/src/libs/metadata/impl/taglib/TagLibImageReader.cpp similarity index 95% rename from src/libs/metadata/impl/TagLibImageReader.cpp rename to src/libs/metadata/impl/taglib/TagLibImageReader.cpp index 8dba0b9b..a5d7f940 100644 --- a/src/libs/metadata/impl/TagLibImageReader.cpp +++ b/src/libs/metadata/impl/taglib/TagLibImageReader.cpp @@ -19,6 +19,8 @@ #include "TagLibImageReader.hpp" +#include "TagLibDefs.hpp" + #include #include #include @@ -34,24 +36,16 @@ #include #include #include -#include #include "core/ILogger.hpp" -#include "core/ITraceLogger.hpp" - -#include "core/String.hpp" #include "metadata/Exception.hpp" -#include "TagLibDefs.hpp" +#include "taglib/Utils.hpp" -namespace lms::metadata +namespace lms::metadata::taglib { namespace { - class ImageParsingFailedException : public Exception - { - }; - Image::Type imageTypeFromfromIDv2(TagLib::ID3v2::AttachedPictureFrame::Type type) { switch (type) @@ -349,44 +343,37 @@ namespace lms::metadata } } // namespace - TagLib::FileRef parseFile(const std::filesystem::path& p) - { - LMS_SCOPED_TRACE_DETAILED("MetaData", "TagLibParseFileForImages"); - - return TagLib::FileRef{ p.c_str(), false }; - } - TagLibImageReader::TagLibImageReader(const std::filesystem::path& p) - : _file{ parseFile(p) } + : _file{ utils::parseFile(p, TagLib::AudioProperties::ReadStyle::Fast, utils::ReadAudioProperties{ false }) } { - if (_file.isNull()) + if (!_file) { LMS_LOG(METADATA, ERROR, "File " << p << ": parsing failed"); - throw ImageParsingFailedException{}; + throw AudioFileParsingException{}; } } void TagLibImageReader::visitImages(ImageVisitor visitor) const { // MP3 - if (TagLib::MPEG::File * mp3File{ dynamic_cast(_file.file()) }) + if (TagLib::MPEG::File * mp3File{ dynamic_cast(_file.get()) }) { if (mp3File->hasID3v2Tag()) visitID3V2Images(*mp3File->ID3v2Tag(), std::move(visitor)); } // MP4 - else if (TagLib::MP4::File * mp4File{ dynamic_cast(_file.file()) }) + else if (TagLib::MP4::File * mp4File{ dynamic_cast(_file.get()) }) { visitMP4Images(*mp4File, std::move(visitor)); } // WMA - else if (TagLib::ASF::File * asfFile{ dynamic_cast(_file.file()) }) + else if (TagLib::ASF::File * asfFile{ dynamic_cast(_file.get()) }) { if (const TagLib::ASF::Tag * tag{ asfFile->tag() }) visitASFImages(*tag, std::move(visitor)); } // FLAC - else if (TagLib::FLAC::File * flacFile{ dynamic_cast(_file.file()) }) + else if (TagLib::FLAC::File * flacFile{ dynamic_cast(_file.get()) }) { if (flacFile->hasID3v2Tag()) // usage discouraged visitID3V2Images(*flacFile->ID3v2Tag(), std::move(visitor)); @@ -394,38 +381,38 @@ namespace lms::metadata visitFLACImages(flacFile->pictureList(), std::move(visitor)); } // Ogg vorbis - else if (TagLib::Ogg::Vorbis::File * vorbisFile{ dynamic_cast(_file.file()) }) + else if (TagLib::Ogg::Vorbis::File * vorbisFile{ dynamic_cast(_file.get()) }) { visitFLACImages(vorbisFile->tag()->pictureList(), std::move(visitor)); } // Ogg Opus - else if (TagLib::Ogg::Opus::File * opusFile{ dynamic_cast(_file.file()) }) + else if (TagLib::Ogg::Opus::File * opusFile{ dynamic_cast(_file.get()) }) { visitFLACImages(opusFile->tag()->pictureList(), std::move(visitor)); } // Aiff - else if (TagLib::RIFF::AIFF::File * aiffFile{ dynamic_cast(_file.file()) }) + else if (TagLib::RIFF::AIFF::File * aiffFile{ dynamic_cast(_file.get()) }) { if (aiffFile->hasID3v2Tag()) visitID3V2Images(*aiffFile->tag(), std::move(visitor)); } // Wav - else if (TagLib::RIFF::WAV::File * wavFile{ dynamic_cast(_file.file()) }) + else if (TagLib::RIFF::WAV::File * wavFile{ dynamic_cast(_file.get()) }) { if (wavFile->hasID3v2Tag()) visitID3V2Images(*wavFile->ID3v2Tag(), std::move(visitor)); } // MPC - else if (TagLib::MPC::File * mpcFile{ dynamic_cast(_file.file()) }) + else if (TagLib::MPC::File * mpcFile{ dynamic_cast(_file.get()) }) { if (mpcFile->hasAPETag()) visitAPEImages(*mpcFile->APETag(), std::move(visitor)); } // WavPack - else if (TagLib::WavPack::File * wavPackFile{ dynamic_cast(_file.file()) }) + else if (TagLib::WavPack::File * wavPackFile{ dynamic_cast(_file.get()) }) { if (wavPackFile->hasAPETag()) visitAPEImages(*wavPackFile->APETag(), std::move(visitor)); } } -} // namespace lms::metadata +} // namespace lms::metadata::taglib diff --git a/src/libs/metadata/impl/TagLibImageReader.hpp b/src/libs/metadata/impl/taglib/TagLibImageReader.hpp similarity index 87% rename from src/libs/metadata/impl/TagLibImageReader.hpp rename to src/libs/metadata/impl/taglib/TagLibImageReader.hpp index 075169af..c0c5bea4 100644 --- a/src/libs/metadata/impl/TagLibImageReader.hpp +++ b/src/libs/metadata/impl/taglib/TagLibImageReader.hpp @@ -21,9 +21,9 @@ #include "IImageReader.hpp" -#include +#include -namespace lms::metadata +namespace lms::metadata::taglib { class TagLibImageReader : public IImageReader { @@ -32,6 +32,6 @@ namespace lms::metadata void visitImages(ImageVisitor visitor) const override; - const TagLib::FileRef _file; + std::unique_ptr _file; }; -} // namespace lms::metadata +} // namespace lms::metadata::taglib diff --git a/src/libs/metadata/impl/TagLibTagReader.cpp b/src/libs/metadata/impl/taglib/TagLibTagReader.cpp similarity index 92% rename from src/libs/metadata/impl/TagLibTagReader.cpp rename to src/libs/metadata/impl/taglib/TagLibTagReader.cpp index 1ea6e8ea..a7ae70e0 100644 --- a/src/libs/metadata/impl/TagLibTagReader.cpp +++ b/src/libs/metadata/impl/taglib/TagLibTagReader.cpp @@ -19,44 +19,49 @@ #include "TagLibTagReader.hpp" -#include #include +#include "TagLibDefs.hpp" + #include +#include #include #include #include -#if TAGLIB_MAJOR_VERSION >= 2 - #include -#endif -#include #include #include #include #include #include +#include #include +#include #include #include #include +#include #include #include #include #include +#if TAGLIB_HAS_DSF + #include +#endif #include "core/ILogger.hpp" -#include "core/ITraceLogger.hpp" #include "core/String.hpp" #include "metadata/Exception.hpp" -#include "TagLibDefs.hpp" +#include "Utils.hpp" -namespace lms::metadata +namespace lms::metadata::taglib { namespace { class TagParsingFailedException : public Exception { + public: + using Exception::Exception; }; // Mapping to internal taglib names and/or common alternative custom names @@ -166,21 +171,6 @@ namespace lms::metadata { TagType::Writer, { "WRITER" } }, }; - TagLib::AudioProperties::ReadStyle readStyleToTagLibReadStyle(ParserReadStyle readStyle) - { - switch (readStyle) - { - case ParserReadStyle::Fast: - return TagLib::AudioProperties::ReadStyle::Fast; - case ParserReadStyle::Average: - return TagLib::AudioProperties::ReadStyle::Average; - case ParserReadStyle::Accurate: - return TagLib::AudioProperties::ReadStyle::Accurate; - } - - throw core::LmsException{ "Cannot convert read style" }; - } - void mergeTagMaps(TagLib::PropertyMap& dst, TagLib::PropertyMap&& src) { for (auto&& [tag, values] : src) @@ -211,35 +201,26 @@ namespace lms::metadata } } } - - TagLib::FileRef parseFile(const std::filesystem::path& p, ParserReadStyle parserReadStyle) - { - LMS_SCOPED_TRACE_DETAILED("MetaData", "TagLibParseFileForMetaData"); - - return TagLib::FileRef{ p.c_str(), true // read audio properties - , - readStyleToTagLibReadStyle(parserReadStyle) }; - } } // namespace TagLibTagReader::TagLibTagReader(const std::filesystem::path& p, ParserReadStyle parserReadStyle, bool debug) - : _file{ parseFile(p, parserReadStyle) } + : _file{ utils::parseFile(p, utils::readStyleToTagLibReadStyle(parserReadStyle), utils::ReadAudioProperties{ true }) } { - if (_file.isNull()) + if (!_file) { LMS_LOG(METADATA, ERROR, "File " << p << ": parsing failed"); - throw TagParsingFailedException{}; + throw AudioFileParsingException{ "Parsing failed" }; } - if (!_file.audioProperties()) + if (!_file->audioProperties()) { LMS_LOG(METADATA, ERROR, "File " << p << ": no audio properties"); - throw TagParsingFailedException{}; + throw AudioFileNoAudioPropertiesException{}; } computeAudioProperties(); - _propertyMap = _file.file()->properties(); + _propertyMap = _file->properties(); if (debug && core::Service::get()->isSeverityActive(core::logging::Severity::DEBUG)) { @@ -319,7 +300,7 @@ namespace lms::metadata }; // WMA - if (TagLib::ASF::File * asfFile{ dynamic_cast(_file.file()) }) + if (TagLib::ASF::File * asfFile{ dynamic_cast(_file.get()) }) { if (const TagLib::ASF::Tag * tag{ asfFile->tag() }) { @@ -365,7 +346,7 @@ namespace lms::metadata } } // MP3 - else if (TagLib::MPEG::File * mp3File{ dynamic_cast(_file.file()) }) + else if (TagLib::MPEG::File * mp3File{ dynamic_cast(_file.get()) }) { if (mp3File->hasID3v2Tag()) processID3v2Tags(*mp3File->ID3v2Tag()); @@ -373,7 +354,7 @@ namespace lms::metadata getAPETags(mp3File->APETag()); } // MP4 - else if (TagLib::MP4::File * mp4File{ dynamic_cast(_file.file()) }) + else if (TagLib::MP4::File * mp4File{ dynamic_cast(_file.get()) }) { // Taglib does not expose rtng in properties if (const TagLib::MP4::Item rtngItem{ mp4File->tag()->item("rtng") }; rtngItem.isValid()) @@ -406,40 +387,42 @@ namespace lms::metadata } } // MPC - else if (TagLib::MPC::File * mpcFile{ dynamic_cast(_file.file()) }) + else if (TagLib::MPC::File * mpcFile{ dynamic_cast(_file.get()) }) { getAPETags(mpcFile->APETag()); } // WavPack - else if (TagLib::WavPack::File * wavPackFile{ dynamic_cast(_file.file()) }) + else if (TagLib::WavPack::File * wavPackFile{ dynamic_cast(_file.get()) }) { getAPETags(wavPackFile->APETag()); } // FLAC - else if (TagLib::FLAC::File * flacFile{ dynamic_cast(_file.file()) }) + else if (TagLib::FLAC::File * flacFile{ dynamic_cast(_file.get()) }) { if (flacFile->hasID3v2Tag()) // discouraged usage processID3v2Tags(*flacFile->ID3v2Tag()); } - else if (TagLib::RIFF::AIFF::File * aiffFile{ dynamic_cast(_file.file()) }) + else if (TagLib::RIFF::AIFF::File * aiffFile{ dynamic_cast(_file.get()) }) { if (aiffFile->hasID3v2Tag()) processID3v2Tags(*aiffFile->tag()); } - else if (TagLib::RIFF::WAV::File * wavFile{ dynamic_cast(_file.file()) }) + else if (TagLib::RIFF::WAV::File * wavFile{ dynamic_cast(_file.get()) }) { if (wavFile->hasID3v2Tag()) processID3v2Tags(*wavFile->ID3v2Tag()); } } + TagLibTagReader::~TagLibTagReader() = default; + void TagLibTagReader::computeAudioProperties() { - const TagLib::AudioProperties* properties{ _file.audioProperties() }; + const TagLib::AudioProperties* properties{ _file->audioProperties() }; // Common properties _audioProperties.bitrate = static_cast(properties->bitrate() * 1000); - _audioProperties.channelCount = static_cast(_file.audioProperties()->channels()); + _audioProperties.channelCount = static_cast(_file->audioProperties()->channels()); _audioProperties.duration = std::chrono::milliseconds{ properties->lengthInMilliseconds() }; _audioProperties.sampleRate = static_cast(properties->sampleRate()); @@ -457,7 +440,7 @@ namespace lms::metadata _audioProperties.bitsPerSample = aiffProperties->bitsPerSample(); else if (const auto* wavProperties{ dynamic_cast(properties) }) _audioProperties.bitsPerSample = wavProperties->bitsPerSample(); -#if TAGLIB_MAJOR_VERSION >= 2 +#if TAGLIB_HAS_DSF else if (const auto* dsfProperties{ dynamic_cast(properties) }) _audioProperties.bitsPerSample = dsfProperties->bitsPerSample(); #endif @@ -534,4 +517,4 @@ namespace lms::metadata }); } } -} // namespace lms::metadata +} // namespace lms::metadata::taglib diff --git a/src/libs/metadata/impl/TagLibTagReader.hpp b/src/libs/metadata/impl/taglib/TagLibTagReader.hpp similarity index 90% rename from src/libs/metadata/impl/TagLibTagReader.hpp rename to src/libs/metadata/impl/taglib/TagLibTagReader.hpp index 43d938a7..07034a38 100644 --- a/src/libs/metadata/impl/TagLibTagReader.hpp +++ b/src/libs/metadata/impl/taglib/TagLibTagReader.hpp @@ -21,20 +21,21 @@ #include #include +#include #include -#include +#include #include #include "ITagReader.hpp" -namespace lms::metadata +namespace lms::metadata::taglib { class TagLibTagReader : public ITagReader { public: TagLibTagReader(const std::filesystem::path& path, ParserReadStyle parserReadStyle, bool debug); - ~TagLibTagReader() = default; + ~TagLibTagReader() override; TagLibTagReader(const TagLibTagReader&) = delete; TagLibTagReader& operator=(const TagLibTagReader&) = delete; @@ -47,9 +48,9 @@ namespace lms::metadata const AudioProperties& getAudioProperties() const override { return _audioProperties; } - const TagLib::FileRef _file; + std::unique_ptr _file; AudioProperties _audioProperties; TagLib::PropertyMap _propertyMap; // case-insensitive keys std::multimap _id3v2Lyrics; }; -} // namespace lms::metadata +} // namespace lms::metadata::taglib diff --git a/src/libs/metadata/impl/taglib/Utils.cpp b/src/libs/metadata/impl/taglib/Utils.cpp new file mode 100644 index 00000000..17e0f93a --- /dev/null +++ b/src/libs/metadata/impl/taglib/Utils.cpp @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "Utils.hpp" + +#include "TagLibDefs.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if TAGLIB_HAS_DSF + #include +#endif + +#include "core/ILogger.hpp" +#include "core/ITraceLogger.hpp" + +namespace lms::metadata::taglib::utils +{ + std::span getSupportedExtensions() + { + static const std::vector supportedExtensions + { + ".mp3", ".mp2", ".aac", ".ogg", ".oga", ".flac", ".spx", ".opus", + ".mpc", ".wv", ".ape", ".tta", ".m4a", ".m4r", ".m4b", ".m4p", + ".3g2", ".m4v", ".wma", ".asf", ".aif", ".aiff", ".afc", ".aifc", + ".wav", +#if TAGLIB_HAS_DSF + ".dsf", ".dff", ".dsdiff" +#endif + }; + + return std::span{ supportedExtensions }; + } + + TagLib::AudioProperties::ReadStyle readStyleToTagLibReadStyle(ParserReadStyle readStyle) + { + switch (readStyle) + { + case ParserReadStyle::Fast: + return TagLib::AudioProperties::ReadStyle::Fast; + case ParserReadStyle::Average: + return TagLib::AudioProperties::ReadStyle::Average; + case ParserReadStyle::Accurate: + return TagLib::AudioProperties::ReadStyle::Accurate; + } + + throw Exception{ "Cannot convert read style" }; + } + + TagLib::FileStream createFileStream(const std::filesystem::path& p) + { + FILE* file{ std::fopen(p.c_str(), "r") }; + if (!file) + { + const std::error_code ec{ errno, std::generic_category() }; + LMS_LOG(METADATA, DEBUG, "fopen failed for " << p << ": " << ec.message()); + throw IOException{ "fopen failed", ec }; + } + + int fd{ ::fileno(file) }; + if (fd == -1) + { + const std::error_code ec{ errno, std::generic_category() }; + LMS_LOG(METADATA, DEBUG, "fileno failed for " << p << ": " << ec.message()); + throw IOException{ "fileno failed", ec }; + } + + return TagLib::FileStream{ fd, true }; + } + + std::unique_ptr parseFileByExtension(TagLib::FileStream* stream, const std::filesystem::path& extension, TagLib::AudioProperties::ReadStyle audioPropertiesStyle, bool readAudioProperties) + { + const std::string ext{ core::stringUtils::stringToUpper(extension.string().substr(1)) }; + + std::unique_ptr file; + + // MP3 + if (ext == "MP3" || ext == "MP2" || ext == "AAC") + file = std::make_unique(stream, TagLib::ID3v2::FrameFactory::instance(), readAudioProperties, audioPropertiesStyle); + // VORBIS + else if (ext == "OGG") + return std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (ext == "OGA") + { + /* .oga can be any audio in the Ogg container. First try FLAC, then Vorbis. */ + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + if (!file->isValid()) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + } + else if (ext == "FLAC") + file = std::make_unique(stream, TagLib::ID3v2::FrameFactory::instance(), readAudioProperties, audioPropertiesStyle); + else if (ext == "SPX") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (ext == "OPUS") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + // APE + else if (ext == "MPC") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (ext == "WV") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (ext == "APE") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + // TRUEAUDIO + else if (ext == "TTA") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + // MP4 + else if (ext == "M4A" || ext == "M4R" || ext == "M4B" || ext == "M4P" || ext == "MP4" || ext == "3G2" || ext == "M4V") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + // ASF + else if (ext == "WMA" || ext == "ASF") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + // RIFF + else if (ext == "AIF" || ext == "AIFF" || ext == "AFC" || ext == "AIFC") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (ext == "WAV") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); +#if TAGLIB_HAS_DSF + else if (ext == "DSF") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (ext == "DFF" || ext == "DSDIFF") + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); +#endif + + if (file && !file->isValid()) + { + LMS_LOG(METADATA, DEBUG, "File " << file << ": failed to parse by extension"); + file.reset(); + } + + return file; + } + + std::unique_ptr parseFileByContent(TagLib::FileStream* stream, TagLib::AudioProperties::ReadStyle audioPropertiesStyle, bool readAudioProperties) + { + std::unique_ptr file; + + if (TagLib::MPEG::File::isSupported(stream)) + file = std::make_unique(stream, TagLib::ID3v2::FrameFactory::instance(), readAudioProperties, audioPropertiesStyle); + // VORBIS + else if (TagLib::Ogg::Vorbis::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (TagLib::Ogg::FLAC::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (TagLib::FLAC::File::isSupported(stream)) + file = std::make_unique(stream, TagLib::ID3v2::FrameFactory::instance(), readAudioProperties, audioPropertiesStyle); + else if (TagLib::Ogg::Speex::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (TagLib::Ogg::Opus::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + // APE + else if (TagLib::MPC::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (TagLib::WavPack::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (TagLib::APE::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + // TRUEAUDIO + else if (TagLib::TrueAudio::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + // MP4 + else if (TagLib::MP4::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + //_ASF + else if (TagLib::ASF::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + // RIFF + else if (TagLib::RIFF::AIFF::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (TagLib::RIFF::WAV::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); +#if TAGLIB_HAS_DSF + else if (DSF::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); + else if (DSDIFF::File::isSupported(stream)) + file = std::make_unique(stream, readAudioProperties, audioPropertiesStyle); +#endif + + if (file && !file->isValid()) + { + LMS_LOG(METADATA, DEBUG, "File " << file << ": failed to parse by content"); + file.reset(); + } + + return file; + } + + std::unique_ptr parseFile(const std::filesystem::path& p, TagLib::AudioProperties::ReadStyle readStyle, ReadAudioProperties readAudioProperties) + { + LMS_SCOPED_TRACE_DETAILED("MetaData", "TagLibParseFile"); + + TagLib::FileStream fileStream{ createFileStream(p) }; + std::unique_ptr file{ parseFileByExtension(&fileStream, p.extension(), readStyle, readAudioProperties.value()) }; + if (!file) + file = parseFileByContent(&fileStream, readStyle, readAudioProperties.value()); + + return file; + } +} // namespace lms::metadata::taglib::utils \ No newline at end of file diff --git a/src/libs/metadata/impl/taglib/Utils.hpp b/src/libs/metadata/impl/taglib/Utils.hpp new file mode 100644 index 00000000..7ddafdbd --- /dev/null +++ b/src/libs/metadata/impl/taglib/Utils.hpp @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include +#include + +#include "core/TaggedType.hpp" +#include "metadata/Types.hpp" + +namespace lms::metadata::taglib::utils +{ + std::span getSupportedExtensions(); + TagLib::AudioProperties::ReadStyle readStyleToTagLibReadStyle(ParserReadStyle readStyle); + + using ReadAudioProperties = core::TaggedBool; + std::unique_ptr parseFile(const std::filesystem::path& p, TagLib::AudioProperties::ReadStyle readStyle, ReadAudioProperties readAudioProperties); +} // namespace lms::metadata::taglib::utils \ No newline at end of file diff --git a/src/libs/metadata/include/metadata/ArtistInfo.hpp b/src/libs/metadata/include/metadata/ArtistInfo.hpp index 5b981af1..75af760f 100644 --- a/src/libs/metadata/include/metadata/ArtistInfo.hpp +++ b/src/libs/metadata/include/metadata/ArtistInfo.hpp @@ -51,6 +51,6 @@ namespace lms::metadata using Exception::Exception; }; - std::span getSupportedInfoFileExtensions(); + std::span getSupportedArtistInfoFiles(); ArtistInfo parseArtistInfo(std::istream& is); } // namespace lms::metadata diff --git a/src/libs/metadata/include/metadata/Exception.hpp b/src/libs/metadata/include/metadata/Exception.hpp index 7a878420..2acdda72 100644 --- a/src/libs/metadata/include/metadata/Exception.hpp +++ b/src/libs/metadata/include/metadata/Exception.hpp @@ -19,6 +19,8 @@ #pragma once +#include + #include "core/Exception.hpp" namespace lms::metadata @@ -29,9 +31,31 @@ namespace lms::metadata using LmsException::LmsException; }; - class ParseException : public Exception + class IOException : public Exception + { + public: + IOException(std::string_view message, std::error_code err) + : Exception{ std::string{ message } + ": " + err.message() } + , _err{ err } + { + } + + std::error_code getErrorCode() const { return _err; } + + private: + std::error_code _err; + }; + + class AudioFileParsingException : public Exception { public: using Exception::Exception; }; + + class AudioFileNoAudioPropertiesException : public AudioFileParsingException + { + public: + using AudioFileParsingException::AudioFileParsingException; + }; + } // namespace lms::metadata \ No newline at end of file diff --git a/src/libs/metadata/include/metadata/IAudioFileParser.hpp b/src/libs/metadata/include/metadata/IAudioFileParser.hpp index 38eb6694..599773f0 100644 --- a/src/libs/metadata/include/metadata/IAudioFileParser.hpp +++ b/src/libs/metadata/include/metadata/IAudioFileParser.hpp @@ -24,6 +24,7 @@ #include #include +#include "metadata/Exception.hpp" #include "metadata/Types.hpp" namespace lms::metadata diff --git a/src/libs/metadata/test/PlayList.cpp b/src/libs/metadata/test/PlayList.cpp index e11c77ae..165963a5 100644 --- a/src/libs/metadata/test/PlayList.cpp +++ b/src/libs/metadata/test/PlayList.cpp @@ -30,6 +30,7 @@ namespace lms::metadata::tests #PLAYLIST:My super playlist 01-Foo.mp3 + #EXTINF:263,Alice in Chains - Don't Follow 02-FooBar.mp3 diff --git a/src/libs/services/scanner/CMakeLists.txt b/src/libs/services/scanner/CMakeLists.txt index 7966d1f5..cd28e4b7 100644 --- a/src/libs/services/scanner/CMakeLists.txt +++ b/src/libs/services/scanner/CMakeLists.txt @@ -2,12 +2,14 @@ add_library(lmsscanner STATIC impl/helpers/ArtistHelpers.cpp impl/scanners/ArtistInfoFileScanner.cpp impl/scanners/AudioFileScanOperation.cpp + impl/scanners/FileScanOperationBase.cpp impl/scanners/AudioFileScanner.cpp impl/scanners/ImageFileScanner.cpp impl/scanners/LyricsFileScanner.cpp impl/scanners/PlayListFileScanner.cpp impl/scanners/Utils.cpp impl/steps/FileScanQueue.cpp + impl/steps/ScanErrorLogger.cpp impl/steps/ScanStepArtistReconciliation.cpp impl/steps/ScanStepAssociateArtistImages.cpp impl/steps/ScanStepAssociateExternalLyrics.cpp diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index 8b47c6fb..c03cebbf 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -51,14 +51,16 @@ #include "steps/ScanStepScanFiles.hpp" #include "steps/ScanStepUpdateLibraryFields.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { using namespace db; namespace { - static constexpr std::string_view currentSettingsName{ "" }; - static constexpr std::string_view lastScanSettingsName{ "last_scan" }; + constexpr std::string_view currentSettingsName{}; + constexpr std::string_view lastScanSettingsName{ "last_scan" }; Wt::WDate getNextMonday(Wt::WDate current) { @@ -352,7 +354,7 @@ namespace lms::scanner _currentScanStepStats.reset(); // must be sync with _curState } - LMS_LOG(DBUPDATER, INFO, "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size()); + LMS_LOG(DBUPDATER, INFO, "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << ", failures = " << stats.failures << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errorsCount << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size()); if (!_abortScan) { @@ -433,11 +435,11 @@ namespace lms::scanner } }; _fileScanners.clear(); - _fileScanners.emplace_back(std::make_unique(_settings, _db)); + _fileScanners.emplace_back(std::make_unique(_db, _settings)); _fileScanners.emplace_back(std::make_unique(_db, _settings)); - _fileScanners.emplace_back(std::make_unique(_db)); - _fileScanners.emplace_back(std::make_unique(_db)); - _fileScanners.emplace_back(std::make_unique(_db)); + _fileScanners.emplace_back(std::make_unique(_db, _settings)); + _fileScanners.emplace_back(std::make_unique(_db, _settings)); + _fileScanners.emplace_back(std::make_unique(_db, _settings)); std::vector fileScanners; std::transform(std::cbegin(_fileScanners), std::cend(_fileScanners), std::back_inserter(fileScanners), [](const std::unique_ptr& scanner) { return scanner.get(); }); diff --git a/src/libs/services/scanner/impl/ScannerStats.cpp b/src/libs/services/scanner/impl/ScannerStats.cpp index 38c179f1..37887424 100644 --- a/src/libs/services/scanner/impl/ScannerStats.cpp +++ b/src/libs/services/scanner/impl/ScannerStats.cpp @@ -21,16 +21,9 @@ namespace lms::scanner { - ScanError::ScanError(const std::filesystem::path& _file, ScanErrorType _error, const std::string& _systemError) - : file{ _file } - , error{ _error } - , systemError{ _systemError } - { - } - std::size_t ScanStats::nbFiles() const { - return skips + additions + updates; + return skips + additions + updates + failures; } std::size_t ScanStats::nbChanges() const diff --git a/src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.cpp b/src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.cpp index 51382c26..718a60eb 100644 --- a/src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.cpp +++ b/src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.cpp @@ -20,6 +20,7 @@ #include "ArtistInfoFileScanner.hpp" #include +#include #include "core/ILogger.hpp" #include "core/String.hpp" @@ -29,45 +30,33 @@ #include "database/MediaLibrary.hpp" #include "database/Session.hpp" #include "metadata/ArtistInfo.hpp" +#include "metadata/Types.hpp" +#include "services/scanner/ScanErrors.hpp" -#include "IFileScanOperation.hpp" -#include "ScanContext.hpp" +#include "FileScanOperationBase.hpp" #include "ScannerSettings.hpp" #include "Utils.hpp" #include "helpers/ArtistHelpers.hpp" -#include "metadata/Types.hpp" namespace lms::scanner { namespace { - class ArtistInfoFileScanOperation : public IFileScanOperation + class ArtistInfoFileScanOperation : public FileScanOperationBase { public: - ArtistInfoFileScanOperation(const FileToScan& file, const ScannerSettings& settings, db::Db& db) - : _file{ file.file } - , _mediaLibrary{ file.mediaLibrary } - , _settings{ settings } - , _db{ db } - { - } + using FileScanOperationBase::FileScanOperationBase; ~ArtistInfoFileScanOperation() override = default; ArtistInfoFileScanOperation(const ArtistInfoFileScanOperation&) = delete; ArtistInfoFileScanOperation& operator=(const ArtistInfoFileScanOperation&) = delete; private: - const std::filesystem::path& getFile() const override { return _file; }; core::LiteralString getName() const override { return "ScanArtistInfoFile"; } void scan() override; - void processResult(ScanContext& context) override; + OperationResult processResult() override; std::string getArtistNameFromArtistInfoFilePath(); - const std::filesystem::path _file; - const MediaLibraryInfo _mediaLibrary; - const ScannerSettings& _settings; - db::Db& _db; - std::optional _parsedArtistInfo; }; @@ -75,91 +64,82 @@ namespace lms::scanner { try { - std::ifstream ifs{ _file }; + std::ifstream ifs{ getFilePath() }; if (!ifs) { - LMS_LOG(DBUPDATER, ERROR, "Cannot open file " << _file); + const std::error_code ec{ errno, std::generic_category() }; + addError(getFilePath(), ec); return; } _parsedArtistInfo = metadata::parseArtistInfo(ifs); if (_parsedArtistInfo->name.empty()) { - LMS_LOG(DBUPDATER, DEBUG, "Discarding artist info in file " << _file << ": no name set"); + addError(getFilePath()); _parsedArtistInfo.reset(); } } catch (const metadata::ArtistInfoParseException& e) { - LMS_LOG(DBUPDATER, ERROR, "Cannot read artist info in file " << _file << ": " << e.what()); + addError(getFilePath()); } } - void ArtistInfoFileScanOperation::processResult(ScanContext& context) + ArtistInfoFileScanOperation::OperationResult ArtistInfoFileScanOperation::processResult() { - ScanStats& stats{ context.stats }; - - const std::optional fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) }; - if (!fileInfo) - { - stats.skips++; - return; - } - - db::Session& dbSession{ _db.getTLSSession() }; - db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, _file) }; + db::Session& dbSession{ getDb().getTLSSession() }; + db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, getFilePath()) }; if (!_parsedArtistInfo) { if (artistInfo) { artistInfo.remove(); - stats.deletions++; - LMS_LOG(DBUPDATER, DEBUG, "Removed artist info file " << _file); + + LMS_LOG(DBUPDATER, DEBUG, "Removed artist info file " << getFilePath()); + return OperationResult::Removed; } - context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadArtistInfoFile); - return; + + return OperationResult::Skipped; } const bool added{ !artistInfo }; if (!artistInfo) { artistInfo = dbSession.create(); - artistInfo.modify()->setAbsoluteFilePath(_file); + artistInfo.modify()->setAbsoluteFilePath(getFilePath()); } - artistInfo.modify()->setScanVersion(_settings.artistInfoScanVersion); + artistInfo.modify()->setScanVersion(getScannerSettings().artistInfoScanVersion); artistInfo.modify()->setName(_parsedArtistInfo->name); artistInfo.modify()->setSortName(_parsedArtistInfo->sortName); - artistInfo.modify()->setLastWriteTime(fileInfo->lastWriteTime); + artistInfo.modify()->setLastWriteTime(getLastWriteTime()); artistInfo.modify()->setType(_parsedArtistInfo->type); artistInfo.modify()->setGender(_parsedArtistInfo->gender); artistInfo.modify()->setDisambiguation(_parsedArtistInfo->disambiguation); artistInfo.modify()->setBiography(_parsedArtistInfo->biography); - db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this - artistInfo.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary)); + db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this + artistInfo.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary)); const metadata::Artist artistMetadata{ _parsedArtistInfo->mbid, _parsedArtistInfo->name, _parsedArtistInfo->sortName.empty() ? std::nullopt : std::make_optional(_parsedArtistInfo->sortName) }; - db::Artist::pointer artist{ helpers::getOrCreateArtist(dbSession, artistMetadata, helpers::AllowFallbackOnMBIDEntry{ _settings.allowArtistMBIDFallback }) }; + db::Artist::pointer artist{ helpers::getOrCreateArtist(dbSession, artistMetadata, helpers::AllowFallbackOnMBIDEntry{ getScannerSettings().allowArtistMBIDFallback }) }; artistInfo.modify()->setArtist(artist); artistInfo.modify()->setMBIDMatched(_parsedArtistInfo->mbid.has_value() && _parsedArtistInfo->mbid == artist->getMBID()); if (added) { - LMS_LOG(DBUPDATER, DEBUG, "Added artist info file " << _file); - stats.additions++; - } - else - { - LMS_LOG(DBUPDATER, DEBUG, "Updated artist info file " << _file); - stats.updates++; + LMS_LOG(DBUPDATER, DEBUG, "Added artist info file " << getFilePath()); + return OperationResult::Added; } + + LMS_LOG(DBUPDATER, DEBUG, "Updated artist info file " << getFilePath()); + return OperationResult::Updated; } } // namespace - ArtistInfoFileScanner::ArtistInfoFileScanner(const ScannerSettings& settings, db::Db& db) - : _settings{ settings } - , _db{ db } + ArtistInfoFileScanner::ArtistInfoFileScanner(db::Db& db, const ScannerSettings& settings) + : _db{ db } + , _settings{ settings } { } @@ -168,45 +148,34 @@ namespace lms::scanner return "Artist info scanner"; } - std::span ArtistInfoFileScanner::getSupportedExtensions() const + std::span ArtistInfoFileScanner::getSupportedFiles() const { - return metadata::getSupportedInfoFileExtensions(); + return metadata::getSupportedArtistInfoFiles(); } - bool ArtistInfoFileScanner::needsScan(ScanContext& context, const FileToScan& file) const + std::span ArtistInfoFileScanner::getSupportedExtensions() const + { + return {}; + } + + bool ArtistInfoFileScanner::needsScan(const FileToScan& file) const { // Special case: only files named "artist.nfo" are compatible with this scanner // Hack here since the scanner framework only handle extensions (the discover count is not accurate) - if (!core::stringUtils::stringCaseInsensitiveEqual(file.file.stem().string(), "artist")) + if (!core::stringUtils::stringCaseInsensitiveEqual(file.filePath.stem().string(), "artist")) return false; - const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) }; - // Should rarely fail as we are currently iterating it - if (!lastWriteTime.isValid()) - { - context.stats.skips++; - return false; - } - - if (context.scanOptions.fullScan) - return true; - db::Session& dbSession{ _db.getTLSSession() }; auto transaction{ dbSession.createReadTransaction() }; - db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, file.file) }; - if (artistInfo - && artistInfo->getLastWriteTime() == lastWriteTime - && artistInfo->getScanVersion() == _settings.artistInfoScanVersion) - { - context.stats.skips++; - return false; - } + db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, file.filePath) }; - return true; + return !artistInfo + || artistInfo->getLastWriteTime() != file.lastWriteTime + || artistInfo->getScanVersion() != _settings.artistInfoScanVersion; } - std::unique_ptr ArtistInfoFileScanner::createScanOperation(const FileToScan& fileToScan) const + std::unique_ptr ArtistInfoFileScanner::createScanOperation(FileToScan&& fileToScan) const { - return std::make_unique(fileToScan, _settings, _db); + return std::make_unique(std::move(fileToScan), _db, _settings); } } // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.hpp b/src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.hpp index b94a11e3..4d0af151 100644 --- a/src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.hpp +++ b/src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.hpp @@ -36,18 +36,19 @@ namespace lms::scanner class ArtistInfoFileScanner : public IFileScanner { public: - ArtistInfoFileScanner(const ScannerSettings& _settings, db::Db& db); + ArtistInfoFileScanner(db::Db& db, const ScannerSettings& _settings); ~ArtistInfoFileScanner() override = default; ArtistInfoFileScanner(const ArtistInfoFileScanner&) = delete; ArtistInfoFileScanner& operator=(const ArtistInfoFileScanner&) = delete; private: core::LiteralString getName() const override; + std::span getSupportedFiles() const override; std::span getSupportedExtensions() const override; - bool needsScan(ScanContext& context, const FileToScan& file) const override; - std::unique_ptr createScanOperation(const FileToScan& fileToScan) const override; + bool needsScan(const FileToScan& file) const override; + std::unique_ptr createScanOperation(FileToScan&& fileToScan) const override; - const ScannerSettings& _settings; db::Db& _db; + const ScannerSettings& _settings; }; } // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp b/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp index 8ca0c14e..8a33e817 100644 --- a/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp +++ b/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp @@ -41,9 +41,10 @@ #include "image/Exception.hpp" #include "image/Image.hpp" #include "metadata/Exception.hpp" +#include "metadata/IAudioFileParser.hpp" +#include "services/scanner/ScanErrors.hpp" #include "IFileScanOperation.hpp" -#include "ScanContext.hpp" #include "ScannerSettings.hpp" #include "Utils.hpp" #include "helpers/ArtistHelpers.hpp" @@ -387,7 +388,7 @@ namespace lms::scanner return db::Advisory::UnSet; } - db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, const FileInfo& fileInfo) + db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, size_t fileSize, const std::filesystem::path& relativePath) { db::Track::FindParameters params; // Add as many fields as possible to limit errors @@ -401,7 +402,7 @@ namespace lms::scanner } if (parsedTrack.position) params.setTrackNumber(*parsedTrack.position); - params.setFileSize(fileInfo.fileSize); + params.setFileSize(fileSize); bool error{}; db::Track::pointer res; @@ -413,7 +414,7 @@ namespace lms::scanner if (res) { - LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << fileInfo.relativePath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath()); + LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << relativePath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath()); error = true; } res = track; @@ -474,6 +475,14 @@ namespace lms::scanner } } // namespace + AudioFileScanOperation::AudioFileScanOperation(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings, metadata::IAudioFileParser& parser) + : FileScanOperationBase{ std::move(fileToScan), db, settings } + , _parser{ parser } + { + } + + AudioFileScanOperation::~AudioFileScanOperation() = default; + void AudioFileScanOperation::scan() { LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanAudioFile"); @@ -481,13 +490,13 @@ namespace lms::scanner try { - _parsedTrack = _parser.parseMetaData(_file); + _parsedTrack = _parser.parseMetaData(getFilePath()); // We fill missing artist mbids with mbids found on other artist roles fillMissingMbids(*_parsedTrack); std::size_t index{}; - _parser.parseImages(_file, [&](const metadata::Image& image) { + _parser.parseImages(getFilePath(), [&](const metadata::Image& image) { try { image::ImageProperties properties{ image::probeImage(image.data) }; @@ -508,46 +517,43 @@ namespace lms::scanner } catch (const image::Exception& e) { - LMS_LOG(DBUPDATER, ERROR, "Failed to parse image in track file " << _file); + addError(getFilePath(), index); } index++; }); } + catch (const metadata::AudioFileNoAudioPropertiesException&) + { + addError(getFilePath()); + } + catch (const metadata::IOException& e) + { + addError(getFilePath(), e.getErrorCode()); + } catch (const metadata::Exception& e) { - LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << _file); + addError(getFilePath()); } } - void AudioFileScanOperation::processResult(ScanContext& context) + AudioFileScanOperation::OperationResult AudioFileScanOperation::processResult() { LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessAudioScanData"); - ScanStats& stats{ context.stats }; - - const std::optional fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) }; - if (!fileInfo) - { - stats.skips++; - return; - } - - db::Session& dbSession{ _db.getTLSSession() }; - db::Track::pointer track{ db::Track::findByPath(dbSession, _file) }; - + db::Session& dbSession{ getDb().getTLSSession() }; + db::Track::pointer track{ db::Track::findByPath(dbSession, getFilePath()) }; if (!_parsedTrack) { if (track) { track.remove(); - stats.deletions++; + return OperationResult::Removed; } - context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadAudioFile); - return; + return OperationResult::Skipped; } - if (_parsedTrack->mbid && (!track || _settings.skipDuplicateTrackMBID)) + if (_parsedTrack->mbid && (!track || getScannerSettings().skipDuplicateTrackMBID)) { std::vector duplicateTracks{ db::Track::findByMBID(dbSession, *_parsedTrack->mbid) }; @@ -558,14 +564,14 @@ namespace lms::scanner std::error_code ec; if (!std::filesystem::exists(otherTrack->getAbsoluteFilePath(), ec)) { - LMS_LOG(DBUPDATER, DEBUG, "Considering track " << _file << " moved from " << otherTrack->getAbsoluteFilePath()); + LMS_LOG(DBUPDATER, DEBUG, "Considering track " << getFilePath() << " moved from " << otherTrack->getAbsoluteFilePath()); track = otherTrack; - track.modify()->setAbsoluteFilePath(_file); + track.modify()->setAbsoluteFilePath(getFilePath()); } } // Skip duplicate track MBID - if (_settings.skipDuplicateTrackMBID) + if (getScannerSettings().skipDuplicateTrackMBID) { for (db::Track::pointer& otherTrack : duplicateTracks) { @@ -574,22 +580,26 @@ namespace lms::scanner continue; // Skip if duplicate files no longer in media root: as it will be removed later, we will end up with no file - if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries), + auto& mediaLibraries{ getScannerSettings().mediaLibraries }; + if (std::none_of(std::cbegin(mediaLibraries), std::cend(mediaLibraries), [&](const MediaLibraryInfo& libraryInfo) { - return core::pathUtils::isPathInRootPath(_file, libraryInfo.rootDirectory, &excludeDirFileName); + return core::pathUtils::isPathInRootPath(getFilePath(), libraryInfo.rootDirectory, &excludeDirFileName); })) { continue; } - LMS_LOG(DBUPDATER, DEBUG, "Skipped " << _file << " (similar MBID in " << otherTrack->getAbsoluteFilePath() << ")"); + LMS_LOG(DBUPDATER, DEBUG, "Skipped " << getFilePath() << ": same MBID already found in " << otherTrack->getAbsoluteFilePath()); // As this MBID already exists, just remove what we just scanned if (track) { track.remove(); - stats.deletions++; + + LMS_LOG(DBUPDATER, DEBUG, "Removed " << getFilePath() << ": same MBID already found in " << otherTrack->getAbsoluteFilePath()); + return OperationResult::Removed; } - return; + + return OperationResult::Skipped; } } } @@ -597,27 +607,25 @@ namespace lms::scanner if (!track) { // maybe the file just moved? - track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, *fileInfo); + track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, getFileSize(), getRelativeFilePath()); if (track) { - LMS_LOG(DBUPDATER, DEBUG, "Considering track " << _file << " moved from " << track->getAbsoluteFilePath()); - track.modify()->setAbsoluteFilePath(_file); + LMS_LOG(DBUPDATER, DEBUG, "Considering track " << getFilePath() << " moved from " << track->getAbsoluteFilePath()); + track.modify()->setAbsoluteFilePath(getFilePath()); } } // We estimate this is an audio file if the duration is not null if (_parsedTrack->audioProperties.duration == std::chrono::milliseconds::zero()) { - LMS_LOG(DBUPDATER, DEBUG, "Skipped " << _file << " (duration is 0)"); + addError(getFilePath()); - // If Track exists here, delete it! if (track) { track.remove(); - stats.deletions++; + return OperationResult::Removed; } - stats.errors.emplace_back(_file, ScanErrorType::BadDuration); - return; + return OperationResult::Skipped; } // ***** Title @@ -626,9 +634,9 @@ namespace lms::scanner title = _parsedTrack->title; else { - // TODO parse file name guess track etc. - // For now juste use file name as title - title = _file.filename().string(); + // TODO parse file name to guess track etc. + // For now, we just use file name as title + title = getFilePath().filename().string(); } // If file already exists, update its data @@ -639,13 +647,13 @@ namespace lms::scanner track = dbSession.create(); added = true; - track.modify()->setAbsoluteFilePath(_file); - track.modify()->setAddedTime(_mediaLibrary.firstScan ? fileInfo->lastWriteTime : Wt::WDateTime::currentDateTime()); // may be erased by encodingTime + track.modify()->setAbsoluteFilePath(getFilePath()); + track.modify()->setAddedTime(getMediaLibrary().firstScan ? getLastWriteTime() : Wt::WDateTime::currentDateTime()); // may be erased by encodingTime } // Track related data assert(track); - track.modify()->setScanVersion(_settings.audioScanVersion); + track.modify()->setScanVersion(getScannerSettings().audioScanVersion); // Audio properties track.modify()->setBitrate(_parsedTrack->audioProperties.bitrate); @@ -654,9 +662,9 @@ namespace lms::scanner track.modify()->setDuration(_parsedTrack->audioProperties.duration); track.modify()->setSampleRate(_parsedTrack->audioProperties.sampleRate); - track.modify()->setRelativeFilePath(fileInfo->relativePath); - track.modify()->setFileSize(fileInfo->fileSize); - track.modify()->setLastWriteTime(fileInfo->lastWriteTime); + track.modify()->setRelativeFilePath(getRelativeFilePath()); + track.modify()->setFileSize(getFileSize()); + track.modify()->setLastWriteTime(getLastWriteTime()); if (_parsedTrack->encodingTime.isValid()) { @@ -672,14 +680,14 @@ namespace lms::scanner track.modify()->setAddedTime(time.isValid() ? Wt::WDateTime{ date, time } : Wt::WDateTime{ date }); } - db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this + db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this track.modify()->setMediaLibrary(mediaLibrary); - db::Directory::pointer directory{ utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary) }; + db::Directory::pointer directory{ utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary) }; track.modify()->setDirectory(directory); track.modify()->clearArtistLinks(); - const helpers::AllowFallbackOnMBIDEntry allowFallback{ _settings.allowArtistMBIDFallback }; + const helpers::AllowFallbackOnMBIDEntry allowFallback{ getScannerSettings().allowArtistMBIDFallback }; createTrackArtistLinks(dbSession, track, db::TrackArtistLinkType::Artist, _parsedTrack->artists, allowFallback); if (_parsedTrack->medium && _parsedTrack->medium->release) createTrackArtistLinks(dbSession, track, db::TrackArtistLinkType::ReleaseArtist, _parsedTrack->medium->release->artists, allowFallback); @@ -732,13 +740,11 @@ namespace lms::scanner if (added) { - LMS_LOG(DBUPDATER, DEBUG, "Added audio file " << _file); - stats.additions++; - } - else - { - LMS_LOG(DBUPDATER, DEBUG, "Updated audio file " << _file); - stats.updates++; + LMS_LOG(DBUPDATER, DEBUG, "Added audio file " << getFilePath()); + return OperationResult::Added; } + + LMS_LOG(DBUPDATER, DEBUG, "Updated audio file " << getFilePath()); + return OperationResult::Updated; } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.hpp b/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.hpp index 3de06544..a6bf3c54 100644 --- a/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.hpp +++ b/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.hpp @@ -25,16 +25,21 @@ #include #include "image/Types.hpp" -#include "metadata/IAudioFileParser.hpp" +#include "metadata/Types.hpp" +#include "FileScanOperationBase.hpp" #include "FileToScan.hpp" -#include "IFileScanner.hpp" namespace lms::db { class Db; } // namespace lms::db +namespace lms::metadata +{ + class IAudioFileParser; +} // namespace lms::metadata + namespace lms::scanner { struct ImageInfo @@ -48,32 +53,20 @@ namespace lms::scanner std::string description; }; - class AudioFileScanOperation : public IFileScanOperation + class AudioFileScanOperation : public FileScanOperationBase { public: - AudioFileScanOperation(const FileToScan& fileToScan, db::Db& db, metadata::IAudioFileParser& parser, const ScannerSettings& settings) - : _file{ fileToScan.file } - , _mediaLibrary{ fileToScan.mediaLibrary } - , _db{ db } - , _parser{ parser } - , _settings{ settings } - { - } - ~AudioFileScanOperation() override = default; + AudioFileScanOperation(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings, metadata::IAudioFileParser& parser); + ~AudioFileScanOperation() override; AudioFileScanOperation(const AudioFileScanOperation&) = delete; AudioFileScanOperation& operator=(const AudioFileScanOperation&) = delete; private: - const std::filesystem::path& getFile() const override { return _file; }; core::LiteralString getName() const override { return "ScanAudioFile"; } void scan() override; - void processResult(ScanContext& context) override; + OperationResult processResult() override; - const std::filesystem::path _file; - const MediaLibraryInfo _mediaLibrary; - db::Db& _db; metadata::IAudioFileParser& _parser; - const ScannerSettings& _settings; std::unique_ptr _parsedTrack; std::vector _parsedImages; }; diff --git a/src/libs/services/scanner/impl/scanners/AudioFileScanner.cpp b/src/libs/services/scanner/impl/scanners/AudioFileScanner.cpp index 14b74efa..476bfea9 100644 --- a/src/libs/services/scanner/impl/scanners/AudioFileScanner.cpp +++ b/src/libs/services/scanner/impl/scanners/AudioFileScanner.cpp @@ -25,9 +25,9 @@ #include "database/MediaLibrary.hpp" #include "database/Session.hpp" #include "database/Track.hpp" +#include "metadata/IAudioFileParser.hpp" #include "AudioFileScanOperation.hpp" -#include "ScanContext.hpp" #include "ScannerSettings.hpp" #include "Utils.hpp" @@ -77,66 +77,29 @@ namespace lms::scanner return "Audio scanner"; } + std::span AudioFileScanner::getSupportedFiles() const + { + return {}; + } + std::span AudioFileScanner::getSupportedExtensions() const { return _metadataParser->getSupportedExtensions(); } - bool AudioFileScanner::needsScan(ScanContext& context, const FileToScan& file) const + bool AudioFileScanner::needsScan(const FileToScan& file) const { - ScanStats& stats{ context.stats }; - - const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) }; - // Should rarely fail as we are currently iterating it - if (!lastWriteTime.isValid()) - { - stats.skips++; - return false; - } - - if (context.scanOptions.fullScan) - return true; - - bool needUpdateLibrary{}; db::Session& dbSession{ _db.getTLSSession() }; + auto transaction{ dbSession.createReadTransaction() }; - { - auto transaction{ dbSession.createReadTransaction() }; - - // Skip file if last write is the same - const db::Track::pointer track{ db::Track::findByPath(dbSession, file.file) }; - if (track - && track->getLastWriteTime() == lastWriteTime - && track->getScanVersion() == _settings.audioScanVersion) - { - // this file may have been moved from one library to another, then we just need to update the media library id instead of a full rescan - const auto trackMediaLibrary{ track->getMediaLibrary() }; - if (trackMediaLibrary && trackMediaLibrary->getId() == file.mediaLibrary.id) - { - stats.skips++; - return false; - } - - needUpdateLibrary = true; - } - } - - if (needUpdateLibrary) - { - auto transaction{ dbSession.createWriteTransaction() }; - - db::Track::pointer track{ db::Track::findByPath(dbSession, file.file) }; - assert(track); - track.modify()->setMediaLibrary(db::MediaLibrary::find(dbSession, file.mediaLibrary.id)); // may be null, will be handled in the next scan anyway - stats.updates++; - return false; - } - - return true; // need to scan + const db::Track::pointer track{ db::Track::findByPath(dbSession, file.filePath) }; + return !track + || track->getLastWriteTime() != file.lastWriteTime + || track->getScanVersion() != _settings.audioScanVersion; } - std::unique_ptr AudioFileScanner::createScanOperation(const FileToScan& fileToScan) const + std::unique_ptr AudioFileScanner::createScanOperation(FileToScan&& fileToScan) const { - return std::make_unique(fileToScan, _db, *_metadataParser, _settings); + return std::make_unique(std::move(fileToScan), _db, _settings, *_metadataParser); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/scanners/AudioFileScanner.hpp b/src/libs/services/scanner/impl/scanners/AudioFileScanner.hpp index dd6a9247..3fa46cbf 100644 --- a/src/libs/services/scanner/impl/scanners/AudioFileScanner.hpp +++ b/src/libs/services/scanner/impl/scanners/AudioFileScanner.hpp @@ -48,9 +48,10 @@ namespace lms::scanner private: core::LiteralString getName() const override; + std::span getSupportedFiles() const override; std::span getSupportedExtensions() const override; - bool needsScan(ScanContext& context, const FileToScan& file) const override; - std::unique_ptr createScanOperation(const FileToScan& fileToScan) const override; + bool needsScan(const FileToScan& file) const override; + std::unique_ptr createScanOperation(FileToScan&& fileToScan) const override; db::Db& _db; const ScannerSettings& _settings; diff --git a/src/libs/services/scanner/impl/scanners/FileScanOperationBase.cpp b/src/libs/services/scanner/impl/scanners/FileScanOperationBase.cpp new file mode 100644 index 00000000..ca3ca660 --- /dev/null +++ b/src/libs/services/scanner/impl/scanners/FileScanOperationBase.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "FileScanOperationBase.hpp" + +#include "services/scanner/ScanErrors.hpp" + +namespace lms::scanner +{ + FileScanOperationBase::FileScanOperationBase(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings) + : _file{ std::move(fileToScan) } + , _db{ db } + , _settings{ settings } + { + } + + FileScanOperationBase::~FileScanOperationBase() = default; +} // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp b/src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp new file mode 100644 index 00000000..6d3b717d --- /dev/null +++ b/src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include "FileToScan.hpp" +#include "IFileScanOperation.hpp" + +namespace lms::db +{ + class Db; +} // namespace lms::db + +namespace lms::scanner +{ + struct ScanError; + struct ScannerSettings; + + class FileScanOperationBase : public IFileScanOperation + { + public: + FileScanOperationBase(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings); + ~FileScanOperationBase() override; + FileScanOperationBase(const FileScanOperationBase&) = delete; + FileScanOperationBase& operator=(const FileScanOperationBase&) = delete; + + protected: + const std::filesystem::path& getFilePath() const override { return _file.filePath; } + const MediaLibraryInfo& getMediaLibrary() const { return _file.mediaLibrary; } + db::Db& getDb() { return _db; } + const ScannerSettings& getScannerSettings() const { return _settings; } + Wt::WDateTime getLastWriteTime() const { return _file.lastWriteTime; } + std::size_t getFileSize() const { return _file.fileSize; } + const std::filesystem::path& getRelativeFilePath() const { return _file.relativePath; } + + template + void addError(CtrArgs&&... args) + { + _errors.emplace_back(std::make_shared(std::forward(args)...)); + } + + const ScanErrorVector& getErrors() override { return _errors; } + + private: + const FileToScan _file; + db::Db& _db; + const ScannerSettings& _settings; + ScanErrorVector _errors; + }; +} // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/scanners/FileToScan.hpp b/src/libs/services/scanner/impl/scanners/FileToScan.hpp index 8a94a0f3..8f44015d 100644 --- a/src/libs/services/scanner/impl/scanners/FileToScan.hpp +++ b/src/libs/services/scanner/impl/scanners/FileToScan.hpp @@ -21,13 +21,18 @@ #include +#include + #include "MediaLibraryInfo.hpp" namespace lms::scanner { struct FileToScan { - std::filesystem::path file; + std::filesystem::path filePath; + std::filesystem::path relativePath; MediaLibraryInfo mediaLibrary; + Wt::WDateTime lastWriteTime; + std::size_t fileSize{}; }; } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/scanners/IFileScanOperation.hpp b/src/libs/services/scanner/impl/scanners/IFileScanOperation.hpp index cbf828af..d7c2c22c 100644 --- a/src/libs/services/scanner/impl/scanners/IFileScanOperation.hpp +++ b/src/libs/services/scanner/impl/scanners/IFileScanOperation.hpp @@ -20,12 +20,14 @@ #pragma once #include +#include +#include #include "core/LiteralString.hpp" namespace lms::scanner { - struct ScanContext; + struct ScanError; class IFileScanOperation { @@ -34,8 +36,23 @@ namespace lms::scanner virtual core::LiteralString getName() const = 0; - virtual const std::filesystem::path& getFile() const = 0; + virtual const std::filesystem::path& getFilePath() const = 0; + + // scan() is called asynchronously by a pool of threads + // processResult() is called sequentially by a single thread virtual void scan() = 0; - virtual void processResult(ScanContext& context) = 0; + + enum class OperationResult + { + Added, + Removed, + Updated, + Skipped, + }; + virtual OperationResult processResult() = 0; + + using ScanErrorVector = std::vector>; + // list of errors collected during scan/result processing (there might be errors without skipping the file) + virtual const ScanErrorVector& getErrors() = 0; }; } // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/scanners/IFileScanner.hpp b/src/libs/services/scanner/impl/scanners/IFileScanner.hpp index 381054f2..81c9bdf2 100644 --- a/src/libs/services/scanner/impl/scanners/IFileScanner.hpp +++ b/src/libs/services/scanner/impl/scanners/IFileScanner.hpp @@ -29,8 +29,6 @@ namespace lms::scanner { class IFileScanOperation; - struct ScanContext; - struct ScannerSettings; class IFileScanner { @@ -38,8 +36,9 @@ namespace lms::scanner virtual ~IFileScanner() = default; virtual core::LiteralString getName() const = 0; + virtual std::span getSupportedFiles() const = 0; virtual std::span getSupportedExtensions() const = 0; - virtual bool needsScan(ScanContext& context, const FileToScan& file) const = 0; - virtual std::unique_ptr createScanOperation(const FileToScan& fileToScan) const = 0; + virtual bool needsScan(const FileToScan& file) const = 0; + virtual std::unique_ptr createScanOperation(FileToScan&& fileToScan) const = 0; }; } // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp b/src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp index d84a3c65..7869e938 100644 --- a/src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp +++ b/src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp @@ -27,34 +27,26 @@ #include "database/MediaLibrary.hpp" #include "database/Session.hpp" #include "image/Exception.hpp" -#include "image/IRawImage.hpp" #include "image/Image.hpp" +#include "services/scanner/ScanErrors.hpp" +#include "FileScanOperationBase.hpp" #include "IFileScanOperation.hpp" -#include "ScanContext.hpp" #include "Utils.hpp" namespace lms::scanner { namespace { - class ImageFileScanOperation : public IFileScanOperation + class ImageFileScanOperation : public FileScanOperationBase { public: - ImageFileScanOperation(const FileToScan& file, db::Db& db) - : _file{ file.file } - , _mediaLibrary{ file.mediaLibrary } - , _db{ db } {} + using FileScanOperationBase::FileScanOperationBase; private: - const std::filesystem::path& getFile() const override { return _file; }; core::LiteralString getName() const override { return "ScanImageFile"; } void scan() override; - void processResult(ScanContext& context) override; - - const std::filesystem::path _file; - const MediaLibraryInfo _mediaLibrary; - db::Db& _db; + OperationResult processResult() override; std::optional _parsedImageProperties; }; @@ -63,67 +55,58 @@ namespace lms::scanner { try { - _parsedImageProperties = image::probeImage(_file); + _parsedImageProperties = image::probeImage(getFilePath()); } catch (const image::Exception& e) { _parsedImageProperties.reset(); - LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file " << _file << ": " << e.what()); + addError(getFilePath()); } } - void ImageFileScanOperation::processResult(ScanContext& context) + ImageFileScanOperation::OperationResult ImageFileScanOperation::processResult() { - ScanStats& stats{ context.stats }; - - const std::optional fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) }; - if (!fileInfo) - { - stats.skips++; - return; - } - - db::Session& dbSession{ _db.getTLSSession() }; - db::Image::pointer image{ db::Image::find(dbSession, _file) }; + db::Session& dbSession{ getDb().getTLSSession() }; + db::Image::pointer image{ db::Image::find(dbSession, getFilePath()) }; if (!_parsedImageProperties) { if (image) { image.remove(); - stats.deletions++; - LMS_LOG(DBUPDATER, DEBUG, "Removed image " << _file); + + LMS_LOG(DBUPDATER, DEBUG, "Removed image " << getFilePath()); + return OperationResult::Removed; } - context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadImageFile); - return; + + return OperationResult::Skipped; } const bool added{ !image }; if (!image) - image = dbSession.create(_file); + image = dbSession.create(getFilePath()); - image.modify()->setLastWriteTime(fileInfo->lastWriteTime); - image.modify()->setFileSize(fileInfo->fileSize); + image.modify()->setLastWriteTime(getLastWriteTime()); + image.modify()->setFileSize(getFileSize()); image.modify()->setHeight(_parsedImageProperties->height); image.modify()->setWidth(_parsedImageProperties->width); - db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this - image.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary)); + db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this + image.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary)); if (added) { - LMS_LOG(DBUPDATER, DEBUG, "Added image " << _file); - stats.additions++; - } - else - { - LMS_LOG(DBUPDATER, DEBUG, "Updated image " << _file); - stats.updates++; + LMS_LOG(DBUPDATER, DEBUG, "Added image " << getFilePath()); + return OperationResult::Added; } + + LMS_LOG(DBUPDATER, DEBUG, "Updated image " << getFilePath()); + return OperationResult::Updated; } } // namespace - ImageFileScanner::ImageFileScanner(db::Db& db) + ImageFileScanner::ImageFileScanner(db::Db& db, ScannerSettings& settings) : _db{ db } + , _settings{ settings } { } @@ -132,41 +115,27 @@ namespace lms::scanner return "Image scanner"; } + std::span ImageFileScanner::getSupportedFiles() const + { + return {}; + } + std::span ImageFileScanner::getSupportedExtensions() const { return image::getSupportedFileExtensions(); } - bool ImageFileScanner::needsScan(ScanContext& context, const FileToScan& file) const + bool ImageFileScanner::needsScan(const FileToScan& file) const { - ScanStats& stats{ context.stats }; + db::Session& dbSession{ _db.getTLSSession() }; + auto transaction{ _db.getTLSSession().createReadTransaction() }; - const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) }; - // Should rarely fail as we are currently iterating it - if (!lastWriteTime.isValid()) - { - stats.skips++; - return false; - } - - if (!context.scanOptions.fullScan) - { - db::Session& dbSession{ _db.getTLSSession() }; - auto transaction{ _db.getTLSSession().createReadTransaction() }; - - const db::Image::pointer image{ db::Image::find(dbSession, file.file) }; - if (image && image->getLastWriteTime() == lastWriteTime) - { - stats.skips++; - return false; - } - } - - return true; // need to scan + const db::Image::pointer image{ db::Image::find(dbSession, file.filePath) }; + return (!image || image->getLastWriteTime() != file.lastWriteTime); } - std::unique_ptr ImageFileScanner::createScanOperation(const FileToScan& fileToScan) const + std::unique_ptr ImageFileScanner::createScanOperation(FileToScan&& fileToScan) const { - return std::make_unique(fileToScan, _db); + return std::make_unique(std::move(fileToScan), _db, _settings); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/scanners/ImageFileScanner.hpp b/src/libs/services/scanner/impl/scanners/ImageFileScanner.hpp index a5ade2a6..4e34b0cb 100644 --- a/src/libs/services/scanner/impl/scanners/ImageFileScanner.hpp +++ b/src/libs/services/scanner/impl/scanners/ImageFileScanner.hpp @@ -36,17 +36,19 @@ namespace lms::scanner class ImageFileScanner : public IFileScanner { public: - ImageFileScanner(db::Db& db); + ImageFileScanner(db::Db& db, ScannerSettings& settings); ~ImageFileScanner() override = default; ImageFileScanner(const ImageFileScanner&) = delete; ImageFileScanner& operator=(const ImageFileScanner&) = delete; private: core::LiteralString getName() const override; + std::span getSupportedFiles() const override; std::span getSupportedExtensions() const override; - bool needsScan(ScanContext& context, const FileToScan& file) const override; - std::unique_ptr createScanOperation(const FileToScan& fileToScan) const override; + bool needsScan(const FileToScan& file) const override; + std::unique_ptr createScanOperation(FileToScan&& fileToScan) const override; db::Db& _db; + ScannerSettings& _settings; }; } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/scanners/LyricsFileScanner.cpp b/src/libs/services/scanner/impl/scanners/LyricsFileScanner.cpp index ef33f67c..f88e3563 100644 --- a/src/libs/services/scanner/impl/scanners/LyricsFileScanner.cpp +++ b/src/libs/services/scanner/impl/scanners/LyricsFileScanner.cpp @@ -22,38 +22,31 @@ #include #include +#include "FileScanOperationBase.hpp" +#include "ScannerSettings.hpp" #include "core/ILogger.hpp" #include "database/Db.hpp" #include "database/MediaLibrary.hpp" #include "database/Session.hpp" #include "database/TrackLyrics.hpp" #include "metadata/Lyrics.hpp" +#include "services/scanner/ScanErrors.hpp" -#include "IFileScanOperation.hpp" -#include "ScanContext.hpp" #include "Utils.hpp" namespace lms::scanner { namespace { - class LyricsFileScanOperation : public IFileScanOperation + class LyricsFileScanOperation : public FileScanOperationBase { public: - LyricsFileScanOperation(const FileToScan& file, db::Db& db) - : _file{ file.file } - , _mediaLibrary{ file.mediaLibrary } - , _db{ db } {} + using FileScanOperationBase::FileScanOperationBase; private: - const std::filesystem::path& getFile() const override { return _file; }; core::LiteralString getName() const override { return "ScanLyricsFile"; } void scan() override; - void processResult(ScanContext& context) override; - - const std::filesystem::path _file; - const MediaLibraryInfo _mediaLibrary; - db::Db& _db; + OperationResult processResult() override; std::optional _parsedLyrics; }; @@ -62,53 +55,50 @@ namespace lms::scanner { try { - std::ifstream ifs{ _file }; + std::ifstream ifs{ getFilePath() }; if (!ifs) - LMS_LOG(DBUPDATER, ERROR, "Cannot open file " << _file); - else - _parsedLyrics = metadata::parseLyrics(ifs); + { + const std::error_code ec{ errno, std::generic_category() }; + + addError(getFilePath(), ec); + return; + } + + _parsedLyrics = metadata::parseLyrics(ifs); } catch (const metadata::Exception& e) { - LMS_LOG(DBUPDATER, ERROR, "Cannot read lyrics in file " << _file << ": " << e.what()); + addError(getFilePath()); } } - void LyricsFileScanOperation::processResult(ScanContext& context) + LyricsFileScanOperation::OperationResult LyricsFileScanOperation::processResult() { - ScanStats& stats{ context.stats }; - - const std::optional fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) }; - if (!fileInfo) - { - stats.skips++; - return; - } - - db::Session& dbSession{ _db.getTLSSession() }; - db::TrackLyrics::pointer trackLyrics{ db::TrackLyrics::find(dbSession, _file) }; + db::Session& dbSession{ getDb().getTLSSession() }; + db::TrackLyrics::pointer trackLyrics{ db::TrackLyrics::find(dbSession, getFilePath()) }; if (!_parsedLyrics) { if (trackLyrics) { trackLyrics.remove(); - stats.deletions++; - LMS_LOG(DBUPDATER, DEBUG, "Removed lyrics file " << _file); + + LMS_LOG(DBUPDATER, DEBUG, "Removed lyrics file " << getFilePath()); + return OperationResult::Removed; } - context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadLyricsFile); - return; + + return OperationResult::Skipped; } const bool added{ !trackLyrics }; if (!trackLyrics) { trackLyrics = dbSession.create(); - trackLyrics.modify()->setAbsoluteFilePath(_file); + trackLyrics.modify()->setAbsoluteFilePath(getFilePath()); } - trackLyrics.modify()->setLastWriteTime(fileInfo->lastWriteTime); - trackLyrics.modify()->setFileSize(fileInfo->fileSize); + trackLyrics.modify()->setLastWriteTime(getLastWriteTime()); + trackLyrics.modify()->setFileSize(getFileSize()); trackLyrics.modify()->setLanguage(!_parsedLyrics->language.empty() ? _parsedLyrics->language : "xxx"); trackLyrics.modify()->setOffset(_parsedLyrics->offset); trackLyrics.modify()->setDisplayTitle(_parsedLyrics->displayTitle); @@ -118,24 +108,23 @@ namespace lms::scanner else trackLyrics.modify()->setUnsynchronizedLines(_parsedLyrics->unsynchronizedLines); - db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this - trackLyrics.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary)); + db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this + trackLyrics.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary)); if (added) { - LMS_LOG(DBUPDATER, DEBUG, "Added external lyrics " << _file); - stats.additions++; - } - else - { - LMS_LOG(DBUPDATER, DEBUG, "Updated external lyrics " << _file); - stats.updates++; + LMS_LOG(DBUPDATER, DEBUG, "Added external lyrics " << getFilePath()); + return OperationResult::Added; } + + LMS_LOG(DBUPDATER, DEBUG, "Updated external lyrics " << getFilePath()); + return OperationResult::Updated; } } // namespace - LyricsFileScanner::LyricsFileScanner(db::Db& db) + LyricsFileScanner::LyricsFileScanner(db::Db& db, ScannerSettings& _settings) : _db{ db } + , _settings{ _settings } { } @@ -144,41 +133,27 @@ namespace lms::scanner return "Lyrics scanner"; } + std::span LyricsFileScanner::getSupportedFiles() const + { + return {}; + } + std::span LyricsFileScanner::getSupportedExtensions() const { return metadata::getSupportedLyricsFileExtensions(); } - bool LyricsFileScanner::needsScan(ScanContext& context, const FileToScan& file) const + bool LyricsFileScanner::needsScan(const FileToScan& file) const { - ScanStats& stats{ context.stats }; + db::Session& dbSession{ _db.getTLSSession() }; + auto transaction{ _db.getTLSSession().createReadTransaction() }; - const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) }; - // Should rarely fail as we are currently iterating it - if (!lastWriteTime.isValid()) - { - stats.skips++; - return false; - } - - if (!context.scanOptions.fullScan) - { - db::Session& dbSession{ _db.getTLSSession() }; - auto transaction{ _db.getTLSSession().createReadTransaction() }; - - const db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(dbSession, file.file) }; - if (lyrics && lyrics->getLastWriteTime() == lastWriteTime) - { - stats.skips++; - return false; - } - } - - return true; // need to scan + const db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(dbSession, file.filePath) }; + return !lyrics || lyrics->getLastWriteTime() != file.lastWriteTime; } - std::unique_ptr LyricsFileScanner::createScanOperation(const FileToScan& fileToScan) const + std::unique_ptr LyricsFileScanner::createScanOperation(FileToScan&& fileToScan) const { - return std::make_unique(fileToScan, _db); + return std::make_unique(std::move(fileToScan), _db, _settings); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/scanners/LyricsFileScanner.hpp b/src/libs/services/scanner/impl/scanners/LyricsFileScanner.hpp index 44fb2e1d..3bcb4f0d 100644 --- a/src/libs/services/scanner/impl/scanners/LyricsFileScanner.hpp +++ b/src/libs/services/scanner/impl/scanners/LyricsFileScanner.hpp @@ -36,17 +36,19 @@ namespace lms::scanner class LyricsFileScanner : public IFileScanner { public: - LyricsFileScanner(db::Db& db); + LyricsFileScanner(db::Db& db, ScannerSettings& settings); ~LyricsFileScanner() override = default; LyricsFileScanner(const LyricsFileScanner&) = delete; LyricsFileScanner& operator=(const LyricsFileScanner&) = delete; private: core::LiteralString getName() const override; + std::span getSupportedFiles() const override; std::span getSupportedExtensions() const override; - bool needsScan(ScanContext& context, const FileToScan& file) const override; - std::unique_ptr createScanOperation(const FileToScan& fileToScan) const override; + bool needsScan(const FileToScan& file) const override; + std::unique_ptr createScanOperation(FileToScan&& fileToScan) const override; db::Db& _db; + ScannerSettings& _settings; }; } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/scanners/PlayListFileScanner.cpp b/src/libs/services/scanner/impl/scanners/PlayListFileScanner.cpp index cfef85dd..ca168dab 100644 --- a/src/libs/services/scanner/impl/scanners/PlayListFileScanner.cpp +++ b/src/libs/services/scanner/impl/scanners/PlayListFileScanner.cpp @@ -30,7 +30,7 @@ #include "metadata/Exception.hpp" #include "metadata/PlayList.hpp" -#include "IFileScanOperation.hpp" +#include "FileScanOperationBase.hpp" #include "ScanContext.hpp" #include "Utils.hpp" @@ -38,26 +38,18 @@ namespace lms::scanner { namespace { - class PlayListFileScanOperation : public IFileScanOperation + class PlayListFileScanOperation : public FileScanOperationBase { public: - PlayListFileScanOperation(const FileToScan& file, db::Db& db) - : _file{ file.file } - , _mediaLibrary{ file.mediaLibrary } - , _db{ db } {} + using FileScanOperationBase::FileScanOperationBase; ~PlayListFileScanOperation() override = default; PlayListFileScanOperation(const PlayListFileScanOperation&) = delete; PlayListFileScanOperation& operator=(const PlayListFileScanOperation&) = delete; private: - const std::filesystem::path& getFile() const override { return _file; }; core::LiteralString getName() const override { return "ScanPlayListFile"; } void scan() override; - void processResult(ScanContext& context) override; - - const std::filesystem::path _file; - const MediaLibraryInfo _mediaLibrary; - db::Db& _db; + OperationResult processResult() override; std::optional _parsedPlayList; }; @@ -66,74 +58,72 @@ namespace lms::scanner { try { - std::ifstream ifs{ _file }; + std::ifstream ifs{ getFilePath() }; if (!ifs) - LMS_LOG(DBUPDATER, ERROR, "Cannot open file " << _file); - else - _parsedPlayList = metadata::parsePlayList(ifs); + { + const std::error_code ec{ errno, std::generic_category() }; + + addError(getFilePath(), ec); + return; + } + + _parsedPlayList = metadata::parsePlayList(ifs); } catch (const metadata::Exception& e) { - LMS_LOG(DBUPDATER, ERROR, "Cannot read playlist in file " << _file << ": " << e.what()); + addError(getFilePath()); } } - void PlayListFileScanOperation::processResult(ScanContext& context) + PlayListFileScanOperation::OperationResult PlayListFileScanOperation::processResult() { - ScanStats& stats{ context.stats }; - - const std::optional fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) }; - if (!fileInfo) - { - stats.skips++; - return; - } - - db::Session& dbSession{ _db.getTLSSession() }; - db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, _file) }; + db::Session& dbSession{ getDb().getTLSSession() }; + db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, getFilePath()) }; if (!_parsedPlayList) { if (playList) { playList.remove(); - stats.deletions++; - LMS_LOG(DBUPDATER, DEBUG, "Removed playlist file " << _file); + + LMS_LOG(DBUPDATER, DEBUG, "Removed playlist file " << getFilePath()); + return OperationResult::Removed; } - context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadPlayListFile); - return; + + return OperationResult::Skipped; } const bool added{ !playList }; if (!playList) - playList = dbSession.create(_file); + playList = dbSession.create(getFilePath()); - playList.modify()->setLastWriteTime(fileInfo->lastWriteTime); - playList.modify()->setFileSize(fileInfo->fileSize); + playList.modify()->setLastWriteTime(getLastWriteTime()); + playList.modify()->setFileSize(getFileSize()); if (!_parsedPlayList->name.empty()) playList.modify()->setName(_parsedPlayList->name); else - playList.modify()->setName(_file.stem().string()); + playList.modify()->setName(getFilePath().stem().string()); playList.modify()->setFiles(_parsedPlayList->files); - db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this - playList.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary)); + db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this + playList.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary)); if (added) { - LMS_LOG(DBUPDATER, DEBUG, "Added playlist file " << _file); - stats.additions++; + LMS_LOG(DBUPDATER, DEBUG, "Added playlist file " << getFilePath()); + return OperationResult::Added; } else { - LMS_LOG(DBUPDATER, DEBUG, "Updated playlist file '" << _file); - stats.updates++; + LMS_LOG(DBUPDATER, DEBUG, "Updated playlist file '" << getFilePath()); + return OperationResult::Updated; } } } // namespace - PlayListFileScanner::PlayListFileScanner(db::Db& db) + PlayListFileScanner::PlayListFileScanner(db::Db& db, ScannerSettings& settings) : _db{ db } + , _settings{ settings } { } @@ -142,41 +132,27 @@ namespace lms::scanner return "PlayList scanner"; } + std::span PlayListFileScanner::getSupportedFiles() const + { + return {}; + } + std::span PlayListFileScanner::getSupportedExtensions() const { return metadata::getSupportedPlayListFileExtensions(); } - bool PlayListFileScanner::needsScan(ScanContext& context, const FileToScan& file) const + bool PlayListFileScanner::needsScan(const FileToScan& file) const { - ScanStats& stats{ context.stats }; + db::Session& dbSession{ _db.getTLSSession() }; + auto transaction{ _db.getTLSSession().createReadTransaction() }; - const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) }; - // Should rarely fail as we are currently iterating it - if (!lastWriteTime.isValid()) - { - stats.skips++; - return false; - } - - if (!context.scanOptions.fullScan) - { - db::Session& dbSession{ _db.getTLSSession() }; - auto transaction{ _db.getTLSSession().createReadTransaction() }; - - const db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, file.file) }; - if (playList && playList->getLastWriteTime() == lastWriteTime) - { - stats.skips++; - return false; - } - } - - return true; // need to scan + const db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, file.filePath) }; + return !playList || playList->getLastWriteTime() != file.lastWriteTime; } - std::unique_ptr PlayListFileScanner::createScanOperation(const FileToScan& fileToScan) const + std::unique_ptr PlayListFileScanner::createScanOperation(FileToScan&& fileToScan) const { - return std::make_unique(fileToScan, _db); + return std::make_unique(std::move(fileToScan), _db, _settings); } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/scanners/PlayListFileScanner.hpp b/src/libs/services/scanner/impl/scanners/PlayListFileScanner.hpp index d61d7e67..db0f2333 100644 --- a/src/libs/services/scanner/impl/scanners/PlayListFileScanner.hpp +++ b/src/libs/services/scanner/impl/scanners/PlayListFileScanner.hpp @@ -36,17 +36,19 @@ namespace lms::scanner class PlayListFileScanner : public IFileScanner { public: - PlayListFileScanner(db::Db& db); + PlayListFileScanner(db::Db& db, ScannerSettings& settings); ~PlayListFileScanner() override = default; PlayListFileScanner(const PlayListFileScanner&) = delete; PlayListFileScanner& operator=(const PlayListFileScanner&) = delete; private: core::LiteralString getName() const override; + std::span getSupportedFiles() const override; std::span getSupportedExtensions() const override; - bool needsScan(ScanContext& context, const FileToScan& file) const override; - std::unique_ptr createScanOperation(const FileToScan& fileToScan) const override; + bool needsScan(const FileToScan& file) const override; + std::unique_ptr createScanOperation(FileToScan&& fileToScan) const override; db::Db& _db; + ScannerSettings& _settings; }; } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/scanners/Utils.cpp b/src/libs/services/scanner/impl/scanners/Utils.cpp index 48e7bbfd..4469a3a1 100644 --- a/src/libs/services/scanner/impl/scanners/Utils.cpp +++ b/src/libs/services/scanner/impl/scanners/Utils.cpp @@ -19,7 +19,8 @@ #include "Utils.hpp" -#include "core/ILogger.hpp" +#include + #include "core/Path.hpp" #include "database/Directory.hpp" #include "database/MediaLibrary.hpp" @@ -27,59 +28,6 @@ namespace lms::scanner::utils { - Wt::WDateTime retrieveFileGetLastWrite(const std::filesystem::path& file) - { - Wt::WDateTime res; - - try - { - res = core::pathUtils::getLastWriteTime(file); - } - catch (core::LmsException& e) - { - LMS_LOG(DBUPDATER, ERROR, "Cannot get last write time: " << e.what()); - } - - return res; - } - - std::optional retrieveFileInfo(const std::filesystem::path& file, const std::filesystem::path& rootPath) - { - std::optional res; - res.emplace(); - - res->lastWriteTime = retrieveFileGetLastWrite(file); - if (!res->lastWriteTime.isValid()) - { - res.reset(); - return res; - } - - { - std::error_code ec; - res->relativePath = std::filesystem::relative(file, rootPath, ec); - if (ec) - { - LMS_LOG(DBUPDATER, ERROR, "Cannot get relative file path for '" << file.string() << "' from '" << rootPath.string() << "': " << ec.message()); - res.reset(); - return res; - } - } - - { - std::error_code ec; - res->fileSize = std::filesystem::file_size(file, ec); - if (ec) - { - LMS_LOG(DBUPDATER, ERROR, "Cannot get file size for '" << file.string() << "': " << ec.message()); - res.reset(); - return res; - } - } - - return res; - } - db::Directory::pointer getOrCreateDirectory(db::Session& session, const std::filesystem::path& path, const db::MediaLibrary::pointer& mediaLibrary) { db::Directory::pointer directory{ db::Directory::find(session, path) }; diff --git a/src/libs/services/scanner/impl/scanners/Utils.hpp b/src/libs/services/scanner/impl/scanners/Utils.hpp index 2adafd3e..710e3309 100644 --- a/src/libs/services/scanner/impl/scanners/Utils.hpp +++ b/src/libs/services/scanner/impl/scanners/Utils.hpp @@ -20,9 +20,6 @@ #pragma once #include -#include - -#include #include "database/Object.hpp" @@ -35,18 +32,8 @@ namespace lms::db namespace lms::scanner { - struct FileInfo - { - Wt::WDateTime lastWriteTime; - std::filesystem::path relativePath; - std::size_t fileSize{}; - }; - namespace utils { - Wt::WDateTime retrieveFileGetLastWrite(const std::filesystem::path& file); - std::optional retrieveFileInfo(const std::filesystem::path& file, const std::filesystem::path& rootPath); - db::ObjectPtr getOrCreateDirectory(db::Session& session, const std::filesystem::path& path, const db::ObjectPtr& mediaLibrary); } // namespace utils } // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/steps/FileScanQueue.cpp b/src/libs/services/scanner/impl/steps/FileScanQueue.cpp index 4c71027e..9b61a808 100644 --- a/src/libs/services/scanner/impl/steps/FileScanQueue.cpp +++ b/src/libs/services/scanner/impl/steps/FileScanQueue.cpp @@ -53,7 +53,7 @@ namespace lms::scanner { { LMS_SCOPED_TRACE_OVERVIEW("Scanner", operation->getName()); - LMS_LOG(DBUPDATER, DEBUG, operation->getName() << ": scanning file " << operation->getFile()); + LMS_LOG(DBUPDATER, DEBUG, operation->getName() << ": scanning file " << operation->getFilePath()); operation->scan(); } diff --git a/src/libs/services/scanner/impl/steps/IScanStep.hpp b/src/libs/services/scanner/impl/steps/IScanStep.hpp index 44adb1f7..7f851e3f 100644 --- a/src/libs/services/scanner/impl/steps/IScanStep.hpp +++ b/src/libs/services/scanner/impl/steps/IScanStep.hpp @@ -20,11 +20,12 @@ #pragma once #include "core/LiteralString.hpp" - -#include "ScanContext.hpp" +#include "services/scanner/ScannerStats.hpp" namespace lms::scanner { + struct ScanContext; + class IScanStep { public: diff --git a/src/libs/services/scanner/impl/steps/ScanErrorLogger.cpp b/src/libs/services/scanner/impl/steps/ScanErrorLogger.cpp new file mode 100644 index 00000000..c42b8523 --- /dev/null +++ b/src/libs/services/scanner/impl/steps/ScanErrorLogger.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanErrorLogger.hpp" + +#include + +#include "core/ILogger.hpp" + +namespace lms::scanner +{ + void ScanErrorLogger::visit([[maybe_unused]] const scanner::ScanError& error) + { + // There should never be a ScanError without a specific type + assert(false); + } + + void ScanErrorLogger::visit(const scanner::IOScanError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to open file " << error.path << ": " << error.err.message()); + } + + void ScanErrorLogger::visit(const scanner::AudioFileScanError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << error.path); + } + + void ScanErrorLogger::visit(const scanner::EmbeddedImageScanError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to parse image in track file " << error.path << " at index " << error.index); + } + + void ScanErrorLogger::visit(const scanner::NoAudioTrackFoundError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << error.path << ": no audio track found"); + } + + void ScanErrorLogger::visit(const scanner::BadAudioDurationError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << error.path << ": duration is 0"); + } + + void ScanErrorLogger::visit(const scanner::ArtistInfoFileScanError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to read artist info file " << error.path); + } + + void ScanErrorLogger::visit(const scanner::MissingArtistNameError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to read artist info file " << error.path << ": missing name"); + } + + void ScanErrorLogger::visit(const scanner::ImageFileScanError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to read image file " << error.path); + } + + void ScanErrorLogger::visit(const scanner::LyricsFileScanError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to read lyrics file " << error.path); + } + + void ScanErrorLogger::visit(const scanner::PlayListFileScanError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to read playlist file " << error.path); + } + + void ScanErrorLogger::visit(const scanner::PlayListFilePathMissingError& error) + { + LMS_LOG(DBUPDATER, DEBUG, "Track " << error.entry << " not found in playlist " << error.path); + } + + void ScanErrorLogger::visit(const scanner::PlayListFileAllPathesMissingError& error) + { + LMS_LOG(DBUPDATER, ERROR, "Failed to parse playlist " << error.path << ": all entries are missing"); + } +} // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/steps/ScanErrorLogger.hpp b/src/libs/services/scanner/impl/steps/ScanErrorLogger.hpp new file mode 100644 index 00000000..fbf45d75 --- /dev/null +++ b/src/libs/services/scanner/impl/steps/ScanErrorLogger.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "services/scanner/ScanErrors.hpp" + +namespace lms::scanner +{ + class ScanErrorLogger : public scanner::ScanErrorVisitor + { + private: + void visit(const scanner::ScanError&) override; + void visit(const scanner::IOScanError& error) override; + void visit(const scanner::AudioFileScanError& error) override; + void visit(const scanner::EmbeddedImageScanError& error) override; + void visit(const scanner::NoAudioTrackFoundError& error) override; + void visit(const scanner::BadAudioDurationError& error) override; + void visit(const scanner::ArtistInfoFileScanError& error) override; + void visit(const scanner::MissingArtistNameError& error) override; + void visit(const scanner::ImageFileScanError& error) override; + void visit(const scanner::LyricsFileScanError& error) override; + void visit(const scanner::PlayListFileScanError& error) override; + void visit(const scanner::PlayListFilePathMissingError& error) override; + void visit(const scanner::PlayListFileAllPathesMissingError& error) override; + }; +} // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/steps/ScanStepArtistReconciliation.cpp b/src/libs/services/scanner/impl/steps/ScanStepArtistReconciliation.cpp index 8882649b..c39bc2ba 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepArtistReconciliation.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepArtistReconciliation.cpp @@ -32,6 +32,7 @@ #include "database/TrackList.hpp" #include "metadata/Types.hpp" +#include "ScanContext.hpp" #include "ScannerSettings.hpp" #include "helpers/ArtistHelpers.hpp" diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp index e770f089..96c16532 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp @@ -37,6 +37,8 @@ #include "database/Session.hpp" #include "database/Track.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { namespace diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateExternalLyrics.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateExternalLyrics.cpp index 19f4a1f3..7765eb58 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateExternalLyrics.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateExternalLyrics.cpp @@ -28,6 +28,8 @@ #include "database/Track.hpp" #include "database/TrackLyrics.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { namespace diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp index 628363e4..7e058cb5 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp @@ -30,7 +30,9 @@ #include "database/Session.hpp" #include "database/Track.hpp" #include "database/TrackList.hpp" +#include "services/scanner/ScanErrors.hpp" +#include "ScanContext.hpp" #include "ScannerSettings.hpp" namespace lms::scanner @@ -57,6 +59,7 @@ namespace lms::scanner db::PlayListFileId lastRetrievedPlayListFileId; std::size_t processedPlayListFileCount{}; const ScannerSettings& settings; + std::vector> errors; }; db::Track::pointer getMatchingTrack(db::Session& session, const std::filesystem::path& filePath, const db::Directory::pointer& playListDirectory) @@ -81,10 +84,7 @@ namespace lms::scanner return true; const db::ReleaseId releaseId{ tracks.front().releaseId }; - if (std::all_of(std::cbegin(tracks) + 1, std::cend(tracks), [=](const TrackInfo& trackInfo) { return trackInfo.releaseId == releaseId; })) - return true; - - return false; + return std::all_of(std::cbegin(tracks) + 1, std::cend(tracks), [=](const TrackInfo& trackInfo) { return trackInfo.releaseId == releaseId; }); } bool trackListNeedsUpdate(db::Session& session, std::string_view name, std::span tracks, const db::TrackList::pointer& trackList) @@ -124,17 +124,28 @@ namespace lms::scanner playListAssociation.playListFileIdId = playListFile->getId(); + std::vector> pendingErrors; + const auto files{ playListFile->getFiles() }; for (const std::filesystem::path& file : files) { // TODO optim: no need to fetch the whole track - db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) }; + const db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) }; if (track) playListAssociation.tracks.push_back(TrackInfo{ .trackId = track->getId(), .releaseId = track->getReleaseId() }); else - LMS_LOG(DBUPDATER, DEBUG, "Track " << file << " not found in playlist " << playListFile->getAbsoluteFilePath()); + { + pendingErrors.emplace_back(std::make_shared(playListFile->getAbsoluteFilePath(), file)); + } } + if (pendingErrors.size() == files.size()) + { + pendingErrors.clear(); + pendingErrors.emplace_back(std::make_shared(playListFile->getAbsoluteFilePath())); + } + searchContext.errors.insert(std::end(searchContext.errors), std::begin(pendingErrors), std::end(pendingErrors)); + if (playListAssociation.tracks.empty() || (searchContext.settings.skipSingleReleasePlayLists && isSingleReleasePlayList(playListAssociation.tracks))) { @@ -237,6 +248,7 @@ namespace lms::scanner .session = session, .lastRetrievedPlayListFileId = {}, .settings = _settings, + .errors = context.stats.errors }; PlayListFileAssociationContainer playListFileAssociations; @@ -246,7 +258,12 @@ namespace lms::scanner return; updatePlayListFiles(session, playListFileAssociations); + context.currentStepStats.processedElems = searchContext.processedPlayListFileCount; + for (const std::shared_ptr& error : searchContext.errors) + addError(context, error); + searchContext.errors.clear(); + _progressCallback(context.currentStepStats); } } diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp index 4ec18e89..d758e192 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp @@ -34,6 +34,8 @@ #include "database/Session.hpp" #include "database/Track.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { namespace diff --git a/src/libs/services/scanner/impl/steps/ScanStepBase.cpp b/src/libs/services/scanner/impl/steps/ScanStepBase.cpp index fa09f4ac..a82d7fb7 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepBase.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepBase.cpp @@ -19,6 +19,11 @@ #include "ScanStepBase.hpp" +#include "core/String.hpp" + +#include "ScanContext.hpp" +#include "scanners/IFileScanner.hpp" + namespace lms::scanner { ScanStepBase::ScanStepBase(InitParams& initParams) @@ -29,7 +34,58 @@ namespace lms::scanner , _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners)) , _lastScanSettings{ initParams.lastScanSettings } { + for (IFileScanner* scanner : _fileScanners) + { + for (const std::filesystem::path& file : scanner->getSupportedFiles()) + { + [[maybe_unused]] auto [it, inserted]{ _scannerByFile.emplace(file, scanner) }; + assert(inserted); + } + + for (const std::filesystem::path& extension : scanner->getSupportedExtensions()) + { + [[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) }; + assert(inserted); + } + } } ScanStepBase::~ScanStepBase() = default; + + IFileScanner* ScanStepBase::selectFileScanner(const std::filesystem::path& filePath) const + { + { + const std::string fileName{ core::stringUtils::stringToLower(filePath.filename().string()) }; + + auto itScanner{ _scannerByFile.find(fileName) }; + if (itScanner != std::cend(_scannerByFile)) + return itScanner->second; + } + + { + const std::string extension{ core::stringUtils::stringToLower(filePath.extension().string()) }; + + auto itScanner{ _scannerByExtension.find(extension) }; + if (itScanner != std::cend(_scannerByExtension)) + return itScanner->second; + } + + return nullptr; + } + + void ScanStepBase::visitFileScanners(const std::function& visitor) const + { + for (IFileScanner* scanner : _fileScanners) + visitor(scanner); + } + + void ScanStepBase::addError(ScanContext& context, std::shared_ptr error) + { + error->accept(_scanErrorLogger); + + context.stats.errorsCount++; + + if (context.stats.errors.size() < ScanStats::maxStoredErrorCount) + context.stats.errors.emplace_back(error); + } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/steps/ScanStepBase.hpp b/src/libs/services/scanner/impl/steps/ScanStepBase.hpp index 0b209415..a467ddab 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepBase.hpp +++ b/src/libs/services/scanner/impl/steps/ScanStepBase.hpp @@ -21,9 +21,11 @@ #include #include +#include #include #include "IScanStep.hpp" +#include "ScanErrorLogger.hpp" namespace lms::db { @@ -35,6 +37,7 @@ namespace lms::scanner class IFileScanner; struct ScannerSettings; struct ScanStepStats; + struct ScanContext; class ScanStepBase : public IScanStep { @@ -57,14 +60,29 @@ namespace lms::scanner protected: const ScannerSettings* getLastScanSettings() const { return _lastScanSettings; } + IFileScanner* selectFileScanner(const std::filesystem::path& filePath) const; + void visitFileScanners(const std::function& visitor) const; + + void addError(ScanContext& context, std::shared_ptr error); + + template + void addError(ScanContext& context, CtrArgs&&... args) + { + auto error{ std::make_shared(std::forward(args)...) }; + addError(context, error); + } const ScannerSettings& _settings; ProgressCallback _progressCallback; bool& _abortScan; db::Db& _db; - std::vector _fileScanners; private: + std::unordered_map _scannerByFile; + std::unordered_map _scannerByExtension; + std::vector _fileScanners; + const ScannerSettings* _lastScanSettings{}; + ScanErrorLogger _scanErrorLogger; }; } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/steps/ScanStepCheckForDuplicatedFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepCheckForDuplicatedFiles.cpp index 0fc8deaa..cea51cc6 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepCheckForDuplicatedFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepCheckForDuplicatedFiles.cpp @@ -24,6 +24,8 @@ #include "database/Session.hpp" #include "database/Track.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { bool ScanStepCheckForDuplicatedFiles::needProcess([[maybe_unused]] const ScanContext& context) const diff --git a/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp index 6f04948b..35a1eeb1 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp @@ -30,8 +30,8 @@ #include "database/Session.hpp" #include "database/Track.hpp" #include "database/TrackLyrics.hpp" -#include "scanners/IFileScanner.hpp" +#include "ScanContext.hpp" #include "ScannerSettings.hpp" namespace lms::scanner @@ -62,22 +62,15 @@ namespace lms::scanner } LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " files to be checked..."); - std::vector supportedFileExtensions; - for (IFileScanner* scanner : _fileScanners) - { - for (const std::filesystem::path& extension : scanner->getSupportedExtensions()) - supportedFileExtensions.emplace_back(extension); - } - - checkForRemovedFiles(context, supportedFileExtensions); - checkForRemovedFiles(context, supportedFileExtensions); - checkForRemovedFiles(context, supportedFileExtensions); - checkForRemovedFiles(context, supportedFileExtensions); - checkForRemovedFiles(context, supportedFileExtensions); + checkForRemovedFiles(context); + checkForRemovedFiles(context); + checkForRemovedFiles(context); + checkForRemovedFiles(context); + checkForRemovedFiles(context); } template - void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context, std::span supportedFileExtensions) + void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context) { using namespace db; @@ -110,7 +103,7 @@ namespace lms::scanner return; } - if (!checkFile(object->getAbsoluteFilePath(), supportedFileExtensions)) + if (!checkFile(object->getAbsoluteFilePath())) objectsToRemove.push_back(object); context.currentStepStats.processedElems++; @@ -132,7 +125,7 @@ namespace lms::scanner } } - bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p, std::span allowedExtensions) + bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p) { try { @@ -140,7 +133,7 @@ namespace lms::scanner // and still belongs to a media directory if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p)) { - LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': missing"); + LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing"); return false; } @@ -149,13 +142,13 @@ namespace lms::scanner return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName); })) { - LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': out of media directory"); + LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory"); return false; } - if (!core::pathUtils::hasFileAnyExtension(p, allowedExtensions)) + if (!selectFileScanner(p)) { - LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': file format no longer handled"); + LMS_LOG(DBUPDATER, DEBUG, "Removing " << p.string() << ": file format no longer handled"); return false; } diff --git a/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.hpp b/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.hpp index d33bec8c..90d4ceae 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.hpp +++ b/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.hpp @@ -20,7 +20,6 @@ #pragma once #include -#include #include "ScanStepBase.hpp" @@ -38,8 +37,8 @@ namespace lms::scanner void process(ScanContext& context) override; template - void checkForRemovedFiles(ScanContext& context, std::span supportedFileExtensions); + void checkForRemovedFiles(ScanContext& context); - bool checkFile(const std::filesystem::path& p, std::span allowedExtensions); + bool checkFile(const std::filesystem::path& p); }; } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/steps/ScanStepCompact.cpp b/src/libs/services/scanner/impl/steps/ScanStepCompact.cpp index bc740555..130d3408 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepCompact.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepCompact.cpp @@ -22,6 +22,8 @@ #include "database/Db.hpp" #include "database/Session.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { bool ScanStepCompact::needProcess(const ScanContext& context) const diff --git a/src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.cpp b/src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.cpp index e4344d95..e2fd438a 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.cpp @@ -23,6 +23,8 @@ #include "database/Db.hpp" #include "database/Session.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { bool ScanStepComputeClusterStats::needProcess(const ScanContext& context) const diff --git a/src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.cpp index 8babbd5b..61056433 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.cpp @@ -24,7 +24,8 @@ #include "MediaLibraryInfo.hpp" #include "ScannerSettings.hpp" -#include "scanners/IFileScanner.hpp" + +#include "ScanContext.hpp" namespace lms::scanner { @@ -38,13 +39,6 @@ namespace lms::scanner { context.stats.totalFileCount = 0; - std::vector supportedFileExtensions; - for (IFileScanner* scanner : _fileScanners) - { - for (const std::filesystem::path& extension : scanner->getSupportedExtensions()) - supportedFileExtensions.emplace_back(extension); - } - for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries) { std::size_t currentDirectoryProcessElemsCount{}; @@ -53,7 +47,8 @@ namespace lms::scanner if (_abortScan) return false; - if (!ec && core::pathUtils::hasFileAnyExtension(path, supportedFileExtensions)) + // we don't report errors here (done in the actual scan step) + if (!ec && selectFileScanner(path)) { context.currentStepStats.processedElems++; currentDirectoryProcessElemsCount++; @@ -64,7 +59,7 @@ namespace lms::scanner }, &excludeDirFileName); - LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in '" << mediaLibrary.rootDirectory << "'"); + LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in " << mediaLibrary.rootDirectory); } context.stats.totalFileCount = context.currentStepStats.processedElems; diff --git a/src/libs/services/scanner/impl/steps/ScanStepOptimize.cpp b/src/libs/services/scanner/impl/steps/ScanStepOptimize.cpp index 97550d22..576f79d9 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepOptimize.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepOptimize.cpp @@ -23,6 +23,8 @@ #include "database/Db.hpp" #include "database/Session.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { bool ScanStepOptimize::needProcess(const ScanContext& context) const diff --git a/src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.cpp b/src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.cpp index 4a5e78ea..360331bd 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.cpp @@ -29,6 +29,8 @@ #include "database/Track.hpp" #include "database/TrackEmbeddedImage.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { bool ScanStepRemoveOrphanedDbEntries::needProcess([[maybe_unused]] const ScanContext& context) const diff --git a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp index 7434c8b3..1ed23520 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp @@ -27,9 +27,12 @@ #include "core/Path.hpp" #include "database/Db.hpp" #include "database/Session.hpp" +#include "scanners/FileToScan.hpp" #include "scanners/IFileScanOperation.hpp" #include "scanners/IFileScanner.hpp" +#include "ScanContext.hpp" + namespace lms::scanner { using namespace db; @@ -45,21 +48,38 @@ namespace lms::scanner return threadCount; } + + FileToScan retrieveFileInfo(const std::filesystem::path& file, const MediaLibraryInfo& mediaLibrary, std::error_code& ec) + { + FileToScan res; + + res.lastWriteTime = core::pathUtils::getLastWriteTime(file, ec); + if (!ec) + res.relativePath = std::filesystem::relative(file, mediaLibrary.rootDirectory, ec); + if (!ec) + res.fileSize = std::filesystem::file_size(file, ec); + + if (!ec) + { + res.filePath = file; + res.mediaLibrary = mediaLibrary; + } + + return res; + } } // namespace ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams) : ScanStepBase{ initParams } , _fileScanQueue{ getScanMetaDataThreadCount(), _abortScan } { - for (IFileScanner* scanner : _fileScanners) - { + visitFileScanners([](IFileScanner* scanner) { + for (const std::filesystem::path& file : scanner->getSupportedFiles()) + LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file " << file); + for (const std::filesystem::path& extension : scanner->getSupportedExtensions()) - { - [[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) }; - assert(inserted); - LMS_LOG(DBUPDATER, INFO, "Registered extension " << extension << " for " << scanner->getName()); - } - } + LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file extension " << extension); + }); LMS_LOG(DBUPDATER, INFO, "Using " << _fileScanQueue.getThreadCount() << " thread(s) for scanning file metadata"); } @@ -94,26 +114,28 @@ namespace lms::scanner if (ec) { - LMS_LOG(DBUPDATER, ERROR, "Cannot scan file " << path << ": " << ec.message()); - context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() }); + addError(context, path, ec); + context.stats.skips++; } - else + else if (IFileScanner * scanner{ selectFileScanner(path) }) { - auto itScanner{ _scannerByExtension.find(core::stringUtils::stringToLower(path.extension().c_str())) }; - if (itScanner != std::cend(_scannerByExtension)) + FileToScan fileToScan{ retrieveFileInfo(path, mediaLibrary, ec) }; + if (ec) { - IFileScanner& scanner{ *itScanner->second }; - - FileToScan fileToScan{ .file = path, .mediaLibrary = mediaLibrary }; - if (scanner.needsScan(context, fileToScan)) + addError(context, path, ec); + context.stats.skips++; + } + else + { + if (context.scanOptions.fullScan || scanner->needsScan(fileToScan)) { - auto scanOperation{ scanner.createScanOperation(fileToScan) }; + auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) }; _fileScanQueue.pushScanRequest(std::move(scanOperation)); } - - context.currentStepStats.processedElems++; - _progressCallback(context.currentStepStats); } + + context.currentStepStats.processedElems++; + _progressCallback(context.currentStepStats); } while (_fileScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2)) @@ -146,9 +168,27 @@ namespace lms::scanner if (_abortScan) return; - LMS_LOG(DBUPDATER, DEBUG, scanOperation->getName() << ": processing result for " << scanOperation->getFile()); - scanOperation->processResult(context); + LMS_LOG(DBUPDATER, DEBUG, scanOperation->getName() << ": processing result for " << scanOperation->getFilePath()); + const IFileScanOperation::OperationResult res{ scanOperation->processResult() }; + switch (res) + { + case IFileScanOperation::OperationResult::Added: + context.stats.additions++; + break; + case IFileScanOperation::OperationResult::Removed: + context.stats.deletions++; + break; + case IFileScanOperation::OperationResult::Skipped: + context.stats.failures++; + break; + case IFileScanOperation::OperationResult::Updated: + context.stats.updates++; + break; + } context.stats.scans++; + + for (const auto& error : scanOperation->getErrors()) + addError(context, error); } } } // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp index 8dc9bd70..90e9ce87 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp +++ b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp @@ -40,10 +40,10 @@ namespace lms::scanner core::LiteralString getStepName() const override { return "Scan files"; } bool needProcess(const ScanContext& context) const override; void process(ScanContext& context) override; + void process(ScanContext& context, const MediaLibraryInfo& mediaLibrary); void processFileScanResults(ScanContext& context, std::span> scanOperations); FileScanQueue _fileScanQueue; - std::unordered_map _scannerByExtension; }; } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/steps/ScanStepUpdateLibraryFields.cpp b/src/libs/services/scanner/impl/steps/ScanStepUpdateLibraryFields.cpp index 35dcda97..1e9b52fd 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepUpdateLibraryFields.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepUpdateLibraryFields.cpp @@ -25,6 +25,7 @@ #include "database/Session.hpp" #include "MediaLibraryInfo.hpp" +#include "ScanContext.hpp" #include "ScannerSettings.hpp" namespace lms::scanner diff --git a/src/libs/services/scanner/include/services/scanner/ScanErrors.hpp b/src/libs/services/scanner/include/services/scanner/ScanErrors.hpp new file mode 100644 index 00000000..b57c44f6 --- /dev/null +++ b/src/libs/services/scanner/include/services/scanner/ScanErrors.hpp @@ -0,0 +1,205 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +namespace lms::scanner +{ + // Forward declarations of all error types + struct ScanError; + struct IOScanError; + struct AudioFileScanError; + struct EmbeddedImageScanError; + struct NoAudioTrackFoundError; + struct BadAudioDurationError; + struct ArtistInfoFileScanError; + struct MissingArtistNameError; + struct ImageFileScanError; + struct LyricsFileScanError; + struct PlayListFileScanError; + struct PlayListFilePathMissingError; + struct PlayListFileAllPathesMissingError; + + // Visitor interface + struct ScanErrorVisitor + { + virtual ~ScanErrorVisitor() = default; + + virtual void visit(const ScanError&) = 0; + virtual void visit(const IOScanError&) = 0; + virtual void visit(const AudioFileScanError&) = 0; + virtual void visit(const EmbeddedImageScanError&) = 0; + virtual void visit(const NoAudioTrackFoundError&) = 0; + virtual void visit(const BadAudioDurationError&) = 0; + virtual void visit(const ArtistInfoFileScanError&) = 0; + virtual void visit(const MissingArtistNameError&) = 0; + virtual void visit(const ImageFileScanError&) = 0; + virtual void visit(const LyricsFileScanError&) = 0; + virtual void visit(const PlayListFileScanError&) = 0; + virtual void visit(const scanner::PlayListFilePathMissingError& error) = 0; + virtual void visit(const scanner::PlayListFileAllPathesMissingError& error) = 0; + }; + + struct ScanError + { + ScanError(const std::filesystem::path& p) + : path{ p } {} + virtual ~ScanError() = default; + virtual void accept(ScanErrorVisitor&) const = 0; + + std::filesystem::path path; // Error that occurs on this path + }; + + struct IOScanError : public ScanError + { + IOScanError(const std::filesystem::path& p, std::error_code e) + : ScanError{ p } + , err{ e } {} + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + + std::error_code err; + }; + + struct AudioFileScanError : public ScanError + { + using ScanError::ScanError; + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + }; + + struct EmbeddedImageScanError : public AudioFileScanError + { + EmbeddedImageScanError(const std::filesystem::path& p, unsigned i) + : AudioFileScanError{ p } + , index{ i } {} + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + + unsigned index; + }; + + struct NoAudioTrackFoundError : public AudioFileScanError + { + using AudioFileScanError::AudioFileScanError; + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + }; + + struct BadAudioDurationError : public AudioFileScanError + { + using AudioFileScanError::AudioFileScanError; + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + }; + + struct ArtistInfoFileScanError : public ScanError + { + using ScanError::ScanError; + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + }; + + struct MissingArtistNameError : public ArtistInfoFileScanError + { + using ArtistInfoFileScanError::ArtistInfoFileScanError; + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + }; + + struct ImageFileScanError : public ScanError + { + using ScanError::ScanError; + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + }; + + struct LyricsFileScanError : public ScanError + { + using ScanError::ScanError; + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + }; + + struct PlayListFileScanError : public ScanError + { + using ScanError::ScanError; + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + }; + + struct PlayListFilePathMissingError : public PlayListFileScanError + { + PlayListFilePathMissingError(const std::filesystem::path& p, const std::filesystem::path& e) + : PlayListFileScanError{ p } + , entry{ e } + { + } + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + + std::filesystem::path entry; + }; + + struct PlayListFileAllPathesMissingError : public PlayListFileScanError + { + using PlayListFileScanError::PlayListFileScanError; + + void accept(ScanErrorVisitor& visitor) const override + { + visitor.visit(*this); + } + }; +} // namespace lms::scanner diff --git a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp index 614d2a96..13951d34 100644 --- a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp +++ b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp @@ -21,40 +21,21 @@ #include -#include +#include #include #include "database/TrackId.hpp" +#include "ScanErrors.hpp" + namespace lms::scanner { - enum class ScanErrorType - { - CannotReadFile, - CannotReadArtistInfoFile, - CannotReadAudioFile, - CannotReadImageFile, - CannotReadLyricsFile, - CannotReadPlayListFile, - NoAudioTrack, - BadDuration, - }; - enum class DuplicateReason { SameHash, SameTrackMBID, }; - struct ScanError - { - std::filesystem::path file; - ScanErrorType error; - std::string systemError; - - ScanError(const std::filesystem::path& file, ScanErrorType error, const std::string& systemError = ""); - }; - struct ScanDuplicate { db::TrackId trackId; @@ -105,15 +86,18 @@ namespace lms::scanner std::size_t totalFileCount{}; // Total number of files (estimated) std::size_t skips{}; // no change since last scan - std::size_t scans{}; // actually scanned filed + std::size_t scans{}; // count of scanned files std::size_t additions{}; // added in DB std::size_t deletions{}; // removed from DB std::size_t updates{}; // updated file in DB + std::size_t failures{}; // scan failure std::size_t featuresFetched{}; // features fetched in DB - std::vector errors; + static constexpr std::size_t maxStoredErrorCount{ 5'000 }; // TODO make this configurable + std::vector> errors; + std::size_t errorsCount{}; // maybe bigger than errors.size() if too many errors std::vector duplicates; std::size_t nbFiles() const; diff --git a/src/lms/CMakeLists.txt b/src/lms/CMakeLists.txt index 19f1790b..b5027141 100644 --- a/src/lms/CMakeLists.txt +++ b/src/lms/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable(lms ui/admin/MediaLibraryModal.cpp ui/admin/TracingView.cpp ui/admin/ScannerController.cpp + ui/admin/ScannerReportResource.cpp ui/admin/ScanSettingsView.cpp ui/admin/UserView.cpp ui/admin/UsersView.cpp diff --git a/src/lms/ui/LmsApplication.cpp b/src/lms/ui/LmsApplication.cpp index b5932496..01ecca92 100644 --- a/src/lms/ui/LmsApplication.cpp +++ b/src/lms/ui/LmsApplication.cpp @@ -542,7 +542,7 @@ namespace lms::ui .arg(static_cast(stats.updates)) .arg(static_cast(stats.deletions)) .arg(static_cast(stats.duplicates.size())) - .arg(static_cast(stats.errors.size()))); + .arg(static_cast(stats.errorsCount))); }); } diff --git a/src/lms/ui/admin/ScannerController.cpp b/src/lms/ui/admin/ScannerController.cpp index dff32263..5cefb88a 100644 --- a/src/lms/ui/admin/ScannerController.cpp +++ b/src/lms/ui/admin/ScannerController.cpp @@ -19,21 +19,19 @@ #include "ScannerController.hpp" -#include -#include #include #include #include #include #include +#include #include "core/Service.hpp" #include "core/String.hpp" -#include "database/Session.hpp" -#include "database/Track.hpp" #include "services/scanner/IScannerService.hpp" #include "LmsApplication.hpp" +#include "ScannerReportResource.hpp" namespace lms::ui { @@ -45,110 +43,6 @@ namespace lms::ui } } // namespace - class ScannerReportResource : public Wt::WResource - { - public: - ScannerReportResource() = default; - ~ScannerReportResource() override - { - beingDeleted(); - } - ScannerReportResource(const ScannerReportResource&) = delete; - ScannerReportResource& operator=(const ScannerReportResource&) = delete; - - void setScanStats(const scanner::ScanStats& stats) - { - if (!_stats) - _stats = std::make_unique(); - - *_stats = stats; - } - - void handleRequest(const Wt::Http::Request&, Wt::Http::Response& response) override - { - if (!_stats) - return; - - auto encodeHttpHeaderField = [](const std::string& fieldName, const std::string& fieldValue) { - // This implements RFC 5987 - return fieldName + "*=UTF-8''" + Wt::Utils::urlEncode(fieldValue); - }; - - const std::string cdp{ encodeHttpHeaderField("filename", "LMS_scan_report_" + core::stringUtils::toISO8601String(_stats->startTime) + ".txt") }; - response.addHeader("Content-Disposition", "attachment; " + cdp); - - response.out() << Wt::WString::tr("Lms.Admin.ScannerController.errors-header").arg(_stats->errors.size()).toUTF8() << std::endl; - - for (const auto& error : _stats->errors) - { - response.out() << error.file.string() << " - " << errorTypeToWString(error.error).toUTF8(); - if (!error.systemError.empty()) - response.out() << ": " << error.systemError; - response.out() << std::endl; - } - - response.out() << std::endl; - - response.out() << Wt::WString::tr("Lms.Admin.ScannerController.duplicates-header").arg(_stats->duplicates.size()).toUTF8() << std::endl; - - { - auto transaction{ LmsApp->getDbSession().createReadTransaction() }; - - for (const auto& duplicate : _stats->duplicates) - { - const auto& track{ db::Track::find(LmsApp->getDbSession(), duplicate.trackId) }; - if (!track) - continue; - - response.out() << track->getAbsoluteFilePath().string(); - if (auto mbid{ track->getTrackMBID() }) - response.out() << " (Track MBID " << mbid->getAsString() << ")"; - - response.out() << " - " << duplicateReasonToWString(duplicate.reason).toUTF8() << '\n'; - } - } - } - - private: - static Wt::WString errorTypeToWString(scanner::ScanErrorType error) - { - switch (error) - { - case scanner::ScanErrorType::CannotReadFile: - return Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-file"); - case scanner::ScanErrorType::CannotReadArtistInfoFile: - return Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-artist-info-file"); - case scanner::ScanErrorType::CannotReadAudioFile: - return Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-audio-file"); - case scanner::ScanErrorType::CannotReadImageFile: - return Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-image-file"); - case scanner::ScanErrorType::CannotReadLyricsFile: - return Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-lyrics-file"); - case scanner::ScanErrorType::CannotReadPlayListFile: - return Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-playlist-file"); - case scanner::ScanErrorType::NoAudioTrack: - return Wt::WString::tr("Lms.Admin.ScannerController.no-audio-track"); - case scanner::ScanErrorType::BadDuration: - return Wt::WString::tr("Lms.Admin.ScannerController.bad-duration"); - } - return "?"; - } - - static Wt::WString duplicateReasonToWString(scanner::DuplicateReason reason) - { - switch (reason) - { - case scanner::DuplicateReason::SameHash: - return Wt::WString::tr("Lms.Admin.ScannerController.same-hash"); - case scanner::DuplicateReason::SameTrackMBID: - return Wt::WString::tr("Lms.Admin.ScannerController.same-mbid"); - } - return "?"; - } - - std::unique_ptr _stats; - }; - ScannerController::ScannerController() : WTemplate{ Wt::WString::tr("Lms.Admin.ScannerController.template") } { @@ -225,7 +119,7 @@ namespace lms::ui .arg(durationToString(status.lastCompleteScanStats->startTime, status.lastCompleteScanStats->stopTime)) .arg(status.lastCompleteScanStats->stopTime.date().toString(Wt::WLocale::currentLocale().dateFormat())) .arg(status.lastCompleteScanStats->stopTime.time().toString(Wt::WLocale::currentLocale().timeFormat())) - .arg(status.lastCompleteScanStats->errors.size()) + .arg(status.lastCompleteScanStats->errorsCount) .arg(status.lastCompleteScanStats->duplicates.size())); _reportResource->setScanStats(*status.lastCompleteScanStats); diff --git a/src/lms/ui/admin/ScannerReportResource.cpp b/src/lms/ui/admin/ScannerReportResource.cpp new file mode 100644 index 00000000..e430757f --- /dev/null +++ b/src/lms/ui/admin/ScannerReportResource.cpp @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScannerReportResource.hpp" + +#include +#include + +#include "core/String.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" + +#include "LmsApplication.hpp" +#include "services/scanner/ScanErrors.hpp" + +namespace lms::ui +{ + namespace + { + class ErrorFormatter : public scanner::ScanErrorVisitor + { + public: + ErrorFormatter(std::ostream& os) + : _os{ os } {} + + private: + void visit(const scanner::ScanError&) override + { + } + + void visit(const scanner::IOScanError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-file").arg(Wt::WString::fromUTF8(error.err.message())).toUTF8() << '\n'; + } + + void visit(const scanner::AudioFileScanError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-audio-file").toUTF8() << '\n'; + } + void visit(const scanner::EmbeddedImageScanError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.bad-embedded-image").arg(error.index).toUTF8() << '\n'; + } + void visit(const scanner::NoAudioTrackFoundError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.no-audio-track").toUTF8() << '\n'; + } + void visit(const scanner::BadAudioDurationError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.bad-duration").toUTF8() << '\n'; + } + void visit(const scanner::ArtistInfoFileScanError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-artist-info-file").toUTF8() << '\n'; + } + void visit(const scanner::MissingArtistNameError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.missing-artist-name").toUTF8() << '\n'; + } + void visit(const scanner::ImageFileScanError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-image-file").toUTF8() << '\n'; + } + void visit(const scanner::LyricsFileScanError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-lyrics-file").toUTF8() << '\n'; + } + void visit(const scanner::PlayListFileScanError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.cannot-read-playlist-file").toUTF8() << '\n'; + } + void visit(const scanner::PlayListFilePathMissingError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.playlist-path-missing").arg(Wt::WString::fromUTF8(error.entry)).toUTF8() << '\n'; + } + void visit(const scanner::PlayListFileAllPathesMissingError& error) override + { + _os << error.path << ": " << Wt::WString::tr("Lms.Admin.ScannerController.playlist-all-pathes-missing").toUTF8() << '\n'; + } + + std::ostream& _os; + }; + } // namespace + + ScannerReportResource::ScannerReportResource() = default; + ScannerReportResource::~ScannerReportResource() + { + beingDeleted(); + } + + void ScannerReportResource::setScanStats(const scanner::ScanStats& stats) + { + if (!_stats) + _stats = std::make_unique(); + + *_stats = stats; + } + + void ScannerReportResource::handleRequest(const Wt::Http::Request&, Wt::Http::Response& response) + { + if (!_stats) + return; + + auto encodeHttpHeaderField = [](const std::string& fieldName, const std::string& fieldValue) { + // This implements RFC 5987 + return fieldName + "*=UTF-8''" + Wt::Utils::urlEncode(fieldValue); + }; + + const std::string cdp{ encodeHttpHeaderField("filename", "LMS_scan_report_" + core::stringUtils::toISO8601String(_stats->startTime) + ".txt") }; + response.addHeader("Content-Disposition", "attachment; " + cdp); + + response.out() << Wt::WString::tr("Lms.Admin.ScannerController.errors-header").arg(_stats->errorsCount).toUTF8() << std::endl; + + ErrorFormatter errorFormatter{ response.out() }; + for (const auto& error : _stats->errors) + error->accept(errorFormatter); + + response.out() << std::endl; + + response.out() << Wt::WString::tr("Lms.Admin.ScannerController.duplicates-header").arg(_stats->duplicates.size()).toUTF8() << std::endl; + + { + auto transaction{ LmsApp->getDbSession().createReadTransaction() }; + + for (const auto& duplicate : _stats->duplicates) + { + const auto& track{ db::Track::find(LmsApp->getDbSession(), duplicate.trackId) }; + if (!track) + continue; + + response.out() << track->getAbsoluteFilePath().string(); + if (auto mbid{ track->getTrackMBID() }) + response.out() << " (Track MBID " << mbid->getAsString() << ")"; + + response.out() << " - " << duplicateReasonToWString(duplicate.reason).toUTF8() << '\n'; + } + } + } + + Wt::WString ScannerReportResource::duplicateReasonToWString(scanner::DuplicateReason reason) + { + switch (reason) + { + case scanner::DuplicateReason::SameHash: + return Wt::WString::tr("Lms.Admin.ScannerController.same-hash"); + case scanner::DuplicateReason::SameTrackMBID: + return Wt::WString::tr("Lms.Admin.ScannerController.same-mbid"); + } + return "?"; + } + +} // namespace lms::ui diff --git a/src/lms/ui/admin/ScannerReportResource.hpp b/src/lms/ui/admin/ScannerReportResource.hpp new file mode 100644 index 00000000..7fa17458 --- /dev/null +++ b/src/lms/ui/admin/ScannerReportResource.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ +#pragma once + +#include + +#include "services/scanner/ScannerStats.hpp" + +namespace lms::ui +{ + + class ScannerReportResource : public Wt::WResource + { + public: + ScannerReportResource(); + ~ScannerReportResource() override; + ScannerReportResource(const ScannerReportResource&) = delete; + ScannerReportResource& operator=(const ScannerReportResource&) = delete; + + void setScanStats(const scanner::ScanStats& stats); + void handleRequest(const Wt::Http::Request& request, Wt::Http::Response& response) override; + + private: + static Wt::WString formatScanError(const scanner::ScanError* error); + static Wt::WString duplicateReasonToWString(scanner::DuplicateReason reason); + std::unique_ptr _stats; + }; + +} // namespace lms::ui diff --git a/src/tools/metadata/LmsMetadata.cpp b/src/tools/metadata/LmsMetadata.cpp index 44904722..f6d15ca4 100644 --- a/src/tools/metadata/LmsMetadata.cpp +++ b/src/tools/metadata/LmsMetadata.cpp @@ -462,7 +462,7 @@ int main(int argc, char* argv[]) { std::filesystem::path file{ inputFile }; - std::cout << "Parsing file '" << file << "'" << std::endl; + std::cout << "Parsing file " << file << std::endl; if (parsers.contains(Parser::Lyrics)) {