diff --git a/src/libs/services/scanner/CMakeLists.txt b/src/libs/services/scanner/CMakeLists.txt index 3aab7cc5..79122c0b 100644 --- a/src/libs/services/scanner/CMakeLists.txt +++ b/src/libs/services/scanner/CMakeLists.txt @@ -25,6 +25,7 @@ add_library(lmsscanner STATIC impl/steps/ScanStepRemoveOrphanedDbEntries.cpp impl/steps/ScanStepScanFiles.cpp impl/steps/ScanStepUpdateLibraryFields.cpp + impl/FileScanners.cpp impl/ScannerService.cpp impl/ScannerStats.cpp ) diff --git a/src/libs/services/scanner/impl/FileScanners.cpp b/src/libs/services/scanner/impl/FileScanners.cpp new file mode 100644 index 00000000..3a43e656 --- /dev/null +++ b/src/libs/services/scanner/impl/FileScanners.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "FileScanners.hpp" + +#include + +#include "core/String.hpp" + +#include "scanners/IFileScanner.hpp" + +namespace lms::scanner +{ + void FileScanners::add(std::unique_ptr scanner) + { + for (const std::filesystem::path& file : scanner->getSupportedFiles()) + { + [[maybe_unused]] auto [it, inserted]{ _scannerByFile.emplace(file, scanner.get()) }; + assert(inserted); + } + + for (const std::filesystem::path& extension : scanner->getSupportedExtensions()) + { + [[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner.get()) }; + assert(inserted); + } + + _fileScanners.emplace_back(std::move(scanner)); + } + + void FileScanners::clear() + { + _fileScanners.clear(); + _scannerByFile.clear(); + _scannerByExtension.clear(); + } + + IFileScanner* FileScanners::select(const std::filesystem::path& filePath) const + { + { + const std::string fileName{ core::stringUtils::stringToLower(filePath.filename().string()) }; + + auto itScanner{ _scannerByFile.find(fileName) }; + if (itScanner != std::cend(_scannerByFile)) + return itScanner->second; + } + + { + const std::string extension{ core::stringUtils::stringToLower(filePath.extension().string()) }; + + auto itScanner{ _scannerByExtension.find(extension) }; + if (itScanner != std::cend(_scannerByExtension)) + return itScanner->second; + } + + return nullptr; + } + + void FileScanners::visit(const std::function& visitor) const + { + for (const auto& scanner : _fileScanners) + visitor(*scanner); + } + +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/FileScanners.hpp b/src/libs/services/scanner/impl/FileScanners.hpp new file mode 100644 index 00000000..8c7e6af2 --- /dev/null +++ b/src/libs/services/scanner/impl/FileScanners.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2025 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +namespace lms::scanner +{ + class IFileScanner; + + class FileScanners + { + public: + void add(std::unique_ptr scanner); + void clear(); + + IFileScanner* select(const std::filesystem::path& filePath) const; + void visit(const std::function& visitor) const; + + private: + std::unordered_map _scannerByFile; + std::unordered_map _scannerByExtension; + std::vector> _fileScanners; + }; +} // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index 41137b52..53d19a17 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -452,14 +452,19 @@ namespace lms::scanner } }; _fileScanners.clear(); - _fileScanners.emplace_back(std::make_unique(_db, _settings)); - _fileScanners.emplace_back(std::make_unique(_db, _settings)); - _fileScanners.emplace_back(std::make_unique(_db, _settings)); - _fileScanners.emplace_back(std::make_unique(_db, _settings)); - _fileScanners.emplace_back(std::make_unique(_db, _settings)); + _fileScanners.add(std::make_unique(_db, _settings)); + _fileScanners.add(std::make_unique(_db, _settings)); + _fileScanners.add(std::make_unique(_db, _settings)); + _fileScanners.add(std::make_unique(_db, _settings)); + _fileScanners.add(std::make_unique(_db, _settings)); - std::vector fileScanners; - std::transform(std::cbegin(_fileScanners), std::cend(_fileScanners), std::back_inserter(fileScanners), [](const std::unique_ptr& scanner) { return scanner.get(); }); + _fileScanners.visit([](const IFileScanner& scanner) { + for (const std::filesystem::path& file : scanner.getSupportedFiles()) + LMS_LOG(DBUPDATER, INFO, scanner.getName() << ": supporting file " << file); + + for (const std::filesystem::path& extension : scanner.getSupportedExtensions()) + LMS_LOG(DBUPDATER, INFO, scanner.getName() << ": supporting file extension " << extension); + }); ScanStepBase::InitParams params{ .jobScheduler = *_jobScheduler, @@ -468,7 +473,7 @@ namespace lms::scanner .progressCallback = progressFunc, .abortScan = _abortScan, .db = _db, - .fileScanners = fileScanners, + .fileScanners = _fileScanners, }; // Order is important: steps are sequential diff --git a/src/libs/services/scanner/impl/ScannerService.hpp b/src/libs/services/scanner/impl/ScannerService.hpp index 35599da9..2ac1481f 100644 --- a/src/libs/services/scanner/impl/ScannerService.hpp +++ b/src/libs/services/scanner/impl/ScannerService.hpp @@ -31,6 +31,7 @@ #include #include +#include "FileScanners.hpp" #include "ScannerSettings.hpp" #include "database/IDb.hpp" #include "database/Session.hpp" @@ -88,7 +89,7 @@ namespace lms::scanner db::IDb& _db; std::unique_ptr _jobScheduler; - std::vector> _fileScanners; + FileScanners _fileScanners; std::vector> _scanSteps; std::mutex _controlMutex; diff --git a/src/libs/services/scanner/impl/steps/JobQueue.hpp b/src/libs/services/scanner/impl/steps/JobQueue.hpp index a7d16c9b..db68727f 100644 --- a/src/libs/services/scanner/impl/steps/JobQueue.hpp +++ b/src/libs/services/scanner/impl/steps/JobQueue.hpp @@ -37,7 +37,9 @@ namespace lms::scanner public: using ProcessFunction = std::function>)>; - JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float drainThreshold); + // processBatchSize -> how many jobs done to notify at once using processJobsDoneFunc + // drainThreshold: fraction of maxQueueSize at which completed jobs are processed + JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t processBatchSize, float drainThreshold); ~JobQueue(); JobQueue(const JobQueue&) = delete; JobQueue& operator=(const JobQueue&) = delete; diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp index 08b36aed..c3200419 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp @@ -363,14 +363,14 @@ namespace lms::scanner _progressCallback(context.currentStepStats); }; - JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F }; + { + JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F }; - db::ArtistId lastRetrievedArtistId{}; - db::IdRange artistIdRange; - while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange)) - queue.push(std::make_unique(_db, searchParams, artistIdRange)); - - queue.finish(); + db::ArtistId lastRetrievedArtistId{}; + db::IdRange artistIdRange; + while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange)) + queue.push(std::make_unique(_db, searchParams, artistIdRange)); + } // process all remaining associations updateArtistPreferredArtworks(session, artistArtworkAssociations, false); diff --git a/src/libs/services/scanner/impl/steps/ScanStepBase.cpp b/src/libs/services/scanner/impl/steps/ScanStepBase.cpp index bde21dde..ba1be78c 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepBase.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepBase.cpp @@ -19,10 +19,7 @@ #include "ScanStepBase.hpp" -#include "core/String.hpp" - #include "ScanContext.hpp" -#include "scanners/IFileScanner.hpp" namespace lms::scanner { @@ -32,54 +29,13 @@ namespace lms::scanner , _abortScan{ initParams.abortScan } , _db{ initParams.db } , _jobScheduler{ initParams.jobScheduler } - , _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners)) + , _fileScanners(initParams.fileScanners) , _lastScanSettings{ initParams.lastScanSettings } { - for (IFileScanner* scanner : _fileScanners) - { - for (const std::filesystem::path& file : scanner->getSupportedFiles()) - { - [[maybe_unused]] auto [it, inserted]{ _scannerByFile.emplace(file, scanner) }; - assert(inserted); - } - - for (const std::filesystem::path& extension : scanner->getSupportedExtensions()) - { - [[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) }; - assert(inserted); - } - } } ScanStepBase::~ScanStepBase() = default; - IFileScanner* ScanStepBase::selectFileScanner(const std::filesystem::path& filePath) const - { - { - const std::string fileName{ core::stringUtils::stringToLower(filePath.filename().string()) }; - - auto itScanner{ _scannerByFile.find(fileName) }; - if (itScanner != std::cend(_scannerByFile)) - return itScanner->second; - } - - { - const std::string extension{ core::stringUtils::stringToLower(filePath.extension().string()) }; - - auto itScanner{ _scannerByExtension.find(extension) }; - if (itScanner != std::cend(_scannerByExtension)) - return itScanner->second; - } - - return nullptr; - } - - void ScanStepBase::visitFileScanners(const std::function& visitor) const - { - for (IFileScanner* scanner : _fileScanners) - visitor(scanner); - } - void ScanStepBase::addError(ScanContext& context, std::shared_ptr error) { error->accept(_scanErrorLogger); diff --git a/src/libs/services/scanner/impl/steps/ScanStepBase.hpp b/src/libs/services/scanner/impl/steps/ScanStepBase.hpp index a544274a..93b35343 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepBase.hpp +++ b/src/libs/services/scanner/impl/steps/ScanStepBase.hpp @@ -20,9 +20,6 @@ #pragma once #include -#include -#include -#include #include "IScanStep.hpp" #include "ScanErrorLogger.hpp" @@ -39,7 +36,7 @@ namespace lms::db namespace lms::scanner { - class IFileScanner; + class FileScanners; struct ScannerSettings; struct ScanStepStats; struct ScanContext; @@ -57,7 +54,7 @@ namespace lms::scanner ProgressCallback progressCallback; bool& abortScan; db::IDb& db; - std::span fileScanners; + const FileScanners& fileScanners; }; ScanStepBase(InitParams& initParams); ~ScanStepBase() override; @@ -67,8 +64,7 @@ namespace lms::scanner protected: core::IJobScheduler& getJobScheduler() { return _jobScheduler; }; const ScannerSettings* getLastScanSettings() const { return _lastScanSettings; } - IFileScanner* selectFileScanner(const std::filesystem::path& filePath) const; - void visitFileScanners(const std::function& visitor) const; + const FileScanners& getFileScanners() const { return _fileScanners; } void addError(ScanContext& context, std::shared_ptr error); @@ -86,9 +82,7 @@ namespace lms::scanner private: core::IJobScheduler& _jobScheduler; - std::unordered_map _scannerByFile; - std::unordered_map _scannerByExtension; - std::vector _fileScanners; + const FileScanners& _fileScanners; const ScannerSettings* _lastScanSettings{}; ScanErrorLogger _scanErrorLogger; diff --git a/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp index d30dae79..dd255c55 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp @@ -19,8 +19,12 @@ #include "ScanStepCheckForRemovedFiles.hpp" +#include +#include +#include #include +#include "core/IJob.hpp" #include "core/ILogger.hpp" #include "core/Path.hpp" #include "database/IDb.hpp" @@ -31,11 +35,153 @@ #include "database/objects/Track.hpp" #include "database/objects/TrackLyrics.hpp" +#include "FileScanners.hpp" +#include "JobQueue.hpp" #include "ScanContext.hpp" #include "ScannerSettings.hpp" +#include "services/scanner/ScannerStats.hpp" namespace lms::scanner { + namespace + { + template + struct FileToCheck + { + IdType objectId; + std::filesystem::path file; + }; + + template + class CheckForRemovedFilesJob : public core::IJob + { + public: + CheckForRemovedFilesJob(const ScannerSettings& settings, const FileScanners& scanners, std::span> filesToCheck) + : _settings{ settings } + , _scanners{ scanners } + , _filesToCheck{ std::cbegin(filesToCheck), std::cend(filesToCheck) } + { + } + + std::size_t getProcessedCount() const { return _processedCount; } + std::span getObjectsToRemove() const { return _objectsToRemove; } + + private: + core::LiteralString getName() const override { return "Check For Removed Files"; } + void run() override + { + for (const FileToCheck& fileToCheck : _filesToCheck) + { + if (!checkFile(fileToCheck.file)) + _objectsToRemove.push_back(fileToCheck.objectId); + } + + _processedCount += _filesToCheck.size(); + } + + bool checkFile(const std::filesystem::path& p) + { + std::error_code ec; + const std::filesystem::directory_entry fileEntry{ p, ec }; + if (ec) + { + // TODO store error? + LMS_LOG(DBUPDATER, ERROR, "Error while checking file " << p << ": " << ec.message()); + return false; + } + + // For each track, make sure the the file still exists + // and still belongs to a media directory + if (!fileEntry.exists() || !fileEntry.is_regular_file()) + { + LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing"); + return false; + } + + if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries), + [&](const MediaLibraryInfo& libraryInfo) { + return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName); + })) + { + LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory"); + return false; + } + + if (!_scanners.select(p)) + { + LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": file format no longer handled"); + return false; + } + + return true; + } + + const ScannerSettings& _settings; + const FileScanners& _scanners; + std::vector> _filesToCheck; + std::vector _objectsToRemove; + std::size_t _processedCount{}; + }; + + template + std::size_t removeObjects(db::Session& session, std::deque& objectIdsToRemove, bool forceFullBatch) + { + std::size_t removedObjectCount{}; + constexpr std::size_t writeBatchSize{ 50 }; + + std::vector ids; + while ((forceFullBatch && objectIdsToRemove.size() >= writeBatchSize) || !objectIdsToRemove.empty()) + { + for (std::size_t i{}; !objectIdsToRemove.empty() && i < writeBatchSize; ++i) + { + ids.push_back(objectIdsToRemove.front()); + objectIdsToRemove.pop_front(); + } + + { + auto transaction{ session.createWriteTransaction() }; + session.destroy(ids); + } + + removedObjectCount += ids.size(); + ids.clear(); + } + + return removedObjectCount; + } + + template + bool fetchNextFilesToCheck(db::Session& session, typename Object::IdType& lastCheckedId, std::vector>& filesToCheck) + { + constexpr std::size_t batchSize{ 200 }; + + filesToCheck.clear(); + filesToCheck.reserve(batchSize); + + auto transaction{ session.createReadTransaction() }; + + while (filesToCheck.size() < batchSize) + { + const typename Object::IdType previousLastCheckedId{ lastCheckedId }; + Object::findAbsoluteFilePath(session, lastCheckedId, batchSize, [&](Object::IdType objectId, const std::filesystem::path& filePath) { + // special case for track lyrics, only check external lyrics + if constexpr (std::is_same_v) + { + if (filePath.empty()) + return; + } + + filesToCheck.emplace_back(objectId, filePath); + }); + + if (previousLastCheckedId == lastCheckedId) + break; + } + + return !filesToCheck.empty(); + } + } // namespace + bool ScanStepCheckForRemovedFiles::needProcess([[maybe_unused]] const ScanContext& context) const { // always check for removed files @@ -67,91 +213,48 @@ namespace lms::scanner template void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context) { - using namespace db; + using ObjectIdType = typename Object::IdType; if (_abortScan) return; - Session& session{ _db.getTLSSession() }; + db::Session& session{ _db.getTLSSession() }; - std::vector objectIdsToRemove; + std::deque objectIdsToRemove; - typename Object::IdType lastCheckedId; - bool endReached{}; - while (!endReached) - { + auto processJobsDone = [&](std::span> jobs) { if (_abortScan) - break; + return; - objectIdsToRemove.clear(); + for (const auto& job : jobs) { - constexpr std::size_t batchSize = 200; + const auto& checkJob{ static_cast&>(*job) }; + std::span objectsToRemove{ checkJob.getObjectsToRemove() }; - auto transaction{ session.createReadTransaction() }; + objectIdsToRemove.insert(std::end(objectIdsToRemove), std::cbegin(objectsToRemove), std::cend(objectsToRemove)); - endReached = true; - Object::findAbsoluteFilePath(session, lastCheckedId, batchSize, [&](Object::IdType objectId, const std::filesystem::path& filePath) { - endReached = false; - - // special case for track lyrics, only check external lyrics - if constexpr (std::is_same_v) - { - if (filePath.empty()) - return; - } - - if (!checkFile(filePath)) - objectIdsToRemove.push_back(objectId); - - context.currentStepStats.processedElems++; - }); + context.currentStepStats.processedElems += checkJob.getProcessedCount(); } if (!objectIdsToRemove.empty()) { - auto transaction{ session.createWriteTransaction() }; - - session.destroy(objectIdsToRemove); + removeObjects(session, objectIdsToRemove, true); context.stats.deletions += objectIdsToRemove.size(); } - _progressCallback(context.currentStepStats); + }; + + { + JobQueue queue{ getJobScheduler(), 50, processJobsDone, 1, 0.85F }; + + ObjectIdType lastCheckedId; + std::vector> filesToCheck; + while (fetchNextFilesToCheck(session, lastCheckedId, filesToCheck)) + queue.push(std::make_unique>(_settings, getFileScanners(), filesToCheck)); } + + // process all remaining objects + removeObjects(session, objectIdsToRemove, false); } - bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p) - { - try - { - // For each track, make sure the the file still exists - // and still belongs to a media directory - if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p)) - { - LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing"); - return false; - } - - if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries), - [&](const MediaLibraryInfo& libraryInfo) { - return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName); - })) - { - LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory"); - return false; - } - - if (!selectFileScanner(p)) - { - LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": file format no longer handled"); - return false; - } - - return true; - } - catch (std::filesystem::filesystem_error& e) - { - LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file " << p << ": " << e.what()); - return false; - } - } } // namespace lms::scanner diff --git a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp index e0183444..29d9c576 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp @@ -31,6 +31,7 @@ #include "scanners/IFileScanOperation.hpp" #include "scanners/IFileScanner.hpp" +#include "FileScanners.hpp" #include "JobQueue.hpp" #include "ScanContext.hpp" @@ -68,18 +69,6 @@ namespace lms::scanner }; } // namespace - ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams) - : ScanStepBase{ initParams } - { - visitFileScanners([](IFileScanner* scanner) { - for (const std::filesystem::path& file : scanner->getSupportedFiles()) - LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file " << file); - - for (const std::filesystem::path& extension : scanner->getSupportedExtensions()) - LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file extension " << extension); - }); - } - bool ScanStepScanFiles::needProcess([[maybe_unused]] const ScanContext& context) const { // Always need to scan files @@ -122,7 +111,7 @@ namespace lms::scanner addError(context, path, ec); context.stats.skips++; } - else if (IFileScanner * scanner{ selectFileScanner(path) }) + else if (IFileScanner * scanner{ getFileScanners().select(path) }) { FileToScan fileToScan; diff --git a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp index 1dfc315b..fe0b2ece 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp +++ b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp @@ -36,7 +36,7 @@ namespace lms::scanner class ScanStepScanFiles : public ScanStepBase { public: - ScanStepScanFiles(InitParams& initParams); + using ScanStepBase::ScanStepBase; private: ScanStep getStep() const override { return ScanStep::ScanFiles; }