diff --git a/src/libs/database/impl/Session.cpp b/src/libs/database/impl/Session.cpp index b3394b14..b01e7268 100644 --- a/src/libs/database/impl/Session.cpp +++ b/src/libs/database/impl/Session.cpp @@ -23,6 +23,7 @@ #include "core/ILogger.hpp" #include "core/ITraceLogger.hpp" + #include "database/objects/Artist.hpp" #include "database/objects/ArtistInfo.hpp" #include "database/objects/Artwork.hpp" @@ -367,17 +368,17 @@ namespace lms::db }); } - std::size_t Session::getTotalFilesCount() + FileStats Session::getFileStats() { - std::size_t res{}; + FileStats stats{}; - res += db::Track::getCount(*this); - res += db::Image::getCount(*this); - res += db::TrackLyrics::getExternalLyricsCount(*this); - res += db::PlayListFile::getCount(*this); - res += db::ArtistInfo::getCount(*this); + stats.trackCount = db::Track::getCount(*this); + stats.artistInfoCount = db::ArtistInfo::getCount(*this); + stats.imageCount = db::Image::getCount(*this); + stats.trackLyricsCount = db::TrackLyrics::getExternalLyricsCount(*this); + stats.playListCount = db::PlayListFile::getCount(*this); - return res; + return stats; } void Session::retrieveEntriesToAnalyze(std::vector& entryList) diff --git a/src/libs/database/include/database/Session.hpp b/src/libs/database/include/database/Session.hpp index b7e45687..90c0448c 100644 --- a/src/libs/database/include/database/Session.hpp +++ b/src/libs/database/include/database/Session.hpp @@ -27,6 +27,7 @@ #include #include "database/Transaction.hpp" +#include "database/Types.hpp" namespace lms::db { @@ -52,8 +53,8 @@ namespace lms::db void retrieveEntriesToAnalyze(std::vector& entryList); void analyzeEntry(const std::string& entry); - bool areAllTablesEmpty(); // need to acquire a read transaction - std::size_t getTotalFilesCount(); // need to acquire a read transaction + bool areAllTablesEmpty(); // need to acquire a read transaction + FileStats getFileStats(); // need to acquire a read transaction void prepareTablesIfNeeded(); // need to run only once at startup bool migrateSchemaIfNeeded(); // returns true if migration was performed diff --git a/src/libs/database/include/database/Types.hpp b/src/libs/database/include/database/Types.hpp index c2103044..9d4e4fc4 100644 --- a/src/libs/database/include/database/Types.hpp +++ b/src/libs/database/include/database/Types.hpp @@ -102,6 +102,17 @@ namespace lms::db } }; + struct FileStats + { + std::size_t trackCount; + std::size_t imageCount; + std::size_t trackLyricsCount; + std::size_t playListCount; + std::size_t artistInfoCount; + + std::size_t getTotalFileCount() const { return trackCount + imageCount + trackLyricsCount + playListCount + artistInfoCount; } + }; + struct YearRange { int begin{}; diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index 3cf55aa8..b944dafb 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -166,17 +166,17 @@ namespace lms::scanner LMS_LOG(DBUPDATER, INFO, "Using " << _jobScheduler->getThreadCount() << " thread(s) for jobs"); _jobScheduler->setShouldAbortCallback([this]() { return _abortScan; }); - std::size_t totalFilesCount{}; + std::size_t totalFileCount{}; { auto& session{ _db.getTLSSession() }; auto transaction{ session.createReadTransaction() }; - totalFilesCount = session.getTotalFilesCount(); + totalFileCount = session.getFileStats().getTotalFileCount(); } // Force optimize in case scanner aborted during a large import, but do this only if there are enough elements in the database // Otherwise, indexes may be not used and queries may be slower and slower while adding more and more elements in the db - LMS_LOG(DBUPDATER, INFO, "Scanned file count = " << totalFilesCount); - if (totalFilesCount >= 1'000) + LMS_LOG(DBUPDATER, INFO, "Scanned file count = " << totalFileCount); + if (totalFileCount >= 1'000) _db.getTLSSession().fullAnalyze(); refreshTracingLoggerStats(); diff --git a/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp index 0266249b..9a6bee1a 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp @@ -194,7 +194,7 @@ namespace lms::scanner { auto transaction{ session.createReadTransaction() }; - context.currentStepStats.totalElems = session.getTotalFilesCount(); + context.currentStepStats.totalElems = session.getFileStats().getTotalFileCount(); } LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " files to be checked..."); diff --git a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp index bfafaef1..6f319ddf 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp @@ -19,12 +19,13 @@ #include "ScanStepScanFiles.hpp" +#include + #include "ScannerSettings.hpp" #include "core/IJob.hpp" #include "core/IJobScheduler.hpp" #include "core/ILogger.hpp" #include "core/ITraceLogger.hpp" -#include "core/Path.hpp" #include "database/IDb.hpp" #include "database/Session.hpp" #include "scanners/FileToScan.hpp" @@ -93,28 +94,60 @@ namespace lms::scanner class FileScanJob : public core::IJob { public: - FileScanJob(std::unique_ptr scanOperation) - : _scanOperation{ std::move(scanOperation) } + FileScanJob(const FileScanners& fileScanners, const MediaLibraryInfo& mediaLibrary, bool fullScan, std::span files) + : _fileScanners{ fileScanners } + , _mediaLibrary{ mediaLibrary } + , _fullScan{ fullScan } + , _files{ std::cbegin(files), std::cend(files) } { } - IFileScanOperation& getScanOperation() + std::size_t getFileCount() const { return _files.size(); }; + + std::span> getScanOperations() { - return *_scanOperation; + return _scanOperations; } private: core::LiteralString getName() const override { - return _scanOperation->getName(); + return "Scan Files"; } void run() override { - _scanOperation->scan(); + for (const auto& file : _files) + { + IFileScanner* scanner{ _fileScanners.select(file.path()) }; + if (!scanner) + continue; + + FileToScan fileToScan; + + fileToScan.filePath = file.path(); + fileToScan.mediaLibrary = _mediaLibrary; + fileToScan.lastWriteTime.setTime_t(Wt::WDateTime{ std::chrono::file_clock::to_sys(file.last_write_time()) }.toTime_t()); // sec resolution, as stored in the database + fileToScan.fileSize = file.file_size(); + + if (_fullScan || scanner->needsScan(fileToScan)) + { + auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) }; + + { + LMS_SCOPED_TRACE_DETAILED("Scanner", scanOperation->getName()); + scanOperation->scan(); + } + _scanOperations.push_back(std::move(scanOperation)); + } + } } - std::unique_ptr _scanOperation; + const FileScanners& _fileScanners; + const MediaLibraryInfo& _mediaLibrary; + const bool _fullScan; + std::vector _files; + std::vector> _scanOperations; }; } // namespace @@ -128,96 +161,124 @@ namespace lms::scanner { for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries) process(context, mediaLibrary); + + context.stats.totalFileCount = context.currentStepStats.processedElems; } void ScanStepScanFiles::process(ScanContext& context, const MediaLibraryInfo& mediaLibrary) { - const std::size_t scanQueueMaxScanRequestCount{ 50 * getJobScheduler().getThreadCount() }; - constexpr std::size_t processFileResultsBatchSize{ 10 }; + constexpr std::size_t filesPerScanJob{ 10 }; + constexpr std::size_t scanQueueMaxSize{ 50 }; + constexpr std::size_t processFileResultsBatchSize{ 1 }; constexpr float drainRatio{ 0.85 }; + std::deque> operations; + auto processDoneJobs = [&](std::span> jobsDone) { + for (const auto& jobDone : jobsDone) + { + auto& fileScanJob{ static_cast(*jobDone) }; + for (std::unique_ptr& scanOperation : fileScanJob.getScanOperations()) + operations.push_back(std::move(scanOperation)); + + context.currentStepStats.processedElems += fileScanJob.getFileCount(); + } + if (!_abortScan) - processFileScanResults(context, jobsDone); + processFileScanOperations(context, operations, true /* force batch */); + + _progressCallback(context.currentStepStats); }; - JobQueue queue{ getJobScheduler(), scanQueueMaxScanRequestCount, processDoneJobs, processFileResultsBatchSize, drainRatio }; + { + JobQueue queue{ getJobScheduler(), scanQueueMaxSize, processDoneJobs, processFileResultsBatchSize, drainRatio }; - std::vector> jobsDone; - std::vector> scanOperations; + std::vector filesToScan; - exploreFilesRecursive( - mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path, const std::filesystem::directory_entry* fileEntry) { - LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile"); + exploreFilesRecursive( + mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path, const std::filesystem::directory_entry* fileEntry) { + LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile"); - assert((ec && !fileEntry) || (!ec && fileEntry)); + assert((ec && !fileEntry) || (!ec && fileEntry)); - if (_abortScan) - return false; // stop iterating + if (_abortScan) + return false; // stop iterating - if (ec) - { - addError(context, path, ec); - context.stats.skips++; - } - else if (IFileScanner * scanner{ getFileScanners().select(path) }) - { - FileToScan fileToScan; - - fileToScan.filePath = path; - fileToScan.mediaLibrary = mediaLibrary; - fileToScan.lastWriteTime.setTime_t(Wt::WDateTime{ std::chrono::file_clock::to_sys(fileEntry->last_write_time()) }.toTime_t()); // sec resolution, as stored in the database - fileToScan.fileSize = fileEntry->file_size(); - - if (context.scanOptions.fullScan || scanner->needsScan(fileToScan)) + if (ec) { - auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) }; - queue.push(std::make_unique(std::move(scanOperation))); + addError(context, path, ec); + context.stats.skips++; + } + else + { + filesToScan.push_back(*fileEntry); + + if (filesToScan.size() >= filesPerScanJob) + { + queue.push(std::make_unique(getFileScanners(), mediaLibrary, context.scanOptions.fullScan, filesToScan)); + filesToScan.clear(); + } } - context.currentStepStats.processedElems++; - _progressCallback(context.currentStepStats); - } + return true; + }, + &excludeDirFileName); - return true; - }, - &excludeDirFileName); + if (!filesToScan.empty()) + queue.push(std::make_unique(getFileScanners(), mediaLibrary, context.scanOptions.fullScan, filesToScan)); - context.stats.totalFileCount = context.currentStepStats.totalElems; + _progressCallback(context.currentStepStats); + } + + // Process remaining objects + processFileScanOperations(context, operations, false /* force batch */); } - void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span> scanJobs) + std::size_t ScanStepScanFiles::processFileScanOperations(ScanContext& context, std::deque>& scanOperations, bool forceBatch) { + std::size_t count{}; + constexpr std::size_t writeBatchSize{ 10 }; + LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ProcessScanResults"); - db::Session& dbSession{ _db.getTLSSession() }; - auto transaction{ dbSession.createWriteTransaction() }; - - for (auto& scanJob : scanJobs) + while ((forceBatch && scanOperations.size() >= writeBatchSize) || !scanOperations.empty()) { - IFileScanOperation& scanOperation{ static_cast(*scanJob).getScanOperation() }; + db::Session& dbSession{ _db.getTLSSession() }; + auto transaction{ dbSession.createWriteTransaction() }; - LMS_LOG(DBUPDATER, DEBUG, scanOperation.getName() << ": processing result for " << scanOperation.getFilePath()); - const IFileScanOperation::OperationResult res{ scanOperation.processResult() }; - switch (res) + for (std::size_t i{}; !scanOperations.empty() && i < writeBatchSize; ++i) { - case IFileScanOperation::OperationResult::Added: - context.stats.additions++; - break; - case IFileScanOperation::OperationResult::Removed: - context.stats.deletions++; - break; - case IFileScanOperation::OperationResult::Skipped: - context.stats.failures++; - break; - case IFileScanOperation::OperationResult::Updated: - context.stats.updates++; - break; + processFileScanOperation(context, *scanOperations.front()); + scanOperations.pop_front(); + count++; } - context.stats.scans++; - - for (const auto& error : scanOperation.getErrors()) - addError(context, error); } + + return count; + } + + void ScanStepScanFiles::processFileScanOperation(ScanContext& context, IFileScanOperation& scanOperation) + { + LMS_LOG(DBUPDATER, DEBUG, scanOperation.getName() << ": processing result for " << scanOperation.getFilePath()); + const IFileScanOperation::OperationResult res{ scanOperation.processResult() }; + switch (res) + { + case IFileScanOperation::OperationResult::Added: + context.stats.additions++; + break; + case IFileScanOperation::OperationResult::Removed: + context.stats.deletions++; + break; + case IFileScanOperation::OperationResult::Skipped: + context.stats.failures++; + break; + case IFileScanOperation::OperationResult::Updated: + context.stats.updates++; + break; + } + context.stats.scans++; + + for (const auto& error : scanOperation.getErrors()) + addError(context, error); } } // namespace lms::scanner \ No newline at end of file diff --git a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp index fe0b2ece..0713454a 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp +++ b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp @@ -19,7 +19,7 @@ #pragma once -#include +#include #include "ScanStepBase.hpp" @@ -30,7 +30,7 @@ namespace lms::core namespace lms::scanner { - class IFileScanner; + class IFileScanOperation; struct MediaLibraryInfo; class ScanStepScanFiles : public ScanStepBase @@ -45,6 +45,7 @@ namespace lms::scanner void process(ScanContext& context) override; void process(ScanContext& context, const MediaLibraryInfo& mediaLibrary); - void processFileScanResults(ScanContext& context, std::span> scanJobs); + std::size_t processFileScanOperations(ScanContext& context, std::deque>& scanOperations, bool forceBatch); + void processFileScanOperation(ScanContext& context, IFileScanOperation& operation); }; } // namespace lms::scanner