From 1800aaf32efbe9b04f4b16618248a9729d4d25d2 Mon Sep 17 00:00:00 2001 From: emeric Date: Mon, 7 Jul 2025 23:07:51 +0200 Subject: [PATCH] Explore files in the filesystem with a single stat for each file --- src/libs/core/impl/Path.cpp | 80 ++++--------------- src/libs/core/include/core/Path.hpp | 8 +- .../impl/scanners/AudioFileScanOperation.cpp | 6 +- .../impl/scanners/FileScanOperationBase.hpp | 1 - .../scanner/impl/scanners/FileToScan.hpp | 1 - .../scanner/impl/steps/ScanStepScanFiles.cpp | 43 +++------- 6 files changed, 32 insertions(+), 107 deletions(-) diff --git a/src/libs/core/impl/Path.cpp b/src/libs/core/impl/Path.cpp index 47f717a3..fc7c188a 100644 --- a/src/libs/core/impl/Path.cpp +++ b/src/libs/core/impl/Path.cpp @@ -20,45 +20,13 @@ #include "core/Path.hpp" #include -#include -#include -#include -#include #include -#include - -#include "core/Crc32Calculator.hpp" -#include "core/Exception.hpp" #include "core/ILogger.hpp" #include "core/String.hpp" namespace lms::core::pathUtils { - std::uint32_t computeCrc32(const std::filesystem::path& p) - { - core::Crc32Calculator crc32; - - std::ifstream ifs{ p, std::ios_base::binary }; - if (ifs) - { - do - { - std::array buffer; - - ifs.read(buffer.data(), buffer.size()); - crc32.processBytes(reinterpret_cast(buffer.data()), ifs.gcount()); - } while (ifs); - } - else - { - LMS_LOG(DBUPDATER, ERROR, "Failed to open file " << p); - throw LmsException("Failed to open file '" + p.string() + "'"); - } - - return crc32.getResult(); - } - bool ensureDirectory(const std::filesystem::path& dir) { if (std::filesystem::exists(dir)) @@ -67,35 +35,14 @@ namespace lms::core::pathUtils return std::filesystem::create_directory(dir); } - FileInfo getFileInfo(const std::filesystem::path& file, std::error_code& ec) - { - FileInfo fileInfo; - - struct stat sb - { - }; - if (stat(file.c_str(), &sb) == -1) - { - ec = std::error_code{ errno, std::generic_category() }; - } - else - { - ec = std::error_code{}; - fileInfo.lastWriteTime = Wt::WDateTime::fromTime_t(sb.st_mtime); - fileInfo.fileSize = sb.st_size; - } - - return fileInfo; - } - - bool exploreFilesRecursive(const std::filesystem::path& directory, std::function cb, const std::filesystem::path* excludeDirFileName) + bool exploreFilesRecursive(const std::filesystem::path& directory, ExploreFileCallback cb, const std::filesystem::path* excludeDirFileName) { std::error_code ec; std::filesystem::directory_iterator itPath{ directory, std::filesystem::directory_options::follow_directory_symlink, ec }; if (ec) { - cb(ec, directory); + cb(ec, directory, nullptr); return true; // try to continue exploring anyway } @@ -105,6 +52,7 @@ namespace lms::core::pathUtils if (std::filesystem::exists(excludePath, ec)) { + // TODO: handle exclude another way + remove this log LMS_LOG(DBUPDATER, DEBUG, "Found " << excludePath << ": skipping directory"); return true; } @@ -117,22 +65,22 @@ namespace lms::core::pathUtils if (ec) { - continueExploring = cb(ec, *itPath); + continueExploring = cb(ec, *itPath, nullptr); } else { - // TODO get status once and then test regular file/directory - if (std::filesystem::is_regular_file(*itPath, ec)) + const std::filesystem::directory_entry& entry{ *itPath }; + + if (entry.is_regular_file()) { - continueExploring = cb(ec, *itPath); - } - else if (std::filesystem::is_directory(*itPath, ec)) - { - if (!ec) - continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName); - else - continueExploring = cb(ec, *itPath); + FileInfo fileInfo; + fileInfo.fileSize = entry.file_size(); + fileInfo.lastWriteTime = Wt::WDateTime{ std::chrono::file_clock::to_sys(entry.last_write_time()) }; + + continueExploring = cb(ec, *itPath, &fileInfo); } + else if (entry.is_directory()) + continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName); } if (!continueExploring) diff --git a/src/libs/core/include/core/Path.hpp b/src/libs/core/include/core/Path.hpp index 704c5e77..ccc4c64b 100644 --- a/src/libs/core/include/core/Path.hpp +++ b/src/libs/core/include/core/Path.hpp @@ -28,8 +28,6 @@ namespace lms::core::pathUtils { - std::uint32_t computeCrc32(const std::filesystem::path& p); - // Make sure the given path is a directory // Create it if needed bool ensureDirectory(const std::filesystem::path& dir); @@ -39,11 +37,9 @@ namespace lms::core::pathUtils Wt::WDateTime lastWriteTime; // Last write time of the file since Epoch std::uint64_t fileSize{}; // Size of the file in bytes }; - // Get the last write time since Epoch - FileInfo getFileInfo(const std::filesystem::path& file, std::error_code& ec); - + using ExploreFileCallback = std::function; // returns false if aborted by user - bool exploreFilesRecursive(const std::filesystem::path& directory, std::function cb, const std::filesystem::path* excludeDirFileName = {}); + bool exploreFilesRecursive(const std::filesystem::path& directory, ExploreFileCallback cb, const std::filesystem::path* excludeDirFileName = {}); // Check if file's extension is one of provided extensions bool hasFileAnyExtension(const std::filesystem::path& file, std::span extensions); diff --git a/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp b/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp index d29a2731..4406e91b 100644 --- a/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp +++ b/src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp @@ -379,7 +379,7 @@ namespace lms::scanner return db::Advisory::UnSet; } - db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, size_t fileSize, const std::filesystem::path& relativePath) + db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, const std::filesystem::path& trackPath, size_t fileSize) { db::Track::FindParameters params; // Add as many fields as possible to limit errors @@ -405,7 +405,7 @@ namespace lms::scanner if (res) { - LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << relativePath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath()); + LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << trackPath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath()); error = true; } res = track; @@ -598,7 +598,7 @@ namespace lms::scanner if (!track) { // maybe the file just moved? - track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, getFileSize(), getRelativeFilePath()); + track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, getFilePath(), getFileSize()); if (track) { LMS_LOG(DBUPDATER, DEBUG, "Considering track " << getFilePath() << " moved from " << track->getAbsoluteFilePath()); diff --git a/src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp b/src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp index 6d3b717d..33482190 100644 --- a/src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp +++ b/src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp @@ -50,7 +50,6 @@ namespace lms::scanner const ScannerSettings& getScannerSettings() const { return _settings; } Wt::WDateTime getLastWriteTime() const { return _file.lastWriteTime; } std::size_t getFileSize() const { return _file.fileSize; } - const std::filesystem::path& getRelativeFilePath() const { return _file.relativePath; } template void addError(CtrArgs&&... args) diff --git a/src/libs/services/scanner/impl/scanners/FileToScan.hpp b/src/libs/services/scanner/impl/scanners/FileToScan.hpp index 8f44015d..4996b26e 100644 --- a/src/libs/services/scanner/impl/scanners/FileToScan.hpp +++ b/src/libs/services/scanner/impl/scanners/FileToScan.hpp @@ -30,7 +30,6 @@ namespace lms::scanner struct FileToScan { std::filesystem::path filePath; - std::filesystem::path relativePath; MediaLibraryInfo mediaLibrary; Wt::WDateTime lastWriteTime; std::size_t fileSize{}; diff --git a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp index 69128b98..5b1f91e7 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp @@ -66,23 +66,6 @@ namespace lms::scanner std::unique_ptr _scanOperation; }; - - FileToScan retrieveFileInfo(const std::filesystem::path& file, const MediaLibraryInfo& mediaLibrary, std::error_code& ec) - { - FileToScan res; - - const core::pathUtils::FileInfo fileInfo{ core::pathUtils::getFileInfo(file, ec) }; - if (!ec) - { - res.filePath = file; - res.mediaLibrary = mediaLibrary; - res.relativePath = std::filesystem::relative(file, mediaLibrary.rootDirectory, ec); - res.lastWriteTime = fileInfo.lastWriteTime; - res.fileSize = fileInfo.fileSize; - } - - return res; - } } // namespace ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams) @@ -126,9 +109,11 @@ namespace lms::scanner std::vector> scanOperations; core::pathUtils::exploreFilesRecursive( - mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) { + mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path, const core::pathUtils::FileInfo* fileInfo) { LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile"); + assert((ec && !fileInfo) || (!ec && fileInfo)); + if (_abortScan) return false; // stop iterating @@ -139,19 +124,17 @@ namespace lms::scanner } else if (IFileScanner * scanner{ selectFileScanner(path) }) { - FileToScan fileToScan{ retrieveFileInfo(path, mediaLibrary, ec) }; - if (ec) + FileToScan fileToScan; + + fileToScan.filePath = path; + fileToScan.mediaLibrary = mediaLibrary; + fileToScan.lastWriteTime.setTime_t(fileInfo->lastWriteTime.toTime_t()); // sec resolution, as stored in the database + fileToScan.fileSize = fileInfo->fileSize; + + if (context.scanOptions.fullScan || scanner->needsScan(fileToScan)) { - addError(context, path, ec); - context.stats.skips++; - } - else - { - if (context.scanOptions.fullScan || scanner->needsScan(fileToScan)) - { - auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) }; - queue.push(std::make_unique(std::move(scanOperation))); - } + auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) }; + queue.push(std::make_unique(std::move(scanOperation))); } context.currentStepStats.processedElems++;