owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Explore files in the filesystem with a single stat for each file

Browse Source
This commit is contained in:
emeric
2025-07-07 23:07:51 +02:00
parent 49afdb3db6
commit 1800aaf32e
6 changed files with 32 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/core/impl/Path.cpp
+14 -66
View File
@@ -20,45 +20,13 @@
#include "core/Path.hpp"
#include <array>
#include <fstream>
#include <sys/stat.h>
#include <sys/types.h>
#include <system_error>
#include <unistd.h>
#include <boost/tokenizer.hpp>
#include "core/Crc32Calculator.hpp"
#include "core/Exception.hpp"
#include "core/ILogger.hpp"
#include "core/String.hpp"
namespace lms::core::pathUtils
{
std::uint32_t computeCrc32(const std::filesystem::path& p)
{
core::Crc32Calculator crc32;
std::ifstream ifs{ p, std::ios_base::binary };
if (ifs)
{
do
{
std::array<char, 1024> buffer;
ifs.read(buffer.data(), buffer.size());
crc32.processBytes(reinterpret_cast<const std::byte*>(buffer.data()), ifs.gcount());
} while (ifs);
}
else
{
LMS_LOG(DBUPDATER, ERROR, "Failed to open file " << p);
throw LmsException("Failed to open file '" + p.string() + "'");
}
return crc32.getResult();
}
bool ensureDirectory(const std::filesystem::path& dir)
{
if (std::filesystem::exists(dir))
@@ -67,35 +35,14 @@ namespace lms::core::pathUtils
return std::filesystem::create_directory(dir);
}
FileInfo getFileInfo(const std::filesystem::path& file, std::error_code& ec)
{
FileInfo fileInfo;
struct stat sb
{
};
if (stat(file.c_str(), &sb) == -1)
{
ec = std::error_code{ errno, std::generic_category() };
}
else
{
ec = std::error_code{};
fileInfo.lastWriteTime = Wt::WDateTime::fromTime_t(sb.st_mtime);
fileInfo.fileSize = sb.st_size;
}
return fileInfo;
}
bool exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path* excludeDirFileName)
bool exploreFilesRecursive(const std::filesystem::path& directory, ExploreFileCallback cb, const std::filesystem::path* excludeDirFileName)
{
std::error_code ec;
std::filesystem::directory_iterator itPath{ directory, std::filesystem::directory_options::follow_directory_symlink, ec };
if (ec)
{
cb(ec, directory);
cb(ec, directory, nullptr);
return true; // try to continue exploring anyway
}
@@ -105,6 +52,7 @@ namespace lms::core::pathUtils
if (std::filesystem::exists(excludePath, ec))
{
// TODO: handle exclude another way + remove this log
LMS_LOG(DBUPDATER, DEBUG, "Found " << excludePath << ": skipping directory");
return true;
}
@@ -117,22 +65,22 @@ namespace lms::core::pathUtils
if (ec)
{
continueExploring = cb(ec, *itPath);
continueExploring = cb(ec, *itPath, nullptr);
}
else
{
// TODO get status once and then test regular file/directory
if (std::filesystem::is_regular_file(*itPath, ec))
const std::filesystem::directory_entry& entry{ *itPath };
if (entry.is_regular_file())
{
continueExploring = cb(ec, *itPath);
}
else if (std::filesystem::is_directory(*itPath, ec))
{
if (!ec)
continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName);
else
continueExploring = cb(ec, *itPath);
FileInfo fileInfo;
fileInfo.fileSize = entry.file_size();
fileInfo.lastWriteTime = Wt::WDateTime{ std::chrono::file_clock::to_sys(entry.last_write_time()) };
continueExploring = cb(ec, *itPath, &fileInfo);
}
else if (entry.is_directory())
continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName);
}
if (!continueExploring)
src/libs/core/include/core/Path.hpp
+2 -6
View File
@@ -28,8 +28,6 @@
namespace lms::core::pathUtils
{
std::uint32_t computeCrc32(const std::filesystem::path& p);
// Make sure the given path is a directory
// Create it if needed
bool ensureDirectory(const std::filesystem::path& dir);
@@ -39,11 +37,9 @@ namespace lms::core::pathUtils
Wt::WDateTime lastWriteTime; // Last write time of the file since Epoch
std::uint64_t fileSize{}; // Size of the file in bytes
};
// Get the last write time since Epoch
FileInfo getFileInfo(const std::filesystem::path& file, std::error_code& ec);
using ExploreFileCallback = std::function<bool(std::error_code, const std::filesystem::path&, const FileInfo* fileInfo)>;
// returns false if aborted by user
bool exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path* excludeDirFileName = {});
bool exploreFilesRecursive(const std::filesystem::path& directory, ExploreFileCallback cb, const std::filesystem::path* excludeDirFileName = {});
// Check if file's extension is one of provided extensions
bool hasFileAnyExtension(const std::filesystem::path& file, std::span<const std::filesystem::path> extensions);
src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp
+3 -3
View File
@@ -379,7 +379,7 @@ namespace lms::scanner
return db::Advisory::UnSet;
}
db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, size_t fileSize, const std::filesystem::path& relativePath)
db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, const std::filesystem::path& trackPath, size_t fileSize)
{
db::Track::FindParameters params;
// Add as many fields as possible to limit errors
@@ -405,7 +405,7 @@ namespace lms::scanner
if (res)
{
LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << relativePath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath());
LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << trackPath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath());
error = true;
}
res = track;
@@ -598,7 +598,7 @@ namespace lms::scanner
if (!track)
{
// maybe the file just moved?
track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, getFileSize(), getRelativeFilePath());
track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, getFilePath(), getFileSize());
if (track)
{
LMS_LOG(DBUPDATER, DEBUG, "Considering track " << getFilePath() << " moved from " << track->getAbsoluteFilePath());
src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp
-1
View File
@@ -50,7 +50,6 @@ namespace lms::scanner
const ScannerSettings& getScannerSettings() const { return _settings; }
Wt::WDateTime getLastWriteTime() const { return _file.lastWriteTime; }
std::size_t getFileSize() const { return _file.fileSize; }
const std::filesystem::path& getRelativeFilePath() const { return _file.relativePath; }
template<typename T, typename... CtrArgs>
void addError(CtrArgs&&... args)
src/libs/services/scanner/impl/scanners/FileToScan.hpp
-1
View File
@@ -30,7 +30,6 @@ namespace lms::scanner
struct FileToScan
{
std::filesystem::path filePath;
std::filesystem::path relativePath;
MediaLibraryInfo mediaLibrary;
Wt::WDateTime lastWriteTime;
std::size_t fileSize{};
src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp
+13 -30
View File
@@ -66,23 +66,6 @@ namespace lms::scanner
std::unique_ptr<IFileScanOperation> _scanOperation;
};
FileToScan retrieveFileInfo(const std::filesystem::path& file, const MediaLibraryInfo& mediaLibrary, std::error_code& ec)
{
FileToScan res;
const core::pathUtils::FileInfo fileInfo{ core::pathUtils::getFileInfo(file, ec) };
if (!ec)
{
res.filePath = file;
res.mediaLibrary = mediaLibrary;
res.relativePath = std::filesystem::relative(file, mediaLibrary.rootDirectory, ec);
res.lastWriteTime = fileInfo.lastWriteTime;
res.fileSize = fileInfo.fileSize;
}
return res;
}
} // namespace
ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
@@ -126,9 +109,11 @@ namespace lms::scanner
std::vector<std::unique_ptr<IFileScanOperation>> scanOperations;
core::pathUtils::exploreFilesRecursive(
mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) {
mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path, const core::pathUtils::FileInfo* fileInfo) {
LMS_SCOPED_TRACE_DETAILED("Scanner", "OnExploreFile");
assert((ec && !fileInfo) || (!ec && fileInfo));
if (_abortScan)
return false; // stop iterating
@@ -139,19 +124,17 @@ namespace lms::scanner
}
else if (IFileScanner * scanner{ selectFileScanner(path) })
{
FileToScan fileToScan{ retrieveFileInfo(path, mediaLibrary, ec) };
if (ec)
FileToScan fileToScan;
fileToScan.filePath = path;
fileToScan.mediaLibrary = mediaLibrary;
fileToScan.lastWriteTime.setTime_t(fileInfo->lastWriteTime.toTime_t()); // sec resolution, as stored in the database
fileToScan.fileSize = fileInfo->fileSize;
if (context.scanOptions.fullScan || scanner->needsScan(fileToScan))
{
addError<IOScanError>(context, path, ec);
context.stats.skips++;
}
else
{
if (context.scanOptions.fullScan || scanner->needsScan(fileToScan))
{
auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) };
queue.push(std::make_unique<FileScanJob>(std::move(scanOperation)));
}
auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) };
queue.push(std::make_unique<FileScanJob>(std::move(scanOperation)));
}
context.currentStepStats.processedElems++;