owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made the step 'check for removed files' faster when nothing was actually removed

Browse Source
This commit is contained in:
emeric
2025-07-18 18:39:50 +02:00
parent aa23350f56
commit f31d366731
12 changed files with 329 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp
+169 -66
View File
@@ -19,8 +19,12 @@
#include "ScanStepCheckForRemovedFiles.hpp"
#include <deque>
#include <filesystem>
#include <span>
#include <vector>
#include "core/IJob.hpp"
#include "core/ILogger.hpp"
#include "core/Path.hpp"
#include "database/IDb.hpp"
@@ -31,11 +35,153 @@
#include "database/objects/Track.hpp"
#include "database/objects/TrackLyrics.hpp"
#include "FileScanners.hpp"
#include "JobQueue.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
#include "services/scanner/ScannerStats.hpp"
namespace lms::scanner
{
namespace
{
template<typename IdType>
struct FileToCheck
{
IdType objectId;
std::filesystem::path file;
};
template<typename IdType>
class CheckForRemovedFilesJob : public core::IJob
{
public:
CheckForRemovedFilesJob(const ScannerSettings& settings, const FileScanners& scanners, std::span<const FileToCheck<IdType>> filesToCheck)
: _settings{ settings }
, _scanners{ scanners }
, _filesToCheck{ std::cbegin(filesToCheck), std::cend(filesToCheck) }
{
}
std::size_t getProcessedCount() const { return _processedCount; }
std::span<const IdType> getObjectsToRemove() const { return _objectsToRemove; }
private:
core::LiteralString getName() const override { return "Check For Removed Files"; }
void run() override
{
for (const FileToCheck<IdType>& fileToCheck : _filesToCheck)
{
if (!checkFile(fileToCheck.file))
_objectsToRemove.push_back(fileToCheck.objectId);
}
_processedCount += _filesToCheck.size();
}
bool checkFile(const std::filesystem::path& p)
{
std::error_code ec;
const std::filesystem::directory_entry fileEntry{ p, ec };
if (ec)
{
// TODO store error?
LMS_LOG(DBUPDATER, ERROR, "Error while checking file " << p << ": " << ec.message());
return false;
}
// For each track, make sure the the file still exists
// and still belongs to a media directory
if (!fileEntry.exists() || !fileEntry.is_regular_file())
{
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing");
return false;
}
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
[&](const MediaLibraryInfo& libraryInfo) {
return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
}))
{
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory");
return false;
}
if (!_scanners.select(p))
{
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": file format no longer handled");
return false;
}
return true;
}
const ScannerSettings& _settings;
const FileScanners& _scanners;
std::vector<FileToCheck<IdType>> _filesToCheck;
std::vector<IdType> _objectsToRemove;
std::size_t _processedCount{};
};
template<typename Object>
std::size_t removeObjects(db::Session& session, std::deque<typename Object::IdType>& objectIdsToRemove, bool forceFullBatch)
{
std::size_t removedObjectCount{};
constexpr std::size_t writeBatchSize{ 50 };
std::vector<typename Object::IdType> ids;
while ((forceFullBatch && objectIdsToRemove.size() >= writeBatchSize) || !objectIdsToRemove.empty())
{
for (std::size_t i{}; !objectIdsToRemove.empty() && i < writeBatchSize; ++i)
{
ids.push_back(objectIdsToRemove.front());
objectIdsToRemove.pop_front();
}
{
auto transaction{ session.createWriteTransaction() };
session.destroy<Object>(ids);
}
removedObjectCount += ids.size();
ids.clear();
}
return removedObjectCount;
}
template<typename Object>
bool fetchNextFilesToCheck(db::Session& session, typename Object::IdType& lastCheckedId, std::vector<FileToCheck<typename Object::IdType>>& filesToCheck)
{
constexpr std::size_t batchSize{ 200 };
filesToCheck.clear();
filesToCheck.reserve(batchSize);
auto transaction{ session.createReadTransaction() };
while (filesToCheck.size() < batchSize)
{
const typename Object::IdType previousLastCheckedId{ lastCheckedId };
Object::findAbsoluteFilePath(session, lastCheckedId, batchSize, [&](Object::IdType objectId, const std::filesystem::path& filePath) {
// special case for track lyrics, only check external lyrics
if constexpr (std::is_same_v<Object, db::TrackLyrics>)
{
if (filePath.empty())
return;
}
filesToCheck.emplace_back(objectId, filePath);
});
if (previousLastCheckedId == lastCheckedId)
break;
}
return !filesToCheck.empty();
}
} // namespace
bool ScanStepCheckForRemovedFiles::needProcess([[maybe_unused]] const ScanContext& context) const
{
// always check for removed files
@@ -67,91 +213,48 @@ namespace lms::scanner
template<typename Object>
void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context)
{
using namespace db;
using ObjectIdType = typename Object::IdType;
if (_abortScan)
return;
Session& session{ _db.getTLSSession() };
db::Session& session{ _db.getTLSSession() };
std::vector<typename Object::IdType> objectIdsToRemove;
std::deque<ObjectIdType> objectIdsToRemove;
typename Object::IdType lastCheckedId;
bool endReached{};
while (!endReached)
{
auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
if (_abortScan)
break;
return;
objectIdsToRemove.clear();
for (const auto& job : jobs)
{
constexpr std::size_t batchSize = 200;
const auto& checkJob{ static_cast<const CheckForRemovedFilesJob<ObjectIdType>&>(*job) };
std::span<const ObjectIdType> objectsToRemove{ checkJob.getObjectsToRemove() };
auto transaction{ session.createReadTransaction() };
objectIdsToRemove.insert(std::end(objectIdsToRemove), std::cbegin(objectsToRemove), std::cend(objectsToRemove));
endReached = true;
Object::findAbsoluteFilePath(session, lastCheckedId, batchSize, [&](Object::IdType objectId, const std::filesystem::path& filePath) {
endReached = false;
// special case for track lyrics, only check external lyrics
if constexpr (std::is_same_v<Object, TrackLyrics>)
{
if (filePath.empty())
return;
}
if (!checkFile(filePath))
objectIdsToRemove.push_back(objectId);
context.currentStepStats.processedElems++;
});
context.currentStepStats.processedElems += checkJob.getProcessedCount();
}
if (!objectIdsToRemove.empty())
{
auto transaction{ session.createWriteTransaction() };
session.destroy<Object>(objectIdsToRemove);
removeObjects<Object>(session, objectIdsToRemove, true);
context.stats.deletions += objectIdsToRemove.size();
}
_progressCallback(context.currentStepStats);
};
{
JobQueue queue{ getJobScheduler(), 50, processJobsDone, 1, 0.85F };
ObjectIdType lastCheckedId;
std::vector<FileToCheck<ObjectIdType>> filesToCheck;
while (fetchNextFilesToCheck<Object>(session, lastCheckedId, filesToCheck))
queue.push(std::make_unique<CheckForRemovedFilesJob<ObjectIdType>>(_settings, getFileScanners(), filesToCheck));
}
// process all remaining objects
removeObjects<Object>(session, objectIdsToRemove, false);
}
bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p)
{
try
{
// For each track, make sure the the file still exists
// and still belongs to a media directory
if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
{
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing");
return false;
}
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
[&](const MediaLibraryInfo& libraryInfo) {
return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
}))
{
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory");
return false;
}
if (!selectFileScanner(p))
{
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": file format no longer handled");
return false;
}
return true;
}
catch (std::filesystem::filesystem_error& e)
{
LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file " << p << ": " << e.what());
return false;
}
}
} // namespace lms::scanner