Made the step 'check for removed files' faster when nothing was actually removed
This commit is contained in:
@@ -25,6 +25,7 @@ add_library(lmsscanner STATIC
|
||||
impl/steps/ScanStepRemoveOrphanedDbEntries.cpp
|
||||
impl/steps/ScanStepScanFiles.cpp
|
||||
impl/steps/ScanStepUpdateLibraryFields.cpp
|
||||
impl/FileScanners.cpp
|
||||
impl/ScannerService.cpp
|
||||
impl/ScannerStats.cpp
|
||||
)
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright (C) 2025 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "FileScanners.hpp"
|
||||
|
||||
#include <cassert>
|
||||
|
||||
#include "core/String.hpp"
|
||||
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
void FileScanners::add(std::unique_ptr<IFileScanner> scanner)
|
||||
{
|
||||
for (const std::filesystem::path& file : scanner->getSupportedFiles())
|
||||
{
|
||||
[[maybe_unused]] auto [it, inserted]{ _scannerByFile.emplace(file, scanner.get()) };
|
||||
assert(inserted);
|
||||
}
|
||||
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
{
|
||||
[[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner.get()) };
|
||||
assert(inserted);
|
||||
}
|
||||
|
||||
_fileScanners.emplace_back(std::move(scanner));
|
||||
}
|
||||
|
||||
void FileScanners::clear()
|
||||
{
|
||||
_fileScanners.clear();
|
||||
_scannerByFile.clear();
|
||||
_scannerByExtension.clear();
|
||||
}
|
||||
|
||||
IFileScanner* FileScanners::select(const std::filesystem::path& filePath) const
|
||||
{
|
||||
{
|
||||
const std::string fileName{ core::stringUtils::stringToLower(filePath.filename().string()) };
|
||||
|
||||
auto itScanner{ _scannerByFile.find(fileName) };
|
||||
if (itScanner != std::cend(_scannerByFile))
|
||||
return itScanner->second;
|
||||
}
|
||||
|
||||
{
|
||||
const std::string extension{ core::stringUtils::stringToLower(filePath.extension().string()) };
|
||||
|
||||
auto itScanner{ _scannerByExtension.find(extension) };
|
||||
if (itScanner != std::cend(_scannerByExtension))
|
||||
return itScanner->second;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void FileScanners::visit(const std::function<void(const IFileScanner&)>& visitor) const
|
||||
{
|
||||
for (const auto& scanner : _fileScanners)
|
||||
visitor(*scanner);
|
||||
}
|
||||
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright (C) 2025 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
#include <functional>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class IFileScanner;
|
||||
|
||||
class FileScanners
|
||||
{
|
||||
public:
|
||||
void add(std::unique_ptr<IFileScanner> scanner);
|
||||
void clear();
|
||||
|
||||
IFileScanner* select(const std::filesystem::path& filePath) const;
|
||||
void visit(const std::function<void(const IFileScanner&)>& visitor) const;
|
||||
|
||||
private:
|
||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByFile;
|
||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
|
||||
std::vector<std::unique_ptr<IFileScanner>> _fileScanners;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -452,14 +452,19 @@ namespace lms::scanner
|
||||
} };
|
||||
|
||||
_fileScanners.clear();
|
||||
_fileScanners.emplace_back(std::make_unique<ArtistInfoFileScanner>(_db, _settings));
|
||||
_fileScanners.emplace_back(std::make_unique<AudioFileScanner>(_db, _settings));
|
||||
_fileScanners.emplace_back(std::make_unique<ImageFileScanner>(_db, _settings));
|
||||
_fileScanners.emplace_back(std::make_unique<LyricsFileScanner>(_db, _settings));
|
||||
_fileScanners.emplace_back(std::make_unique<PlayListFileScanner>(_db, _settings));
|
||||
_fileScanners.add(std::make_unique<ArtistInfoFileScanner>(_db, _settings));
|
||||
_fileScanners.add(std::make_unique<AudioFileScanner>(_db, _settings));
|
||||
_fileScanners.add(std::make_unique<ImageFileScanner>(_db, _settings));
|
||||
_fileScanners.add(std::make_unique<LyricsFileScanner>(_db, _settings));
|
||||
_fileScanners.add(std::make_unique<PlayListFileScanner>(_db, _settings));
|
||||
|
||||
std::vector<IFileScanner*> fileScanners;
|
||||
std::transform(std::cbegin(_fileScanners), std::cend(_fileScanners), std::back_inserter(fileScanners), [](const std::unique_ptr<IFileScanner>& scanner) { return scanner.get(); });
|
||||
_fileScanners.visit([](const IFileScanner& scanner) {
|
||||
for (const std::filesystem::path& file : scanner.getSupportedFiles())
|
||||
LMS_LOG(DBUPDATER, INFO, scanner.getName() << ": supporting file " << file);
|
||||
|
||||
for (const std::filesystem::path& extension : scanner.getSupportedExtensions())
|
||||
LMS_LOG(DBUPDATER, INFO, scanner.getName() << ": supporting file extension " << extension);
|
||||
});
|
||||
|
||||
ScanStepBase::InitParams params{
|
||||
.jobScheduler = *_jobScheduler,
|
||||
@@ -468,7 +473,7 @@ namespace lms::scanner
|
||||
.progressCallback = progressFunc,
|
||||
.abortScan = _abortScan,
|
||||
.db = _db,
|
||||
.fileScanners = fileScanners,
|
||||
.fileScanners = _fileScanners,
|
||||
};
|
||||
|
||||
// Order is important: steps are sequential
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
#include <Wt/WIOService.h>
|
||||
#include <Wt/WSignal.h>
|
||||
|
||||
#include "FileScanners.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
#include "database/IDb.hpp"
|
||||
#include "database/Session.hpp"
|
||||
@@ -88,7 +89,7 @@ namespace lms::scanner
|
||||
db::IDb& _db;
|
||||
std::unique_ptr<core::IJobScheduler> _jobScheduler;
|
||||
|
||||
std::vector<std::unique_ptr<IFileScanner>> _fileScanners;
|
||||
FileScanners _fileScanners;
|
||||
std::vector<std::unique_ptr<IScanStep>> _scanSteps;
|
||||
|
||||
std::mutex _controlMutex;
|
||||
|
||||
@@ -37,7 +37,9 @@ namespace lms::scanner
|
||||
public:
|
||||
using ProcessFunction = std::function<void(std::span<std::unique_ptr<core::IJob>>)>;
|
||||
|
||||
JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float drainThreshold);
|
||||
// processBatchSize -> how many jobs done to notify at once using processJobsDoneFunc
|
||||
// drainThreshold: fraction of maxQueueSize at which completed jobs are processed
|
||||
JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t processBatchSize, float drainThreshold);
|
||||
~JobQueue();
|
||||
JobQueue(const JobQueue&) = delete;
|
||||
JobQueue& operator=(const JobQueue&) = delete;
|
||||
|
||||
@@ -363,14 +363,14 @@ namespace lms::scanner
|
||||
_progressCallback(context.currentStepStats);
|
||||
};
|
||||
|
||||
JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
|
||||
{
|
||||
JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
|
||||
|
||||
db::ArtistId lastRetrievedArtistId{};
|
||||
db::IdRange<db::ArtistId> artistIdRange;
|
||||
while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange))
|
||||
queue.push(std::make_unique<ComputeArtistArtworkAssociationsJob>(_db, searchParams, artistIdRange));
|
||||
|
||||
queue.finish();
|
||||
db::ArtistId lastRetrievedArtistId{};
|
||||
db::IdRange<db::ArtistId> artistIdRange;
|
||||
while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange))
|
||||
queue.push(std::make_unique<ComputeArtistArtworkAssociationsJob>(_db, searchParams, artistIdRange));
|
||||
}
|
||||
|
||||
// process all remaining associations
|
||||
updateArtistPreferredArtworks(session, artistArtworkAssociations, false);
|
||||
|
||||
@@ -19,10 +19,7 @@
|
||||
|
||||
#include "ScanStepBase.hpp"
|
||||
|
||||
#include "core/String.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
@@ -32,54 +29,13 @@ namespace lms::scanner
|
||||
, _abortScan{ initParams.abortScan }
|
||||
, _db{ initParams.db }
|
||||
, _jobScheduler{ initParams.jobScheduler }
|
||||
, _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners))
|
||||
, _fileScanners(initParams.fileScanners)
|
||||
, _lastScanSettings{ initParams.lastScanSettings }
|
||||
{
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
{
|
||||
for (const std::filesystem::path& file : scanner->getSupportedFiles())
|
||||
{
|
||||
[[maybe_unused]] auto [it, inserted]{ _scannerByFile.emplace(file, scanner) };
|
||||
assert(inserted);
|
||||
}
|
||||
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
{
|
||||
[[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) };
|
||||
assert(inserted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ScanStepBase::~ScanStepBase() = default;
|
||||
|
||||
IFileScanner* ScanStepBase::selectFileScanner(const std::filesystem::path& filePath) const
|
||||
{
|
||||
{
|
||||
const std::string fileName{ core::stringUtils::stringToLower(filePath.filename().string()) };
|
||||
|
||||
auto itScanner{ _scannerByFile.find(fileName) };
|
||||
if (itScanner != std::cend(_scannerByFile))
|
||||
return itScanner->second;
|
||||
}
|
||||
|
||||
{
|
||||
const std::string extension{ core::stringUtils::stringToLower(filePath.extension().string()) };
|
||||
|
||||
auto itScanner{ _scannerByExtension.find(extension) };
|
||||
if (itScanner != std::cend(_scannerByExtension))
|
||||
return itScanner->second;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void ScanStepBase::visitFileScanners(const std::function<void(IFileScanner*)>& visitor) const
|
||||
{
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
visitor(scanner);
|
||||
}
|
||||
|
||||
void ScanStepBase::addError(ScanContext& context, std::shared_ptr<ScanError> error)
|
||||
{
|
||||
error->accept(_scanErrorLogger);
|
||||
|
||||
@@ -20,9 +20,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
#include <span>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "IScanStep.hpp"
|
||||
#include "ScanErrorLogger.hpp"
|
||||
@@ -39,7 +36,7 @@ namespace lms::db
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class IFileScanner;
|
||||
class FileScanners;
|
||||
struct ScannerSettings;
|
||||
struct ScanStepStats;
|
||||
struct ScanContext;
|
||||
@@ -57,7 +54,7 @@ namespace lms::scanner
|
||||
ProgressCallback progressCallback;
|
||||
bool& abortScan;
|
||||
db::IDb& db;
|
||||
std::span<IFileScanner*> fileScanners;
|
||||
const FileScanners& fileScanners;
|
||||
};
|
||||
ScanStepBase(InitParams& initParams);
|
||||
~ScanStepBase() override;
|
||||
@@ -67,8 +64,7 @@ namespace lms::scanner
|
||||
protected:
|
||||
core::IJobScheduler& getJobScheduler() { return _jobScheduler; };
|
||||
const ScannerSettings* getLastScanSettings() const { return _lastScanSettings; }
|
||||
IFileScanner* selectFileScanner(const std::filesystem::path& filePath) const;
|
||||
void visitFileScanners(const std::function<void(IFileScanner*)>& visitor) const;
|
||||
const FileScanners& getFileScanners() const { return _fileScanners; }
|
||||
|
||||
void addError(ScanContext& context, std::shared_ptr<ScanError> error);
|
||||
|
||||
@@ -86,9 +82,7 @@ namespace lms::scanner
|
||||
|
||||
private:
|
||||
core::IJobScheduler& _jobScheduler;
|
||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByFile;
|
||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
|
||||
std::vector<IFileScanner*> _fileScanners;
|
||||
const FileScanners& _fileScanners;
|
||||
|
||||
const ScannerSettings* _lastScanSettings{};
|
||||
ScanErrorLogger _scanErrorLogger;
|
||||
|
||||
@@ -19,8 +19,12 @@
|
||||
|
||||
#include "ScanStepCheckForRemovedFiles.hpp"
|
||||
|
||||
#include <deque>
|
||||
#include <filesystem>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
#include "core/IJob.hpp"
|
||||
#include "core/ILogger.hpp"
|
||||
#include "core/Path.hpp"
|
||||
#include "database/IDb.hpp"
|
||||
@@ -31,11 +35,153 @@
|
||||
#include "database/objects/Track.hpp"
|
||||
#include "database/objects/TrackLyrics.hpp"
|
||||
|
||||
#include "FileScanners.hpp"
|
||||
#include "JobQueue.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
#include "services/scanner/ScannerStats.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
{
|
||||
template<typename IdType>
|
||||
struct FileToCheck
|
||||
{
|
||||
IdType objectId;
|
||||
std::filesystem::path file;
|
||||
};
|
||||
|
||||
template<typename IdType>
|
||||
class CheckForRemovedFilesJob : public core::IJob
|
||||
{
|
||||
public:
|
||||
CheckForRemovedFilesJob(const ScannerSettings& settings, const FileScanners& scanners, std::span<const FileToCheck<IdType>> filesToCheck)
|
||||
: _settings{ settings }
|
||||
, _scanners{ scanners }
|
||||
, _filesToCheck{ std::cbegin(filesToCheck), std::cend(filesToCheck) }
|
||||
{
|
||||
}
|
||||
|
||||
std::size_t getProcessedCount() const { return _processedCount; }
|
||||
std::span<const IdType> getObjectsToRemove() const { return _objectsToRemove; }
|
||||
|
||||
private:
|
||||
core::LiteralString getName() const override { return "Check For Removed Files"; }
|
||||
void run() override
|
||||
{
|
||||
for (const FileToCheck<IdType>& fileToCheck : _filesToCheck)
|
||||
{
|
||||
if (!checkFile(fileToCheck.file))
|
||||
_objectsToRemove.push_back(fileToCheck.objectId);
|
||||
}
|
||||
|
||||
_processedCount += _filesToCheck.size();
|
||||
}
|
||||
|
||||
bool checkFile(const std::filesystem::path& p)
|
||||
{
|
||||
std::error_code ec;
|
||||
const std::filesystem::directory_entry fileEntry{ p, ec };
|
||||
if (ec)
|
||||
{
|
||||
// TODO store error?
|
||||
LMS_LOG(DBUPDATER, ERROR, "Error while checking file " << p << ": " << ec.message());
|
||||
return false;
|
||||
}
|
||||
|
||||
// For each track, make sure the the file still exists
|
||||
// and still belongs to a media directory
|
||||
if (!fileEntry.exists() || !fileEntry.is_regular_file())
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
|
||||
[&](const MediaLibraryInfo& libraryInfo) {
|
||||
return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
|
||||
}))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!_scanners.select(p))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": file format no longer handled");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
const ScannerSettings& _settings;
|
||||
const FileScanners& _scanners;
|
||||
std::vector<FileToCheck<IdType>> _filesToCheck;
|
||||
std::vector<IdType> _objectsToRemove;
|
||||
std::size_t _processedCount{};
|
||||
};
|
||||
|
||||
template<typename Object>
|
||||
std::size_t removeObjects(db::Session& session, std::deque<typename Object::IdType>& objectIdsToRemove, bool forceFullBatch)
|
||||
{
|
||||
std::size_t removedObjectCount{};
|
||||
constexpr std::size_t writeBatchSize{ 50 };
|
||||
|
||||
std::vector<typename Object::IdType> ids;
|
||||
while ((forceFullBatch && objectIdsToRemove.size() >= writeBatchSize) || !objectIdsToRemove.empty())
|
||||
{
|
||||
for (std::size_t i{}; !objectIdsToRemove.empty() && i < writeBatchSize; ++i)
|
||||
{
|
||||
ids.push_back(objectIdsToRemove.front());
|
||||
objectIdsToRemove.pop_front();
|
||||
}
|
||||
|
||||
{
|
||||
auto transaction{ session.createWriteTransaction() };
|
||||
session.destroy<Object>(ids);
|
||||
}
|
||||
|
||||
removedObjectCount += ids.size();
|
||||
ids.clear();
|
||||
}
|
||||
|
||||
return removedObjectCount;
|
||||
}
|
||||
|
||||
template<typename Object>
|
||||
bool fetchNextFilesToCheck(db::Session& session, typename Object::IdType& lastCheckedId, std::vector<FileToCheck<typename Object::IdType>>& filesToCheck)
|
||||
{
|
||||
constexpr std::size_t batchSize{ 200 };
|
||||
|
||||
filesToCheck.clear();
|
||||
filesToCheck.reserve(batchSize);
|
||||
|
||||
auto transaction{ session.createReadTransaction() };
|
||||
|
||||
while (filesToCheck.size() < batchSize)
|
||||
{
|
||||
const typename Object::IdType previousLastCheckedId{ lastCheckedId };
|
||||
Object::findAbsoluteFilePath(session, lastCheckedId, batchSize, [&](Object::IdType objectId, const std::filesystem::path& filePath) {
|
||||
// special case for track lyrics, only check external lyrics
|
||||
if constexpr (std::is_same_v<Object, db::TrackLyrics>)
|
||||
{
|
||||
if (filePath.empty())
|
||||
return;
|
||||
}
|
||||
|
||||
filesToCheck.emplace_back(objectId, filePath);
|
||||
});
|
||||
|
||||
if (previousLastCheckedId == lastCheckedId)
|
||||
break;
|
||||
}
|
||||
|
||||
return !filesToCheck.empty();
|
||||
}
|
||||
} // namespace
|
||||
|
||||
bool ScanStepCheckForRemovedFiles::needProcess([[maybe_unused]] const ScanContext& context) const
|
||||
{
|
||||
// always check for removed files
|
||||
@@ -67,91 +213,48 @@ namespace lms::scanner
|
||||
template<typename Object>
|
||||
void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context)
|
||||
{
|
||||
using namespace db;
|
||||
using ObjectIdType = typename Object::IdType;
|
||||
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
Session& session{ _db.getTLSSession() };
|
||||
db::Session& session{ _db.getTLSSession() };
|
||||
|
||||
std::vector<typename Object::IdType> objectIdsToRemove;
|
||||
std::deque<ObjectIdType> objectIdsToRemove;
|
||||
|
||||
typename Object::IdType lastCheckedId;
|
||||
bool endReached{};
|
||||
while (!endReached)
|
||||
{
|
||||
auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
|
||||
if (_abortScan)
|
||||
break;
|
||||
return;
|
||||
|
||||
objectIdsToRemove.clear();
|
||||
for (const auto& job : jobs)
|
||||
{
|
||||
constexpr std::size_t batchSize = 200;
|
||||
const auto& checkJob{ static_cast<const CheckForRemovedFilesJob<ObjectIdType>&>(*job) };
|
||||
std::span<const ObjectIdType> objectsToRemove{ checkJob.getObjectsToRemove() };
|
||||
|
||||
auto transaction{ session.createReadTransaction() };
|
||||
objectIdsToRemove.insert(std::end(objectIdsToRemove), std::cbegin(objectsToRemove), std::cend(objectsToRemove));
|
||||
|
||||
endReached = true;
|
||||
Object::findAbsoluteFilePath(session, lastCheckedId, batchSize, [&](Object::IdType objectId, const std::filesystem::path& filePath) {
|
||||
endReached = false;
|
||||
|
||||
// special case for track lyrics, only check external lyrics
|
||||
if constexpr (std::is_same_v<Object, TrackLyrics>)
|
||||
{
|
||||
if (filePath.empty())
|
||||
return;
|
||||
}
|
||||
|
||||
if (!checkFile(filePath))
|
||||
objectIdsToRemove.push_back(objectId);
|
||||
|
||||
context.currentStepStats.processedElems++;
|
||||
});
|
||||
context.currentStepStats.processedElems += checkJob.getProcessedCount();
|
||||
}
|
||||
|
||||
if (!objectIdsToRemove.empty())
|
||||
{
|
||||
auto transaction{ session.createWriteTransaction() };
|
||||
|
||||
session.destroy<Object>(objectIdsToRemove);
|
||||
removeObjects<Object>(session, objectIdsToRemove, true);
|
||||
context.stats.deletions += objectIdsToRemove.size();
|
||||
}
|
||||
|
||||
_progressCallback(context.currentStepStats);
|
||||
};
|
||||
|
||||
{
|
||||
JobQueue queue{ getJobScheduler(), 50, processJobsDone, 1, 0.85F };
|
||||
|
||||
ObjectIdType lastCheckedId;
|
||||
std::vector<FileToCheck<ObjectIdType>> filesToCheck;
|
||||
while (fetchNextFilesToCheck<Object>(session, lastCheckedId, filesToCheck))
|
||||
queue.push(std::make_unique<CheckForRemovedFilesJob<ObjectIdType>>(_settings, getFileScanners(), filesToCheck));
|
||||
}
|
||||
|
||||
// process all remaining objects
|
||||
removeObjects<Object>(session, objectIdsToRemove, false);
|
||||
}
|
||||
|
||||
bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p)
|
||||
{
|
||||
try
|
||||
{
|
||||
// For each track, make sure the the file still exists
|
||||
// and still belongs to a media directory
|
||||
if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
|
||||
[&](const MediaLibraryInfo& libraryInfo) {
|
||||
return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
|
||||
}))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!selectFileScanner(p))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": file format no longer handled");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
catch (std::filesystem::filesystem_error& e)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file " << p << ": " << e.what());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
#include "scanners/IFileScanOperation.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
#include "FileScanners.hpp"
|
||||
#include "JobQueue.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
@@ -68,18 +69,6 @@ namespace lms::scanner
|
||||
};
|
||||
} // namespace
|
||||
|
||||
ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
|
||||
: ScanStepBase{ initParams }
|
||||
{
|
||||
visitFileScanners([](IFileScanner* scanner) {
|
||||
for (const std::filesystem::path& file : scanner->getSupportedFiles())
|
||||
LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file " << file);
|
||||
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file extension " << extension);
|
||||
});
|
||||
}
|
||||
|
||||
bool ScanStepScanFiles::needProcess([[maybe_unused]] const ScanContext& context) const
|
||||
{
|
||||
// Always need to scan files
|
||||
@@ -122,7 +111,7 @@ namespace lms::scanner
|
||||
addError<IOScanError>(context, path, ec);
|
||||
context.stats.skips++;
|
||||
}
|
||||
else if (IFileScanner * scanner{ selectFileScanner(path) })
|
||||
else if (IFileScanner * scanner{ getFileScanners().select(path) })
|
||||
{
|
||||
FileToScan fileToScan;
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ namespace lms::scanner
|
||||
class ScanStepScanFiles : public ScanStepBase
|
||||
{
|
||||
public:
|
||||
ScanStepScanFiles(InitParams& initParams);
|
||||
using ScanStepBase::ScanStepBase;
|
||||
|
||||
private:
|
||||
ScanStep getStep() const override { return ScanStep::ScanFiles; }
|
||||
|
||||
Reference in New Issue
Block a user