Made the step 'check for removed files' faster when nothing was actually removed
This commit is contained in:
@@ -25,6 +25,7 @@ add_library(lmsscanner STATIC
|
|||||||
impl/steps/ScanStepRemoveOrphanedDbEntries.cpp
|
impl/steps/ScanStepRemoveOrphanedDbEntries.cpp
|
||||||
impl/steps/ScanStepScanFiles.cpp
|
impl/steps/ScanStepScanFiles.cpp
|
||||||
impl/steps/ScanStepUpdateLibraryFields.cpp
|
impl/steps/ScanStepUpdateLibraryFields.cpp
|
||||||
|
impl/FileScanners.cpp
|
||||||
impl/ScannerService.cpp
|
impl/ScannerService.cpp
|
||||||
impl/ScannerStats.cpp
|
impl/ScannerStats.cpp
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2025 Emeric Poupon
|
||||||
|
*
|
||||||
|
* This file is part of LMS.
|
||||||
|
*
|
||||||
|
* LMS is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* LMS is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "FileScanners.hpp"
|
||||||
|
|
||||||
|
#include <cassert>
|
||||||
|
|
||||||
|
#include "core/String.hpp"
|
||||||
|
|
||||||
|
#include "scanners/IFileScanner.hpp"
|
||||||
|
|
||||||
|
namespace lms::scanner
|
||||||
|
{
|
||||||
|
void FileScanners::add(std::unique_ptr<IFileScanner> scanner)
|
||||||
|
{
|
||||||
|
for (const std::filesystem::path& file : scanner->getSupportedFiles())
|
||||||
|
{
|
||||||
|
[[maybe_unused]] auto [it, inserted]{ _scannerByFile.emplace(file, scanner.get()) };
|
||||||
|
assert(inserted);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||||
|
{
|
||||||
|
[[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner.get()) };
|
||||||
|
assert(inserted);
|
||||||
|
}
|
||||||
|
|
||||||
|
_fileScanners.emplace_back(std::move(scanner));
|
||||||
|
}
|
||||||
|
|
||||||
|
void FileScanners::clear()
|
||||||
|
{
|
||||||
|
_fileScanners.clear();
|
||||||
|
_scannerByFile.clear();
|
||||||
|
_scannerByExtension.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
IFileScanner* FileScanners::select(const std::filesystem::path& filePath) const
|
||||||
|
{
|
||||||
|
{
|
||||||
|
const std::string fileName{ core::stringUtils::stringToLower(filePath.filename().string()) };
|
||||||
|
|
||||||
|
auto itScanner{ _scannerByFile.find(fileName) };
|
||||||
|
if (itScanner != std::cend(_scannerByFile))
|
||||||
|
return itScanner->second;
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const std::string extension{ core::stringUtils::stringToLower(filePath.extension().string()) };
|
||||||
|
|
||||||
|
auto itScanner{ _scannerByExtension.find(extension) };
|
||||||
|
if (itScanner != std::cend(_scannerByExtension))
|
||||||
|
return itScanner->second;
|
||||||
|
}
|
||||||
|
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
void FileScanners::visit(const std::function<void(const IFileScanner&)>& visitor) const
|
||||||
|
{
|
||||||
|
for (const auto& scanner : _fileScanners)
|
||||||
|
visitor(*scanner);
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace lms::scanner
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (C) 2025 Emeric Poupon
|
||||||
|
*
|
||||||
|
* This file is part of LMS.
|
||||||
|
*
|
||||||
|
* LMS is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* LMS is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <filesystem>
|
||||||
|
#include <functional>
|
||||||
|
#include <unordered_map>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
namespace lms::scanner
|
||||||
|
{
|
||||||
|
class IFileScanner;
|
||||||
|
|
||||||
|
class FileScanners
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
void add(std::unique_ptr<IFileScanner> scanner);
|
||||||
|
void clear();
|
||||||
|
|
||||||
|
IFileScanner* select(const std::filesystem::path& filePath) const;
|
||||||
|
void visit(const std::function<void(const IFileScanner&)>& visitor) const;
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByFile;
|
||||||
|
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
|
||||||
|
std::vector<std::unique_ptr<IFileScanner>> _fileScanners;
|
||||||
|
};
|
||||||
|
} // namespace lms::scanner
|
||||||
@@ -452,14 +452,19 @@ namespace lms::scanner
|
|||||||
} };
|
} };
|
||||||
|
|
||||||
_fileScanners.clear();
|
_fileScanners.clear();
|
||||||
_fileScanners.emplace_back(std::make_unique<ArtistInfoFileScanner>(_db, _settings));
|
_fileScanners.add(std::make_unique<ArtistInfoFileScanner>(_db, _settings));
|
||||||
_fileScanners.emplace_back(std::make_unique<AudioFileScanner>(_db, _settings));
|
_fileScanners.add(std::make_unique<AudioFileScanner>(_db, _settings));
|
||||||
_fileScanners.emplace_back(std::make_unique<ImageFileScanner>(_db, _settings));
|
_fileScanners.add(std::make_unique<ImageFileScanner>(_db, _settings));
|
||||||
_fileScanners.emplace_back(std::make_unique<LyricsFileScanner>(_db, _settings));
|
_fileScanners.add(std::make_unique<LyricsFileScanner>(_db, _settings));
|
||||||
_fileScanners.emplace_back(std::make_unique<PlayListFileScanner>(_db, _settings));
|
_fileScanners.add(std::make_unique<PlayListFileScanner>(_db, _settings));
|
||||||
|
|
||||||
std::vector<IFileScanner*> fileScanners;
|
_fileScanners.visit([](const IFileScanner& scanner) {
|
||||||
std::transform(std::cbegin(_fileScanners), std::cend(_fileScanners), std::back_inserter(fileScanners), [](const std::unique_ptr<IFileScanner>& scanner) { return scanner.get(); });
|
for (const std::filesystem::path& file : scanner.getSupportedFiles())
|
||||||
|
LMS_LOG(DBUPDATER, INFO, scanner.getName() << ": supporting file " << file);
|
||||||
|
|
||||||
|
for (const std::filesystem::path& extension : scanner.getSupportedExtensions())
|
||||||
|
LMS_LOG(DBUPDATER, INFO, scanner.getName() << ": supporting file extension " << extension);
|
||||||
|
});
|
||||||
|
|
||||||
ScanStepBase::InitParams params{
|
ScanStepBase::InitParams params{
|
||||||
.jobScheduler = *_jobScheduler,
|
.jobScheduler = *_jobScheduler,
|
||||||
@@ -468,7 +473,7 @@ namespace lms::scanner
|
|||||||
.progressCallback = progressFunc,
|
.progressCallback = progressFunc,
|
||||||
.abortScan = _abortScan,
|
.abortScan = _abortScan,
|
||||||
.db = _db,
|
.db = _db,
|
||||||
.fileScanners = fileScanners,
|
.fileScanners = _fileScanners,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Order is important: steps are sequential
|
// Order is important: steps are sequential
|
||||||
|
|||||||
@@ -31,6 +31,7 @@
|
|||||||
#include <Wt/WIOService.h>
|
#include <Wt/WIOService.h>
|
||||||
#include <Wt/WSignal.h>
|
#include <Wt/WSignal.h>
|
||||||
|
|
||||||
|
#include "FileScanners.hpp"
|
||||||
#include "ScannerSettings.hpp"
|
#include "ScannerSettings.hpp"
|
||||||
#include "database/IDb.hpp"
|
#include "database/IDb.hpp"
|
||||||
#include "database/Session.hpp"
|
#include "database/Session.hpp"
|
||||||
@@ -88,7 +89,7 @@ namespace lms::scanner
|
|||||||
db::IDb& _db;
|
db::IDb& _db;
|
||||||
std::unique_ptr<core::IJobScheduler> _jobScheduler;
|
std::unique_ptr<core::IJobScheduler> _jobScheduler;
|
||||||
|
|
||||||
std::vector<std::unique_ptr<IFileScanner>> _fileScanners;
|
FileScanners _fileScanners;
|
||||||
std::vector<std::unique_ptr<IScanStep>> _scanSteps;
|
std::vector<std::unique_ptr<IScanStep>> _scanSteps;
|
||||||
|
|
||||||
std::mutex _controlMutex;
|
std::mutex _controlMutex;
|
||||||
|
|||||||
@@ -37,7 +37,9 @@ namespace lms::scanner
|
|||||||
public:
|
public:
|
||||||
using ProcessFunction = std::function<void(std::span<std::unique_ptr<core::IJob>>)>;
|
using ProcessFunction = std::function<void(std::span<std::unique_ptr<core::IJob>>)>;
|
||||||
|
|
||||||
JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t batchSize, float drainThreshold);
|
// processBatchSize -> how many jobs done to notify at once using processJobsDoneFunc
|
||||||
|
// drainThreshold: fraction of maxQueueSize at which completed jobs are processed
|
||||||
|
JobQueue(core::IJobScheduler& scheduler, std::size_t maxQueueSize, ProcessFunction processJobsDoneFunc, std::size_t processBatchSize, float drainThreshold);
|
||||||
~JobQueue();
|
~JobQueue();
|
||||||
JobQueue(const JobQueue&) = delete;
|
JobQueue(const JobQueue&) = delete;
|
||||||
JobQueue& operator=(const JobQueue&) = delete;
|
JobQueue& operator=(const JobQueue&) = delete;
|
||||||
|
|||||||
@@ -363,14 +363,14 @@ namespace lms::scanner
|
|||||||
_progressCallback(context.currentStepStats);
|
_progressCallback(context.currentStepStats);
|
||||||
};
|
};
|
||||||
|
|
||||||
JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
|
{
|
||||||
|
JobQueue queue{ getJobScheduler(), 20, processJobsDone, 1, 0.85F };
|
||||||
|
|
||||||
db::ArtistId lastRetrievedArtistId{};
|
db::ArtistId lastRetrievedArtistId{};
|
||||||
db::IdRange<db::ArtistId> artistIdRange;
|
db::IdRange<db::ArtistId> artistIdRange;
|
||||||
while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange))
|
while (fetchNextArtistIdRange(session, lastRetrievedArtistId, artistIdRange))
|
||||||
queue.push(std::make_unique<ComputeArtistArtworkAssociationsJob>(_db, searchParams, artistIdRange));
|
queue.push(std::make_unique<ComputeArtistArtworkAssociationsJob>(_db, searchParams, artistIdRange));
|
||||||
|
}
|
||||||
queue.finish();
|
|
||||||
|
|
||||||
// process all remaining associations
|
// process all remaining associations
|
||||||
updateArtistPreferredArtworks(session, artistArtworkAssociations, false);
|
updateArtistPreferredArtworks(session, artistArtworkAssociations, false);
|
||||||
|
|||||||
@@ -19,10 +19,7 @@
|
|||||||
|
|
||||||
#include "ScanStepBase.hpp"
|
#include "ScanStepBase.hpp"
|
||||||
|
|
||||||
#include "core/String.hpp"
|
|
||||||
|
|
||||||
#include "ScanContext.hpp"
|
#include "ScanContext.hpp"
|
||||||
#include "scanners/IFileScanner.hpp"
|
|
||||||
|
|
||||||
namespace lms::scanner
|
namespace lms::scanner
|
||||||
{
|
{
|
||||||
@@ -32,54 +29,13 @@ namespace lms::scanner
|
|||||||
, _abortScan{ initParams.abortScan }
|
, _abortScan{ initParams.abortScan }
|
||||||
, _db{ initParams.db }
|
, _db{ initParams.db }
|
||||||
, _jobScheduler{ initParams.jobScheduler }
|
, _jobScheduler{ initParams.jobScheduler }
|
||||||
, _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners))
|
, _fileScanners(initParams.fileScanners)
|
||||||
, _lastScanSettings{ initParams.lastScanSettings }
|
, _lastScanSettings{ initParams.lastScanSettings }
|
||||||
{
|
{
|
||||||
for (IFileScanner* scanner : _fileScanners)
|
|
||||||
{
|
|
||||||
for (const std::filesystem::path& file : scanner->getSupportedFiles())
|
|
||||||
{
|
|
||||||
[[maybe_unused]] auto [it, inserted]{ _scannerByFile.emplace(file, scanner) };
|
|
||||||
assert(inserted);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
|
||||||
{
|
|
||||||
[[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) };
|
|
||||||
assert(inserted);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ScanStepBase::~ScanStepBase() = default;
|
ScanStepBase::~ScanStepBase() = default;
|
||||||
|
|
||||||
IFileScanner* ScanStepBase::selectFileScanner(const std::filesystem::path& filePath) const
|
|
||||||
{
|
|
||||||
{
|
|
||||||
const std::string fileName{ core::stringUtils::stringToLower(filePath.filename().string()) };
|
|
||||||
|
|
||||||
auto itScanner{ _scannerByFile.find(fileName) };
|
|
||||||
if (itScanner != std::cend(_scannerByFile))
|
|
||||||
return itScanner->second;
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
const std::string extension{ core::stringUtils::stringToLower(filePath.extension().string()) };
|
|
||||||
|
|
||||||
auto itScanner{ _scannerByExtension.find(extension) };
|
|
||||||
if (itScanner != std::cend(_scannerByExtension))
|
|
||||||
return itScanner->second;
|
|
||||||
}
|
|
||||||
|
|
||||||
return nullptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
void ScanStepBase::visitFileScanners(const std::function<void(IFileScanner*)>& visitor) const
|
|
||||||
{
|
|
||||||
for (IFileScanner* scanner : _fileScanners)
|
|
||||||
visitor(scanner);
|
|
||||||
}
|
|
||||||
|
|
||||||
void ScanStepBase::addError(ScanContext& context, std::shared_ptr<ScanError> error)
|
void ScanStepBase::addError(ScanContext& context, std::shared_ptr<ScanError> error)
|
||||||
{
|
{
|
||||||
error->accept(_scanErrorLogger);
|
error->accept(_scanErrorLogger);
|
||||||
|
|||||||
@@ -20,9 +20,6 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <span>
|
|
||||||
#include <unordered_map>
|
|
||||||
#include <vector>
|
|
||||||
|
|
||||||
#include "IScanStep.hpp"
|
#include "IScanStep.hpp"
|
||||||
#include "ScanErrorLogger.hpp"
|
#include "ScanErrorLogger.hpp"
|
||||||
@@ -39,7 +36,7 @@ namespace lms::db
|
|||||||
|
|
||||||
namespace lms::scanner
|
namespace lms::scanner
|
||||||
{
|
{
|
||||||
class IFileScanner;
|
class FileScanners;
|
||||||
struct ScannerSettings;
|
struct ScannerSettings;
|
||||||
struct ScanStepStats;
|
struct ScanStepStats;
|
||||||
struct ScanContext;
|
struct ScanContext;
|
||||||
@@ -57,7 +54,7 @@ namespace lms::scanner
|
|||||||
ProgressCallback progressCallback;
|
ProgressCallback progressCallback;
|
||||||
bool& abortScan;
|
bool& abortScan;
|
||||||
db::IDb& db;
|
db::IDb& db;
|
||||||
std::span<IFileScanner*> fileScanners;
|
const FileScanners& fileScanners;
|
||||||
};
|
};
|
||||||
ScanStepBase(InitParams& initParams);
|
ScanStepBase(InitParams& initParams);
|
||||||
~ScanStepBase() override;
|
~ScanStepBase() override;
|
||||||
@@ -67,8 +64,7 @@ namespace lms::scanner
|
|||||||
protected:
|
protected:
|
||||||
core::IJobScheduler& getJobScheduler() { return _jobScheduler; };
|
core::IJobScheduler& getJobScheduler() { return _jobScheduler; };
|
||||||
const ScannerSettings* getLastScanSettings() const { return _lastScanSettings; }
|
const ScannerSettings* getLastScanSettings() const { return _lastScanSettings; }
|
||||||
IFileScanner* selectFileScanner(const std::filesystem::path& filePath) const;
|
const FileScanners& getFileScanners() const { return _fileScanners; }
|
||||||
void visitFileScanners(const std::function<void(IFileScanner*)>& visitor) const;
|
|
||||||
|
|
||||||
void addError(ScanContext& context, std::shared_ptr<ScanError> error);
|
void addError(ScanContext& context, std::shared_ptr<ScanError> error);
|
||||||
|
|
||||||
@@ -86,9 +82,7 @@ namespace lms::scanner
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
core::IJobScheduler& _jobScheduler;
|
core::IJobScheduler& _jobScheduler;
|
||||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByFile;
|
const FileScanners& _fileScanners;
|
||||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
|
|
||||||
std::vector<IFileScanner*> _fileScanners;
|
|
||||||
|
|
||||||
const ScannerSettings* _lastScanSettings{};
|
const ScannerSettings* _lastScanSettings{};
|
||||||
ScanErrorLogger _scanErrorLogger;
|
ScanErrorLogger _scanErrorLogger;
|
||||||
|
|||||||
@@ -19,8 +19,12 @@
|
|||||||
|
|
||||||
#include "ScanStepCheckForRemovedFiles.hpp"
|
#include "ScanStepCheckForRemovedFiles.hpp"
|
||||||
|
|
||||||
|
#include <deque>
|
||||||
|
#include <filesystem>
|
||||||
|
#include <span>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
#include "core/IJob.hpp"
|
||||||
#include "core/ILogger.hpp"
|
#include "core/ILogger.hpp"
|
||||||
#include "core/Path.hpp"
|
#include "core/Path.hpp"
|
||||||
#include "database/IDb.hpp"
|
#include "database/IDb.hpp"
|
||||||
@@ -31,11 +35,153 @@
|
|||||||
#include "database/objects/Track.hpp"
|
#include "database/objects/Track.hpp"
|
||||||
#include "database/objects/TrackLyrics.hpp"
|
#include "database/objects/TrackLyrics.hpp"
|
||||||
|
|
||||||
|
#include "FileScanners.hpp"
|
||||||
|
#include "JobQueue.hpp"
|
||||||
#include "ScanContext.hpp"
|
#include "ScanContext.hpp"
|
||||||
#include "ScannerSettings.hpp"
|
#include "ScannerSettings.hpp"
|
||||||
|
#include "services/scanner/ScannerStats.hpp"
|
||||||
|
|
||||||
namespace lms::scanner
|
namespace lms::scanner
|
||||||
{
|
{
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
template<typename IdType>
|
||||||
|
struct FileToCheck
|
||||||
|
{
|
||||||
|
IdType objectId;
|
||||||
|
std::filesystem::path file;
|
||||||
|
};
|
||||||
|
|
||||||
|
template<typename IdType>
|
||||||
|
class CheckForRemovedFilesJob : public core::IJob
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
CheckForRemovedFilesJob(const ScannerSettings& settings, const FileScanners& scanners, std::span<const FileToCheck<IdType>> filesToCheck)
|
||||||
|
: _settings{ settings }
|
||||||
|
, _scanners{ scanners }
|
||||||
|
, _filesToCheck{ std::cbegin(filesToCheck), std::cend(filesToCheck) }
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
std::size_t getProcessedCount() const { return _processedCount; }
|
||||||
|
std::span<const IdType> getObjectsToRemove() const { return _objectsToRemove; }
|
||||||
|
|
||||||
|
private:
|
||||||
|
core::LiteralString getName() const override { return "Check For Removed Files"; }
|
||||||
|
void run() override
|
||||||
|
{
|
||||||
|
for (const FileToCheck<IdType>& fileToCheck : _filesToCheck)
|
||||||
|
{
|
||||||
|
if (!checkFile(fileToCheck.file))
|
||||||
|
_objectsToRemove.push_back(fileToCheck.objectId);
|
||||||
|
}
|
||||||
|
|
||||||
|
_processedCount += _filesToCheck.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool checkFile(const std::filesystem::path& p)
|
||||||
|
{
|
||||||
|
std::error_code ec;
|
||||||
|
const std::filesystem::directory_entry fileEntry{ p, ec };
|
||||||
|
if (ec)
|
||||||
|
{
|
||||||
|
// TODO store error?
|
||||||
|
LMS_LOG(DBUPDATER, ERROR, "Error while checking file " << p << ": " << ec.message());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// For each track, make sure the the file still exists
|
||||||
|
// and still belongs to a media directory
|
||||||
|
if (!fileEntry.exists() || !fileEntry.is_regular_file())
|
||||||
|
{
|
||||||
|
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
|
||||||
|
[&](const MediaLibraryInfo& libraryInfo) {
|
||||||
|
return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
|
||||||
|
}))
|
||||||
|
{
|
||||||
|
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!_scanners.select(p))
|
||||||
|
{
|
||||||
|
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": file format no longer handled");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ScannerSettings& _settings;
|
||||||
|
const FileScanners& _scanners;
|
||||||
|
std::vector<FileToCheck<IdType>> _filesToCheck;
|
||||||
|
std::vector<IdType> _objectsToRemove;
|
||||||
|
std::size_t _processedCount{};
|
||||||
|
};
|
||||||
|
|
||||||
|
template<typename Object>
|
||||||
|
std::size_t removeObjects(db::Session& session, std::deque<typename Object::IdType>& objectIdsToRemove, bool forceFullBatch)
|
||||||
|
{
|
||||||
|
std::size_t removedObjectCount{};
|
||||||
|
constexpr std::size_t writeBatchSize{ 50 };
|
||||||
|
|
||||||
|
std::vector<typename Object::IdType> ids;
|
||||||
|
while ((forceFullBatch && objectIdsToRemove.size() >= writeBatchSize) || !objectIdsToRemove.empty())
|
||||||
|
{
|
||||||
|
for (std::size_t i{}; !objectIdsToRemove.empty() && i < writeBatchSize; ++i)
|
||||||
|
{
|
||||||
|
ids.push_back(objectIdsToRemove.front());
|
||||||
|
objectIdsToRemove.pop_front();
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
auto transaction{ session.createWriteTransaction() };
|
||||||
|
session.destroy<Object>(ids);
|
||||||
|
}
|
||||||
|
|
||||||
|
removedObjectCount += ids.size();
|
||||||
|
ids.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
return removedObjectCount;
|
||||||
|
}
|
||||||
|
|
||||||
|
template<typename Object>
|
||||||
|
bool fetchNextFilesToCheck(db::Session& session, typename Object::IdType& lastCheckedId, std::vector<FileToCheck<typename Object::IdType>>& filesToCheck)
|
||||||
|
{
|
||||||
|
constexpr std::size_t batchSize{ 200 };
|
||||||
|
|
||||||
|
filesToCheck.clear();
|
||||||
|
filesToCheck.reserve(batchSize);
|
||||||
|
|
||||||
|
auto transaction{ session.createReadTransaction() };
|
||||||
|
|
||||||
|
while (filesToCheck.size() < batchSize)
|
||||||
|
{
|
||||||
|
const typename Object::IdType previousLastCheckedId{ lastCheckedId };
|
||||||
|
Object::findAbsoluteFilePath(session, lastCheckedId, batchSize, [&](Object::IdType objectId, const std::filesystem::path& filePath) {
|
||||||
|
// special case for track lyrics, only check external lyrics
|
||||||
|
if constexpr (std::is_same_v<Object, db::TrackLyrics>)
|
||||||
|
{
|
||||||
|
if (filePath.empty())
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
filesToCheck.emplace_back(objectId, filePath);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (previousLastCheckedId == lastCheckedId)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return !filesToCheck.empty();
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
bool ScanStepCheckForRemovedFiles::needProcess([[maybe_unused]] const ScanContext& context) const
|
bool ScanStepCheckForRemovedFiles::needProcess([[maybe_unused]] const ScanContext& context) const
|
||||||
{
|
{
|
||||||
// always check for removed files
|
// always check for removed files
|
||||||
@@ -67,91 +213,48 @@ namespace lms::scanner
|
|||||||
template<typename Object>
|
template<typename Object>
|
||||||
void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context)
|
void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context)
|
||||||
{
|
{
|
||||||
using namespace db;
|
using ObjectIdType = typename Object::IdType;
|
||||||
|
|
||||||
if (_abortScan)
|
if (_abortScan)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
Session& session{ _db.getTLSSession() };
|
db::Session& session{ _db.getTLSSession() };
|
||||||
|
|
||||||
std::vector<typename Object::IdType> objectIdsToRemove;
|
std::deque<ObjectIdType> objectIdsToRemove;
|
||||||
|
|
||||||
typename Object::IdType lastCheckedId;
|
auto processJobsDone = [&](std::span<std::unique_ptr<core::IJob>> jobs) {
|
||||||
bool endReached{};
|
|
||||||
while (!endReached)
|
|
||||||
{
|
|
||||||
if (_abortScan)
|
if (_abortScan)
|
||||||
break;
|
return;
|
||||||
|
|
||||||
objectIdsToRemove.clear();
|
for (const auto& job : jobs)
|
||||||
{
|
{
|
||||||
constexpr std::size_t batchSize = 200;
|
const auto& checkJob{ static_cast<const CheckForRemovedFilesJob<ObjectIdType>&>(*job) };
|
||||||
|
std::span<const ObjectIdType> objectsToRemove{ checkJob.getObjectsToRemove() };
|
||||||
|
|
||||||
auto transaction{ session.createReadTransaction() };
|
objectIdsToRemove.insert(std::end(objectIdsToRemove), std::cbegin(objectsToRemove), std::cend(objectsToRemove));
|
||||||
|
|
||||||
endReached = true;
|
context.currentStepStats.processedElems += checkJob.getProcessedCount();
|
||||||
Object::findAbsoluteFilePath(session, lastCheckedId, batchSize, [&](Object::IdType objectId, const std::filesystem::path& filePath) {
|
|
||||||
endReached = false;
|
|
||||||
|
|
||||||
// special case for track lyrics, only check external lyrics
|
|
||||||
if constexpr (std::is_same_v<Object, TrackLyrics>)
|
|
||||||
{
|
|
||||||
if (filePath.empty())
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!checkFile(filePath))
|
|
||||||
objectIdsToRemove.push_back(objectId);
|
|
||||||
|
|
||||||
context.currentStepStats.processedElems++;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!objectIdsToRemove.empty())
|
if (!objectIdsToRemove.empty())
|
||||||
{
|
{
|
||||||
auto transaction{ session.createWriteTransaction() };
|
removeObjects<Object>(session, objectIdsToRemove, true);
|
||||||
|
|
||||||
session.destroy<Object>(objectIdsToRemove);
|
|
||||||
context.stats.deletions += objectIdsToRemove.size();
|
context.stats.deletions += objectIdsToRemove.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
_progressCallback(context.currentStepStats);
|
_progressCallback(context.currentStepStats);
|
||||||
|
};
|
||||||
|
|
||||||
|
{
|
||||||
|
JobQueue queue{ getJobScheduler(), 50, processJobsDone, 1, 0.85F };
|
||||||
|
|
||||||
|
ObjectIdType lastCheckedId;
|
||||||
|
std::vector<FileToCheck<ObjectIdType>> filesToCheck;
|
||||||
|
while (fetchNextFilesToCheck<Object>(session, lastCheckedId, filesToCheck))
|
||||||
|
queue.push(std::make_unique<CheckForRemovedFilesJob<ObjectIdType>>(_settings, getFileScanners(), filesToCheck));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// process all remaining objects
|
||||||
|
removeObjects<Object>(session, objectIdsToRemove, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
// For each track, make sure the the file still exists
|
|
||||||
// and still belongs to a media directory
|
|
||||||
if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
|
|
||||||
{
|
|
||||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
|
|
||||||
[&](const MediaLibraryInfo& libraryInfo) {
|
|
||||||
return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
|
|
||||||
}))
|
|
||||||
{
|
|
||||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!selectFileScanner(p))
|
|
||||||
{
|
|
||||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": file format no longer handled");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch (std::filesystem::filesystem_error& e)
|
|
||||||
{
|
|
||||||
LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file " << p << ": " << e.what());
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} // namespace lms::scanner
|
} // namespace lms::scanner
|
||||||
|
|||||||
@@ -31,6 +31,7 @@
|
|||||||
#include "scanners/IFileScanOperation.hpp"
|
#include "scanners/IFileScanOperation.hpp"
|
||||||
#include "scanners/IFileScanner.hpp"
|
#include "scanners/IFileScanner.hpp"
|
||||||
|
|
||||||
|
#include "FileScanners.hpp"
|
||||||
#include "JobQueue.hpp"
|
#include "JobQueue.hpp"
|
||||||
#include "ScanContext.hpp"
|
#include "ScanContext.hpp"
|
||||||
|
|
||||||
@@ -68,18 +69,6 @@ namespace lms::scanner
|
|||||||
};
|
};
|
||||||
} // namespace
|
} // namespace
|
||||||
|
|
||||||
ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
|
|
||||||
: ScanStepBase{ initParams }
|
|
||||||
{
|
|
||||||
visitFileScanners([](IFileScanner* scanner) {
|
|
||||||
for (const std::filesystem::path& file : scanner->getSupportedFiles())
|
|
||||||
LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file " << file);
|
|
||||||
|
|
||||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
|
||||||
LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file extension " << extension);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
bool ScanStepScanFiles::needProcess([[maybe_unused]] const ScanContext& context) const
|
bool ScanStepScanFiles::needProcess([[maybe_unused]] const ScanContext& context) const
|
||||||
{
|
{
|
||||||
// Always need to scan files
|
// Always need to scan files
|
||||||
@@ -122,7 +111,7 @@ namespace lms::scanner
|
|||||||
addError<IOScanError>(context, path, ec);
|
addError<IOScanError>(context, path, ec);
|
||||||
context.stats.skips++;
|
context.stats.skips++;
|
||||||
}
|
}
|
||||||
else if (IFileScanner * scanner{ selectFileScanner(path) })
|
else if (IFileScanner * scanner{ getFileScanners().select(path) })
|
||||||
{
|
{
|
||||||
FileToScan fileToScan;
|
FileToScan fileToScan;
|
||||||
|
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ namespace lms::scanner
|
|||||||
class ScanStepScanFiles : public ScanStepBase
|
class ScanStepScanFiles : public ScanStepBase
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
ScanStepScanFiles(InitParams& initParams);
|
using ScanStepBase::ScanStepBase;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
ScanStep getStep() const override { return ScanStep::ScanFiles; }
|
ScanStep getStep() const override { return ScanStep::ScanFiles; }
|
||||||
|
|||||||
Reference in New Issue
Block a user