More details for scan error reports #676
This commit is contained in:
@@ -53,7 +53,7 @@ namespace lms::scanner
|
||||
{
|
||||
{
|
||||
LMS_SCOPED_TRACE_OVERVIEW("Scanner", operation->getName());
|
||||
LMS_LOG(DBUPDATER, DEBUG, operation->getName() << ": scanning file " << operation->getFile());
|
||||
LMS_LOG(DBUPDATER, DEBUG, operation->getName() << ": scanning file " << operation->getFilePath());
|
||||
operation->scan();
|
||||
}
|
||||
|
||||
|
||||
@@ -20,11 +20,12 @@
|
||||
#pragma once
|
||||
|
||||
#include "core/LiteralString.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
#include "services/scanner/ScannerStats.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
struct ScanContext;
|
||||
|
||||
class IScanStep
|
||||
{
|
||||
public:
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Copyright (C) 2025 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "ScanErrorLogger.hpp"
|
||||
|
||||
#include <cassert>
|
||||
|
||||
#include "core/ILogger.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
void ScanErrorLogger::visit([[maybe_unused]] const scanner::ScanError& error)
|
||||
{
|
||||
// There should never be a ScanError without a specific type
|
||||
assert(false);
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::IOScanError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to open file " << error.path << ": " << error.err.message());
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::AudioFileScanError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << error.path);
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::EmbeddedImageScanError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to parse image in track file " << error.path << " at index " << error.index);
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::NoAudioTrackFoundError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << error.path << ": no audio track found");
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::BadAudioDurationError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << error.path << ": duration is 0");
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::ArtistInfoFileScanError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to read artist info file " << error.path);
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::MissingArtistNameError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to read artist info file " << error.path << ": missing name");
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::ImageFileScanError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to read image file " << error.path);
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::LyricsFileScanError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to read lyrics file " << error.path);
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::PlayListFileScanError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to read playlist file " << error.path);
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::PlayListFilePathMissingError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Track " << error.entry << " not found in playlist " << error.path);
|
||||
}
|
||||
|
||||
void ScanErrorLogger::visit(const scanner::PlayListFileAllPathesMissingError& error)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Failed to parse playlist " << error.path << ": all entries are missing");
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Copyright (C) 2025 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "services/scanner/ScanErrors.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
class ScanErrorLogger : public scanner::ScanErrorVisitor
|
||||
{
|
||||
private:
|
||||
void visit(const scanner::ScanError&) override;
|
||||
void visit(const scanner::IOScanError& error) override;
|
||||
void visit(const scanner::AudioFileScanError& error) override;
|
||||
void visit(const scanner::EmbeddedImageScanError& error) override;
|
||||
void visit(const scanner::NoAudioTrackFoundError& error) override;
|
||||
void visit(const scanner::BadAudioDurationError& error) override;
|
||||
void visit(const scanner::ArtistInfoFileScanError& error) override;
|
||||
void visit(const scanner::MissingArtistNameError& error) override;
|
||||
void visit(const scanner::ImageFileScanError& error) override;
|
||||
void visit(const scanner::LyricsFileScanError& error) override;
|
||||
void visit(const scanner::PlayListFileScanError& error) override;
|
||||
void visit(const scanner::PlayListFilePathMissingError& error) override;
|
||||
void visit(const scanner::PlayListFileAllPathesMissingError& error) override;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
@@ -32,6 +32,7 @@
|
||||
#include "database/TrackList.hpp"
|
||||
#include "metadata/Types.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
#include "helpers/ArtistHelpers.hpp"
|
||||
|
||||
|
||||
@@ -37,6 +37,8 @@
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
|
||||
@@ -28,6 +28,8 @@
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackLyrics.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
|
||||
@@ -30,7 +30,9 @@
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackList.hpp"
|
||||
#include "services/scanner/ScanErrors.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
@@ -57,6 +59,7 @@ namespace lms::scanner
|
||||
db::PlayListFileId lastRetrievedPlayListFileId;
|
||||
std::size_t processedPlayListFileCount{};
|
||||
const ScannerSettings& settings;
|
||||
std::vector<std::shared_ptr<ScanError>> errors;
|
||||
};
|
||||
|
||||
db::Track::pointer getMatchingTrack(db::Session& session, const std::filesystem::path& filePath, const db::Directory::pointer& playListDirectory)
|
||||
@@ -81,10 +84,7 @@ namespace lms::scanner
|
||||
return true;
|
||||
|
||||
const db::ReleaseId releaseId{ tracks.front().releaseId };
|
||||
if (std::all_of(std::cbegin(tracks) + 1, std::cend(tracks), [=](const TrackInfo& trackInfo) { return trackInfo.releaseId == releaseId; }))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
return std::all_of(std::cbegin(tracks) + 1, std::cend(tracks), [=](const TrackInfo& trackInfo) { return trackInfo.releaseId == releaseId; });
|
||||
}
|
||||
|
||||
bool trackListNeedsUpdate(db::Session& session, std::string_view name, std::span<const TrackInfo> tracks, const db::TrackList::pointer& trackList)
|
||||
@@ -124,17 +124,28 @@ namespace lms::scanner
|
||||
|
||||
playListAssociation.playListFileIdId = playListFile->getId();
|
||||
|
||||
std::vector<std::shared_ptr<ScanError>> pendingErrors;
|
||||
|
||||
const auto files{ playListFile->getFiles() };
|
||||
for (const std::filesystem::path& file : files)
|
||||
{
|
||||
// TODO optim: no need to fetch the whole track
|
||||
db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) };
|
||||
const db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) };
|
||||
if (track)
|
||||
playListAssociation.tracks.push_back(TrackInfo{ .trackId = track->getId(), .releaseId = track->getReleaseId() });
|
||||
else
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Track " << file << " not found in playlist " << playListFile->getAbsoluteFilePath());
|
||||
{
|
||||
pendingErrors.emplace_back(std::make_shared<PlayListFilePathMissingError>(playListFile->getAbsoluteFilePath(), file));
|
||||
}
|
||||
}
|
||||
|
||||
if (pendingErrors.size() == files.size())
|
||||
{
|
||||
pendingErrors.clear();
|
||||
pendingErrors.emplace_back(std::make_shared<PlayListFileAllPathesMissingError>(playListFile->getAbsoluteFilePath()));
|
||||
}
|
||||
searchContext.errors.insert(std::end(searchContext.errors), std::begin(pendingErrors), std::end(pendingErrors));
|
||||
|
||||
if (playListAssociation.tracks.empty()
|
||||
|| (searchContext.settings.skipSingleReleasePlayLists && isSingleReleasePlayList(playListAssociation.tracks)))
|
||||
{
|
||||
@@ -237,6 +248,7 @@ namespace lms::scanner
|
||||
.session = session,
|
||||
.lastRetrievedPlayListFileId = {},
|
||||
.settings = _settings,
|
||||
.errors = context.stats.errors
|
||||
};
|
||||
|
||||
PlayListFileAssociationContainer playListFileAssociations;
|
||||
@@ -246,7 +258,12 @@ namespace lms::scanner
|
||||
return;
|
||||
|
||||
updatePlayListFiles(session, playListFileAssociations);
|
||||
|
||||
context.currentStepStats.processedElems = searchContext.processedPlayListFileCount;
|
||||
for (const std::shared_ptr<ScanError>& error : searchContext.errors)
|
||||
addError(context, error);
|
||||
searchContext.errors.clear();
|
||||
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,8 @@
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
namespace
|
||||
|
||||
@@ -19,6 +19,11 @@
|
||||
|
||||
#include "ScanStepBase.hpp"
|
||||
|
||||
#include "core/String.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
ScanStepBase::ScanStepBase(InitParams& initParams)
|
||||
@@ -29,7 +34,58 @@ namespace lms::scanner
|
||||
, _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners))
|
||||
, _lastScanSettings{ initParams.lastScanSettings }
|
||||
{
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
{
|
||||
for (const std::filesystem::path& file : scanner->getSupportedFiles())
|
||||
{
|
||||
[[maybe_unused]] auto [it, inserted]{ _scannerByFile.emplace(file, scanner) };
|
||||
assert(inserted);
|
||||
}
|
||||
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
{
|
||||
[[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) };
|
||||
assert(inserted);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ScanStepBase::~ScanStepBase() = default;
|
||||
|
||||
IFileScanner* ScanStepBase::selectFileScanner(const std::filesystem::path& filePath) const
|
||||
{
|
||||
{
|
||||
const std::string fileName{ core::stringUtils::stringToLower(filePath.filename().string()) };
|
||||
|
||||
auto itScanner{ _scannerByFile.find(fileName) };
|
||||
if (itScanner != std::cend(_scannerByFile))
|
||||
return itScanner->second;
|
||||
}
|
||||
|
||||
{
|
||||
const std::string extension{ core::stringUtils::stringToLower(filePath.extension().string()) };
|
||||
|
||||
auto itScanner{ _scannerByExtension.find(extension) };
|
||||
if (itScanner != std::cend(_scannerByExtension))
|
||||
return itScanner->second;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void ScanStepBase::visitFileScanners(const std::function<void(IFileScanner*)>& visitor) const
|
||||
{
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
visitor(scanner);
|
||||
}
|
||||
|
||||
void ScanStepBase::addError(ScanContext& context, std::shared_ptr<ScanError> error)
|
||||
{
|
||||
error->accept(_scanErrorLogger);
|
||||
|
||||
context.stats.errorsCount++;
|
||||
|
||||
if (context.stats.errors.size() < ScanStats::maxStoredErrorCount)
|
||||
context.stats.errors.emplace_back(error);
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
|
||||
@@ -21,9 +21,11 @@
|
||||
|
||||
#include <functional>
|
||||
#include <span>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "IScanStep.hpp"
|
||||
#include "ScanErrorLogger.hpp"
|
||||
|
||||
namespace lms::db
|
||||
{
|
||||
@@ -35,6 +37,7 @@ namespace lms::scanner
|
||||
class IFileScanner;
|
||||
struct ScannerSettings;
|
||||
struct ScanStepStats;
|
||||
struct ScanContext;
|
||||
|
||||
class ScanStepBase : public IScanStep
|
||||
{
|
||||
@@ -57,14 +60,29 @@ namespace lms::scanner
|
||||
|
||||
protected:
|
||||
const ScannerSettings* getLastScanSettings() const { return _lastScanSettings; }
|
||||
IFileScanner* selectFileScanner(const std::filesystem::path& filePath) const;
|
||||
void visitFileScanners(const std::function<void(IFileScanner*)>& visitor) const;
|
||||
|
||||
void addError(ScanContext& context, std::shared_ptr<ScanError> error);
|
||||
|
||||
template<typename T, typename... CtrArgs>
|
||||
void addError(ScanContext& context, CtrArgs&&... args)
|
||||
{
|
||||
auto error{ std::make_shared<T>(std::forward<CtrArgs>(args)...) };
|
||||
addError(context, error);
|
||||
}
|
||||
|
||||
const ScannerSettings& _settings;
|
||||
ProgressCallback _progressCallback;
|
||||
bool& _abortScan;
|
||||
db::Db& _db;
|
||||
std::vector<IFileScanner*> _fileScanners;
|
||||
|
||||
private:
|
||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByFile;
|
||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
|
||||
std::vector<IFileScanner*> _fileScanners;
|
||||
|
||||
const ScannerSettings* _lastScanSettings{};
|
||||
ScanErrorLogger _scanErrorLogger;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
|
||||
@@ -24,6 +24,8 @@
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
bool ScanStepCheckForDuplicatedFiles::needProcess([[maybe_unused]] const ScanContext& context) const
|
||||
|
||||
@@ -30,8 +30,8 @@
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackLyrics.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
@@ -62,22 +62,15 @@ namespace lms::scanner
|
||||
}
|
||||
LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " files to be checked...");
|
||||
|
||||
std::vector<std::filesystem::path> supportedFileExtensions;
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
{
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
supportedFileExtensions.emplace_back(extension);
|
||||
}
|
||||
|
||||
checkForRemovedFiles<db::Track>(context, supportedFileExtensions);
|
||||
checkForRemovedFiles<db::Image>(context, supportedFileExtensions);
|
||||
checkForRemovedFiles<db::TrackLyrics>(context, supportedFileExtensions);
|
||||
checkForRemovedFiles<db::PlayListFile>(context, supportedFileExtensions);
|
||||
checkForRemovedFiles<db::ArtistInfo>(context, supportedFileExtensions);
|
||||
checkForRemovedFiles<db::Track>(context);
|
||||
checkForRemovedFiles<db::Image>(context);
|
||||
checkForRemovedFiles<db::TrackLyrics>(context);
|
||||
checkForRemovedFiles<db::PlayListFile>(context);
|
||||
checkForRemovedFiles<db::ArtistInfo>(context);
|
||||
}
|
||||
|
||||
template<typename Object>
|
||||
void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context, std::span<const std::filesystem::path> supportedFileExtensions)
|
||||
void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context)
|
||||
{
|
||||
using namespace db;
|
||||
|
||||
@@ -110,7 +103,7 @@ namespace lms::scanner
|
||||
return;
|
||||
}
|
||||
|
||||
if (!checkFile(object->getAbsoluteFilePath(), supportedFileExtensions))
|
||||
if (!checkFile(object->getAbsoluteFilePath()))
|
||||
objectsToRemove.push_back(object);
|
||||
|
||||
context.currentStepStats.processedElems++;
|
||||
@@ -132,7 +125,7 @@ namespace lms::scanner
|
||||
}
|
||||
}
|
||||
|
||||
bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p, std::span<const std::filesystem::path> allowedExtensions)
|
||||
bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -140,7 +133,7 @@ namespace lms::scanner
|
||||
// and still belongs to a media directory
|
||||
if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': missing");
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing");
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -149,13 +142,13 @@ namespace lms::scanner
|
||||
return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
|
||||
}))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': out of media directory");
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!core::pathUtils::hasFileAnyExtension(p, allowedExtensions))
|
||||
if (!selectFileScanner(p))
|
||||
{
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': file format no longer handled");
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p.string() << ": file format no longer handled");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
#include <span>
|
||||
|
||||
#include "ScanStepBase.hpp"
|
||||
|
||||
@@ -38,8 +37,8 @@ namespace lms::scanner
|
||||
void process(ScanContext& context) override;
|
||||
|
||||
template<typename Object>
|
||||
void checkForRemovedFiles(ScanContext& context, std::span<const std::filesystem::path> supportedFileExtensions);
|
||||
void checkForRemovedFiles(ScanContext& context);
|
||||
|
||||
bool checkFile(const std::filesystem::path& p, std::span<const std::filesystem::path> allowedExtensions);
|
||||
bool checkFile(const std::filesystem::path& p);
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
|
||||
@@ -22,6 +22,8 @@
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Session.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
bool ScanStepCompact::needProcess(const ScanContext& context) const
|
||||
|
||||
@@ -23,6 +23,8 @@
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Session.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
bool ScanStepComputeClusterStats::needProcess(const ScanContext& context) const
|
||||
|
||||
@@ -24,7 +24,8 @@
|
||||
|
||||
#include "MediaLibraryInfo.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
@@ -38,13 +39,6 @@ namespace lms::scanner
|
||||
{
|
||||
context.stats.totalFileCount = 0;
|
||||
|
||||
std::vector<std::filesystem::path> supportedFileExtensions;
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
{
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
supportedFileExtensions.emplace_back(extension);
|
||||
}
|
||||
|
||||
for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
|
||||
{
|
||||
std::size_t currentDirectoryProcessElemsCount{};
|
||||
@@ -53,7 +47,8 @@ namespace lms::scanner
|
||||
if (_abortScan)
|
||||
return false;
|
||||
|
||||
if (!ec && core::pathUtils::hasFileAnyExtension(path, supportedFileExtensions))
|
||||
// we don't report errors here (done in the actual scan step)
|
||||
if (!ec && selectFileScanner(path))
|
||||
{
|
||||
context.currentStepStats.processedElems++;
|
||||
currentDirectoryProcessElemsCount++;
|
||||
@@ -64,7 +59,7 @@ namespace lms::scanner
|
||||
},
|
||||
&excludeDirFileName);
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in '" << mediaLibrary.rootDirectory << "'");
|
||||
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in " << mediaLibrary.rootDirectory);
|
||||
}
|
||||
|
||||
context.stats.totalFileCount = context.currentStepStats.processedElems;
|
||||
|
||||
@@ -23,6 +23,8 @@
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Session.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
bool ScanStepOptimize::needProcess(const ScanContext& context) const
|
||||
|
||||
@@ -29,6 +29,8 @@
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackEmbeddedImage.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
bool ScanStepRemoveOrphanedDbEntries::needProcess([[maybe_unused]] const ScanContext& context) const
|
||||
|
||||
@@ -27,9 +27,12 @@
|
||||
#include "core/Path.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "scanners/FileToScan.hpp"
|
||||
#include "scanners/IFileScanOperation.hpp"
|
||||
#include "scanners/IFileScanner.hpp"
|
||||
|
||||
#include "ScanContext.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
{
|
||||
using namespace db;
|
||||
@@ -45,21 +48,38 @@ namespace lms::scanner
|
||||
|
||||
return threadCount;
|
||||
}
|
||||
|
||||
FileToScan retrieveFileInfo(const std::filesystem::path& file, const MediaLibraryInfo& mediaLibrary, std::error_code& ec)
|
||||
{
|
||||
FileToScan res;
|
||||
|
||||
res.lastWriteTime = core::pathUtils::getLastWriteTime(file, ec);
|
||||
if (!ec)
|
||||
res.relativePath = std::filesystem::relative(file, mediaLibrary.rootDirectory, ec);
|
||||
if (!ec)
|
||||
res.fileSize = std::filesystem::file_size(file, ec);
|
||||
|
||||
if (!ec)
|
||||
{
|
||||
res.filePath = file;
|
||||
res.mediaLibrary = mediaLibrary;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
|
||||
: ScanStepBase{ initParams }
|
||||
, _fileScanQueue{ getScanMetaDataThreadCount(), _abortScan }
|
||||
{
|
||||
for (IFileScanner* scanner : _fileScanners)
|
||||
{
|
||||
visitFileScanners([](IFileScanner* scanner) {
|
||||
for (const std::filesystem::path& file : scanner->getSupportedFiles())
|
||||
LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file " << file);
|
||||
|
||||
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
|
||||
{
|
||||
[[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) };
|
||||
assert(inserted);
|
||||
LMS_LOG(DBUPDATER, INFO, "Registered extension " << extension << " for " << scanner->getName());
|
||||
}
|
||||
}
|
||||
LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file extension " << extension);
|
||||
});
|
||||
|
||||
LMS_LOG(DBUPDATER, INFO, "Using " << _fileScanQueue.getThreadCount() << " thread(s) for scanning file metadata");
|
||||
}
|
||||
@@ -94,26 +114,28 @@ namespace lms::scanner
|
||||
|
||||
if (ec)
|
||||
{
|
||||
LMS_LOG(DBUPDATER, ERROR, "Cannot scan file " << path << ": " << ec.message());
|
||||
context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
|
||||
addError<IOScanError>(context, path, ec);
|
||||
context.stats.skips++;
|
||||
}
|
||||
else
|
||||
else if (IFileScanner * scanner{ selectFileScanner(path) })
|
||||
{
|
||||
auto itScanner{ _scannerByExtension.find(core::stringUtils::stringToLower(path.extension().c_str())) };
|
||||
if (itScanner != std::cend(_scannerByExtension))
|
||||
FileToScan fileToScan{ retrieveFileInfo(path, mediaLibrary, ec) };
|
||||
if (ec)
|
||||
{
|
||||
IFileScanner& scanner{ *itScanner->second };
|
||||
|
||||
FileToScan fileToScan{ .file = path, .mediaLibrary = mediaLibrary };
|
||||
if (scanner.needsScan(context, fileToScan))
|
||||
addError<IOScanError>(context, path, ec);
|
||||
context.stats.skips++;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (context.scanOptions.fullScan || scanner->needsScan(fileToScan))
|
||||
{
|
||||
auto scanOperation{ scanner.createScanOperation(fileToScan) };
|
||||
auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) };
|
||||
_fileScanQueue.pushScanRequest(std::move(scanOperation));
|
||||
}
|
||||
|
||||
context.currentStepStats.processedElems++;
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
|
||||
context.currentStepStats.processedElems++;
|
||||
_progressCallback(context.currentStepStats);
|
||||
}
|
||||
|
||||
while (_fileScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2))
|
||||
@@ -146,9 +168,27 @@ namespace lms::scanner
|
||||
if (_abortScan)
|
||||
return;
|
||||
|
||||
LMS_LOG(DBUPDATER, DEBUG, scanOperation->getName() << ": processing result for " << scanOperation->getFile());
|
||||
scanOperation->processResult(context);
|
||||
LMS_LOG(DBUPDATER, DEBUG, scanOperation->getName() << ": processing result for " << scanOperation->getFilePath());
|
||||
const IFileScanOperation::OperationResult res{ scanOperation->processResult() };
|
||||
switch (res)
|
||||
{
|
||||
case IFileScanOperation::OperationResult::Added:
|
||||
context.stats.additions++;
|
||||
break;
|
||||
case IFileScanOperation::OperationResult::Removed:
|
||||
context.stats.deletions++;
|
||||
break;
|
||||
case IFileScanOperation::OperationResult::Skipped:
|
||||
context.stats.failures++;
|
||||
break;
|
||||
case IFileScanOperation::OperationResult::Updated:
|
||||
context.stats.updates++;
|
||||
break;
|
||||
}
|
||||
context.stats.scans++;
|
||||
|
||||
for (const auto& error : scanOperation->getErrors())
|
||||
addError(context, error);
|
||||
}
|
||||
}
|
||||
} // namespace lms::scanner
|
||||
@@ -40,10 +40,10 @@ namespace lms::scanner
|
||||
core::LiteralString getStepName() const override { return "Scan files"; }
|
||||
bool needProcess(const ScanContext& context) const override;
|
||||
void process(ScanContext& context) override;
|
||||
|
||||
void process(ScanContext& context, const MediaLibraryInfo& mediaLibrary);
|
||||
void processFileScanResults(ScanContext& context, std::span<std::unique_ptr<IFileScanOperation>> scanOperations);
|
||||
|
||||
FileScanQueue _fileScanQueue;
|
||||
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
|
||||
};
|
||||
} // namespace lms::scanner
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
#include "database/Session.hpp"
|
||||
|
||||
#include "MediaLibraryInfo.hpp"
|
||||
#include "ScanContext.hpp"
|
||||
#include "ScannerSettings.hpp"
|
||||
|
||||
namespace lms::scanner
|
||||
|
||||
Reference in New Issue
Block a user