owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More details for scan error reports #676

Browse Source
This commit is contained in:
emeric
2025-05-26 23:33:52 +02:00
parent 6d6ea4920b
commit 571c0bc5f4
86 changed files with 1794 additions and 950 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/scanner/CMakeLists.txt
+2
View File
@@ -2,12 +2,14 @@ add_library(lmsscanner STATIC
impl/helpers/ArtistHelpers.cpp
impl/scanners/ArtistInfoFileScanner.cpp
impl/scanners/AudioFileScanOperation.cpp
impl/scanners/FileScanOperationBase.cpp
impl/scanners/AudioFileScanner.cpp
impl/scanners/ImageFileScanner.cpp
impl/scanners/LyricsFileScanner.cpp
impl/scanners/PlayListFileScanner.cpp
impl/scanners/Utils.cpp
impl/steps/FileScanQueue.cpp
impl/steps/ScanErrorLogger.cpp
impl/steps/ScanStepArtistReconciliation.cpp
impl/steps/ScanStepAssociateArtistImages.cpp
impl/steps/ScanStepAssociateExternalLyrics.cpp
src/libs/services/scanner/impl/ScannerService.cpp
+9 -7
View File
@@ -51,14 +51,16 @@
#include "steps/ScanStepScanFiles.hpp"
#include "steps/ScanStepUpdateLibraryFields.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
using namespace db;
namespace
{
static constexpr std::string_view currentSettingsName{ "" };
static constexpr std::string_view lastScanSettingsName{ "last_scan" };
constexpr std::string_view currentSettingsName{};
constexpr std::string_view lastScanSettingsName{ "last_scan" };
Wt::WDate getNextMonday(Wt::WDate current)
{
@@ -352,7 +354,7 @@ namespace lms::scanner
_currentScanStepStats.reset(); // must be sync with _curState
}
LMS_LOG(DBUPDATER, INFO, "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size());
LMS_LOG(DBUPDATER, INFO, "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << ", failures = " << stats.failures << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errorsCount << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size());
if (!_abortScan)
{
@@ -433,11 +435,11 @@ namespace lms::scanner
} };
_fileScanners.clear();
_fileScanners.emplace_back(std::make_unique<ArtistInfoFileScanner>(_settings, _db));
_fileScanners.emplace_back(std::make_unique<ArtistInfoFileScanner>(_db, _settings));
_fileScanners.emplace_back(std::make_unique<AudioFileScanner>(_db, _settings));
_fileScanners.emplace_back(std::make_unique<ImageFileScanner>(_db));
_fileScanners.emplace_back(std::make_unique<LyricsFileScanner>(_db));
_fileScanners.emplace_back(std::make_unique<PlayListFileScanner>(_db));
_fileScanners.emplace_back(std::make_unique<ImageFileScanner>(_db, _settings));
_fileScanners.emplace_back(std::make_unique<LyricsFileScanner>(_db, _settings));
_fileScanners.emplace_back(std::make_unique<PlayListFileScanner>(_db, _settings));
std::vector<IFileScanner*> fileScanners;
std::transform(std::cbegin(_fileScanners), std::cend(_fileScanners), std::back_inserter(fileScanners), [](const std::unique_ptr<IFileScanner>& scanner) { return scanner.get(); });
src/libs/services/scanner/impl/ScannerStats.cpp
+1 -8
View File
@@ -21,16 +21,9 @@
namespace lms::scanner
{
ScanError::ScanError(const std::filesystem::path& _file, ScanErrorType _error, const std::string& _systemError)
: file{ _file }
, error{ _error }
, systemError{ _systemError }
{
}
std::size_t ScanStats::nbFiles() const
{
return skips + additions + updates;
return skips + additions + updates + failures;
}
std::size_t ScanStats::nbChanges() const
src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.cpp
+49 -80
View File
@@ -20,6 +20,7 @@
#include "ArtistInfoFileScanner.hpp"
#include <fstream>
#include <system_error>
#include "core/ILogger.hpp"
#include "core/String.hpp"
@@ -29,45 +30,33 @@
#include "database/MediaLibrary.hpp"
#include "database/Session.hpp"
#include "metadata/ArtistInfo.hpp"
#include "metadata/Types.hpp"
#include "services/scanner/ScanErrors.hpp"
#include "IFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "FileScanOperationBase.hpp"
#include "ScannerSettings.hpp"
#include "Utils.hpp"
#include "helpers/ArtistHelpers.hpp"
#include "metadata/Types.hpp"
namespace lms::scanner
{
namespace
{
class ArtistInfoFileScanOperation : public IFileScanOperation
class ArtistInfoFileScanOperation : public FileScanOperationBase
{
public:
ArtistInfoFileScanOperation(const FileToScan& file, const ScannerSettings& settings, db::Db& db)
: _file{ file.file }
, _mediaLibrary{ file.mediaLibrary }
, _settings{ settings }
, _db{ db }
{
}
using FileScanOperationBase::FileScanOperationBase;
~ArtistInfoFileScanOperation() override = default;
ArtistInfoFileScanOperation(const ArtistInfoFileScanOperation&) = delete;
ArtistInfoFileScanOperation& operator=(const ArtistInfoFileScanOperation&) = delete;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanArtistInfoFile"; }
void scan() override;
void processResult(ScanContext& context) override;
OperationResult processResult() override;
std::string getArtistNameFromArtistInfoFilePath();
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
const ScannerSettings& _settings;
db::Db& _db;
std::optional<metadata::ArtistInfo> _parsedArtistInfo;
};
@@ -75,91 +64,82 @@ namespace lms::scanner
{
try
{
std::ifstream ifs{ _file };
std::ifstream ifs{ getFilePath() };
if (!ifs)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot open file " << _file);
const std::error_code ec{ errno, std::generic_category() };
addError<IOScanError>(getFilePath(), ec);
return;
}
_parsedArtistInfo = metadata::parseArtistInfo(ifs);
if (_parsedArtistInfo->name.empty())
{
LMS_LOG(DBUPDATER, DEBUG, "Discarding artist info in file " << _file << ": no name set");
addError<MissingArtistNameError>(getFilePath());
_parsedArtistInfo.reset();
}
}
catch (const metadata::ArtistInfoParseException& e)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot read artist info in file " << _file << ": " << e.what());
addError<ArtistInfoFileScanError>(getFilePath());
}
}
void ArtistInfoFileScanOperation::processResult(ScanContext& context)
ArtistInfoFileScanOperation::OperationResult ArtistInfoFileScanOperation::processResult()
{
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, getFilePath()) };
if (!_parsedArtistInfo)
{
if (artistInfo)
{
artistInfo.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed artist info file " << _file);
LMS_LOG(DBUPDATER, DEBUG, "Removed artist info file " << getFilePath());
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadArtistInfoFile);
return;
return OperationResult::Skipped;
}
const bool added{ !artistInfo };
if (!artistInfo)
{
artistInfo = dbSession.create<db::ArtistInfo>();
artistInfo.modify()->setAbsoluteFilePath(_file);
artistInfo.modify()->setAbsoluteFilePath(getFilePath());
}
artistInfo.modify()->setScanVersion(_settings.artistInfoScanVersion);
artistInfo.modify()->setScanVersion(getScannerSettings().artistInfoScanVersion);
artistInfo.modify()->setName(_parsedArtistInfo->name);
artistInfo.modify()->setSortName(_parsedArtistInfo->sortName);
artistInfo.modify()->setLastWriteTime(fileInfo->lastWriteTime);
artistInfo.modify()->setLastWriteTime(getLastWriteTime());
artistInfo.modify()->setType(_parsedArtistInfo->type);
artistInfo.modify()->setGender(_parsedArtistInfo->gender);
artistInfo.modify()->setDisambiguation(_parsedArtistInfo->disambiguation);
artistInfo.modify()->setBiography(_parsedArtistInfo->biography);
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
artistInfo.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
artistInfo.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary));
const metadata::Artist artistMetadata{ _parsedArtistInfo->mbid, _parsedArtistInfo->name, _parsedArtistInfo->sortName.empty() ? std::nullopt : std::make_optional<std::string>(_parsedArtistInfo->sortName) };
db::Artist::pointer artist{ helpers::getOrCreateArtist(dbSession, artistMetadata, helpers::AllowFallbackOnMBIDEntry{ _settings.allowArtistMBIDFallback }) };
db::Artist::pointer artist{ helpers::getOrCreateArtist(dbSession, artistMetadata, helpers::AllowFallbackOnMBIDEntry{ getScannerSettings().allowArtistMBIDFallback }) };
artistInfo.modify()->setArtist(artist);
artistInfo.modify()->setMBIDMatched(_parsedArtistInfo->mbid.has_value() && _parsedArtistInfo->mbid == artist->getMBID());
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added artist info file " << _file);
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated artist info file " << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Added artist info file " << getFilePath());
return OperationResult::Added;
}
LMS_LOG(DBUPDATER, DEBUG, "Updated artist info file " << getFilePath());
return OperationResult::Updated;
}
} // namespace
ArtistInfoFileScanner::ArtistInfoFileScanner(const ScannerSettings& settings, db::Db& db)
: _settings{ settings }
, _db{ db }
ArtistInfoFileScanner::ArtistInfoFileScanner(db::Db& db, const ScannerSettings& settings)
: _db{ db }
, _settings{ settings }
{
}
@@ -168,45 +148,34 @@ namespace lms::scanner
return "Artist info scanner";
}
std::span<const std::filesystem::path> ArtistInfoFileScanner::getSupportedExtensions() const
std::span<const std::filesystem::path> ArtistInfoFileScanner::getSupportedFiles() const
{
return metadata::getSupportedInfoFileExtensions();
return metadata::getSupportedArtistInfoFiles();
}
bool ArtistInfoFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
std::span<const std::filesystem::path> ArtistInfoFileScanner::getSupportedExtensions() const
{
return {};
}
bool ArtistInfoFileScanner::needsScan(const FileToScan& file) const
{
// Special case: only files named "artist.nfo" are compatible with this scanner
// Hack here since the scanner framework only handle extensions (the discover count is not accurate)
if (!core::stringUtils::stringCaseInsensitiveEqual(file.file.stem().string(), "artist"))
if (!core::stringUtils::stringCaseInsensitiveEqual(file.filePath.stem().string(), "artist"))
return false;
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
context.stats.skips++;
return false;
}
if (context.scanOptions.fullScan)
return true;
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ dbSession.createReadTransaction() };
db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, file.file) };
if (artistInfo
&& artistInfo->getLastWriteTime() == lastWriteTime
&& artistInfo->getScanVersion() == _settings.artistInfoScanVersion)
{
context.stats.skips++;
return false;
}
db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, file.filePath) };
return true;
return !artistInfo
|| artistInfo->getLastWriteTime() != file.lastWriteTime
|| artistInfo->getScanVersion() != _settings.artistInfoScanVersion;
}
std::unique_ptr<IFileScanOperation> ArtistInfoFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> ArtistInfoFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<ArtistInfoFileScanOperation>(fileToScan, _settings, _db);
return std::make_unique<ArtistInfoFileScanOperation>(std::move(fileToScan), _db, _settings);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.hpp
+5 -4
View File
@@ -36,18 +36,19 @@ namespace lms::scanner
class ArtistInfoFileScanner : public IFileScanner
{
public:
ArtistInfoFileScanner(const ScannerSettings& _settings, db::Db& db);
ArtistInfoFileScanner(db::Db& db, const ScannerSettings& _settings);
~ArtistInfoFileScanner() override = default;
ArtistInfoFileScanner(const ArtistInfoFileScanner&) = delete;
ArtistInfoFileScanner& operator=(const ArtistInfoFileScanner&) = delete;
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
const ScannerSettings& _settings;
db::Db& _db;
const ScannerSettings& _settings;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp
+66 -60
View File
@@ -41,9 +41,10 @@
#include "image/Exception.hpp"
#include "image/Image.hpp"
#include "metadata/Exception.hpp"
#include "metadata/IAudioFileParser.hpp"
#include "services/scanner/ScanErrors.hpp"
#include "IFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
#include "Utils.hpp"
#include "helpers/ArtistHelpers.hpp"
@@ -387,7 +388,7 @@ namespace lms::scanner
return db::Advisory::UnSet;
}
db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, const FileInfo& fileInfo)
db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, size_t fileSize, const std::filesystem::path& relativePath)
{
db::Track::FindParameters params;
// Add as many fields as possible to limit errors
@@ -401,7 +402,7 @@ namespace lms::scanner
}
if (parsedTrack.position)
params.setTrackNumber(*parsedTrack.position);
params.setFileSize(fileInfo.fileSize);
params.setFileSize(fileSize);
bool error{};
db::Track::pointer res;
@@ -413,7 +414,7 @@ namespace lms::scanner
if (res)
{
LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << fileInfo.relativePath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath());
LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << relativePath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath());
error = true;
}
res = track;
@@ -474,6 +475,14 @@ namespace lms::scanner
}
} // namespace
AudioFileScanOperation::AudioFileScanOperation(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings, metadata::IAudioFileParser& parser)
: FileScanOperationBase{ std::move(fileToScan), db, settings }
, _parser{ parser }
{
}
AudioFileScanOperation::~AudioFileScanOperation() = default;
void AudioFileScanOperation::scan()
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanAudioFile");
@@ -481,13 +490,13 @@ namespace lms::scanner
try
{
_parsedTrack = _parser.parseMetaData(_file);
_parsedTrack = _parser.parseMetaData(getFilePath());
// We fill missing artist mbids with mbids found on other artist roles
fillMissingMbids(*_parsedTrack);
std::size_t index{};
_parser.parseImages(_file, [&](const metadata::Image& image) {
_parser.parseImages(getFilePath(), [&](const metadata::Image& image) {
try
{
image::ImageProperties properties{ image::probeImage(image.data) };
@@ -508,46 +517,43 @@ namespace lms::scanner
}
catch (const image::Exception& e)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to parse image in track file " << _file);
addError<EmbeddedImageScanError>(getFilePath(), index);
}
index++;
});
}
catch (const metadata::AudioFileNoAudioPropertiesException&)
{
addError<NoAudioTrackFoundError>(getFilePath());
}
catch (const metadata::IOException& e)
{
addError<IOScanError>(getFilePath(), e.getErrorCode());
}
catch (const metadata::Exception& e)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << _file);
addError<AudioFileScanError>(getFilePath());
}
}
void AudioFileScanOperation::processResult(ScanContext& context)
AudioFileScanOperation::OperationResult AudioFileScanOperation::processResult()
{
LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessAudioScanData");
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::Track::pointer track{ db::Track::findByPath(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::Track::pointer track{ db::Track::findByPath(dbSession, getFilePath()) };
if (!_parsedTrack)
{
if (track)
{
track.remove();
stats.deletions++;
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadAudioFile);
return;
return OperationResult::Skipped;
}
if (_parsedTrack->mbid && (!track || _settings.skipDuplicateTrackMBID))
if (_parsedTrack->mbid && (!track || getScannerSettings().skipDuplicateTrackMBID))
{
std::vector<db::Track::pointer> duplicateTracks{ db::Track::findByMBID(dbSession, *_parsedTrack->mbid) };
@@ -558,14 +564,14 @@ namespace lms::scanner
std::error_code ec;
if (!std::filesystem::exists(otherTrack->getAbsoluteFilePath(), ec))
{
LMS_LOG(DBUPDATER, DEBUG, "Considering track " << _file << " moved from " << otherTrack->getAbsoluteFilePath());
LMS_LOG(DBUPDATER, DEBUG, "Considering track " << getFilePath() << " moved from " << otherTrack->getAbsoluteFilePath());
track = otherTrack;
track.modify()->setAbsoluteFilePath(_file);
track.modify()->setAbsoluteFilePath(getFilePath());
}
}
// Skip duplicate track MBID
if (_settings.skipDuplicateTrackMBID)
if (getScannerSettings().skipDuplicateTrackMBID)
{
for (db::Track::pointer& otherTrack : duplicateTracks)
{
@@ -574,22 +580,26 @@ namespace lms::scanner
continue;
// Skip if duplicate files no longer in media root: as it will be removed later, we will end up with no file
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
auto& mediaLibraries{ getScannerSettings().mediaLibraries };
if (std::none_of(std::cbegin(mediaLibraries), std::cend(mediaLibraries),
[&](const MediaLibraryInfo& libraryInfo) {
return core::pathUtils::isPathInRootPath(_file, libraryInfo.rootDirectory, &excludeDirFileName);
return core::pathUtils::isPathInRootPath(getFilePath(), libraryInfo.rootDirectory, &excludeDirFileName);
}))
{
continue;
}
LMS_LOG(DBUPDATER, DEBUG, "Skipped " << _file << " (similar MBID in " << otherTrack->getAbsoluteFilePath() << ")");
LMS_LOG(DBUPDATER, DEBUG, "Skipped " << getFilePath() << ": same MBID already found in " << otherTrack->getAbsoluteFilePath());
// As this MBID already exists, just remove what we just scanned
if (track)
{
track.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed " << getFilePath() << ": same MBID already found in " << otherTrack->getAbsoluteFilePath());
return OperationResult::Removed;
}
return;
return OperationResult::Skipped;
}
}
}
@@ -597,27 +607,25 @@ namespace lms::scanner
if (!track)
{
// maybe the file just moved?
track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, *fileInfo);
track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, getFileSize(), getRelativeFilePath());
if (track)
{
LMS_LOG(DBUPDATER, DEBUG, "Considering track " << _file << " moved from " << track->getAbsoluteFilePath());
track.modify()->setAbsoluteFilePath(_file);
LMS_LOG(DBUPDATER, DEBUG, "Considering track " << getFilePath() << " moved from " << track->getAbsoluteFilePath());
track.modify()->setAbsoluteFilePath(getFilePath());
}
}
// We estimate this is an audio file if the duration is not null
if (_parsedTrack->audioProperties.duration == std::chrono::milliseconds::zero())
{
LMS_LOG(DBUPDATER, DEBUG, "Skipped " << _file << " (duration is 0)");
addError<BadAudioDurationError>(getFilePath());
// If Track exists here, delete it!
if (track)
{
track.remove();
stats.deletions++;
return OperationResult::Removed;
}
stats.errors.emplace_back(_file, ScanErrorType::BadDuration);
return;
return OperationResult::Skipped;
}
// ***** Title
@@ -626,9 +634,9 @@ namespace lms::scanner
title = _parsedTrack->title;
else
{
// TODO parse file name guess track etc.
// For now juste use file name as title
title = _file.filename().string();
// TODO parse file name to guess track etc.
// For now, we just use file name as title
title = getFilePath().filename().string();
}
// If file already exists, update its data
@@ -639,13 +647,13 @@ namespace lms::scanner
track = dbSession.create<db::Track>();
added = true;
track.modify()->setAbsoluteFilePath(_file);
track.modify()->setAddedTime(_mediaLibrary.firstScan ? fileInfo->lastWriteTime : Wt::WDateTime::currentDateTime()); // may be erased by encodingTime
track.modify()->setAbsoluteFilePath(getFilePath());
track.modify()->setAddedTime(getMediaLibrary().firstScan ? getLastWriteTime() : Wt::WDateTime::currentDateTime()); // may be erased by encodingTime
}
// Track related data
assert(track);
track.modify()->setScanVersion(_settings.audioScanVersion);
track.modify()->setScanVersion(getScannerSettings().audioScanVersion);
// Audio properties
track.modify()->setBitrate(_parsedTrack->audioProperties.bitrate);
@@ -654,9 +662,9 @@ namespace lms::scanner
track.modify()->setDuration(_parsedTrack->audioProperties.duration);
track.modify()->setSampleRate(_parsedTrack->audioProperties.sampleRate);
track.modify()->setRelativeFilePath(fileInfo->relativePath);
track.modify()->setFileSize(fileInfo->fileSize);
track.modify()->setLastWriteTime(fileInfo->lastWriteTime);
track.modify()->setRelativeFilePath(getRelativeFilePath());
track.modify()->setFileSize(getFileSize());
track.modify()->setLastWriteTime(getLastWriteTime());
if (_parsedTrack->encodingTime.isValid())
{
@@ -672,14 +680,14 @@ namespace lms::scanner
track.modify()->setAddedTime(time.isValid() ? Wt::WDateTime{ date, time } : Wt::WDateTime{ date });
}
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
track.modify()->setMediaLibrary(mediaLibrary);
db::Directory::pointer directory{ utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary) };
db::Directory::pointer directory{ utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary) };
track.modify()->setDirectory(directory);
track.modify()->clearArtistLinks();
const helpers::AllowFallbackOnMBIDEntry allowFallback{ _settings.allowArtistMBIDFallback };
const helpers::AllowFallbackOnMBIDEntry allowFallback{ getScannerSettings().allowArtistMBIDFallback };
createTrackArtistLinks(dbSession, track, db::TrackArtistLinkType::Artist, _parsedTrack->artists, allowFallback);
if (_parsedTrack->medium && _parsedTrack->medium->release)
createTrackArtistLinks(dbSession, track, db::TrackArtistLinkType::ReleaseArtist, _parsedTrack->medium->release->artists, allowFallback);
@@ -732,13 +740,11 @@ namespace lms::scanner
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added audio file " << _file);
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated audio file " << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Added audio file " << getFilePath());
return OperationResult::Added;
}
LMS_LOG(DBUPDATER, DEBUG, "Updated audio file " << getFilePath());
return OperationResult::Updated;
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/AudioFileScanOperation.hpp
+11 -18
View File
@@ -25,16 +25,21 @@
#include <vector>
#include "image/Types.hpp"
#include "metadata/IAudioFileParser.hpp"
#include "metadata/Types.hpp"
#include "FileScanOperationBase.hpp"
#include "FileToScan.hpp"
#include "IFileScanner.hpp"
namespace lms::db
{
class Db;
} // namespace lms::db
namespace lms::metadata
{
class IAudioFileParser;
} // namespace lms::metadata
namespace lms::scanner
{
struct ImageInfo
@@ -48,32 +53,20 @@ namespace lms::scanner
std::string description;
};
class AudioFileScanOperation : public IFileScanOperation
class AudioFileScanOperation : public FileScanOperationBase
{
public:
AudioFileScanOperation(const FileToScan& fileToScan, db::Db& db, metadata::IAudioFileParser& parser, const ScannerSettings& settings)
: _file{ fileToScan.file }
, _mediaLibrary{ fileToScan.mediaLibrary }
, _db{ db }
, _parser{ parser }
, _settings{ settings }
{
}
~AudioFileScanOperation() override = default;
AudioFileScanOperation(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings, metadata::IAudioFileParser& parser);
~AudioFileScanOperation() override;
AudioFileScanOperation(const AudioFileScanOperation&) = delete;
AudioFileScanOperation& operator=(const AudioFileScanOperation&) = delete;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanAudioFile"; }
void scan() override;
void processResult(ScanContext& context) override;
OperationResult processResult() override;
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
db::Db& _db;
metadata::IAudioFileParser& _parser;
const ScannerSettings& _settings;
std::unique_ptr<metadata::Track> _parsedTrack;
std::vector<ImageInfo> _parsedImages;
};
src/libs/services/scanner/impl/scanners/AudioFileScanner.cpp
+14 -51
View File
@@ -25,9 +25,9 @@
#include "database/MediaLibrary.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "metadata/IAudioFileParser.hpp"
#include "AudioFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
#include "Utils.hpp"
@@ -77,66 +77,29 @@ namespace lms::scanner
return "Audio scanner";
}
std::span<const std::filesystem::path> AudioFileScanner::getSupportedFiles() const
{
return {};
}
std::span<const std::filesystem::path> AudioFileScanner::getSupportedExtensions() const
{
return _metadataParser->getSupportedExtensions();
}
bool AudioFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
bool AudioFileScanner::needsScan(const FileToScan& file) const
{
ScanStats& stats{ context.stats };
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
stats.skips++;
return false;
}
if (context.scanOptions.fullScan)
return true;
bool needUpdateLibrary{};
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ dbSession.createReadTransaction() };
{
auto transaction{ dbSession.createReadTransaction() };
// Skip file if last write is the same
const db::Track::pointer track{ db::Track::findByPath(dbSession, file.file) };
if (track
&& track->getLastWriteTime() == lastWriteTime
&& track->getScanVersion() == _settings.audioScanVersion)
{
// this file may have been moved from one library to another, then we just need to update the media library id instead of a full rescan
const auto trackMediaLibrary{ track->getMediaLibrary() };
if (trackMediaLibrary && trackMediaLibrary->getId() == file.mediaLibrary.id)
{
stats.skips++;
return false;
}
needUpdateLibrary = true;
}
}
if (needUpdateLibrary)
{
auto transaction{ dbSession.createWriteTransaction() };
db::Track::pointer track{ db::Track::findByPath(dbSession, file.file) };
assert(track);
track.modify()->setMediaLibrary(db::MediaLibrary::find(dbSession, file.mediaLibrary.id)); // may be null, will be handled in the next scan anyway
stats.updates++;
return false;
}
return true; // need to scan
const db::Track::pointer track{ db::Track::findByPath(dbSession, file.filePath) };
return !track
|| track->getLastWriteTime() != file.lastWriteTime
|| track->getScanVersion() != _settings.audioScanVersion;
}
std::unique_ptr<IFileScanOperation> AudioFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> AudioFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<AudioFileScanOperation>(fileToScan, _db, *_metadataParser, _settings);
return std::make_unique<AudioFileScanOperation>(std::move(fileToScan), _db, _settings, *_metadataParser);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/AudioFileScanner.hpp
+3 -2
View File
@@ -48,9 +48,10 @@ namespace lms::scanner
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
db::Db& _db;
const ScannerSettings& _settings;
src/libs/services/scanner/impl/scanners/FileScanOperationBase.cpp
+34
View File
@@ -0,0 +1,34 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "FileScanOperationBase.hpp"
#include "services/scanner/ScanErrors.hpp"
namespace lms::scanner
{
FileScanOperationBase::FileScanOperationBase(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings)
: _file{ std::move(fileToScan) }
, _db{ db }
, _settings{ settings }
{
}
FileScanOperationBase::~FileScanOperationBase() = default;
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp
+69
View File
@@ -0,0 +1,69 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <memory>
#include <vector>
#include "FileToScan.hpp"
#include "IFileScanOperation.hpp"
namespace lms::db
{
class Db;
} // namespace lms::db
namespace lms::scanner
{
struct ScanError;
struct ScannerSettings;
class FileScanOperationBase : public IFileScanOperation
{
public:
FileScanOperationBase(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings);
~FileScanOperationBase() override;
FileScanOperationBase(const FileScanOperationBase&) = delete;
FileScanOperationBase& operator=(const FileScanOperationBase&) = delete;
protected:
const std::filesystem::path& getFilePath() const override { return _file.filePath; }
const MediaLibraryInfo& getMediaLibrary() const { return _file.mediaLibrary; }
db::Db& getDb() { return _db; }
const ScannerSettings& getScannerSettings() const { return _settings; }
Wt::WDateTime getLastWriteTime() const { return _file.lastWriteTime; }
std::size_t getFileSize() const { return _file.fileSize; }
const std::filesystem::path& getRelativeFilePath() const { return _file.relativePath; }
template<typename T, typename... CtrArgs>
void addError(CtrArgs&&... args)
{
_errors.emplace_back(std::make_shared<T>(std::forward<CtrArgs>(args)...));
}
const ScanErrorVector& getErrors() override { return _errors; }
private:
const FileToScan _file;
db::Db& _db;
const ScannerSettings& _settings;
ScanErrorVector _errors;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/FileToScan.hpp
+6 -1
View File
@@ -21,13 +21,18 @@
#include <filesystem>
#include <Wt/WDateTime.h>
#include "MediaLibraryInfo.hpp"
namespace lms::scanner
{
struct FileToScan
{
std::filesystem::path file;
std::filesystem::path filePath;
std::filesystem::path relativePath;
MediaLibraryInfo mediaLibrary;
Wt::WDateTime lastWriteTime;
std::size_t fileSize{};
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/IFileScanOperation.hpp
+20 -3
View File
@@ -20,12 +20,14 @@
#pragma once
#include <filesystem>
#include <memory>
#include <vector>
#include "core/LiteralString.hpp"
namespace lms::scanner
{
struct ScanContext;
struct ScanError;
class IFileScanOperation
{
@@ -34,8 +36,23 @@ namespace lms::scanner
virtual core::LiteralString getName() const = 0;
virtual const std::filesystem::path& getFile() const = 0;
virtual const std::filesystem::path& getFilePath() const = 0;
// scan() is called asynchronously by a pool of threads
// processResult() is called sequentially by a single thread
virtual void scan() = 0;
virtual void processResult(ScanContext& context) = 0;
enum class OperationResult
{
Added,
Removed,
Updated,
Skipped,
};
virtual OperationResult processResult() = 0;
using ScanErrorVector = std::vector<std::shared_ptr<ScanError>>;
// list of errors collected during scan/result processing (there might be errors without skipping the file)
virtual const ScanErrorVector& getErrors() = 0;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/IFileScanner.hpp
+3 -4
View File
@@ -29,8 +29,6 @@
namespace lms::scanner
{
class IFileScanOperation;
struct ScanContext;
struct ScannerSettings;
class IFileScanner
{
@@ -38,8 +36,9 @@ namespace lms::scanner
virtual ~IFileScanner() = default;
virtual core::LiteralString getName() const = 0;
virtual std::span<const std::filesystem::path> getSupportedFiles() const = 0;
virtual std::span<const std::filesystem::path> getSupportedExtensions() const = 0;
virtual bool needsScan(ScanContext& context, const FileToScan& file) const = 0;
virtual std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const = 0;
virtual bool needsScan(const FileToScan& file) const = 0;
virtual std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const = 0;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp
+39 -70
View File
@@ -27,34 +27,26 @@
#include "database/MediaLibrary.hpp"
#include "database/Session.hpp"
#include "image/Exception.hpp"
#include "image/IRawImage.hpp"
#include "image/Image.hpp"
#include "services/scanner/ScanErrors.hpp"
#include "FileScanOperationBase.hpp"
#include "IFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "Utils.hpp"
namespace lms::scanner
{
namespace
{
class ImageFileScanOperation : public IFileScanOperation
class ImageFileScanOperation : public FileScanOperationBase
{
public:
ImageFileScanOperation(const FileToScan& file, db::Db& db)
: _file{ file.file }
, _mediaLibrary{ file.mediaLibrary }
, _db{ db } {}
using FileScanOperationBase::FileScanOperationBase;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanImageFile"; }
void scan() override;
void processResult(ScanContext& context) override;
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
db::Db& _db;
OperationResult processResult() override;
std::optional<image::ImageProperties> _parsedImageProperties;
};
@@ -63,67 +55,58 @@ namespace lms::scanner
{
try
{
_parsedImageProperties = image::probeImage(_file);
_parsedImageProperties = image::probeImage(getFilePath());
}
catch (const image::Exception& e)
{
_parsedImageProperties.reset();
LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file " << _file << ": " << e.what());
addError<ImageFileScanError>(getFilePath());
}
}
void ImageFileScanOperation::processResult(ScanContext& context)
ImageFileScanOperation::OperationResult ImageFileScanOperation::processResult()
{
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::Image::pointer image{ db::Image::find(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::Image::pointer image{ db::Image::find(dbSession, getFilePath()) };
if (!_parsedImageProperties)
{
if (image)
{
image.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed image " << _file);
LMS_LOG(DBUPDATER, DEBUG, "Removed image " << getFilePath());
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadImageFile);
return;
return OperationResult::Skipped;
}
const bool added{ !image };
if (!image)
image = dbSession.create<db::Image>(_file);
image = dbSession.create<db::Image>(getFilePath());
image.modify()->setLastWriteTime(fileInfo->lastWriteTime);
image.modify()->setFileSize(fileInfo->fileSize);
image.modify()->setLastWriteTime(getLastWriteTime());
image.modify()->setFileSize(getFileSize());
image.modify()->setHeight(_parsedImageProperties->height);
image.modify()->setWidth(_parsedImageProperties->width);
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
image.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
image.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary));
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added image " << _file);
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated image " << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Added image " << getFilePath());
return OperationResult::Added;
}
LMS_LOG(DBUPDATER, DEBUG, "Updated image " << getFilePath());
return OperationResult::Updated;
}
} // namespace
ImageFileScanner::ImageFileScanner(db::Db& db)
ImageFileScanner::ImageFileScanner(db::Db& db, ScannerSettings& settings)
: _db{ db }
, _settings{ settings }
{
}
@@ -132,41 +115,27 @@ namespace lms::scanner
return "Image scanner";
}
std::span<const std::filesystem::path> ImageFileScanner::getSupportedFiles() const
{
return {};
}
std::span<const std::filesystem::path> ImageFileScanner::getSupportedExtensions() const
{
return image::getSupportedFileExtensions();
}
bool ImageFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
bool ImageFileScanner::needsScan(const FileToScan& file) const
{
ScanStats& stats{ context.stats };
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
stats.skips++;
return false;
}
if (!context.scanOptions.fullScan)
{
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const db::Image::pointer image{ db::Image::find(dbSession, file.file) };
if (image && image->getLastWriteTime() == lastWriteTime)
{
stats.skips++;
return false;
}
}
return true; // need to scan
const db::Image::pointer image{ db::Image::find(dbSession, file.filePath) };
return (!image || image->getLastWriteTime() != file.lastWriteTime);
}
std::unique_ptr<IFileScanOperation> ImageFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> ImageFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<ImageFileScanOperation>(fileToScan, _db);
return std::make_unique<ImageFileScanOperation>(std::move(fileToScan), _db, _settings);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/ImageFileScanner.hpp
+5 -3
View File
@@ -36,17 +36,19 @@ namespace lms::scanner
class ImageFileScanner : public IFileScanner
{
public:
ImageFileScanner(db::Db& db);
ImageFileScanner(db::Db& db, ScannerSettings& settings);
~ImageFileScanner() override = default;
ImageFileScanner(const ImageFileScanner&) = delete;
ImageFileScanner& operator=(const ImageFileScanner&) = delete;
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
db::Db& _db;
ScannerSettings& _settings;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/LyricsFileScanner.cpp
+48 -73
View File
@@ -22,38 +22,31 @@
#include <fstream>
#include <optional>
#include "FileScanOperationBase.hpp"
#include "ScannerSettings.hpp"
#include "core/ILogger.hpp"
#include "database/Db.hpp"
#include "database/MediaLibrary.hpp"
#include "database/Session.hpp"
#include "database/TrackLyrics.hpp"
#include "metadata/Lyrics.hpp"
#include "services/scanner/ScanErrors.hpp"
#include "IFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "Utils.hpp"
namespace lms::scanner
{
namespace
{
class LyricsFileScanOperation : public IFileScanOperation
class LyricsFileScanOperation : public FileScanOperationBase
{
public:
LyricsFileScanOperation(const FileToScan& file, db::Db& db)
: _file{ file.file }
, _mediaLibrary{ file.mediaLibrary }
, _db{ db } {}
using FileScanOperationBase::FileScanOperationBase;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanLyricsFile"; }
void scan() override;
void processResult(ScanContext& context) override;
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
db::Db& _db;
OperationResult processResult() override;
std::optional<metadata::Lyrics> _parsedLyrics;
};
@@ -62,53 +55,50 @@ namespace lms::scanner
{
try
{
std::ifstream ifs{ _file };
std::ifstream ifs{ getFilePath() };
if (!ifs)
LMS_LOG(DBUPDATER, ERROR, "Cannot open file " << _file);
else
_parsedLyrics = metadata::parseLyrics(ifs);
{
const std::error_code ec{ errno, std::generic_category() };
addError<IOScanError>(getFilePath(), ec);
return;
}
_parsedLyrics = metadata::parseLyrics(ifs);
}
catch (const metadata::Exception& e)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot read lyrics in file " << _file << ": " << e.what());
addError<LyricsFileScanError>(getFilePath());
}
}
void LyricsFileScanOperation::processResult(ScanContext& context)
LyricsFileScanOperation::OperationResult LyricsFileScanOperation::processResult()
{
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::TrackLyrics::pointer trackLyrics{ db::TrackLyrics::find(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::TrackLyrics::pointer trackLyrics{ db::TrackLyrics::find(dbSession, getFilePath()) };
if (!_parsedLyrics)
{
if (trackLyrics)
{
trackLyrics.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed lyrics file " << _file);
LMS_LOG(DBUPDATER, DEBUG, "Removed lyrics file " << getFilePath());
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadLyricsFile);
return;
return OperationResult::Skipped;
}
const bool added{ !trackLyrics };
if (!trackLyrics)
{
trackLyrics = dbSession.create<db::TrackLyrics>();
trackLyrics.modify()->setAbsoluteFilePath(_file);
trackLyrics.modify()->setAbsoluteFilePath(getFilePath());
}
trackLyrics.modify()->setLastWriteTime(fileInfo->lastWriteTime);
trackLyrics.modify()->setFileSize(fileInfo->fileSize);
trackLyrics.modify()->setLastWriteTime(getLastWriteTime());
trackLyrics.modify()->setFileSize(getFileSize());
trackLyrics.modify()->setLanguage(!_parsedLyrics->language.empty() ? _parsedLyrics->language : "xxx");
trackLyrics.modify()->setOffset(_parsedLyrics->offset);
trackLyrics.modify()->setDisplayTitle(_parsedLyrics->displayTitle);
@@ -118,24 +108,23 @@ namespace lms::scanner
else
trackLyrics.modify()->setUnsynchronizedLines(_parsedLyrics->unsynchronizedLines);
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
trackLyrics.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
trackLyrics.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary));
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added external lyrics " << _file);
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated external lyrics " << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Added external lyrics " << getFilePath());
return OperationResult::Added;
}
LMS_LOG(DBUPDATER, DEBUG, "Updated external lyrics " << getFilePath());
return OperationResult::Updated;
}
} // namespace
LyricsFileScanner::LyricsFileScanner(db::Db& db)
LyricsFileScanner::LyricsFileScanner(db::Db& db, ScannerSettings& _settings)
: _db{ db }
, _settings{ _settings }
{
}
@@ -144,41 +133,27 @@ namespace lms::scanner
return "Lyrics scanner";
}
std::span<const std::filesystem::path> LyricsFileScanner::getSupportedFiles() const
{
return {};
}
std::span<const std::filesystem::path> LyricsFileScanner::getSupportedExtensions() const
{
return metadata::getSupportedLyricsFileExtensions();
}
bool LyricsFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
bool LyricsFileScanner::needsScan(const FileToScan& file) const
{
ScanStats& stats{ context.stats };
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
stats.skips++;
return false;
}
if (!context.scanOptions.fullScan)
{
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(dbSession, file.file) };
if (lyrics && lyrics->getLastWriteTime() == lastWriteTime)
{
stats.skips++;
return false;
}
}
return true; // need to scan
const db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(dbSession, file.filePath) };
return !lyrics || lyrics->getLastWriteTime() != file.lastWriteTime;
}
std::unique_ptr<IFileScanOperation> LyricsFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> LyricsFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<LyricsFileScanOperation>(fileToScan, _db);
return std::make_unique<LyricsFileScanOperation>(std::move(fileToScan), _db, _settings);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/LyricsFileScanner.hpp
+5 -3
View File
@@ -36,17 +36,19 @@ namespace lms::scanner
class LyricsFileScanner : public IFileScanner
{
public:
LyricsFileScanner(db::Db& db);
LyricsFileScanner(db::Db& db, ScannerSettings& settings);
~LyricsFileScanner() override = default;
LyricsFileScanner(const LyricsFileScanner&) = delete;
LyricsFileScanner& operator=(const LyricsFileScanner&) = delete;
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
db::Db& _db;
ScannerSettings& _settings;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/PlayListFileScanner.cpp
+46 -70
View File
@@ -30,7 +30,7 @@
#include "metadata/Exception.hpp"
#include "metadata/PlayList.hpp"
#include "IFileScanOperation.hpp"
#include "FileScanOperationBase.hpp"
#include "ScanContext.hpp"
#include "Utils.hpp"
@@ -38,26 +38,18 @@ namespace lms::scanner
{
namespace
{
class PlayListFileScanOperation : public IFileScanOperation
class PlayListFileScanOperation : public FileScanOperationBase
{
public:
PlayListFileScanOperation(const FileToScan& file, db::Db& db)
: _file{ file.file }
, _mediaLibrary{ file.mediaLibrary }
, _db{ db } {}
using FileScanOperationBase::FileScanOperationBase;
~PlayListFileScanOperation() override = default;
PlayListFileScanOperation(const PlayListFileScanOperation&) = delete;
PlayListFileScanOperation& operator=(const PlayListFileScanOperation&) = delete;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanPlayListFile"; }
void scan() override;
void processResult(ScanContext& context) override;
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
db::Db& _db;
OperationResult processResult() override;
std::optional<metadata::PlayList> _parsedPlayList;
};
@@ -66,74 +58,72 @@ namespace lms::scanner
{
try
{
std::ifstream ifs{ _file };
std::ifstream ifs{ getFilePath() };
if (!ifs)
LMS_LOG(DBUPDATER, ERROR, "Cannot open file " << _file);
else
_parsedPlayList = metadata::parsePlayList(ifs);
{
const std::error_code ec{ errno, std::generic_category() };
addError<IOScanError>(getFilePath(), ec);
return;
}
_parsedPlayList = metadata::parsePlayList(ifs);
}
catch (const metadata::Exception& e)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot read playlist in file " << _file << ": " << e.what());
addError<PlayListFileScanError>(getFilePath());
}
}
void PlayListFileScanOperation::processResult(ScanContext& context)
PlayListFileScanOperation::OperationResult PlayListFileScanOperation::processResult()
{
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, getFilePath()) };
if (!_parsedPlayList)
{
if (playList)
{
playList.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed playlist file " << _file);
LMS_LOG(DBUPDATER, DEBUG, "Removed playlist file " << getFilePath());
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadPlayListFile);
return;
return OperationResult::Skipped;
}
const bool added{ !playList };
if (!playList)
playList = dbSession.create<db::PlayListFile>(_file);
playList = dbSession.create<db::PlayListFile>(getFilePath());
playList.modify()->setLastWriteTime(fileInfo->lastWriteTime);
playList.modify()->setFileSize(fileInfo->fileSize);
playList.modify()->setLastWriteTime(getLastWriteTime());
playList.modify()->setFileSize(getFileSize());
if (!_parsedPlayList->name.empty())
playList.modify()->setName(_parsedPlayList->name);
else
playList.modify()->setName(_file.stem().string());
playList.modify()->setName(getFilePath().stem().string());
playList.modify()->setFiles(_parsedPlayList->files);
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
playList.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
playList.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary));
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added playlist file " << _file);
stats.additions++;
LMS_LOG(DBUPDATER, DEBUG, "Added playlist file " << getFilePath());
return OperationResult::Added;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated playlist file '" << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Updated playlist file '" << getFilePath());
return OperationResult::Updated;
}
}
} // namespace
PlayListFileScanner::PlayListFileScanner(db::Db& db)
PlayListFileScanner::PlayListFileScanner(db::Db& db, ScannerSettings& settings)
: _db{ db }
, _settings{ settings }
{
}
@@ -142,41 +132,27 @@ namespace lms::scanner
return "PlayList scanner";
}
std::span<const std::filesystem::path> PlayListFileScanner::getSupportedFiles() const
{
return {};
}
std::span<const std::filesystem::path> PlayListFileScanner::getSupportedExtensions() const
{
return metadata::getSupportedPlayListFileExtensions();
}
bool PlayListFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
bool PlayListFileScanner::needsScan(const FileToScan& file) const
{
ScanStats& stats{ context.stats };
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
stats.skips++;
return false;
}
if (!context.scanOptions.fullScan)
{
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, file.file) };
if (playList && playList->getLastWriteTime() == lastWriteTime)
{
stats.skips++;
return false;
}
}
return true; // need to scan
const db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, file.filePath) };
return !playList || playList->getLastWriteTime() != file.lastWriteTime;
}
std::unique_ptr<IFileScanOperation> PlayListFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> PlayListFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<PlayListFileScanOperation>(fileToScan, _db);
return std::make_unique<PlayListFileScanOperation>(std::move(fileToScan), _db, _settings);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/PlayListFileScanner.hpp
+5 -3
View File
@@ -36,17 +36,19 @@ namespace lms::scanner
class PlayListFileScanner : public IFileScanner
{
public:
PlayListFileScanner(db::Db& db);
PlayListFileScanner(db::Db& db, ScannerSettings& settings);
~PlayListFileScanner() override = default;
PlayListFileScanner(const PlayListFileScanner&) = delete;
PlayListFileScanner& operator=(const PlayListFileScanner&) = delete;
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
db::Db& _db;
ScannerSettings& _settings;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/Utils.cpp
+2 -54
View File
@@ -19,7 +19,8 @@
#include "Utils.hpp"
#include "core/ILogger.hpp"
#include <system_error>
#include "core/Path.hpp"
#include "database/Directory.hpp"
#include "database/MediaLibrary.hpp"
@@ -27,59 +28,6 @@
namespace lms::scanner::utils
{
Wt::WDateTime retrieveFileGetLastWrite(const std::filesystem::path& file)
{
Wt::WDateTime res;
try
{
res = core::pathUtils::getLastWriteTime(file);
}
catch (core::LmsException& e)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot get last write time: " << e.what());
}
return res;
}
std::optional<FileInfo> retrieveFileInfo(const std::filesystem::path& file, const std::filesystem::path& rootPath)
{
std::optional<FileInfo> res;
res.emplace();
res->lastWriteTime = retrieveFileGetLastWrite(file);
if (!res->lastWriteTime.isValid())
{
res.reset();
return res;
}
{
std::error_code ec;
res->relativePath = std::filesystem::relative(file, rootPath, ec);
if (ec)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot get relative file path for '" << file.string() << "' from '" << rootPath.string() << "': " << ec.message());
res.reset();
return res;
}
}
{
std::error_code ec;
res->fileSize = std::filesystem::file_size(file, ec);
if (ec)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot get file size for '" << file.string() << "': " << ec.message());
res.reset();
return res;
}
}
return res;
}
db::Directory::pointer getOrCreateDirectory(db::Session& session, const std::filesystem::path& path, const db::MediaLibrary::pointer& mediaLibrary)
{
db::Directory::pointer directory{ db::Directory::find(session, path) };
src/libs/services/scanner/impl/scanners/Utils.hpp
-13
View File
@@ -20,9 +20,6 @@
#pragma once
#include <filesystem>
#include <optional>
#include <Wt/WDateTime.h>
#include "database/Object.hpp"
@@ -35,18 +32,8 @@ namespace lms::db
namespace lms::scanner
{
struct FileInfo
{
Wt::WDateTime lastWriteTime;
std::filesystem::path relativePath;
std::size_t fileSize{};
};
namespace utils
{
Wt::WDateTime retrieveFileGetLastWrite(const std::filesystem::path& file);
std::optional<FileInfo> retrieveFileInfo(const std::filesystem::path& file, const std::filesystem::path& rootPath);
db::ObjectPtr<db::Directory> getOrCreateDirectory(db::Session& session, const std::filesystem::path& path, const db::ObjectPtr<db::MediaLibrary>& mediaLibrary);
} // namespace utils
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/FileScanQueue.cpp
+1 -1
View File
@@ -53,7 +53,7 @@ namespace lms::scanner
{
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", operation->getName());
LMS_LOG(DBUPDATER, DEBUG, operation->getName() << ": scanning file " << operation->getFile());
LMS_LOG(DBUPDATER, DEBUG, operation->getName() << ": scanning file " << operation->getFilePath());
operation->scan();
}
src/libs/services/scanner/impl/steps/IScanStep.hpp
+3 -2
View File
@@ -20,11 +20,12 @@
#pragma once
#include "core/LiteralString.hpp"
#include "ScanContext.hpp"
#include "services/scanner/ScannerStats.hpp"
namespace lms::scanner
{
struct ScanContext;
class IScanStep
{
public:
src/libs/services/scanner/impl/steps/ScanErrorLogger.cpp
+93
View File
@@ -0,0 +1,93 @@
/*
* Copyright (C) 2025 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanErrorLogger.hpp"
#include <cassert>
#include "core/ILogger.hpp"
namespace lms::scanner
{
void ScanErrorLogger::visit([[maybe_unused]] const scanner::ScanError& error)
{
// There should never be a ScanError without a specific type
assert(false);
}
void ScanErrorLogger::visit(const scanner::IOScanError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to open file " << error.path << ": " << error.err.message());
}
void ScanErrorLogger::visit(const scanner::AudioFileScanError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << error.path);
}
void ScanErrorLogger::visit(const scanner::EmbeddedImageScanError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to parse image in track file " << error.path << " at index " << error.index);
}
void ScanErrorLogger::visit(const scanner::NoAudioTrackFoundError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << error.path << ": no audio track found");
}
void ScanErrorLogger::visit(const scanner::BadAudioDurationError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << error.path << ": duration is 0");
}
void ScanErrorLogger::visit(const scanner::ArtistInfoFileScanError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to read artist info file " << error.path);
}
void ScanErrorLogger::visit(const scanner::MissingArtistNameError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to read artist info file " << error.path << ": missing name");
}
void ScanErrorLogger::visit(const scanner::ImageFileScanError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to read image file " << error.path);
}
void ScanErrorLogger::visit(const scanner::LyricsFileScanError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to read lyrics file " << error.path);
}
void ScanErrorLogger::visit(const scanner::PlayListFileScanError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to read playlist file " << error.path);
}
void ScanErrorLogger::visit(const scanner::PlayListFilePathMissingError& error)
{
LMS_LOG(DBUPDATER, DEBUG, "Track " << error.entry << " not found in playlist " << error.path);
}
void ScanErrorLogger::visit(const scanner::PlayListFileAllPathesMissingError& error)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to parse playlist " << error.path << ": all entries are missing");
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanErrorLogger.hpp
+43
View File
@@ -0,0 +1,43 @@
/*
* Copyright (C) 2025 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "services/scanner/ScanErrors.hpp"
namespace lms::scanner
{
class ScanErrorLogger : public scanner::ScanErrorVisitor
{
private:
void visit(const scanner::ScanError&) override;
void visit(const scanner::IOScanError& error) override;
void visit(const scanner::AudioFileScanError& error) override;
void visit(const scanner::EmbeddedImageScanError& error) override;
void visit(const scanner::NoAudioTrackFoundError& error) override;
void visit(const scanner::BadAudioDurationError& error) override;
void visit(const scanner::ArtistInfoFileScanError& error) override;
void visit(const scanner::MissingArtistNameError& error) override;
void visit(const scanner::ImageFileScanError& error) override;
void visit(const scanner::LyricsFileScanError& error) override;
void visit(const scanner::PlayListFileScanError& error) override;
void visit(const scanner::PlayListFilePathMissingError& error) override;
void visit(const scanner::PlayListFileAllPathesMissingError& error) override;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepArtistReconciliation.cpp
+1
View File
@@ -32,6 +32,7 @@
#include "database/TrackList.hpp"
#include "metadata/Types.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
#include "helpers/ArtistHelpers.hpp"
src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp
+2
View File
@@ -37,6 +37,8 @@
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
namespace
src/libs/services/scanner/impl/steps/ScanStepAssociateExternalLyrics.cpp
+2
View File
@@ -28,6 +28,8 @@
#include "database/Track.hpp"
#include "database/TrackLyrics.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
namespace
src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp
+23 -6
View File
@@ -30,7 +30,9 @@
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/TrackList.hpp"
#include "services/scanner/ScanErrors.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
namespace lms::scanner
@@ -57,6 +59,7 @@ namespace lms::scanner
db::PlayListFileId lastRetrievedPlayListFileId;
std::size_t processedPlayListFileCount{};
const ScannerSettings& settings;
std::vector<std::shared_ptr<ScanError>> errors;
};
db::Track::pointer getMatchingTrack(db::Session& session, const std::filesystem::path& filePath, const db::Directory::pointer& playListDirectory)
@@ -81,10 +84,7 @@ namespace lms::scanner
return true;
const db::ReleaseId releaseId{ tracks.front().releaseId };
if (std::all_of(std::cbegin(tracks) + 1, std::cend(tracks), [=](const TrackInfo& trackInfo) { return trackInfo.releaseId == releaseId; }))
return true;
return false;
return std::all_of(std::cbegin(tracks) + 1, std::cend(tracks), [=](const TrackInfo& trackInfo) { return trackInfo.releaseId == releaseId; });
}
bool trackListNeedsUpdate(db::Session& session, std::string_view name, std::span<const TrackInfo> tracks, const db::TrackList::pointer& trackList)
@@ -124,17 +124,28 @@ namespace lms::scanner
playListAssociation.playListFileIdId = playListFile->getId();
std::vector<std::shared_ptr<ScanError>> pendingErrors;
const auto files{ playListFile->getFiles() };
for (const std::filesystem::path& file : files)
{
// TODO optim: no need to fetch the whole track
db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) };
const db::Track::pointer track{ getMatchingTrack(searchContext.session, file, playListFile->getDirectory()) };
if (track)
playListAssociation.tracks.push_back(TrackInfo{ .trackId = track->getId(), .releaseId = track->getReleaseId() });
else
LMS_LOG(DBUPDATER, DEBUG, "Track " << file << " not found in playlist " << playListFile->getAbsoluteFilePath());
{
pendingErrors.emplace_back(std::make_shared<PlayListFilePathMissingError>(playListFile->getAbsoluteFilePath(), file));
}
}
if (pendingErrors.size() == files.size())
{
pendingErrors.clear();
pendingErrors.emplace_back(std::make_shared<PlayListFileAllPathesMissingError>(playListFile->getAbsoluteFilePath()));
}
searchContext.errors.insert(std::end(searchContext.errors), std::begin(pendingErrors), std::end(pendingErrors));
if (playListAssociation.tracks.empty()
|| (searchContext.settings.skipSingleReleasePlayLists && isSingleReleasePlayList(playListAssociation.tracks)))
{
@@ -237,6 +248,7 @@ namespace lms::scanner
.session = session,
.lastRetrievedPlayListFileId = {},
.settings = _settings,
.errors = context.stats.errors
};
PlayListFileAssociationContainer playListFileAssociations;
@@ -246,7 +258,12 @@ namespace lms::scanner
return;
updatePlayListFiles(session, playListFileAssociations);
context.currentStepStats.processedElems = searchContext.processedPlayListFileCount;
for (const std::shared_ptr<ScanError>& error : searchContext.errors)
addError(context, error);
searchContext.errors.clear();
_progressCallback(context.currentStepStats);
}
}
src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp
+2
View File
@@ -34,6 +34,8 @@
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
namespace
src/libs/services/scanner/impl/steps/ScanStepBase.cpp
+56
View File
@@ -19,6 +19,11 @@
#include "ScanStepBase.hpp"
#include "core/String.hpp"
#include "ScanContext.hpp"
#include "scanners/IFileScanner.hpp"
namespace lms::scanner
{
ScanStepBase::ScanStepBase(InitParams& initParams)
@@ -29,7 +34,58 @@ namespace lms::scanner
, _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners))
, _lastScanSettings{ initParams.lastScanSettings }
{
for (IFileScanner* scanner : _fileScanners)
{
for (const std::filesystem::path& file : scanner->getSupportedFiles())
{
[[maybe_unused]] auto [it, inserted]{ _scannerByFile.emplace(file, scanner) };
assert(inserted);
}
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
{
[[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) };
assert(inserted);
}
}
}
ScanStepBase::~ScanStepBase() = default;
IFileScanner* ScanStepBase::selectFileScanner(const std::filesystem::path& filePath) const
{
{
const std::string fileName{ core::stringUtils::stringToLower(filePath.filename().string()) };
auto itScanner{ _scannerByFile.find(fileName) };
if (itScanner != std::cend(_scannerByFile))
return itScanner->second;
}
{
const std::string extension{ core::stringUtils::stringToLower(filePath.extension().string()) };
auto itScanner{ _scannerByExtension.find(extension) };
if (itScanner != std::cend(_scannerByExtension))
return itScanner->second;
}
return nullptr;
}
void ScanStepBase::visitFileScanners(const std::function<void(IFileScanner*)>& visitor) const
{
for (IFileScanner* scanner : _fileScanners)
visitor(scanner);
}
void ScanStepBase::addError(ScanContext& context, std::shared_ptr<ScanError> error)
{
error->accept(_scanErrorLogger);
context.stats.errorsCount++;
if (context.stats.errors.size() < ScanStats::maxStoredErrorCount)
context.stats.errors.emplace_back(error);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepBase.hpp
+19 -1
View File
@@ -21,9 +21,11 @@
#include <functional>
#include <span>
#include <unordered_map>
#include <vector>
#include "IScanStep.hpp"
#include "ScanErrorLogger.hpp"
namespace lms::db
{
@@ -35,6 +37,7 @@ namespace lms::scanner
class IFileScanner;
struct ScannerSettings;
struct ScanStepStats;
struct ScanContext;
class ScanStepBase : public IScanStep
{
@@ -57,14 +60,29 @@ namespace lms::scanner
protected:
const ScannerSettings* getLastScanSettings() const { return _lastScanSettings; }
IFileScanner* selectFileScanner(const std::filesystem::path& filePath) const;
void visitFileScanners(const std::function<void(IFileScanner*)>& visitor) const;
void addError(ScanContext& context, std::shared_ptr<ScanError> error);
template<typename T, typename... CtrArgs>
void addError(ScanContext& context, CtrArgs&&... args)
{
auto error{ std::make_shared<T>(std::forward<CtrArgs>(args)...) };
addError(context, error);
}
const ScannerSettings& _settings;
ProgressCallback _progressCallback;
bool& _abortScan;
db::Db& _db;
std::vector<IFileScanner*> _fileScanners;
private:
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByFile;
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
std::vector<IFileScanner*> _fileScanners;
const ScannerSettings* _lastScanSettings{};
ScanErrorLogger _scanErrorLogger;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepCheckForDuplicatedFiles.cpp
+2
View File
@@ -24,6 +24,8 @@
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
bool ScanStepCheckForDuplicatedFiles::needProcess([[maybe_unused]] const ScanContext& context) const
src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp
+13 -20
View File
@@ -30,8 +30,8 @@
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "database/TrackLyrics.hpp"
#include "scanners/IFileScanner.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
namespace lms::scanner
@@ -62,22 +62,15 @@ namespace lms::scanner
}
LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " files to be checked...");
std::vector<std::filesystem::path> supportedFileExtensions;
for (IFileScanner* scanner : _fileScanners)
{
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
supportedFileExtensions.emplace_back(extension);
}
checkForRemovedFiles<db::Track>(context, supportedFileExtensions);
checkForRemovedFiles<db::Image>(context, supportedFileExtensions);
checkForRemovedFiles<db::TrackLyrics>(context, supportedFileExtensions);
checkForRemovedFiles<db::PlayListFile>(context, supportedFileExtensions);
checkForRemovedFiles<db::ArtistInfo>(context, supportedFileExtensions);
checkForRemovedFiles<db::Track>(context);
checkForRemovedFiles<db::Image>(context);
checkForRemovedFiles<db::TrackLyrics>(context);
checkForRemovedFiles<db::PlayListFile>(context);
checkForRemovedFiles<db::ArtistInfo>(context);
}
template<typename Object>
void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context, std::span<const std::filesystem::path> supportedFileExtensions)
void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context)
{
using namespace db;
@@ -110,7 +103,7 @@ namespace lms::scanner
return;
}
if (!checkFile(object->getAbsoluteFilePath(), supportedFileExtensions))
if (!checkFile(object->getAbsoluteFilePath()))
objectsToRemove.push_back(object);
context.currentStepStats.processedElems++;
@@ -132,7 +125,7 @@ namespace lms::scanner
}
}
bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p, std::span<const std::filesystem::path> allowedExtensions)
bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p)
{
try
{
@@ -140,7 +133,7 @@ namespace lms::scanner
// and still belongs to a media directory
if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
{
LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': missing");
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": missing");
return false;
}
@@ -149,13 +142,13 @@ namespace lms::scanner
return core::pathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
}))
{
LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': out of media directory");
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p << ": out of media directory");
return false;
}
if (!core::pathUtils::hasFileAnyExtension(p, allowedExtensions))
if (!selectFileScanner(p))
{
LMS_LOG(DBUPDATER, DEBUG, "Removing '" << p.string() << "': file format no longer handled");
LMS_LOG(DBUPDATER, DEBUG, "Removing " << p.string() << ": file format no longer handled");
return false;
}
src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.hpp
+2 -3
View File
@@ -20,7 +20,6 @@
#pragma once
#include <filesystem>
#include <span>
#include "ScanStepBase.hpp"
@@ -38,8 +37,8 @@ namespace lms::scanner
void process(ScanContext& context) override;
template<typename Object>
void checkForRemovedFiles(ScanContext& context, std::span<const std::filesystem::path> supportedFileExtensions);
void checkForRemovedFiles(ScanContext& context);
bool checkFile(const std::filesystem::path& p, std::span<const std::filesystem::path> allowedExtensions);
bool checkFile(const std::filesystem::path& p);
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepCompact.cpp
+2
View File
@@ -22,6 +22,8 @@
#include "database/Db.hpp"
#include "database/Session.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
bool ScanStepCompact::needProcess(const ScanContext& context) const
src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.cpp
+2
View File
@@ -23,6 +23,8 @@
#include "database/Db.hpp"
#include "database/Session.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
bool ScanStepComputeClusterStats::needProcess(const ScanContext& context) const
src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.cpp
+5 -10
View File
@@ -24,7 +24,8 @@
#include "MediaLibraryInfo.hpp"
#include "ScannerSettings.hpp"
#include "scanners/IFileScanner.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
@@ -38,13 +39,6 @@ namespace lms::scanner
{
context.stats.totalFileCount = 0;
std::vector<std::filesystem::path> supportedFileExtensions;
for (IFileScanner* scanner : _fileScanners)
{
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
supportedFileExtensions.emplace_back(extension);
}
for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
{
std::size_t currentDirectoryProcessElemsCount{};
@@ -53,7 +47,8 @@ namespace lms::scanner
if (_abortScan)
return false;
if (!ec && core::pathUtils::hasFileAnyExtension(path, supportedFileExtensions))
// we don't report errors here (done in the actual scan step)
if (!ec && selectFileScanner(path))
{
context.currentStepStats.processedElems++;
currentDirectoryProcessElemsCount++;
@@ -64,7 +59,7 @@ namespace lms::scanner
},
&excludeDirFileName);
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in '" << mediaLibrary.rootDirectory << "'");
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in " << mediaLibrary.rootDirectory);
}
context.stats.totalFileCount = context.currentStepStats.processedElems;
src/libs/services/scanner/impl/steps/ScanStepOptimize.cpp
+2
View File
@@ -23,6 +23,8 @@
#include "database/Db.hpp"
#include "database/Session.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
bool ScanStepOptimize::needProcess(const ScanContext& context) const
src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.cpp
+2
View File
@@ -29,6 +29,8 @@
#include "database/Track.hpp"
#include "database/TrackEmbeddedImage.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
bool ScanStepRemoveOrphanedDbEntries::needProcess([[maybe_unused]] const ScanContext& context) const
src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp
+63 -23
View File
@@ -27,9 +27,12 @@
#include "core/Path.hpp"
#include "database/Db.hpp"
#include "database/Session.hpp"
#include "scanners/FileToScan.hpp"
#include "scanners/IFileScanOperation.hpp"
#include "scanners/IFileScanner.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
using namespace db;
@@ -45,21 +48,38 @@ namespace lms::scanner
return threadCount;
}
FileToScan retrieveFileInfo(const std::filesystem::path& file, const MediaLibraryInfo& mediaLibrary, std::error_code& ec)
{
FileToScan res;
res.lastWriteTime = core::pathUtils::getLastWriteTime(file, ec);
if (!ec)
res.relativePath = std::filesystem::relative(file, mediaLibrary.rootDirectory, ec);
if (!ec)
res.fileSize = std::filesystem::file_size(file, ec);
if (!ec)
{
res.filePath = file;
res.mediaLibrary = mediaLibrary;
}
return res;
}
} // namespace
ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
: ScanStepBase{ initParams }
, _fileScanQueue{ getScanMetaDataThreadCount(), _abortScan }
{
for (IFileScanner* scanner : _fileScanners)
{
visitFileScanners([](IFileScanner* scanner) {
for (const std::filesystem::path& file : scanner->getSupportedFiles())
LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file " << file);
for (const std::filesystem::path& extension : scanner->getSupportedExtensions())
{
[[maybe_unused]] auto [it, inserted]{ _scannerByExtension.emplace(extension, scanner) };
assert(inserted);
LMS_LOG(DBUPDATER, INFO, "Registered extension " << extension << " for " << scanner->getName());
}
}
LMS_LOG(DBUPDATER, INFO, scanner->getName() << ": supporting file extension " << extension);
});
LMS_LOG(DBUPDATER, INFO, "Using " << _fileScanQueue.getThreadCount() << " thread(s) for scanning file metadata");
}
@@ -94,26 +114,28 @@ namespace lms::scanner
if (ec)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot scan file " << path << ": " << ec.message());
context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
addError<IOScanError>(context, path, ec);
context.stats.skips++;
}
else
else if (IFileScanner * scanner{ selectFileScanner(path) })
{
auto itScanner{ _scannerByExtension.find(core::stringUtils::stringToLower(path.extension().c_str())) };
if (itScanner != std::cend(_scannerByExtension))
FileToScan fileToScan{ retrieveFileInfo(path, mediaLibrary, ec) };
if (ec)
{
IFileScanner& scanner{ *itScanner->second };
FileToScan fileToScan{ .file = path, .mediaLibrary = mediaLibrary };
if (scanner.needsScan(context, fileToScan))
addError<IOScanError>(context, path, ec);
context.stats.skips++;
}
else
{
if (context.scanOptions.fullScan || scanner->needsScan(fileToScan))
{
auto scanOperation{ scanner.createScanOperation(fileToScan) };
auto scanOperation{ scanner->createScanOperation(std::move(fileToScan)) };
_fileScanQueue.pushScanRequest(std::move(scanOperation));
}
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
while (_fileScanQueue.getResultsCount() > (scanQueueMaxScanRequestCount / 2))
@@ -146,9 +168,27 @@ namespace lms::scanner
if (_abortScan)
return;
LMS_LOG(DBUPDATER, DEBUG, scanOperation->getName() << ": processing result for " << scanOperation->getFile());
scanOperation->processResult(context);
LMS_LOG(DBUPDATER, DEBUG, scanOperation->getName() << ": processing result for " << scanOperation->getFilePath());
const IFileScanOperation::OperationResult res{ scanOperation->processResult() };
switch (res)
{
case IFileScanOperation::OperationResult::Added:
context.stats.additions++;
break;
case IFileScanOperation::OperationResult::Removed:
context.stats.deletions++;
break;
case IFileScanOperation::OperationResult::Skipped:
context.stats.failures++;
break;
case IFileScanOperation::OperationResult::Updated:
context.stats.updates++;
break;
}
context.stats.scans++;
for (const auto& error : scanOperation->getErrors())
addError(context, error);
}
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp
+1 -1
View File
@@ -40,10 +40,10 @@ namespace lms::scanner
core::LiteralString getStepName() const override { return "Scan files"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
void process(ScanContext& context, const MediaLibraryInfo& mediaLibrary);
void processFileScanResults(ScanContext& context, std::span<std::unique_ptr<IFileScanOperation>> scanOperations);
FileScanQueue _fileScanQueue;
std::unordered_map<std::filesystem::path, IFileScanner*> _scannerByExtension;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepUpdateLibraryFields.cpp
+1
View File
@@ -25,6 +25,7 @@
#include "database/Session.hpp"
#include "MediaLibraryInfo.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
namespace lms::scanner
src/libs/services/scanner/include/services/scanner/ScanErrors.hpp
+205
View File
@@ -0,0 +1,205 @@
/*
* Copyright (C) 2025 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <filesystem>
#include <system_error>
namespace lms::scanner
{
// Forward declarations of all error types
struct ScanError;
struct IOScanError;
struct AudioFileScanError;
struct EmbeddedImageScanError;
struct NoAudioTrackFoundError;
struct BadAudioDurationError;
struct ArtistInfoFileScanError;
struct MissingArtistNameError;
struct ImageFileScanError;
struct LyricsFileScanError;
struct PlayListFileScanError;
struct PlayListFilePathMissingError;
struct PlayListFileAllPathesMissingError;
// Visitor interface
struct ScanErrorVisitor
{
virtual ~ScanErrorVisitor() = default;
virtual void visit(const ScanError&) = 0;
virtual void visit(const IOScanError&) = 0;
virtual void visit(const AudioFileScanError&) = 0;
virtual void visit(const EmbeddedImageScanError&) = 0;
virtual void visit(const NoAudioTrackFoundError&) = 0;
virtual void visit(const BadAudioDurationError&) = 0;
virtual void visit(const ArtistInfoFileScanError&) = 0;
virtual void visit(const MissingArtistNameError&) = 0;
virtual void visit(const ImageFileScanError&) = 0;
virtual void visit(const LyricsFileScanError&) = 0;
virtual void visit(const PlayListFileScanError&) = 0;
virtual void visit(const scanner::PlayListFilePathMissingError& error) = 0;
virtual void visit(const scanner::PlayListFileAllPathesMissingError& error) = 0;
};
struct ScanError
{
ScanError(const std::filesystem::path& p)
: path{ p } {}
virtual ~ScanError() = default;
virtual void accept(ScanErrorVisitor&) const = 0;
std::filesystem::path path; // Error that occurs on this path
};
struct IOScanError : public ScanError
{
IOScanError(const std::filesystem::path& p, std::error_code e)
: ScanError{ p }
, err{ e } {}
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
std::error_code err;
};
struct AudioFileScanError : public ScanError
{
using ScanError::ScanError;
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
};
struct EmbeddedImageScanError : public AudioFileScanError
{
EmbeddedImageScanError(const std::filesystem::path& p, unsigned i)
: AudioFileScanError{ p }
, index{ i } {}
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
unsigned index;
};
struct NoAudioTrackFoundError : public AudioFileScanError
{
using AudioFileScanError::AudioFileScanError;
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
};
struct BadAudioDurationError : public AudioFileScanError
{
using AudioFileScanError::AudioFileScanError;
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
};
struct ArtistInfoFileScanError : public ScanError
{
using ScanError::ScanError;
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
};
struct MissingArtistNameError : public ArtistInfoFileScanError
{
using ArtistInfoFileScanError::ArtistInfoFileScanError;
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
};
struct ImageFileScanError : public ScanError
{
using ScanError::ScanError;
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
};
struct LyricsFileScanError : public ScanError
{
using ScanError::ScanError;
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
};
struct PlayListFileScanError : public ScanError
{
using ScanError::ScanError;
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
};
struct PlayListFilePathMissingError : public PlayListFileScanError
{
PlayListFilePathMissingError(const std::filesystem::path& p, const std::filesystem::path& e)
: PlayListFileScanError{ p }
, entry{ e }
{
}
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
std::filesystem::path entry;
};
struct PlayListFileAllPathesMissingError : public PlayListFileScanError
{
using PlayListFileScanError::PlayListFileScanError;
void accept(ScanErrorVisitor& visitor) const override
{
visitor.visit(*this);
}
};
} // namespace lms::scanner
src/libs/services/scanner/include/services/scanner/ScannerStats.hpp
+8 -24
View File
@@ -21,40 +21,21 @@
#include <Wt/WDateTime.h>
#include <filesystem>
#include <memory>
#include <vector>
#include "database/TrackId.hpp"
#include "ScanErrors.hpp"
namespace lms::scanner
{
enum class ScanErrorType
{
CannotReadFile,
CannotReadArtistInfoFile,
CannotReadAudioFile,
CannotReadImageFile,
CannotReadLyricsFile,
CannotReadPlayListFile,
NoAudioTrack,
BadDuration,
};
enum class DuplicateReason
{
SameHash,
SameTrackMBID,
};
struct ScanError
{
std::filesystem::path file;
ScanErrorType error;
std::string systemError;
ScanError(const std::filesystem::path& file, ScanErrorType error, const std::string& systemError = "");
};
struct ScanDuplicate
{
db::TrackId trackId;
@@ -105,15 +86,18 @@ namespace lms::scanner
std::size_t totalFileCount{}; // Total number of files (estimated)
std::size_t skips{}; // no change since last scan
std::size_t scans{}; // actually scanned filed
std::size_t scans{}; // count of scanned files
std::size_t additions{}; // added in DB
std::size_t deletions{}; // removed from DB
std::size_t updates{}; // updated file in DB
std::size_t failures{}; // scan failure
std::size_t featuresFetched{}; // features fetched in DB
std::vector<ScanError> errors;
static constexpr std::size_t maxStoredErrorCount{ 5'000 }; // TODO make this configurable
std::vector<std::shared_ptr<ScanError>> errors;
std::size_t errorsCount{}; // maybe bigger than errors.size() if too many errors
std::vector<ScanDuplicate> duplicates;
std::size_t nbFiles() const;