owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More details for scan error reports #676

Browse Source
This commit is contained in:
emeric
2025-05-26 23:33:52 +02:00
parent 6d6ea4920b
commit 571c0bc5f4
86 changed files with 1794 additions and 950 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.cpp
+49 -80
View File
@@ -20,6 +20,7 @@
#include "ArtistInfoFileScanner.hpp"
#include <fstream>
#include <system_error>
#include "core/ILogger.hpp"
#include "core/String.hpp"
@@ -29,45 +30,33 @@
#include "database/MediaLibrary.hpp"
#include "database/Session.hpp"
#include "metadata/ArtistInfo.hpp"
#include "metadata/Types.hpp"
#include "services/scanner/ScanErrors.hpp"
#include "IFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "FileScanOperationBase.hpp"
#include "ScannerSettings.hpp"
#include "Utils.hpp"
#include "helpers/ArtistHelpers.hpp"
#include "metadata/Types.hpp"
namespace lms::scanner
{
namespace
{
class ArtistInfoFileScanOperation : public IFileScanOperation
class ArtistInfoFileScanOperation : public FileScanOperationBase
{
public:
ArtistInfoFileScanOperation(const FileToScan& file, const ScannerSettings& settings, db::Db& db)
: _file{ file.file }
, _mediaLibrary{ file.mediaLibrary }
, _settings{ settings }
, _db{ db }
{
}
using FileScanOperationBase::FileScanOperationBase;
~ArtistInfoFileScanOperation() override = default;
ArtistInfoFileScanOperation(const ArtistInfoFileScanOperation&) = delete;
ArtistInfoFileScanOperation& operator=(const ArtistInfoFileScanOperation&) = delete;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanArtistInfoFile"; }
void scan() override;
void processResult(ScanContext& context) override;
OperationResult processResult() override;
std::string getArtistNameFromArtistInfoFilePath();
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
const ScannerSettings& _settings;
db::Db& _db;
std::optional<metadata::ArtistInfo> _parsedArtistInfo;
};
@@ -75,91 +64,82 @@ namespace lms::scanner
{
try
{
std::ifstream ifs{ _file };
std::ifstream ifs{ getFilePath() };
if (!ifs)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot open file " << _file);
const std::error_code ec{ errno, std::generic_category() };
addError<IOScanError>(getFilePath(), ec);
return;
}
_parsedArtistInfo = metadata::parseArtistInfo(ifs);
if (_parsedArtistInfo->name.empty())
{
LMS_LOG(DBUPDATER, DEBUG, "Discarding artist info in file " << _file << ": no name set");
addError<MissingArtistNameError>(getFilePath());
_parsedArtistInfo.reset();
}
}
catch (const metadata::ArtistInfoParseException& e)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot read artist info in file " << _file << ": " << e.what());
addError<ArtistInfoFileScanError>(getFilePath());
}
}
void ArtistInfoFileScanOperation::processResult(ScanContext& context)
ArtistInfoFileScanOperation::OperationResult ArtistInfoFileScanOperation::processResult()
{
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, getFilePath()) };
if (!_parsedArtistInfo)
{
if (artistInfo)
{
artistInfo.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed artist info file " << _file);
LMS_LOG(DBUPDATER, DEBUG, "Removed artist info file " << getFilePath());
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadArtistInfoFile);
return;
return OperationResult::Skipped;
}
const bool added{ !artistInfo };
if (!artistInfo)
{
artistInfo = dbSession.create<db::ArtistInfo>();
artistInfo.modify()->setAbsoluteFilePath(_file);
artistInfo.modify()->setAbsoluteFilePath(getFilePath());
}
artistInfo.modify()->setScanVersion(_settings.artistInfoScanVersion);
artistInfo.modify()->setScanVersion(getScannerSettings().artistInfoScanVersion);
artistInfo.modify()->setName(_parsedArtistInfo->name);
artistInfo.modify()->setSortName(_parsedArtistInfo->sortName);
artistInfo.modify()->setLastWriteTime(fileInfo->lastWriteTime);
artistInfo.modify()->setLastWriteTime(getLastWriteTime());
artistInfo.modify()->setType(_parsedArtistInfo->type);
artistInfo.modify()->setGender(_parsedArtistInfo->gender);
artistInfo.modify()->setDisambiguation(_parsedArtistInfo->disambiguation);
artistInfo.modify()->setBiography(_parsedArtistInfo->biography);
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
artistInfo.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
artistInfo.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary));
const metadata::Artist artistMetadata{ _parsedArtistInfo->mbid, _parsedArtistInfo->name, _parsedArtistInfo->sortName.empty() ? std::nullopt : std::make_optional<std::string>(_parsedArtistInfo->sortName) };
db::Artist::pointer artist{ helpers::getOrCreateArtist(dbSession, artistMetadata, helpers::AllowFallbackOnMBIDEntry{ _settings.allowArtistMBIDFallback }) };
db::Artist::pointer artist{ helpers::getOrCreateArtist(dbSession, artistMetadata, helpers::AllowFallbackOnMBIDEntry{ getScannerSettings().allowArtistMBIDFallback }) };
artistInfo.modify()->setArtist(artist);
artistInfo.modify()->setMBIDMatched(_parsedArtistInfo->mbid.has_value() && _parsedArtistInfo->mbid == artist->getMBID());
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added artist info file " << _file);
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated artist info file " << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Added artist info file " << getFilePath());
return OperationResult::Added;
}
LMS_LOG(DBUPDATER, DEBUG, "Updated artist info file " << getFilePath());
return OperationResult::Updated;
}
} // namespace
ArtistInfoFileScanner::ArtistInfoFileScanner(const ScannerSettings& settings, db::Db& db)
: _settings{ settings }
, _db{ db }
ArtistInfoFileScanner::ArtistInfoFileScanner(db::Db& db, const ScannerSettings& settings)
: _db{ db }
, _settings{ settings }
{
}
@@ -168,45 +148,34 @@ namespace lms::scanner
return "Artist info scanner";
}
std::span<const std::filesystem::path> ArtistInfoFileScanner::getSupportedExtensions() const
std::span<const std::filesystem::path> ArtistInfoFileScanner::getSupportedFiles() const
{
return metadata::getSupportedInfoFileExtensions();
return metadata::getSupportedArtistInfoFiles();
}
bool ArtistInfoFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
std::span<const std::filesystem::path> ArtistInfoFileScanner::getSupportedExtensions() const
{
return {};
}
bool ArtistInfoFileScanner::needsScan(const FileToScan& file) const
{
// Special case: only files named "artist.nfo" are compatible with this scanner
// Hack here since the scanner framework only handle extensions (the discover count is not accurate)
if (!core::stringUtils::stringCaseInsensitiveEqual(file.file.stem().string(), "artist"))
if (!core::stringUtils::stringCaseInsensitiveEqual(file.filePath.stem().string(), "artist"))
return false;
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
context.stats.skips++;
return false;
}
if (context.scanOptions.fullScan)
return true;
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ dbSession.createReadTransaction() };
db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, file.file) };
if (artistInfo
&& artistInfo->getLastWriteTime() == lastWriteTime
&& artistInfo->getScanVersion() == _settings.artistInfoScanVersion)
{
context.stats.skips++;
return false;
}
db::ArtistInfo::pointer artistInfo{ db::ArtistInfo::find(dbSession, file.filePath) };
return true;
return !artistInfo
|| artistInfo->getLastWriteTime() != file.lastWriteTime
|| artistInfo->getScanVersion() != _settings.artistInfoScanVersion;
}
std::unique_ptr<IFileScanOperation> ArtistInfoFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> ArtistInfoFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<ArtistInfoFileScanOperation>(fileToScan, _settings, _db);
return std::make_unique<ArtistInfoFileScanOperation>(std::move(fileToScan), _db, _settings);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/ArtistInfoFileScanner.hpp
+5 -4
View File
@@ -36,18 +36,19 @@ namespace lms::scanner
class ArtistInfoFileScanner : public IFileScanner
{
public:
ArtistInfoFileScanner(const ScannerSettings& _settings, db::Db& db);
ArtistInfoFileScanner(db::Db& db, const ScannerSettings& _settings);
~ArtistInfoFileScanner() override = default;
ArtistInfoFileScanner(const ArtistInfoFileScanner&) = delete;
ArtistInfoFileScanner& operator=(const ArtistInfoFileScanner&) = delete;
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
const ScannerSettings& _settings;
db::Db& _db;
const ScannerSettings& _settings;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp
+66 -60
View File
@@ -41,9 +41,10 @@
#include "image/Exception.hpp"
#include "image/Image.hpp"
#include "metadata/Exception.hpp"
#include "metadata/IAudioFileParser.hpp"
#include "services/scanner/ScanErrors.hpp"
#include "IFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
#include "Utils.hpp"
#include "helpers/ArtistHelpers.hpp"
@@ -387,7 +388,7 @@ namespace lms::scanner
return db::Advisory::UnSet;
}
db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, const FileInfo& fileInfo)
db::Track::pointer findMovedTrackBySizeAndMetaData(db::Session& session, const metadata::Track& parsedTrack, size_t fileSize, const std::filesystem::path& relativePath)
{
db::Track::FindParameters params;
// Add as many fields as possible to limit errors
@@ -401,7 +402,7 @@ namespace lms::scanner
}
if (parsedTrack.position)
params.setTrackNumber(*parsedTrack.position);
params.setFileSize(fileInfo.fileSize);
params.setFileSize(fileSize);
bool error{};
db::Track::pointer res;
@@ -413,7 +414,7 @@ namespace lms::scanner
if (res)
{
LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << fileInfo.relativePath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath());
LMS_LOG(DBUPDATER, DEBUG, "Found too many candidates for file move. New file = " << relativePath << ", candidate = " << track->getAbsoluteFilePath() << ", previous candidate = " << res->getAbsoluteFilePath());
error = true;
}
res = track;
@@ -474,6 +475,14 @@ namespace lms::scanner
}
} // namespace
AudioFileScanOperation::AudioFileScanOperation(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings, metadata::IAudioFileParser& parser)
: FileScanOperationBase{ std::move(fileToScan), db, settings }
, _parser{ parser }
{
}
AudioFileScanOperation::~AudioFileScanOperation() = default;
void AudioFileScanOperation::scan()
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanAudioFile");
@@ -481,13 +490,13 @@ namespace lms::scanner
try
{
_parsedTrack = _parser.parseMetaData(_file);
_parsedTrack = _parser.parseMetaData(getFilePath());
// We fill missing artist mbids with mbids found on other artist roles
fillMissingMbids(*_parsedTrack);
std::size_t index{};
_parser.parseImages(_file, [&](const metadata::Image& image) {
_parser.parseImages(getFilePath(), [&](const metadata::Image& image) {
try
{
image::ImageProperties properties{ image::probeImage(image.data) };
@@ -508,46 +517,43 @@ namespace lms::scanner
}
catch (const image::Exception& e)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to parse image in track file " << _file);
addError<EmbeddedImageScanError>(getFilePath(), index);
}
index++;
});
}
catch (const metadata::AudioFileNoAudioPropertiesException&)
{
addError<NoAudioTrackFoundError>(getFilePath());
}
catch (const metadata::IOException& e)
{
addError<IOScanError>(getFilePath(), e.getErrorCode());
}
catch (const metadata::Exception& e)
{
LMS_LOG(DBUPDATER, ERROR, "Failed to parse audio file " << _file);
addError<AudioFileScanError>(getFilePath());
}
}
void AudioFileScanOperation::processResult(ScanContext& context)
AudioFileScanOperation::OperationResult AudioFileScanOperation::processResult()
{
LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessAudioScanData");
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::Track::pointer track{ db::Track::findByPath(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::Track::pointer track{ db::Track::findByPath(dbSession, getFilePath()) };
if (!_parsedTrack)
{
if (track)
{
track.remove();
stats.deletions++;
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadAudioFile);
return;
return OperationResult::Skipped;
}
if (_parsedTrack->mbid && (!track || _settings.skipDuplicateTrackMBID))
if (_parsedTrack->mbid && (!track || getScannerSettings().skipDuplicateTrackMBID))
{
std::vector<db::Track::pointer> duplicateTracks{ db::Track::findByMBID(dbSession, *_parsedTrack->mbid) };
@@ -558,14 +564,14 @@ namespace lms::scanner
std::error_code ec;
if (!std::filesystem::exists(otherTrack->getAbsoluteFilePath(), ec))
{
LMS_LOG(DBUPDATER, DEBUG, "Considering track " << _file << " moved from " << otherTrack->getAbsoluteFilePath());
LMS_LOG(DBUPDATER, DEBUG, "Considering track " << getFilePath() << " moved from " << otherTrack->getAbsoluteFilePath());
track = otherTrack;
track.modify()->setAbsoluteFilePath(_file);
track.modify()->setAbsoluteFilePath(getFilePath());
}
}
// Skip duplicate track MBID
if (_settings.skipDuplicateTrackMBID)
if (getScannerSettings().skipDuplicateTrackMBID)
{
for (db::Track::pointer& otherTrack : duplicateTracks)
{
@@ -574,22 +580,26 @@ namespace lms::scanner
continue;
// Skip if duplicate files no longer in media root: as it will be removed later, we will end up with no file
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
auto& mediaLibraries{ getScannerSettings().mediaLibraries };
if (std::none_of(std::cbegin(mediaLibraries), std::cend(mediaLibraries),
[&](const MediaLibraryInfo& libraryInfo) {
return core::pathUtils::isPathInRootPath(_file, libraryInfo.rootDirectory, &excludeDirFileName);
return core::pathUtils::isPathInRootPath(getFilePath(), libraryInfo.rootDirectory, &excludeDirFileName);
}))
{
continue;
}
LMS_LOG(DBUPDATER, DEBUG, "Skipped " << _file << " (similar MBID in " << otherTrack->getAbsoluteFilePath() << ")");
LMS_LOG(DBUPDATER, DEBUG, "Skipped " << getFilePath() << ": same MBID already found in " << otherTrack->getAbsoluteFilePath());
// As this MBID already exists, just remove what we just scanned
if (track)
{
track.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed " << getFilePath() << ": same MBID already found in " << otherTrack->getAbsoluteFilePath());
return OperationResult::Removed;
}
return;
return OperationResult::Skipped;
}
}
}
@@ -597,27 +607,25 @@ namespace lms::scanner
if (!track)
{
// maybe the file just moved?
track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, *fileInfo);
track = findMovedTrackBySizeAndMetaData(dbSession, *_parsedTrack, getFileSize(), getRelativeFilePath());
if (track)
{
LMS_LOG(DBUPDATER, DEBUG, "Considering track " << _file << " moved from " << track->getAbsoluteFilePath());
track.modify()->setAbsoluteFilePath(_file);
LMS_LOG(DBUPDATER, DEBUG, "Considering track " << getFilePath() << " moved from " << track->getAbsoluteFilePath());
track.modify()->setAbsoluteFilePath(getFilePath());
}
}
// We estimate this is an audio file if the duration is not null
if (_parsedTrack->audioProperties.duration == std::chrono::milliseconds::zero())
{
LMS_LOG(DBUPDATER, DEBUG, "Skipped " << _file << " (duration is 0)");
addError<BadAudioDurationError>(getFilePath());
// If Track exists here, delete it!
if (track)
{
track.remove();
stats.deletions++;
return OperationResult::Removed;
}
stats.errors.emplace_back(_file, ScanErrorType::BadDuration);
return;
return OperationResult::Skipped;
}
// ***** Title
@@ -626,9 +634,9 @@ namespace lms::scanner
title = _parsedTrack->title;
else
{
// TODO parse file name guess track etc.
// For now juste use file name as title
title = _file.filename().string();
// TODO parse file name to guess track etc.
// For now, we just use file name as title
title = getFilePath().filename().string();
}
// If file already exists, update its data
@@ -639,13 +647,13 @@ namespace lms::scanner
track = dbSession.create<db::Track>();
added = true;
track.modify()->setAbsoluteFilePath(_file);
track.modify()->setAddedTime(_mediaLibrary.firstScan ? fileInfo->lastWriteTime : Wt::WDateTime::currentDateTime()); // may be erased by encodingTime
track.modify()->setAbsoluteFilePath(getFilePath());
track.modify()->setAddedTime(getMediaLibrary().firstScan ? getLastWriteTime() : Wt::WDateTime::currentDateTime()); // may be erased by encodingTime
}
// Track related data
assert(track);
track.modify()->setScanVersion(_settings.audioScanVersion);
track.modify()->setScanVersion(getScannerSettings().audioScanVersion);
// Audio properties
track.modify()->setBitrate(_parsedTrack->audioProperties.bitrate);
@@ -654,9 +662,9 @@ namespace lms::scanner
track.modify()->setDuration(_parsedTrack->audioProperties.duration);
track.modify()->setSampleRate(_parsedTrack->audioProperties.sampleRate);
track.modify()->setRelativeFilePath(fileInfo->relativePath);
track.modify()->setFileSize(fileInfo->fileSize);
track.modify()->setLastWriteTime(fileInfo->lastWriteTime);
track.modify()->setRelativeFilePath(getRelativeFilePath());
track.modify()->setFileSize(getFileSize());
track.modify()->setLastWriteTime(getLastWriteTime());
if (_parsedTrack->encodingTime.isValid())
{
@@ -672,14 +680,14 @@ namespace lms::scanner
track.modify()->setAddedTime(time.isValid() ? Wt::WDateTime{ date, time } : Wt::WDateTime{ date });
}
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
track.modify()->setMediaLibrary(mediaLibrary);
db::Directory::pointer directory{ utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary) };
db::Directory::pointer directory{ utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary) };
track.modify()->setDirectory(directory);
track.modify()->clearArtistLinks();
const helpers::AllowFallbackOnMBIDEntry allowFallback{ _settings.allowArtistMBIDFallback };
const helpers::AllowFallbackOnMBIDEntry allowFallback{ getScannerSettings().allowArtistMBIDFallback };
createTrackArtistLinks(dbSession, track, db::TrackArtistLinkType::Artist, _parsedTrack->artists, allowFallback);
if (_parsedTrack->medium && _parsedTrack->medium->release)
createTrackArtistLinks(dbSession, track, db::TrackArtistLinkType::ReleaseArtist, _parsedTrack->medium->release->artists, allowFallback);
@@ -732,13 +740,11 @@ namespace lms::scanner
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added audio file " << _file);
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated audio file " << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Added audio file " << getFilePath());
return OperationResult::Added;
}
LMS_LOG(DBUPDATER, DEBUG, "Updated audio file " << getFilePath());
return OperationResult::Updated;
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/AudioFileScanOperation.hpp
+11 -18
View File
@@ -25,16 +25,21 @@
#include <vector>
#include "image/Types.hpp"
#include "metadata/IAudioFileParser.hpp"
#include "metadata/Types.hpp"
#include "FileScanOperationBase.hpp"
#include "FileToScan.hpp"
#include "IFileScanner.hpp"
namespace lms::db
{
class Db;
} // namespace lms::db
namespace lms::metadata
{
class IAudioFileParser;
} // namespace lms::metadata
namespace lms::scanner
{
struct ImageInfo
@@ -48,32 +53,20 @@ namespace lms::scanner
std::string description;
};
class AudioFileScanOperation : public IFileScanOperation
class AudioFileScanOperation : public FileScanOperationBase
{
public:
AudioFileScanOperation(const FileToScan& fileToScan, db::Db& db, metadata::IAudioFileParser& parser, const ScannerSettings& settings)
: _file{ fileToScan.file }
, _mediaLibrary{ fileToScan.mediaLibrary }
, _db{ db }
, _parser{ parser }
, _settings{ settings }
{
}
~AudioFileScanOperation() override = default;
AudioFileScanOperation(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings, metadata::IAudioFileParser& parser);
~AudioFileScanOperation() override;
AudioFileScanOperation(const AudioFileScanOperation&) = delete;
AudioFileScanOperation& operator=(const AudioFileScanOperation&) = delete;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanAudioFile"; }
void scan() override;
void processResult(ScanContext& context) override;
OperationResult processResult() override;
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
db::Db& _db;
metadata::IAudioFileParser& _parser;
const ScannerSettings& _settings;
std::unique_ptr<metadata::Track> _parsedTrack;
std::vector<ImageInfo> _parsedImages;
};
src/libs/services/scanner/impl/scanners/AudioFileScanner.cpp
+14 -51
View File
@@ -25,9 +25,9 @@
#include "database/MediaLibrary.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
#include "metadata/IAudioFileParser.hpp"
#include "AudioFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "ScannerSettings.hpp"
#include "Utils.hpp"
@@ -77,66 +77,29 @@ namespace lms::scanner
return "Audio scanner";
}
std::span<const std::filesystem::path> AudioFileScanner::getSupportedFiles() const
{
return {};
}
std::span<const std::filesystem::path> AudioFileScanner::getSupportedExtensions() const
{
return _metadataParser->getSupportedExtensions();
}
bool AudioFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
bool AudioFileScanner::needsScan(const FileToScan& file) const
{
ScanStats& stats{ context.stats };
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
stats.skips++;
return false;
}
if (context.scanOptions.fullScan)
return true;
bool needUpdateLibrary{};
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ dbSession.createReadTransaction() };
{
auto transaction{ dbSession.createReadTransaction() };
// Skip file if last write is the same
const db::Track::pointer track{ db::Track::findByPath(dbSession, file.file) };
if (track
&& track->getLastWriteTime() == lastWriteTime
&& track->getScanVersion() == _settings.audioScanVersion)
{
// this file may have been moved from one library to another, then we just need to update the media library id instead of a full rescan
const auto trackMediaLibrary{ track->getMediaLibrary() };
if (trackMediaLibrary && trackMediaLibrary->getId() == file.mediaLibrary.id)
{
stats.skips++;
return false;
}
needUpdateLibrary = true;
}
}
if (needUpdateLibrary)
{
auto transaction{ dbSession.createWriteTransaction() };
db::Track::pointer track{ db::Track::findByPath(dbSession, file.file) };
assert(track);
track.modify()->setMediaLibrary(db::MediaLibrary::find(dbSession, file.mediaLibrary.id)); // may be null, will be handled in the next scan anyway
stats.updates++;
return false;
}
return true; // need to scan
const db::Track::pointer track{ db::Track::findByPath(dbSession, file.filePath) };
return !track
|| track->getLastWriteTime() != file.lastWriteTime
|| track->getScanVersion() != _settings.audioScanVersion;
}
std::unique_ptr<IFileScanOperation> AudioFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> AudioFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<AudioFileScanOperation>(fileToScan, _db, *_metadataParser, _settings);
return std::make_unique<AudioFileScanOperation>(std::move(fileToScan), _db, _settings, *_metadataParser);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/AudioFileScanner.hpp
+3 -2
View File
@@ -48,9 +48,10 @@ namespace lms::scanner
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
db::Db& _db;
const ScannerSettings& _settings;
src/libs/services/scanner/impl/scanners/FileScanOperationBase.cpp
+34
View File
@@ -0,0 +1,34 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "FileScanOperationBase.hpp"
#include "services/scanner/ScanErrors.hpp"
namespace lms::scanner
{
FileScanOperationBase::FileScanOperationBase(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings)
: _file{ std::move(fileToScan) }
, _db{ db }
, _settings{ settings }
{
}
FileScanOperationBase::~FileScanOperationBase() = default;
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/FileScanOperationBase.hpp
+69
View File
@@ -0,0 +1,69 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <memory>
#include <vector>
#include "FileToScan.hpp"
#include "IFileScanOperation.hpp"
namespace lms::db
{
class Db;
} // namespace lms::db
namespace lms::scanner
{
struct ScanError;
struct ScannerSettings;
class FileScanOperationBase : public IFileScanOperation
{
public:
FileScanOperationBase(FileToScan&& fileToScan, db::Db& db, const ScannerSettings& settings);
~FileScanOperationBase() override;
FileScanOperationBase(const FileScanOperationBase&) = delete;
FileScanOperationBase& operator=(const FileScanOperationBase&) = delete;
protected:
const std::filesystem::path& getFilePath() const override { return _file.filePath; }
const MediaLibraryInfo& getMediaLibrary() const { return _file.mediaLibrary; }
db::Db& getDb() { return _db; }
const ScannerSettings& getScannerSettings() const { return _settings; }
Wt::WDateTime getLastWriteTime() const { return _file.lastWriteTime; }
std::size_t getFileSize() const { return _file.fileSize; }
const std::filesystem::path& getRelativeFilePath() const { return _file.relativePath; }
template<typename T, typename... CtrArgs>
void addError(CtrArgs&&... args)
{
_errors.emplace_back(std::make_shared<T>(std::forward<CtrArgs>(args)...));
}
const ScanErrorVector& getErrors() override { return _errors; }
private:
const FileToScan _file;
db::Db& _db;
const ScannerSettings& _settings;
ScanErrorVector _errors;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/FileToScan.hpp
+6 -1
View File
@@ -21,13 +21,18 @@
#include <filesystem>
#include <Wt/WDateTime.h>
#include "MediaLibraryInfo.hpp"
namespace lms::scanner
{
struct FileToScan
{
std::filesystem::path file;
std::filesystem::path filePath;
std::filesystem::path relativePath;
MediaLibraryInfo mediaLibrary;
Wt::WDateTime lastWriteTime;
std::size_t fileSize{};
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/IFileScanOperation.hpp
+20 -3
View File
@@ -20,12 +20,14 @@
#pragma once
#include <filesystem>
#include <memory>
#include <vector>
#include "core/LiteralString.hpp"
namespace lms::scanner
{
struct ScanContext;
struct ScanError;
class IFileScanOperation
{
@@ -34,8 +36,23 @@ namespace lms::scanner
virtual core::LiteralString getName() const = 0;
virtual const std::filesystem::path& getFile() const = 0;
virtual const std::filesystem::path& getFilePath() const = 0;
// scan() is called asynchronously by a pool of threads
// processResult() is called sequentially by a single thread
virtual void scan() = 0;
virtual void processResult(ScanContext& context) = 0;
enum class OperationResult
{
Added,
Removed,
Updated,
Skipped,
};
virtual OperationResult processResult() = 0;
using ScanErrorVector = std::vector<std::shared_ptr<ScanError>>;
// list of errors collected during scan/result processing (there might be errors without skipping the file)
virtual const ScanErrorVector& getErrors() = 0;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/IFileScanner.hpp
+3 -4
View File
@@ -29,8 +29,6 @@
namespace lms::scanner
{
class IFileScanOperation;
struct ScanContext;
struct ScannerSettings;
class IFileScanner
{
@@ -38,8 +36,9 @@ namespace lms::scanner
virtual ~IFileScanner() = default;
virtual core::LiteralString getName() const = 0;
virtual std::span<const std::filesystem::path> getSupportedFiles() const = 0;
virtual std::span<const std::filesystem::path> getSupportedExtensions() const = 0;
virtual bool needsScan(ScanContext& context, const FileToScan& file) const = 0;
virtual std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const = 0;
virtual bool needsScan(const FileToScan& file) const = 0;
virtual std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const = 0;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/ImageFileScanner.cpp
+39 -70
View File
@@ -27,34 +27,26 @@
#include "database/MediaLibrary.hpp"
#include "database/Session.hpp"
#include "image/Exception.hpp"
#include "image/IRawImage.hpp"
#include "image/Image.hpp"
#include "services/scanner/ScanErrors.hpp"
#include "FileScanOperationBase.hpp"
#include "IFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "Utils.hpp"
namespace lms::scanner
{
namespace
{
class ImageFileScanOperation : public IFileScanOperation
class ImageFileScanOperation : public FileScanOperationBase
{
public:
ImageFileScanOperation(const FileToScan& file, db::Db& db)
: _file{ file.file }
, _mediaLibrary{ file.mediaLibrary }
, _db{ db } {}
using FileScanOperationBase::FileScanOperationBase;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanImageFile"; }
void scan() override;
void processResult(ScanContext& context) override;
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
db::Db& _db;
OperationResult processResult() override;
std::optional<image::ImageProperties> _parsedImageProperties;
};
@@ -63,67 +55,58 @@ namespace lms::scanner
{
try
{
_parsedImageProperties = image::probeImage(_file);
_parsedImageProperties = image::probeImage(getFilePath());
}
catch (const image::Exception& e)
{
_parsedImageProperties.reset();
LMS_LOG(DBUPDATER, ERROR, "Cannot read image in file " << _file << ": " << e.what());
addError<ImageFileScanError>(getFilePath());
}
}
void ImageFileScanOperation::processResult(ScanContext& context)
ImageFileScanOperation::OperationResult ImageFileScanOperation::processResult()
{
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::Image::pointer image{ db::Image::find(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::Image::pointer image{ db::Image::find(dbSession, getFilePath()) };
if (!_parsedImageProperties)
{
if (image)
{
image.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed image " << _file);
LMS_LOG(DBUPDATER, DEBUG, "Removed image " << getFilePath());
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadImageFile);
return;
return OperationResult::Skipped;
}
const bool added{ !image };
if (!image)
image = dbSession.create<db::Image>(_file);
image = dbSession.create<db::Image>(getFilePath());
image.modify()->setLastWriteTime(fileInfo->lastWriteTime);
image.modify()->setFileSize(fileInfo->fileSize);
image.modify()->setLastWriteTime(getLastWriteTime());
image.modify()->setFileSize(getFileSize());
image.modify()->setHeight(_parsedImageProperties->height);
image.modify()->setWidth(_parsedImageProperties->width);
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
image.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
image.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary));
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added image " << _file);
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated image " << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Added image " << getFilePath());
return OperationResult::Added;
}
LMS_LOG(DBUPDATER, DEBUG, "Updated image " << getFilePath());
return OperationResult::Updated;
}
} // namespace
ImageFileScanner::ImageFileScanner(db::Db& db)
ImageFileScanner::ImageFileScanner(db::Db& db, ScannerSettings& settings)
: _db{ db }
, _settings{ settings }
{
}
@@ -132,41 +115,27 @@ namespace lms::scanner
return "Image scanner";
}
std::span<const std::filesystem::path> ImageFileScanner::getSupportedFiles() const
{
return {};
}
std::span<const std::filesystem::path> ImageFileScanner::getSupportedExtensions() const
{
return image::getSupportedFileExtensions();
}
bool ImageFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
bool ImageFileScanner::needsScan(const FileToScan& file) const
{
ScanStats& stats{ context.stats };
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
stats.skips++;
return false;
}
if (!context.scanOptions.fullScan)
{
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const db::Image::pointer image{ db::Image::find(dbSession, file.file) };
if (image && image->getLastWriteTime() == lastWriteTime)
{
stats.skips++;
return false;
}
}
return true; // need to scan
const db::Image::pointer image{ db::Image::find(dbSession, file.filePath) };
return (!image || image->getLastWriteTime() != file.lastWriteTime);
}
std::unique_ptr<IFileScanOperation> ImageFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> ImageFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<ImageFileScanOperation>(fileToScan, _db);
return std::make_unique<ImageFileScanOperation>(std::move(fileToScan), _db, _settings);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/ImageFileScanner.hpp
+5 -3
View File
@@ -36,17 +36,19 @@ namespace lms::scanner
class ImageFileScanner : public IFileScanner
{
public:
ImageFileScanner(db::Db& db);
ImageFileScanner(db::Db& db, ScannerSettings& settings);
~ImageFileScanner() override = default;
ImageFileScanner(const ImageFileScanner&) = delete;
ImageFileScanner& operator=(const ImageFileScanner&) = delete;
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
db::Db& _db;
ScannerSettings& _settings;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/LyricsFileScanner.cpp
+48 -73
View File
@@ -22,38 +22,31 @@
#include <fstream>
#include <optional>
#include "FileScanOperationBase.hpp"
#include "ScannerSettings.hpp"
#include "core/ILogger.hpp"
#include "database/Db.hpp"
#include "database/MediaLibrary.hpp"
#include "database/Session.hpp"
#include "database/TrackLyrics.hpp"
#include "metadata/Lyrics.hpp"
#include "services/scanner/ScanErrors.hpp"
#include "IFileScanOperation.hpp"
#include "ScanContext.hpp"
#include "Utils.hpp"
namespace lms::scanner
{
namespace
{
class LyricsFileScanOperation : public IFileScanOperation
class LyricsFileScanOperation : public FileScanOperationBase
{
public:
LyricsFileScanOperation(const FileToScan& file, db::Db& db)
: _file{ file.file }
, _mediaLibrary{ file.mediaLibrary }
, _db{ db } {}
using FileScanOperationBase::FileScanOperationBase;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanLyricsFile"; }
void scan() override;
void processResult(ScanContext& context) override;
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
db::Db& _db;
OperationResult processResult() override;
std::optional<metadata::Lyrics> _parsedLyrics;
};
@@ -62,53 +55,50 @@ namespace lms::scanner
{
try
{
std::ifstream ifs{ _file };
std::ifstream ifs{ getFilePath() };
if (!ifs)
LMS_LOG(DBUPDATER, ERROR, "Cannot open file " << _file);
else
_parsedLyrics = metadata::parseLyrics(ifs);
{
const std::error_code ec{ errno, std::generic_category() };
addError<IOScanError>(getFilePath(), ec);
return;
}
_parsedLyrics = metadata::parseLyrics(ifs);
}
catch (const metadata::Exception& e)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot read lyrics in file " << _file << ": " << e.what());
addError<LyricsFileScanError>(getFilePath());
}
}
void LyricsFileScanOperation::processResult(ScanContext& context)
LyricsFileScanOperation::OperationResult LyricsFileScanOperation::processResult()
{
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::TrackLyrics::pointer trackLyrics{ db::TrackLyrics::find(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::TrackLyrics::pointer trackLyrics{ db::TrackLyrics::find(dbSession, getFilePath()) };
if (!_parsedLyrics)
{
if (trackLyrics)
{
trackLyrics.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed lyrics file " << _file);
LMS_LOG(DBUPDATER, DEBUG, "Removed lyrics file " << getFilePath());
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadLyricsFile);
return;
return OperationResult::Skipped;
}
const bool added{ !trackLyrics };
if (!trackLyrics)
{
trackLyrics = dbSession.create<db::TrackLyrics>();
trackLyrics.modify()->setAbsoluteFilePath(_file);
trackLyrics.modify()->setAbsoluteFilePath(getFilePath());
}
trackLyrics.modify()->setLastWriteTime(fileInfo->lastWriteTime);
trackLyrics.modify()->setFileSize(fileInfo->fileSize);
trackLyrics.modify()->setLastWriteTime(getLastWriteTime());
trackLyrics.modify()->setFileSize(getFileSize());
trackLyrics.modify()->setLanguage(!_parsedLyrics->language.empty() ? _parsedLyrics->language : "xxx");
trackLyrics.modify()->setOffset(_parsedLyrics->offset);
trackLyrics.modify()->setDisplayTitle(_parsedLyrics->displayTitle);
@@ -118,24 +108,23 @@ namespace lms::scanner
else
trackLyrics.modify()->setUnsynchronizedLines(_parsedLyrics->unsynchronizedLines);
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
trackLyrics.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
trackLyrics.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary));
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added external lyrics " << _file);
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated external lyrics " << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Added external lyrics " << getFilePath());
return OperationResult::Added;
}
LMS_LOG(DBUPDATER, DEBUG, "Updated external lyrics " << getFilePath());
return OperationResult::Updated;
}
} // namespace
LyricsFileScanner::LyricsFileScanner(db::Db& db)
LyricsFileScanner::LyricsFileScanner(db::Db& db, ScannerSettings& _settings)
: _db{ db }
, _settings{ _settings }
{
}
@@ -144,41 +133,27 @@ namespace lms::scanner
return "Lyrics scanner";
}
std::span<const std::filesystem::path> LyricsFileScanner::getSupportedFiles() const
{
return {};
}
std::span<const std::filesystem::path> LyricsFileScanner::getSupportedExtensions() const
{
return metadata::getSupportedLyricsFileExtensions();
}
bool LyricsFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
bool LyricsFileScanner::needsScan(const FileToScan& file) const
{
ScanStats& stats{ context.stats };
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
stats.skips++;
return false;
}
if (!context.scanOptions.fullScan)
{
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(dbSession, file.file) };
if (lyrics && lyrics->getLastWriteTime() == lastWriteTime)
{
stats.skips++;
return false;
}
}
return true; // need to scan
const db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(dbSession, file.filePath) };
return !lyrics || lyrics->getLastWriteTime() != file.lastWriteTime;
}
std::unique_ptr<IFileScanOperation> LyricsFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> LyricsFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<LyricsFileScanOperation>(fileToScan, _db);
return std::make_unique<LyricsFileScanOperation>(std::move(fileToScan), _db, _settings);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/LyricsFileScanner.hpp
+5 -3
View File
@@ -36,17 +36,19 @@ namespace lms::scanner
class LyricsFileScanner : public IFileScanner
{
public:
LyricsFileScanner(db::Db& db);
LyricsFileScanner(db::Db& db, ScannerSettings& settings);
~LyricsFileScanner() override = default;
LyricsFileScanner(const LyricsFileScanner&) = delete;
LyricsFileScanner& operator=(const LyricsFileScanner&) = delete;
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
db::Db& _db;
ScannerSettings& _settings;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/PlayListFileScanner.cpp
+46 -70
View File
@@ -30,7 +30,7 @@
#include "metadata/Exception.hpp"
#include "metadata/PlayList.hpp"
#include "IFileScanOperation.hpp"
#include "FileScanOperationBase.hpp"
#include "ScanContext.hpp"
#include "Utils.hpp"
@@ -38,26 +38,18 @@ namespace lms::scanner
{
namespace
{
class PlayListFileScanOperation : public IFileScanOperation
class PlayListFileScanOperation : public FileScanOperationBase
{
public:
PlayListFileScanOperation(const FileToScan& file, db::Db& db)
: _file{ file.file }
, _mediaLibrary{ file.mediaLibrary }
, _db{ db } {}
using FileScanOperationBase::FileScanOperationBase;
~PlayListFileScanOperation() override = default;
PlayListFileScanOperation(const PlayListFileScanOperation&) = delete;
PlayListFileScanOperation& operator=(const PlayListFileScanOperation&) = delete;
private:
const std::filesystem::path& getFile() const override { return _file; };
core::LiteralString getName() const override { return "ScanPlayListFile"; }
void scan() override;
void processResult(ScanContext& context) override;
const std::filesystem::path _file;
const MediaLibraryInfo _mediaLibrary;
db::Db& _db;
OperationResult processResult() override;
std::optional<metadata::PlayList> _parsedPlayList;
};
@@ -66,74 +58,72 @@ namespace lms::scanner
{
try
{
std::ifstream ifs{ _file };
std::ifstream ifs{ getFilePath() };
if (!ifs)
LMS_LOG(DBUPDATER, ERROR, "Cannot open file " << _file);
else
_parsedPlayList = metadata::parsePlayList(ifs);
{
const std::error_code ec{ errno, std::generic_category() };
addError<IOScanError>(getFilePath(), ec);
return;
}
_parsedPlayList = metadata::parsePlayList(ifs);
}
catch (const metadata::Exception& e)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot read playlist in file " << _file << ": " << e.what());
addError<PlayListFileScanError>(getFilePath());
}
}
void PlayListFileScanOperation::processResult(ScanContext& context)
PlayListFileScanOperation::OperationResult PlayListFileScanOperation::processResult()
{
ScanStats& stats{ context.stats };
const std::optional<FileInfo> fileInfo{ utils::retrieveFileInfo(_file, _mediaLibrary.rootDirectory) };
if (!fileInfo)
{
stats.skips++;
return;
}
db::Session& dbSession{ _db.getTLSSession() };
db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, _file) };
db::Session& dbSession{ getDb().getTLSSession() };
db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, getFilePath()) };
if (!_parsedPlayList)
{
if (playList)
{
playList.remove();
stats.deletions++;
LMS_LOG(DBUPDATER, DEBUG, "Removed playlist file " << _file);
LMS_LOG(DBUPDATER, DEBUG, "Removed playlist file " << getFilePath());
return OperationResult::Removed;
}
context.stats.errors.emplace_back(_file, ScanErrorType::CannotReadPlayListFile);
return;
return OperationResult::Skipped;
}
const bool added{ !playList };
if (!playList)
playList = dbSession.create<db::PlayListFile>(_file);
playList = dbSession.create<db::PlayListFile>(getFilePath());
playList.modify()->setLastWriteTime(fileInfo->lastWriteTime);
playList.modify()->setFileSize(fileInfo->fileSize);
playList.modify()->setLastWriteTime(getLastWriteTime());
playList.modify()->setFileSize(getFileSize());
if (!_parsedPlayList->name.empty())
playList.modify()->setName(_parsedPlayList->name);
else
playList.modify()->setName(_file.stem().string());
playList.modify()->setName(getFilePath().stem().string());
playList.modify()->setFiles(_parsedPlayList->files);
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, _mediaLibrary.id) }; // may be null if settings are updated in // => next scan will correct this
playList.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, _file.parent_path(), mediaLibrary));
db::MediaLibrary::pointer mediaLibrary{ db::MediaLibrary::find(dbSession, getMediaLibrary().id) }; // may be null if settings are updated in // => next scan will correct this
playList.modify()->setDirectory(utils::getOrCreateDirectory(dbSession, getFilePath().parent_path(), mediaLibrary));
if (added)
{
LMS_LOG(DBUPDATER, DEBUG, "Added playlist file " << _file);
stats.additions++;
LMS_LOG(DBUPDATER, DEBUG, "Added playlist file " << getFilePath());
return OperationResult::Added;
}
else
{
LMS_LOG(DBUPDATER, DEBUG, "Updated playlist file '" << _file);
stats.updates++;
LMS_LOG(DBUPDATER, DEBUG, "Updated playlist file '" << getFilePath());
return OperationResult::Updated;
}
}
} // namespace
PlayListFileScanner::PlayListFileScanner(db::Db& db)
PlayListFileScanner::PlayListFileScanner(db::Db& db, ScannerSettings& settings)
: _db{ db }
, _settings{ settings }
{
}
@@ -142,41 +132,27 @@ namespace lms::scanner
return "PlayList scanner";
}
std::span<const std::filesystem::path> PlayListFileScanner::getSupportedFiles() const
{
return {};
}
std::span<const std::filesystem::path> PlayListFileScanner::getSupportedExtensions() const
{
return metadata::getSupportedPlayListFileExtensions();
}
bool PlayListFileScanner::needsScan(ScanContext& context, const FileToScan& file) const
bool PlayListFileScanner::needsScan(const FileToScan& file) const
{
ScanStats& stats{ context.stats };
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const Wt::WDateTime lastWriteTime{ utils::retrieveFileGetLastWrite(file.file) };
// Should rarely fail as we are currently iterating it
if (!lastWriteTime.isValid())
{
stats.skips++;
return false;
}
if (!context.scanOptions.fullScan)
{
db::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createReadTransaction() };
const db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, file.file) };
if (playList && playList->getLastWriteTime() == lastWriteTime)
{
stats.skips++;
return false;
}
}
return true; // need to scan
const db::PlayListFile::pointer playList{ db::PlayListFile::find(dbSession, file.filePath) };
return !playList || playList->getLastWriteTime() != file.lastWriteTime;
}
std::unique_ptr<IFileScanOperation> PlayListFileScanner::createScanOperation(const FileToScan& fileToScan) const
std::unique_ptr<IFileScanOperation> PlayListFileScanner::createScanOperation(FileToScan&& fileToScan) const
{
return std::make_unique<PlayListFileScanOperation>(fileToScan, _db);
return std::make_unique<PlayListFileScanOperation>(std::move(fileToScan), _db, _settings);
}
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/PlayListFileScanner.hpp
+5 -3
View File
@@ -36,17 +36,19 @@ namespace lms::scanner
class PlayListFileScanner : public IFileScanner
{
public:
PlayListFileScanner(db::Db& db);
PlayListFileScanner(db::Db& db, ScannerSettings& settings);
~PlayListFileScanner() override = default;
PlayListFileScanner(const PlayListFileScanner&) = delete;
PlayListFileScanner& operator=(const PlayListFileScanner&) = delete;
private:
core::LiteralString getName() const override;
std::span<const std::filesystem::path> getSupportedFiles() const override;
std::span<const std::filesystem::path> getSupportedExtensions() const override;
bool needsScan(ScanContext& context, const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(const FileToScan& fileToScan) const override;
bool needsScan(const FileToScan& file) const override;
std::unique_ptr<IFileScanOperation> createScanOperation(FileToScan&& fileToScan) const override;
db::Db& _db;
ScannerSettings& _settings;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/scanners/Utils.cpp
+2 -54
View File
@@ -19,7 +19,8 @@
#include "Utils.hpp"
#include "core/ILogger.hpp"
#include <system_error>
#include "core/Path.hpp"
#include "database/Directory.hpp"
#include "database/MediaLibrary.hpp"
@@ -27,59 +28,6 @@
namespace lms::scanner::utils
{
Wt::WDateTime retrieveFileGetLastWrite(const std::filesystem::path& file)
{
Wt::WDateTime res;
try
{
res = core::pathUtils::getLastWriteTime(file);
}
catch (core::LmsException& e)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot get last write time: " << e.what());
}
return res;
}
std::optional<FileInfo> retrieveFileInfo(const std::filesystem::path& file, const std::filesystem::path& rootPath)
{
std::optional<FileInfo> res;
res.emplace();
res->lastWriteTime = retrieveFileGetLastWrite(file);
if (!res->lastWriteTime.isValid())
{
res.reset();
return res;
}
{
std::error_code ec;
res->relativePath = std::filesystem::relative(file, rootPath, ec);
if (ec)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot get relative file path for '" << file.string() << "' from '" << rootPath.string() << "': " << ec.message());
res.reset();
return res;
}
}
{
std::error_code ec;
res->fileSize = std::filesystem::file_size(file, ec);
if (ec)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot get file size for '" << file.string() << "': " << ec.message());
res.reset();
return res;
}
}
return res;
}
db::Directory::pointer getOrCreateDirectory(db::Session& session, const std::filesystem::path& path, const db::MediaLibrary::pointer& mediaLibrary)
{
db::Directory::pointer directory{ db::Directory::find(session, path) };
src/libs/services/scanner/impl/scanners/Utils.hpp
-13
View File
@@ -20,9 +20,6 @@
#pragma once
#include <filesystem>
#include <optional>
#include <Wt/WDateTime.h>
#include "database/Object.hpp"
@@ -35,18 +32,8 @@ namespace lms::db
namespace lms::scanner
{
struct FileInfo
{
Wt::WDateTime lastWriteTime;
std::filesystem::path relativePath;
std::size_t fileSize{};
};
namespace utils
{
Wt::WDateTime retrieveFileGetLastWrite(const std::filesystem::path& file);
std::optional<FileInfo> retrieveFileInfo(const std::filesystem::path& file, const std::filesystem::path& rootPath);
db::ObjectPtr<db::Directory> getOrCreateDirectory(db::Session& session, const std::filesystem::path& path, const db::ObjectPtr<db::MediaLibrary>& mediaLibrary);
} // namespace utils
} // namespace lms::scanner