owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial support for multi library, still WIP

Browse Source
This commit is contained in:
emeric
2024-01-25 00:01:11 +01:00
parent f5f577af52
commit 0ed3f1ba79
85 changed files with 3076 additions and 1445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/scanner/impl/IScanStep.hpp
+15 -15
View File
@@ -20,26 +20,26 @@
#pragma once
#include <string_view>
#include <vector>
#include "services/scanner/ScannerStats.hpp"
namespace Scanner
{
class IScanStep
{
public:
virtual ~IScanStep() = default;
class IScanStep
{
public:
virtual ~IScanStep() = default;
virtual ScanStep getStep() const = 0;
virtual std::string_view getStepName() const = 0;
virtual ScanStep getStep() const = 0;
virtual std::string_view getStepName() const = 0;
struct ScanContext
{
const std::filesystem::path directory;
const bool forceScan;
ScanStats stats;
ScanStepStats currentStepStats;
};
virtual void process(ScanContext& context) = 0;
};
struct ScanContext
{
const bool forceScan;
ScanStats stats;
ScanStepStats currentStepStats;
};
virtual void process(ScanContext& context) = 0;
};
}
src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp
+20 -11
View File
@@ -18,6 +18,7 @@
*/
#include "ScanStepDiscoverFiles.hpp"
#include "utils/ILogger.hpp"
#include "utils/Path.hpp"
@@ -26,22 +27,30 @@ namespace Scanner
void ScanStepDiscoverFiles::process(ScanContext& context)
{
context.stats.filesScanned = 0;
PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path)
{
if (_abortScan)
return false;
if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
{
std::size_t currentDirectoryProcessElemsCount{};
PathUtils::exploreFilesRecursive(mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path)
{
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
if (_abortScan)
return false;
return true;
}, &excludeDirFileName);
if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
{
context.currentStepStats.processedElems++;
currentDirectoryProcessElemsCount++;
_progressCallback(context.currentStepStats);
}
return true;
}, &excludeDirFileName);
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in '" << mediaLibrary.rootDirectory << "'");
}
context.stats.filesScanned = context.currentStepStats.processedElems;
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'");
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.filesScanned << " files in all directories");
}
}
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp
+5 -1
View File
@@ -183,7 +183,11 @@ namespace Scanner
return false;
}
if (!PathUtils::isPathInRootPath(p, _settings.mediaDirectory, &excludeDirFileName))
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
[&](const ScannerSettings::MediaLibraryInfo& libraryInfo)
{
return PathUtils::isPathInRootPath(p, libraryInfo.rootDirectory, &excludeDirFileName);
}))
{
LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': out of media directory");
return false;
src/libs/services/scanner/impl/ScanStepScanFiles.cpp
+60 -27
View File
@@ -23,6 +23,7 @@
#include "database/Artist.hpp"
#include "database/Cluster.hpp"
#include "database/Db.hpp"
#include "database/MediaLibrary.hpp"
#include "database/Release.hpp"
#include "database/Session.hpp"
#include "database/Track.hpp"
@@ -232,33 +233,36 @@ namespace Scanner
context.currentStepStats.totalElems = context.stats.filesScanned;
PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path)
{
if (_abortScan)
return false;
if (ec)
for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
{
PathUtils::exploreFilesRecursive(mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot process entry '" << path.string() << "': " << ec.message());
context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
}
else if (PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
{
scanAudioFile(path, context);
if (_abortScan)
return false;
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
if (ec)
{
LMS_LOG(DBUPDATER, ERROR, "Cannot process entry '" << path.string() << "': " << ec.message());
context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
}
else if (PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
{
scanAudioFile(path, context, mediaLibrary);
// optimize the database during scan (if we import a very large database, it may be too late to do it once at end)
if ((context.currentStepStats.processedElems % 1'000) == 0)
_db.getTLSSession().optimize();
}
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
return true;
}, &excludeDirFileName);
// optimize the database during scan (if we import a very large database, it may be too late to do it once at end)
if ((context.currentStepStats.processedElems % 1'000) == 0)
_db.getTLSSession().optimize();
}
return true;
}, &excludeDirFileName);
}
}
void ScanStepScanFiles::scanAudioFile(const std::filesystem::path& file, ScanContext& context)
void ScanStepScanFiles::scanAudioFile(const std::filesystem::path& file, ScanContext& context, const ScannerSettings::MediaLibraryInfo& libraryInfo)
{
ScanStats& stats{ context.stats };
Wt::WDateTime lastWriteTime;
@@ -273,6 +277,7 @@ namespace Scanner
return;
}
bool needUpdateLibrary{};
if (!context.forceScan)
{
// Skip file if last write is the same
@@ -281,14 +286,35 @@ namespace Scanner
const Track::pointer track{ Track::findByPath(dbSession, file) };
if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t()
&& track->getScanVersion() == _settings.scanVersion)
if (track
&& track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t()
&& track->getScanVersion() == _settings.scanVersion
)
{
stats.skips++;
return;
// this file may have been moved from one library to another, then we just need to update the media library id instead of a full rescan
auto trackMediaLibrary{ track->getMediaLibrary() };
if (trackMediaLibrary && trackMediaLibrary->getId() == libraryInfo.id)
{
stats.skips++;
return;
}
needUpdateLibrary = true;
}
}
if (needUpdateLibrary)
{
Database::Session& dbSession{ _db.getTLSSession() };
auto transaction{ _db.getTLSSession().createWriteTransaction() };
Track::pointer track{ Track::findByPath(dbSession, file) };
assert(track);
track.modify()->setMediaLibrary(Database::MediaLibrary::find(dbSession, libraryInfo.id)); // may be null, will be handled in the next scan anyway
stats.updates++;
return;
}
std::optional<MetaData::Track> trackInfo{ _metadataParser->parse(file) };
if (!trackInfo)
{
@@ -330,8 +356,14 @@ namespace Scanner
continue;
// Skip if duplicate files no longer in media root: as it will be removed later, we will end up with no file
if (!PathUtils::isPathInRootPath(file, _settings.mediaDirectory, &excludeDirFileName))
if (std::none_of(std::cbegin(_settings.mediaLibraries), std::cend(_settings.mediaLibraries),
[&](const ScannerSettings::MediaLibraryInfo& libraryInfo)
{
return PathUtils::isPathInRootPath(file, libraryInfo.rootDirectory, &excludeDirFileName);
}))
{
continue;
}
LMS_LOG(DBUPDATER, DEBUG, "Skipped '" << file.string() << "' (similar MBID in '" << otherTrack->getPath().string() << "')");
// As this MBID already exists, just remove what we just scanned
@@ -389,6 +421,7 @@ namespace Scanner
// Track related data
assert(track);
track.modify()->setMediaLibrary(MediaLibrary::find(dbSession, libraryInfo.id)); // may be null if settings are updated in // => next scan will correct this
track.modify()->clearArtistLinks();
// Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files
for (const Artist::pointer& artist : getOrCreateArtists(dbSession, trackInfo->artists, false))
@@ -450,7 +483,7 @@ namespace Scanner
// If a file has an OriginalDate but no date, set it to ease filtering
if (!trackInfo->date.isValid() && trackInfo->originalDate.isValid())
track.modify()->setDate(trackInfo->originalDate);
// If a file has an OriginalYear but no Year, set it to ease filtering
if (!trackInfo->year && trackInfo->originalYear)
track.modify()->setYear(trackInfo->originalYear);
src/libs/services/scanner/impl/ScanStepScanFiles.hpp
+1 -1
View File
@@ -38,7 +38,7 @@ namespace Scanner
std::string_view getStepName() const override { return "Scanning files"; }
void process(ScanContext& context) override;
void scanAudioFile(const std::filesystem::path& file, ScanContext& context);
void scanAudioFile(const std::filesystem::path& file, ScanContext& context, const ScannerSettings::MediaLibraryInfo& libraryInfo);
std::unique_ptr<MetaData::IParser> _metadataParser;
const std::vector<std::string> _extraTagsToParse{ "GENRE", "MOOD", "LANGUAGE", "ALBUMGROUPING" };
src/libs/services/scanner/impl/ScannerService.cpp
+21 -3
View File
@@ -22,7 +22,7 @@
#include <ctime>
#include <boost/asio/placeholders.hpp>
#include "database/Cluster.hpp"
#include "database/MediaLibrary.hpp"
#include "database/TrackFeatures.hpp"
#include "database/ScanSettings.hpp"
#include "utils/Exception.hpp"
@@ -113,6 +113,12 @@ namespace Scanner
void ScannerService::abortScan()
{
bool isRunning{};
{
std::scoped_lock lock{ _statusMutex };
isRunning = _curState == State::InProgress;
}
LMS_LOG(DBUPDATER, DEBUG, "Aborting scan...");
std::scoped_lock lock{ _controlMutex };
@@ -125,6 +131,9 @@ namespace Scanner
_abortScan = false;
_ioService.start();
if (isRunning)
_events.scanAborted.emit();
}
void ScannerService::requestImmediateScan(bool force)
@@ -139,6 +148,11 @@ namespace Scanner
});
}
void ScannerService::requestStop()
{
abortScan();
}
void ScannerService::requestReload()
{
abortScan();
@@ -261,7 +275,7 @@ namespace Scanner
refreshScanSettings();
IScanStep::ScanContext scanContext{ _settings.mediaDirectory, forceScan, ScanStats {}, ScanStepStats {} };
IScanStep::ScanContext scanContext{ forceScan, ScanStats {}, ScanStepStats {} };
ScanStats& stats{ scanContext.stats };
stats.startTime = Wt::WDateTime::currentDateTime();
@@ -359,7 +373,11 @@ namespace Scanner
std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions),
[](const std::filesystem::path& extension) { return std::filesystem::path{ StringUtils::stringToLower(extension.string()) }; });
}
newSettings.mediaDirectory = scanSettings->getMediaDirectory();
MediaLibrary::find(_dbSession, [&](const MediaLibrary::pointer& mediaLibrary)
{
newSettings.mediaLibraries.push_back(ScannerSettings::MediaLibraryInfo{ mediaLibrary->getId(), mediaLibrary->getPath().lexically_normal() });
});
{
const auto& tags{ scanSettings->getExtraTagsToScan() };
src/libs/services/scanner/impl/ScannerService.hpp
+2 -1
View File
@@ -48,7 +48,8 @@ namespace Scanner
ScannerService(const ScannerService&) = delete;
ScannerService& operator=(const ScannerService&) = delete;
private:
void requestStop() override;
void requestReload() override;
void requestImmediateScan(bool force) override;
src/libs/services/scanner/impl/ScannerSettings.hpp
+29 -20
View File
@@ -23,29 +23,38 @@
#include <string>
#include <vector>
#include <Wt/WDateTime.h>
#include "database/MediaLibraryId.hpp"
#include "database/ScanSettings.hpp"
namespace Scanner
{
struct ScannerSettings
{
std::size_t scanVersion {};
Wt::WTime startTime;
Database::ScanSettings::UpdatePeriod updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
std::vector<std::filesystem::path> supportedExtensions;
std::filesystem::path mediaDirectory;
bool skipDuplicateMBID {};
std::vector<std::string> extraTags;
struct ScannerSettings
{
std::size_t scanVersion{};
Wt::WTime startTime;
Database::ScanSettings::UpdatePeriod updatePeriod{ Database::ScanSettings::UpdatePeriod::Never };
std::vector<std::filesystem::path> supportedExtensions;
bool skipDuplicateMBID{};
std::vector<std::string> extraTags;
bool operator==(const ScannerSettings& rhs) const
{
return scanVersion == rhs.scanVersion
&& startTime == rhs.startTime
&& updatePeriod == rhs.updatePeriod
&& supportedExtensions == rhs.supportedExtensions
&& mediaDirectory == rhs.mediaDirectory
&& skipDuplicateMBID == rhs.skipDuplicateMBID
&& extraTags == rhs.extraTags;
}
};
struct MediaLibraryInfo
{
Database::MediaLibraryId id;
std::filesystem::path rootDirectory;
bool operator==(const MediaLibraryInfo& other) const { return id == other.id && rootDirectory == other.rootDirectory; }
};
std::vector<MediaLibraryInfo> mediaLibraries;
bool operator==(const ScannerSettings& rhs) const
{
return scanVersion == rhs.scanVersion
&& startTime == rhs.startTime
&& updatePeriod == rhs.updatePeriod
&& supportedExtensions == rhs.supportedExtensions
&& mediaLibraries == rhs.mediaLibraries
&& skipDuplicateMBID == rhs.skipDuplicateMBID
&& extraTags == rhs.extraTags;
}
};
}