owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a way to set custom tag delimiters for artists and for other fields, fixes #417

Browse Source
This commit is contained in:
emeric
2024-03-01 23:58:49 +01:00
parent 194b9981c1
commit a6d11f4b4b
45 changed files with 2261 additions and 1121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/scanner/impl/ScanStepScanFiles.cpp
+42 -21
View File
@@ -19,7 +19,6 @@
#include "ScanStepScanFiles.hpp"
#include "metadata/IParser.hpp"
#include "database/Artist.hpp"
#include "database/Cluster.hpp"
#include "database/Db.hpp"
@@ -29,15 +28,17 @@
#include "database/Track.hpp"
#include "database/TrackFeatures.hpp"
#include "database/TrackArtistLink.hpp"
#include "metadata/Exception.hpp"
#include "metadata/IParser.hpp"
#include "utils/Exception.hpp"
#include "utils/IConfig.hpp"
#include "utils/ILogger.hpp"
#include "utils/Path.hpp"
using namespace Database;
namespace Scanner
{
using namespace Database;
namespace
{
Artist::pointer createArtist(Session& session, const MetaData::Artist& artistInfo)
@@ -179,25 +180,34 @@ namespace Scanner
return Release::pointer{};
}
std::vector<Cluster::pointer> getOrCreateClusters(Session& session, const MetaData::Tags& tags)
std::vector<Cluster::pointer> getOrCreateClusters(Session& session, const MetaData::Track& track)
{
std::vector<Cluster::pointer> clusters;
for (const auto& [tag, values] : tags)
auto getOrCreateClusters {[&](std::string tag, std::span<const std::string> values)
{
auto clusterType = ClusterType::find(session, tag);
if (!clusterType)
clusterType = session.create<ClusterType>(tag);
for (const auto& clusterName : values)
for (const auto& value : values)
{
auto cluster = clusterType->getCluster(clusterName);
auto cluster{ clusterType->getCluster(value) };
if (!cluster)
cluster = session.create<Cluster>(clusterType, clusterName);
cluster = session.create<Cluster>(clusterType, value);
clusters.push_back(cluster);
}
}
}};
// TODO: migrate these fields in dedicated tables in DB
getOrCreateClusters("GENRE", track.genres);
getOrCreateClusters("MOOD", track.genres);
getOrCreateClusters("LANGUAGE", track.languages);
getOrCreateClusters("GROUPING", track.groupings);
for (const auto& [tag, values] : track.userExtraTags)
getOrCreateClusters(tag, values);
return clusters;
}
@@ -232,7 +242,7 @@ namespace Scanner
, _scanContextRunner{ _scanContext, threadCount }
{}
void ScanStepScanFiles::MetadataScanQueue::pushScanRequest(const std::filesystem::path path)
void ScanStepScanFiles::MetadataScanQueue::pushScanRequest(const std::filesystem::path& path)
{
{
std::scoped_lock lock{ _mutex };
@@ -241,12 +251,21 @@ namespace Scanner
_scanContext.post([=, this]
{
std::optional<MetaData::Track> trackMetaData{ _metadataParser.parse(path) };
std::unique_ptr<MetaData::Track> track;
try
{
track = _metadataParser.parse(path);
}
catch(const MetaData::Exception& e)
{
LMS_LOG(DBUPDATER, INFO, "Failed to parse '" << path.string() << "'");
}
{
std::scoped_lock lock{ _mutex };
_scanResults.emplace_back(std::make_unique<MetaDataScanResult>(path, std::move(trackMetaData)));
_scanResults.emplace_back(MetaDataScanResult{ std::move(path), std::move(track) });
_ongoingScanCount -= 1;
}
_condVar.notify_all();
@@ -259,7 +278,7 @@ namespace Scanner
return _scanResults.size();
}
size_t ScanStepScanFiles::MetadataScanQueue::popResults(std::vector<std::unique_ptr<MetaDataScanResult>>& results, std::size_t maxCount)
size_t ScanStepScanFiles::MetadataScanQueue::popResults(std::vector<MetaDataScanResult>& results, std::size_t maxCount)
{
results.clear();
results.reserve(maxCount);
@@ -285,7 +304,7 @@ namespace Scanner
ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
: ScanStepBase{ initParams }
, _metadataParser{ MetaData::createParser(MetaData::ParserType::TagLib, getParserReadStyle()) } // For now, always use TagLib
, _metadataParser{ MetaData::createParser(MetaData::ParserBackend::TagLib, getParserReadStyle()) } // For now, always use TagLib
, _metadataScanQueue{ *_metadataParser, getScanMetaDataThreadCount() }
{
LMS_LOG(DBUPDATER, INFO, "Using " << _metadataScanQueue.getThreadCount() << " thread(s) for scanning file metadata");
@@ -300,9 +319,11 @@ namespace Scanner
std::vector<std::string> tagsToParse{ _extraTagsToParse };
tagsToParse.insert(std::end(tagsToParse), std::cbegin(_settings.extraTags), std::cend(_settings.extraTags));
_metadataParser->setUserExtraTags(tagsToParse);
_metadataParser->setArtistTagDelimiters(_settings.artistTagDelimiters);
_metadataParser->setDefaultTagDelimiters(_settings.defaultTagDelimiters);
}
std::vector<std::unique_ptr<MetaDataScanResult>> scanResults;
std::vector<MetaDataScanResult> scanResults;
context.currentStepStats.totalElems = context.stats.filesScanned;
for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
@@ -400,21 +421,21 @@ namespace Scanner
return true; // need to scan
}
void ScanStepScanFiles::processMetaDataScanResults(ScanContext& context, std::span<const std::unique_ptr<MetaDataScanResult>> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo)
void ScanStepScanFiles::processMetaDataScanResults(ScanContext& context, std::span<const MetaDataScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo)
{
Database::Session& dbSession{ _db.getTLSSession() };
auto transaction{ dbSession.createWriteTransaction() };
for (const auto& scanResult : scanResults)
for (const MetaDataScanResult& scanResult : scanResults)
{
if (_abortScan)
return;
if (scanResult->trackMetaData)
if (scanResult.trackMetaData)
{
context.stats.scans++;
processFileMetaData(context, scanResult->path, *scanResult->trackMetaData, libraryInfo);
processFileMetaData(context, scanResult.path, *scanResult.trackMetaData, libraryInfo);
// optimize the database during scan (if we import a very large database, it may be too late to do it once at end)
if ((context.stats.scans % 1'000) == 0)
@@ -422,7 +443,7 @@ namespace Scanner
}
else
{
context.stats.errors.emplace_back(scanResult->path, ScanErrorType::CannotParseFile);
context.stats.errors.emplace_back(scanResult.path, ScanErrorType::CannotParseFile);
}
}
}
@@ -583,7 +604,7 @@ namespace Scanner
track.modify()->setTotalTrack(trackMetadata.medium ? trackMetadata.medium->trackCount : std::nullopt);
track.modify()->setReleaseReplayGain(trackMetadata.medium ? trackMetadata.medium->replayGain : std::nullopt);
track.modify()->setDiscSubtitle(trackMetadata.medium ? trackMetadata.medium->name : "");
track.modify()->setClusters(getOrCreateClusters(dbSession, trackMetadata.userExtraTags));
track.modify()->setClusters(getOrCreateClusters(dbSession, trackMetadata));
track.modify()->setLastWriteTime(lastWriteTime);
track.modify()->setName(title);
track.modify()->setDuration(trackMetadata.duration);
src/libs/services/scanner/impl/ScanStepScanFiles.hpp
+7 -7
View File
@@ -47,13 +47,13 @@ namespace Scanner
struct MetaDataScanResult
{
std::filesystem::path path;
std::optional<MetaData::Track> trackMetaData;
std::unique_ptr<MetaData::Track> trackMetaData;
};
void processMetaDataScanResults(ScanContext& context, std::span<const std::unique_ptr<MetaDataScanResult>> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo);
void processMetaDataScanResults(ScanContext& context, std::span<const MetaDataScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo);
void processFileMetaData(ScanContext& context, const std::filesystem::path& file, const MetaData::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo);
std::unique_ptr<MetaData::IParser> _metadataParser;
const std::vector<std::string> _extraTagsToParse{ "GENRE", "MOOD", "LANGUAGE", "ALBUMGROUPING" };
const std::vector<std::string> _extraTagsToParse;
class MetadataScanQueue
{
@@ -62,10 +62,10 @@ namespace Scanner
std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); }
void pushScanRequest(const std::filesystem::path path);
void pushScanRequest(const std::filesystem::path& path);
std::size_t getResultsCount() const;
size_t popResults(std::vector<std::unique_ptr<MetaDataScanResult>>& results, std::size_t maxCount);
size_t popResults(std::vector<MetaDataScanResult>& results, std::size_t maxCount);
void wait(std::size_t maxScanRequestCount = 0); // wait until ongoing scan request count <= maxScanRequestCount
@@ -76,11 +76,11 @@ namespace Scanner
mutable std::mutex _mutex ;
std::size_t _ongoingScanCount{};
std::deque<std::unique_ptr<MetaDataScanResult>> _scanResults;
std::deque<MetaDataScanResult> _scanResults;
std::condition_variable _condVar;
};
MetadataScanQueue _metadataScanQueue;
std::deque<std::unique_ptr<MetaDataScanResult>> _metaDataScanResults;
std::deque<MetaDataScanResult> _metaDataScanResults;
};
}
src/libs/services/scanner/impl/ScannerService.cpp
+3
View File
@@ -383,6 +383,9 @@ namespace Scanner
const auto& tags{ scanSettings->getExtraTagsToScan() };
std::transform(std::cbegin(tags), std::cend(tags), std::back_inserter(newSettings.extraTags), [](std::string_view tag) { return std::string{ tag };});
}
newSettings.artistTagDelimiters = scanSettings->getArtistTagDelimiters();
newSettings.defaultTagDelimiters = scanSettings->getDefaultTagDelimiters();
}
return newSettings;
src/libs/services/scanner/impl/ScannerSettings.hpp
+11 -18
View File
@@ -30,31 +30,24 @@ namespace Scanner
{
struct ScannerSettings
{
std::size_t scanVersion{};
Wt::WTime startTime;
Database::ScanSettings::UpdatePeriod updatePeriod{ Database::ScanSettings::UpdatePeriod::Never };
std::vector<std::filesystem::path> supportedExtensions;
bool skipDuplicateMBID{};
std::vector<std::string> extraTags;
std::size_t scanVersion{};
Wt::WTime startTime;
Database::ScanSettings::UpdatePeriod updatePeriod{ Database::ScanSettings::UpdatePeriod::Never };
std::vector<std::filesystem::path> supportedExtensions;
bool skipDuplicateMBID{};
std::vector<std::string> extraTags;
std::vector<std::string> artistTagDelimiters;
std::vector<std::string> defaultTagDelimiters;
struct MediaLibraryInfo
{
Database::MediaLibraryId id;
std::filesystem::path rootDirectory;
bool operator==(const MediaLibraryInfo& other) const { return id == other.id && rootDirectory == other.rootDirectory; }
bool operator<=>(const MediaLibraryInfo& other) const = default;
};
std::vector<MediaLibraryInfo> mediaLibraries;
std::vector<MediaLibraryInfo> mediaLibraries;
bool operator==(const ScannerSettings& rhs) const
{
return scanVersion == rhs.scanVersion
&& startTime == rhs.startTime
&& updatePeriod == rhs.updatePeriod
&& supportedExtensions == rhs.supportedExtensions
&& mediaLibraries == rhs.mediaLibraries
&& skipDuplicateMBID == rhs.skipDuplicateMBID
&& extraTags == rhs.extraTags;
}
bool operator<=>(const ScannerSettings& rhs) const = default;
};
}