From bdd678f168c3d204ba6e6feaaa0e8c7fd4c0af3e Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 9 Mar 2024 18:58:52 +0100 Subject: [PATCH] Added some profiling events for scanner --- src/libs/metadata/impl/TagLibTagReader.cpp | 14 ++++- src/libs/metadata/impl/TagLibTagReader.hpp | 2 +- .../scanner/impl/ScanStepScanFiles.cpp | 63 ++++++++++++------- .../scanner/impl/ScanStepScanFiles.hpp | 3 +- 4 files changed, 55 insertions(+), 27 deletions(-) diff --git a/src/libs/metadata/impl/TagLibTagReader.cpp b/src/libs/metadata/impl/TagLibTagReader.cpp index 97ebb9d2..6096e575 100644 --- a/src/libs/metadata/impl/TagLibTagReader.cpp +++ b/src/libs/metadata/impl/TagLibTagReader.cpp @@ -35,6 +35,7 @@ #include "metadata/Exception.hpp" #include "utils/ILogger.hpp" +#include "utils/IProfiler.hpp" #include "utils/String.hpp" namespace MetaData @@ -169,12 +170,19 @@ namespace MetaData dst[tag] = std::move(values); } } + + TagLib::FileRef parseFile(const std::filesystem::path& p, ParserReadStyle parserReadStyle) + { + LMS_SCOPED_PROFILE_DETAILED("MetaData", "TagLibParseFile"); + + return TagLib::FileRef{ p.string().c_str() + , true // read audio properties + , readStyleToTagLibReadStyle(parserReadStyle) }; + } } TagLibTagReader::TagLibTagReader(const std::filesystem::path& p, ParserReadStyle parserReadStyle, bool debug) - : _file{ p.string().c_str() - , true // read audio properties - , readStyleToTagLibReadStyle(parserReadStyle) } + : _file{ parseFile(p, parserReadStyle) } { if (_file.isNull()) { diff --git a/src/libs/metadata/impl/TagLibTagReader.hpp b/src/libs/metadata/impl/TagLibTagReader.hpp index cc3dd304..a6db24e1 100644 --- a/src/libs/metadata/impl/TagLibTagReader.hpp +++ b/src/libs/metadata/impl/TagLibTagReader.hpp @@ -48,7 +48,7 @@ namespace MetaData std::size_t getBitsPerSample() const override; std::size_t getSampleRate() const override; - TagLib::FileRef _file; + const TagLib::FileRef _file; TagLib::PropertyMap _propertyMap; // case-insensitive keys bool _hasEmbeddedCover{}; }; diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp index 622ed495..2e2c7eb3 100644 --- a/src/libs/services/scanner/impl/ScanStepScanFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp @@ -34,6 +34,7 @@ #include "utils/IConfig.hpp" #include "utils/ILogger.hpp" #include "utils/Path.hpp" +#include "utils/IProfiler.hpp" namespace Scanner { @@ -188,7 +189,7 @@ namespace Scanner { std::vector clusters; - auto getOrCreateClusters {[&](std::string tag, std::span values) + auto getOrCreateClusters{ [&](std::string tag, std::span values) { auto clusterType = ClusterType::find(session, tag); if (!clusterType) @@ -202,7 +203,7 @@ namespace Scanner clusters.push_back(cluster); } - }}; + } }; // TODO: migrate these fields in dedicated tables in DB getOrCreateClusters("GENRE", track.genres); @@ -236,14 +237,15 @@ namespace Scanner if (threadCount == 0) threadCount = std::max(std::thread::hardware_concurrency() / 2, 1); - + return threadCount; } } // namespace - ScanStepScanFiles::MetadataScanQueue::MetadataScanQueue(MetaData::IParser& parser, std::size_t threadCount) + ScanStepScanFiles::MetadataScanQueue::MetadataScanQueue(MetaData::IParser& parser, std::size_t threadCount, bool& abort) : _metadataParser{ parser } , _scanContextRunner{ _scanContext, threadCount } + , _abort{ abort } {} void ScanStepScanFiles::MetadataScanQueue::pushScanRequest(const std::filesystem::path& path) @@ -255,23 +257,34 @@ namespace Scanner _scanContext.post([=, this] { + LMS_SCOPED_PROFILE_OVERVIEW("Scanner", "AudioFileParseJob"); + std::unique_ptr track; - try - { - track = _metadataParser.parse(path); - } - catch(const MetaData::Exception& e) - { - LMS_LOG(DBUPDATER, INFO, "Failed to parse '" << path.string() << "'"); - } - + if (_abort) { std::scoped_lock lock{ _mutex }; - - _scanResults.emplace_back(MetaDataScanResult{ std::move(path), std::move(track) }); _ongoingScanCount -= 1; } + else + { + try + { + track = _metadataParser.parse(path); + } + catch (const MetaData::Exception& e) + { + LMS_LOG(DBUPDATER, INFO, "Failed to parse '" << path.string() << "'"); + } + + { + std::scoped_lock lock{ _mutex }; + + _scanResults.emplace_back(MetaDataScanResult{ std::move(path), std::move(track) }); + _ongoingScanCount -= 1; + } + } + _condVar.notify_all(); }); } @@ -286,10 +299,10 @@ namespace Scanner { results.clear(); results.reserve(maxCount); - + { std::scoped_lock lock{ _mutex }; - + while (results.size() < maxCount && !_scanResults.empty()) { results.push_back(std::move(_scanResults.front())); @@ -302,22 +315,24 @@ namespace Scanner void ScanStepScanFiles::MetadataScanQueue::wait(std::size_t maxScanRequestCount) { + LMS_SCOPED_PROFILE_OVERVIEW("Scanner", "WaitParseResults"); + std::unique_lock lock{ _mutex }; - _condVar.wait(lock, [=, this] {return _ongoingScanCount <= maxScanRequestCount;}); + _condVar.wait(lock, [=, this] { return _ongoingScanCount <= maxScanRequestCount; }); } ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams) : ScanStepBase{ initParams } , _metadataParser{ MetaData::createParser(MetaData::ParserBackend::TagLib, getParserReadStyle()) } // For now, always use TagLib - , _metadataScanQueue{ *_metadataParser, getScanMetaDataThreadCount() } + , _metadataScanQueue{ *_metadataParser, getScanMetaDataThreadCount(), _abortScan } { LMS_LOG(DBUPDATER, INFO, "Using " << _metadataScanQueue.getThreadCount() << " thread(s) for scanning file metadata"); } void ScanStepScanFiles::process(ScanContext& context) { - const std::size_t scanQueueMaxScanRequestCount{ 20 * _metadataScanQueue.getThreadCount() }; - const std::size_t processMetaDataBatchSize{ 10 }; + const std::size_t scanQueueMaxScanRequestCount{ 100 * _metadataScanQueue.getThreadCount() }; + const std::size_t processMetaDataBatchSize{ 5 }; { std::vector tagsToParse{ _extraTagsToParse }; @@ -334,6 +349,8 @@ namespace Scanner { PathUtils::exploreFilesRecursive(mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) { + LMS_SCOPED_PROFILE_DETAILED("Scanner", "OnExploreFile"); + if (_abortScan) return false; @@ -367,7 +384,7 @@ namespace Scanner while (_metadataScanQueue.popResults(scanResults, processMetaDataBatchSize) > 0) processMetaDataScanResults(context, scanResults, mediaLibrary); } - } + } bool ScanStepScanFiles::checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo) { @@ -427,6 +444,8 @@ namespace Scanner void ScanStepScanFiles::processMetaDataScanResults(ScanContext& context, std::span scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo) { + LMS_SCOPED_PROFILE_OVERVIEW("Scanner", "ProcessScanResults"); + Database::Session& dbSession{ _db.getTLSSession() }; auto transaction{ dbSession.createWriteTransaction() }; diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp index 0669c418..23fbe661 100644 --- a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp @@ -58,7 +58,7 @@ namespace Scanner class MetadataScanQueue { public: - MetadataScanQueue(MetaData::IParser& parser, std::size_t threadCount); + MetadataScanQueue(MetaData::IParser& parser, std::size_t threadCount, bool& abort); std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); } @@ -78,6 +78,7 @@ namespace Scanner std::size_t _ongoingScanCount{}; std::deque _scanResults; std::condition_variable _condVar; + bool& _abort; }; MetadataScanQueue _metadataScanQueue;