Significantly improved scanner performance (now have a dedicated pool of threads for scanning file metadata)

2024-02-10 18:39:55 +01:00
parent de4a0cf583
commit 6b1fd9c841
5 changed files with 261 additions and 98 deletions
@@ -19,11 +19,16 @@

 #pragma once

+#include <condition_variable>
+#include <deque>
 #include <filesystem>
+#include <mutex>
+#include <span>
 #include <string>
 #include <vector>

 #include "metadata/IParser.hpp"
+#include "utils/IOContextRunner.hpp"
 #include "ScanStepBase.hpp"

 namespace Scanner
@@ -38,9 +43,44 @@ namespace Scanner
        std::string_view getStepName() const override { return "Scanning files"; }
        void process(ScanContext& context) override;

-        void scanAudioFile(const std::filesystem::path& file, ScanContext& context, const ScannerSettings::MediaLibraryInfo& libraryInfo);
+        bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);
+        struct MetaDataScanResult
+        {
+            std::filesystem::path path;
+            std::optional<MetaData::Track> trackMetaData;
+        };
+        void processMetaDataScanResults(ScanContext& context, std::span<const std::unique_ptr<MetaDataScanResult>> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo);
+        void processFileMetaData(ScanContext& context, const std::filesystem::path& file, const MetaData::Track& trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo);

        std::unique_ptr<MetaData::IParser>  _metadataParser;
        const std::vector<std::string>      _extraTagsToParse{ "GENRE", "MOOD", "LANGUAGE", "ALBUMGROUPING" };
+
+        class MetadataScanQueue
+        {
+            public:
+                MetadataScanQueue(MetaData::IParser& parser, std::size_t threadCount);
+
+                std::size_t getThreadCount() const { return _scanContextRunner.getThreadCount(); }
+
+                void pushScanRequest(const std::filesystem::path path);
+
+                std::size_t getResultsCount() const;
+                size_t popResults(std::vector<std::unique_ptr<MetaDataScanResult>>& results, std::size_t maxCount);
+
+                void wait(std::size_t maxScanRequestCount = 0); // wait until ongoing scan request count <= maxScanRequestCount
+
+            private:
+                MetaData::IParser& _metadataParser;
+                boost::asio::io_context _scanContext;
+                IOContextRunner _scanContextRunner;
+
+                mutable std::mutex _mutex ;
+                std::size_t _ongoingScanCount{};
+                std::deque<std::unique_ptr<MetaDataScanResult>> _scanResults;
+                std::condition_variable _condVar;
+        };
+        MetadataScanQueue _metadataScanQueue;
+
+        std::deque<std::unique_ptr<MetaDataScanResult>> _metaDataScanResults;
    };
 }