Moved code to scan files in parallel at upper level to ease reuse

2025-07-02 21:18:04 +02:00
parent c9395fefa5
commit 376316da14
18 changed files with 467 additions and 210 deletions
@@ -10,6 +10,7 @@ add_library(lmscore STATIC
 	impl/ChildProcessManager.cpp
 	impl/Config.cpp
 	impl/FileResourceHandler.cpp
+	impl/JobScheduler.cpp
 	impl/IOContextRunner.cpp
 	impl/Logger.cpp
 	impl/MimeTypes.cpp
@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2025 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "JobScheduler.hpp"
+
+#include <boost/asio/post.hpp>
+
+#include "core/ITraceLogger.hpp"
+
+#include "core/IJob.hpp"
+
+namespace lms::core
+{
+    std::unique_ptr<IJobScheduler> createJobScheduler(core::LiteralString name, std::size_t threadCount)
+    {
+        return std::make_unique<JobScheduler>(name, threadCount);
+    }
+
+    JobScheduler::JobScheduler(core::LiteralString name, std::size_t threadCount)
+        : _name{ name }
+        , _ioContextRunner{ _ioContext, threadCount, name.str() }
+    {
+    }
+
+    JobScheduler::~JobScheduler() = default;
+
+    void JobScheduler::setShouldAbortCallback(ShouldAbortCallback callback)
+    {
+        _abortCallback = callback;
+    }
+
+    std::size_t JobScheduler::getThreadCount() const
+    {
+        return _ioContextRunner.getThreadCount();
+    }
+
+    void JobScheduler::scheduleJob(std::unique_ptr<IJob> job)
+    {
+        {
+            std::scoped_lock lock{ _mutex };
+            _ongoingJobCount += 1;
+        }
+
+        auto jobHandler{ [job = std::move(job), this]() mutable {
+            if (_abortCallback && _abortCallback())
+            {
+                std::scoped_lock lock{ _mutex };
+                _ongoingJobCount -= 1;
+            }
+            else
+            {
+                {
+                    LMS_SCOPED_TRACE_OVERVIEW(_name, job->getName());
+                    job->run();
+                }
+
+                {
+                    std::scoped_lock lock{ _mutex };
+
+                    _doneJobs.emplace_back(std::move(job));
+                    _ongoingJobCount -= 1;
+                }
+            }
+
+            _condVar.notify_all();
+        } };
+
+        boost::asio::post(_ioContext, std::move(jobHandler));
+    }
+
+    std::size_t JobScheduler::getJobsDoneCount() const
+    {
+        std::scoped_lock lock{ _mutex };
+        return _doneJobs.size();
+    }
+
+    size_t JobScheduler::popJobsDone(std::vector<std::unique_ptr<IJob>>& doneJobs, std::size_t maxCount)
+    {
+        doneJobs.clear();
+        doneJobs.reserve(maxCount);
+
+        {
+            std::scoped_lock lock{ _mutex };
+
+            while (doneJobs.size() < maxCount && !_doneJobs.empty())
+            {
+                doneJobs.push_back(std::move(_doneJobs.front()));
+                _doneJobs.pop_front();
+            }
+        }
+
+        return doneJobs.size();
+    }
+
+    void JobScheduler::waitUntilJobCountAtMost(std::size_t maxOngoingJobs)
+    {
+        if (_ongoingJobCount <= maxOngoingJobs)
+            return;
+
+        {
+            LMS_SCOPED_TRACE_OVERVIEW(_name, "WaitJobs");
+
+            std::unique_lock lock{ _mutex };
+            _condVar.wait(lock, [=, this] { return _ongoingJobCount <= maxOngoingJobs; });
+        }
+    }
+
+    void JobScheduler::wait()
+    {
+        waitUntilJobCountAtMost(0);
+    }
+
+} // namespace lms::core
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2025 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <atomic>
+#include <condition_variable>
+#include <deque>
+#include <mutex>
+
+#include "core/IJobScheduler.hpp"
+#include "core/IOContextRunner.hpp"
+
+namespace lms::core
+{
+    class JobScheduler : public IJobScheduler
+    {
+    public:
+        JobScheduler(core::LiteralString name, std::size_t threadCount);
+        ~JobScheduler() override;
+        JobScheduler(const JobScheduler&) = delete;
+        JobScheduler& operator=(const JobScheduler&) = delete;
+
+    private:
+        void setShouldAbortCallback(ShouldAbortCallback callback) override;
+
+        std::size_t getThreadCount() const override;
+        void scheduleJob(std::unique_ptr<IJob> job) override;
+
+        std::size_t getJobsDoneCount() const override;
+        size_t popJobsDone(std::vector<std::unique_ptr<IJob>>& jobs, std::size_t maxCount) override;
+
+        void waitUntilJobCountAtMost(std::size_t maxOngoingJobs) override;
+        void wait() override;
+
+        core::LiteralString _name;
+        boost::asio::io_context _ioContext;
+        core::IOContextRunner _ioContextRunner;
+
+        ShouldAbortCallback _abortCallback;
+
+        mutable std::mutex _mutex;
+        std::atomic<std::size_t> _ongoingJobCount;
+        std::deque<std::unique_ptr<IJob>> _doneJobs;
+        std::condition_variable _condVar;
+    };
+} // namespace lms::core
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2025 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "core/LiteralString.hpp"
+
+namespace lms::core
+{
+    class IJob
+    {
+    public:
+        virtual ~IJob() = default;
+
+        virtual LiteralString getName() const = 0;
+        virtual void run() = 0;
+    };
+} // namespace lms::core
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2025 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "core/LiteralString.hpp"
+
+namespace lms::core
+{
+    class IJob;
+    class IJobScheduler
+    {
+    public:
+        virtual ~IJobScheduler() = default;
+
+        using ShouldAbortCallback = std::function<bool()>;
+        virtual void setShouldAbortCallback(ShouldAbortCallback callback) = 0;
+
+        virtual std::size_t getThreadCount() const = 0;
+        virtual void scheduleJob(std::unique_ptr<IJob> job) = 0;
+
+        virtual std::size_t getJobsDoneCount() const = 0;
+        virtual size_t popJobsDone(std::vector<std::unique_ptr<IJob>>& jobs, std::size_t maxCount) = 0;
+
+        virtual void waitUntilJobCountAtMost(std::size_t maxOngoingJobs) = 0;
+        virtual void wait() = 0;
+    };
+
+    std::unique_ptr<IJobScheduler> createJobScheduler(core::LiteralString name, std::size_t threadCount);
+} // namespace lms::core
@@ -2,6 +2,7 @@ include(GoogleTest)

 add_executable(test-core
 	EnumSet.cpp
+	JobScheduler.cpp
 	LiteralString.cpp
 	PartialDateTime.cpp
 	Path.cpp
@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2025 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <gtest/gtest.h>
+
+#include <thread>
+
+#include "core/IJob.hpp"
+#include "core/IJobScheduler.hpp"
+
+namespace lms::core
+{
+    namespace
+    {
+        class TestJob : public IJob
+        {
+        public:
+            TestJob(std::atomic<std::size_t>& count)
+                : workCount(count) {}
+
+        private:
+            LiteralString getName() const override { return "test"; };
+            void run() override
+            {
+                // Simulate some work
+                std::this_thread::sleep_for(std::chrono::milliseconds(10));
+                ++workCount;
+            }
+
+            std::atomic<std::size_t>& workCount;
+        };
+
+    } // namespace
+    TEST(JobScheduler, basic)
+    {
+        std::atomic<std::size_t> workCount{ 0 };
+
+        auto scheduler{ createJobScheduler("TestScheduler", 2) };
+        ASSERT_NE(scheduler, nullptr);
+
+        for (int i = 0; i < 10; ++i)
+            scheduler->scheduleJob(std::make_unique<TestJob>(workCount));
+
+        // Wait for all jobs to complete
+        scheduler->wait();
+        EXPECT_EQ(workCount.load(), 10);
+
+        std::vector<std::unique_ptr<IJob>> doneJobs;
+        scheduler->popJobsDone(doneJobs, 10);
+        EXPECT_EQ(doneJobs.size(), 10);
+    }
+
+    TEST(JobScheduler, abort)
+    {
+        std::atomic<std::size_t> workCount{ 0 };
+
+        auto scheduler{ createJobScheduler("TestScheduler", 2) };
+        ASSERT_NE(scheduler, nullptr);
+
+        scheduler->setShouldAbortCallback([] { return true; });
+
+        for (int i = 0; i < 10; ++i)
+            scheduler->scheduleJob(std::make_unique<TestJob>(workCount));
+
+        // Wait for all jobs to complete
+        scheduler->wait();
+        EXPECT_EQ(workCount.load(), 0); // nothing was done
+
+        std::vector<std::unique_ptr<IJob>> doneJobs;
+        scheduler->popJobsDone(doneJobs, 10);
+        EXPECT_EQ(doneJobs.size(), 0);
+    }
+
+} // namespace lms::core