Changed db optimization strategy: now perform a full analyze (only after scan if enough changes has been made, and during application startup). Also perform a vacuum if needed during startup. Added an init screen for the web interface since it can take a while to complete

src/libs/services/scanner/CMakeLists.txt

@@ -2,6 +2,7 @@
 add_library(lmsscanner SHARED
 	impl/ScannerService.cpp
 	impl/ScannerStats.cpp
+	impl/ScanStepAnalyze.cpp
 	impl/ScanStepCheckDuplicatedDbFiles.cpp
 	impl/ScanStepComputeClusterStats.cpp
 	impl/ScanStepDiscoverFiles.cpp

src/libs/services/scanner/impl/IScanStep.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
-#include <string_view>
 #include <vector>
 
+#include "core/LiteralString.hpp"
 #include "services/scanner/ScannerStats.hpp"
 
 namespace lms::scanner
@@ -32,7 +32,7 @@ namespace lms::scanner
         virtual ~IScanStep() = default;
 
         virtual ScanStep getStep() const = 0;
-        virtual std::string_view getStepName() const = 0;
+        virtual core::LiteralString getStepName() const = 0;
 
         struct ScanContext
         {

src/libs/services/scanner/impl/ScanStepAnalyze.cpp

@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepAnalyze.hpp"
+
+#include "core/ILogger.hpp"
+#include "database/Db.hpp"
+#include "database/Session.hpp"
+
+namespace lms::scanner
+{
+    void ScanStepAnalyze::process(ScanContext& context)
+    {
+        ScanStats& stats{ context.stats };
+
+        if (stats.nbChanges() > (stats.nbFiles() / 5))
+        {
+            LMS_LOG(DBUPDATER, INFO, "Database changed substantially: triggering full analyze");
+
+            auto& session{ _db.getTLSSession() };
+
+            std::vector<std::string> entries;
+            session.retrieveEntriesToAnalyze(entries);
+            context.currentStepStats.totalElems = entries.size();
+
+            for (const std::string& entry : entries)
+            {
+                if (_abortScan)
+                    break;
+
+                _db.getTLSSession().analyzeEntry(entry);
+                context.currentStepStats.processedElems++;
+                _progressCallback(context.currentStepStats);
+            }
+        }
+    }
+}

src/libs/services/scanner/impl/ScanStepAnalyze.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+	class ScanStepAnalyze : public ScanStepBase
+	{
+		public:
+			using ScanStepBase::ScanStepBase;
+
+		private:
+			ScanStep getStep() const override { return ScanStep::Analyze; }
+			core::LiteralString getStepName() const override { return "Analyze"; }
+			void process(ScanContext& context) override;
+	};
+}

src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp

@@ -29,7 +29,7 @@ namespace lms::scanner
 			using ScanStepBase::ScanStepBase;
 
 		private:
-			std::string_view getStepName() const override { return "Checking for duplicated files"; }
+			core::LiteralString getStepName() const override { return "Checking for duplicated files"; }
 			ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; }
 			void process(ScanContext& context) override;
 	};

src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp

@@ -79,6 +79,7 @@ namespace lms::scanner
                     }
 
                     context.currentStepStats.processedElems++;
+                    _progressCallback(context.currentStepStats);
                 }
 
                 return true;

src/libs/services/scanner/impl/ScanStepComputeClusterStats.hpp

@@ -30,7 +30,7 @@ namespace lms::scanner
 
     private:
         ScanStep getStep() const override { return ScanStep::ComputeClusterStats; }
-        std::string_view getStepName() const override { return "Compute cluster stats"; }
+        core::LiteralString getStepName() const override { return "Compute cluster stats"; }
         void process(ScanContext& context) override;
     };
 }

src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp

@@ -30,7 +30,7 @@ namespace lms::scanner
 
 		private:
 			ScanStep getStep() const override { return ScanStep::DiscoveringFiles; }
-			std::string_view getStepName() const override { return "DiscoveringFiles"; }
+			core::LiteralString getStepName() const override { return "Discovering files"; }
 			void process(ScanContext& context) override;
 	};
 }

src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp

@@ -31,7 +31,7 @@ namespace lms::scanner
 			using ScanStepBase::ScanStepBase;
 
 		private:
-			std::string_view getStepName() const override { return "Checking orphaned entries"; }
+			core::LiteralString getStepName() const override { return "Checking orphaned entries"; }
 			ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; }
 			void process(ScanContext& context) override;
 

src/libs/services/scanner/impl/ScanStepScanFiles.cpp

@@ -518,10 +518,6 @@ namespace lms::scanner
                 context.stats.scans++;
 
                 processFileMetaData(context, scanResult.path, *scanResult.trackMetaData, libraryInfo);
-
-                // optimize the database during scan (if we import a very large database, it may be too late to do it once at end)
-                if ((context.stats.scans % 1'000) == 0)
-                    _db.getTLSSession().optimize();
             }
             else
             {

src/libs/services/scanner/impl/ScanStepScanFiles.hpp

@@ -40,7 +40,7 @@ namespace lms::scanner
 
     private:
         ScanStep getStep() const override { return ScanStep::ScanningFiles; }
-        std::string_view getStepName() const override { return "Scanning files"; }
+        core::LiteralString getStepName() const override { return "Scanning files"; }
         void process(ScanContext& context) override;
 
         bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);

src/libs/services/scanner/impl/ScannerService.cpp

@@ -25,15 +25,17 @@
 #include "database/TrackFeatures.hpp"
 #include "database/ScanSettings.hpp"
 #include "core/Exception.hpp"
+#include "core/Path.hpp"
 #include "core/IConfig.hpp"
 #include "core/ILogger.hpp"
-#include "core/Path.hpp"
+#include "core/ITraceLogger.hpp"
 
+#include "ScanStepAnalyze.hpp"
 #include "ScanStepCheckDuplicatedDbFiles.hpp"
+#include "ScanStepComputeClusterStats.hpp"
 #include "ScanStepDiscoverFiles.hpp"
 #include "ScanStepRemoveOrphanDbFiles.hpp"
 #include "ScanStepScanFiles.hpp"
-#include "ScanStepComputeClusterStats.hpp"
 
 namespace lms::scanner
 {
@@ -259,6 +261,8 @@ namespace lms::scanner
 
     void ScannerService::scan(bool forceScan)
     {
+        LMS_SCOPED_TRACE_OVERVIEW("Scanner", "Scan");
+
         _events.scanStarted.emit();
 
         {
@@ -278,6 +282,8 @@ namespace lms::scanner
 
         for (auto& scanStep : _scanSteps)
         {
+            LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
+
             LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
             scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() };
 
@@ -289,8 +295,6 @@ namespace lms::scanner
 
         LMS_LOG(DBUPDATER, INFO, "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ",  duplicates = " << stats.duplicates.size());
 
-        _db.getTLSSession().analyze();
-
         if (!_abortScan)
         {
             stats.stopTime = Wt::WDateTime::currentDateTime();
@@ -348,6 +352,7 @@ namespace lms::scanner
         _scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
         _scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
         _scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
+        _scanSteps.push_back(std::make_unique<ScanStepAnalyze>(params));
     }
 
     ScannerSettings ScannerService::readSettings()

src/libs/services/scanner/include/services/scanner/ScannerStats.hpp

@@ -66,8 +66,9 @@ namespace lms::scanner
         FetchingTrackFeatures,
         ReloadingSimilarityEngine,
         ComputeClusterStats,
+        Analyze,
     };
-    static inline constexpr unsigned ScanProgressStepCount{ 7 };
+    static inline constexpr unsigned ScanProgressStepCount{ 8 };
 
     // reduced scan stats
     struct ScanStepStats