Changed db optimization strategy: now perform a full analyze (only after scan if enough changes has been made, and during application startup). Also perform a vacuum if needed during startup. Added an init screen for the web interface since it can take a while to complete

approot/messages.xml

@@ -105,6 +105,7 @@
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Not scheduled</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Scheduled on {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">Scanning: step {1}/{2}</message>
+<message id="Lms.Admin.ScannerController.step-analyze">Analyzing database... {1}/{2} entries ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Checking for duplicate files... {1} files</message>
 <message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Checking files... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Computing stats... {1}%</message>

approot/messages_fr.xml

@@ -105,6 +105,7 @@
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Non planifié</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Planifié le {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">En cours de scan : étape {1}/{2}</message>
+<message id="Lms.Admin.ScannerController.step-analyze">Analyze de la base de données... {1}/{2} entrées ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Vérification des fichiers dupliqués... {1} fichiers</message>
 <message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Vérification des fichiers... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calcul des statistiques... {1}%</message>

approot/messages_it.xml

@@ -105,6 +105,7 @@
 <message id="Lms.Admin.ScannerController.status-not-scheduled">Non pianificato</message>
 <message id="Lms.Admin.ScannerController.status-scheduled">Pianificato il {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">Scansione: passo {1}/{2}</message>
+<message id="Lms.Admin.ScannerController.step-analyze">Analisi del database... {1}/{2} voci ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Controllo duplicati... {1} files</message>
 <message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Controllo file... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calcolo statistiche... {1}%</message>

approot/messages_zh.xml

@@ -106,6 +106,7 @@
 <message id="Lms.Admin.ScannerController.status-scheduled">计划于 {1}</message>
 <message id="Lms.Admin.ScannerController.status-in-progress">扫描中: 阶段 {1}/{2}</message>
 
+
 <message id="Lms.Admin.ScannerController.step-checking-for-missing-files">检查文件中... {1}%</message>
 
 <message id="Lms.Admin.ScannerController.step-discovering-files">检索文件中: {1} 文件</message>

src/libs/core/include/core/LiteralString.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include <cstddef>
+#include <ostream>
 #include <string>
 #include <string_view>
 
@@ -41,6 +42,12 @@ namespace lms::core
     private:
         std::string_view _str;
     };
+
+    inline std::ostream& operator<<(std::ostream& os, const LiteralString& str)
+    {
+        os << str.str();
+        return os;
+    }
 }
 
 namespace std

src/libs/database/impl/Db.cpp

@@ -25,8 +25,9 @@
 #include "database/Session.hpp"
 #include "database/User.hpp"
 #include "core/IConfig.hpp"
-#include "core/Service.hpp"
 #include "core/ILogger.hpp"
+#include "core/ITraceLogger.hpp"
+#include "core/Service.hpp"
 
 namespace lms::db
 {
@@ -49,7 +50,7 @@ namespace lms::db
                 prepare();
             }
 
-            ~Connection()
+            ~Connection() override
             {
                 // make use of per-connection usage stats to optimize
                 optimize();
@@ -66,16 +67,15 @@ namespace lms::db
             void prepare()
             {
                 LMS_LOG(DB, DEBUG, "Setting per-connection settings...");
-                executeSql("pragma journal_mode=WAL");
+                executeSql("PRAGMA journal_mode=WAL");
-                executeSql("pragma synchronous=normal");
+                executeSql("PRAGMA synchronous=normal");
-                executeSql("pragma analysis_limit=2000"); // to help make analyze command faster, 1000 does not seem to be enough to speed up all queries
                 LMS_LOG(DB, DEBUG, "Setting per-connection settings done!");
             }
 
             void optimize()
             {
                 LMS_LOG(DB, DEBUG, "connection close: Running pragma optimize...");
-                executeSql("pragma optimize");
+                executeSql("PRAGMA optimize");
                 LMS_LOG(DB, DEBUG, "connection close: pragma optimize complete");
             }
 

src/libs/database/impl/Session.cpp

@@ -46,6 +46,7 @@
 #include "EnumSetTraits.hpp"
 #include "PathTraits.hpp"
 #include "Migration.hpp"
+#include "Utils.hpp"
 
 namespace lms::db
 {
@@ -120,7 +121,7 @@ namespace lms::db
         return ReadTransaction{ _session };
     }
 
-    void Session::prepareTables()
+    void Session::prepareTablesIfNeeded()
     {
         LMS_LOG(DB, INFO, "Preparing tables...");
 
@@ -140,127 +141,165 @@ namespace lms::db
                 throw e;
             }
         }
+    }
 
+    void Session::migrateIfNeeded()
+    {
         Migration::doDbMigration(*this);
 
-        // Indexes
+        // TODO: move this elsewhere
-        {
-            LMS_SCOPED_TRACE_OVERVIEW("Database", "IndexCreation");
-
-            auto transaction{ createWriteTransaction() };
-            _session.execute("CREATE INDEX IF NOT EXISTS artist_id_idx ON artist(id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
-            _session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)");
-            _session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS auth_token_user_idx ON auth_token(user_id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS auth_token_expiry_idx ON auth_token(expiry)");
-            _session.execute("CREATE INDEX IF NOT EXISTS auth_token_value_idx ON auth_token(value)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)");
-            _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)");
-            _session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS listen_user_backend_idx ON listen(user_id,backend)");
-            _session.execute("CREATE INDEX IF NOT EXISTS listen_track_user_backend_idx ON listen(track_id,user_id,backend)");
-            _session.execute("CREATE INDEX IF NOT EXISTS listen_user_track_backend_date_time_idx ON listen(user_id,track_id,backend,date_time)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS release_id_idx ON release(id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
-            _session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)");
-            _session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)");
-            _session.execute("CREATE INDEX IF NOT EXISTS release_type_name_idx ON release_type(name)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS track_id_idx ON track(id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_absolute_path_idx ON track(absolute_file_path)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_recording_mbid_idx ON track(recording_mbid)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_file_last_write_idx ON track(file_last_write)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_date_idx ON track(date)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_original_date_idx ON track(original_date)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS track_media_library_idx ON track(media_library_id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_media_library_release_idx ON track(media_library_id, release_id)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)");
-            _session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_idx ON track_artist_link(artist_id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_type_idx ON track_artist_link(artist_id,type)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_idx ON track_artist_link(track_id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_type_idx ON track_artist_link(track_id,type)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_type_idx ON track_artist_link(type)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_idx ON track_bookmark(user_id)");
-            _session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_track_idx ON track_bookmark(user_id,track_id)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS starred_artist_user_backend_idx ON starred_artist(user_id,backend)");
-            _session.execute("CREATE INDEX IF NOT EXISTS starred_artist_artist_user_backend_idx ON starred_artist(artist_id,user_id,backend)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS starred_release_user_backend_idx ON starred_release(user_id,backend)");
-            _session.execute("CREATE INDEX IF NOT EXISTS starred_release_release_user_backend_idx ON starred_release(release_id,user_id,backend)");
-
-            _session.execute("CREATE INDEX IF NOT EXISTS starred_track_user_backend_idx ON starred_track(user_id,backend)");
-            _session.execute("CREATE INDEX IF NOT EXISTS starred_track_track_user_backend_idx ON starred_track(track_id,user_id,backend)");
-        }
-
-        // Singletons
         {
             auto uniqueTransaction{ createWriteTransaction() };
             ScanSettings::init(*this);
         }
     }
 
+    void Session::createIndexesIfNeeded()
+    {
+        LMS_SCOPED_TRACE_OVERVIEW("Database", "IndexCreation");
+
+        auto transaction{ createWriteTransaction() };
+        _session.execute("CREATE INDEX IF NOT EXISTS artist_id_idx ON artist(id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
+        _session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)");
+        _session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS auth_token_user_idx ON auth_token(user_id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS auth_token_expiry_idx ON auth_token(expiry)");
+        _session.execute("CREATE INDEX IF NOT EXISTS auth_token_value_idx ON auth_token(value)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)");
+        _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)");
+        _session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS listen_user_backend_idx ON listen(user_id,backend)");
+        _session.execute("CREATE INDEX IF NOT EXISTS listen_track_user_backend_idx ON listen(track_id,user_id,backend)");
+        _session.execute("CREATE INDEX IF NOT EXISTS listen_user_track_backend_date_time_idx ON listen(user_id,track_id,backend,date_time)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS release_id_idx ON release(id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
+        _session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)");
+        _session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)");
+        _session.execute("CREATE INDEX IF NOT EXISTS release_type_name_idx ON release_type(name)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS track_id_idx ON track(id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_absolute_path_idx ON track(absolute_file_path)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_recording_mbid_idx ON track(recording_mbid)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_file_last_write_idx ON track(file_last_write)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_date_idx ON track(date)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_original_date_idx ON track(original_date)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS track_media_library_idx ON track(media_library_id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_media_library_release_idx ON track(media_library_id, release_id)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)");
+        _session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_idx ON track_artist_link(artist_id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_type_idx ON track_artist_link(artist_id,type)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_idx ON track_artist_link(track_id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_type_idx ON track_artist_link(track_id,type)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_type_idx ON track_artist_link(type)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_idx ON track_bookmark(user_id)");
+        _session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_track_idx ON track_bookmark(user_id,track_id)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS starred_artist_user_backend_idx ON starred_artist(user_id,backend)");
+        _session.execute("CREATE INDEX IF NOT EXISTS starred_artist_artist_user_backend_idx ON starred_artist(artist_id,user_id,backend)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS starred_release_user_backend_idx ON starred_release(user_id,backend)");
+        _session.execute("CREATE INDEX IF NOT EXISTS starred_release_release_user_backend_idx ON starred_release(release_id,user_id,backend)");
+
+        _session.execute("CREATE INDEX IF NOT EXISTS starred_track_user_backend_idx ON starred_track(user_id,backend)");
+        _session.execute("CREATE INDEX IF NOT EXISTS starred_track_track_user_backend_idx ON starred_track(track_id,user_id,backend)");
+    }
+
+    void Session::vacuumIfNeeded()
+    {
+        long pageCount{};
+        long freeListCount{};
+
+        {
+            auto transaction{ createReadTransaction() };
+            pageCount = utils::fetchQuerySingleResult(_session.query<long>("SELECT page_count FROM pragma_page_count"));
+            freeListCount = utils::fetchQuerySingleResult(_session.query<long>("SELECT freelist_count FROM pragma_freelist_count"));
+        }
+
+        LMS_LOG(DB, INFO, "page stats: page_count = " << pageCount << ", freelist_count = " << freeListCount);
+        if (freeListCount >= (pageCount / 10))
+        {
+            LMS_SCOPED_TRACE_OVERVIEW("Database", "Vacuum");
+            LMS_LOG(DB, INFO, "Performing vacuum... This may take a while...");
+
+            // We manually take a lock here since vacuum cannot be inside a transaction
+            {
+                std::unique_lock lock{ _db.getMutex() };
+                _db.executeSql("VACUUM");
+            }
+
+            LMS_LOG(DB, INFO, "Vacuum complete!");
+        }
+    }
+
     void Session::refreshTracingLoggerStats()
     {
         auto* traceLogger{ core::Service<core::tracing::ITraceLogger>::get() };
         if (!traceLogger)
             return;
 
-        auto& dbSession{ _db.getTLSSession() };
+        auto transaction{ createReadTransaction() };
-        auto transaction{ dbSession.createReadTransaction() };
 
-        traceLogger->setMetadata("db_artist_count", std::to_string(db::Artist::getCount(dbSession)));
+        traceLogger->setMetadata("db_artist_count", std::to_string(db::Artist::getCount(*this)));
-        traceLogger->setMetadata("db_cluster_count", std::to_string(db::Cluster::getCount(dbSession)));
+        traceLogger->setMetadata("db_cluster_count", std::to_string(db::Cluster::getCount(*this)));
-        traceLogger->setMetadata("db_cluster_type_count", std::to_string(db::ClusterType::getCount(dbSession)));
+        traceLogger->setMetadata("db_cluster_type_count", std::to_string(db::ClusterType::getCount(*this)));
-        traceLogger->setMetadata("db_starred_artist_count", std::to_string(db::StarredArtist::getCount(dbSession)));
+        traceLogger->setMetadata("db_starred_artist_count", std::to_string(db::StarredArtist::getCount(*this)));
-        traceLogger->setMetadata("db_starred_release_count", std::to_string(db::StarredRelease::getCount(dbSession)));
+        traceLogger->setMetadata("db_starred_release_count", std::to_string(db::StarredRelease::getCount(*this)));
-        traceLogger->setMetadata("db_starred_track_count", std::to_string(db::StarredTrack::getCount(dbSession)));
+        traceLogger->setMetadata("db_starred_track_count", std::to_string(db::StarredTrack::getCount(*this)));
-        traceLogger->setMetadata("db_track_bookmark_count", std::to_string(db::TrackBookmark::getCount(dbSession)));
+        traceLogger->setMetadata("db_track_bookmark_count", std::to_string(db::TrackBookmark::getCount(*this)));
-        traceLogger->setMetadata("db_listen_count", std::to_string(db::Listen::getCount(dbSession)));
+        traceLogger->setMetadata("db_listen_count", std::to_string(db::Listen::getCount(*this)));
-        traceLogger->setMetadata("db_release_count", std::to_string(db::Release::getCount(dbSession)));
+        traceLogger->setMetadata("db_release_count", std::to_string(db::Release::getCount(*this)));
-        traceLogger->setMetadata("db_track_count", std::to_string(db::Track::getCount(dbSession)));
+        traceLogger->setMetadata("db_track_count", std::to_string(db::Track::getCount(*this)));
     }
 
-    void Session::analyze()
+    void Session::fullAnalyze()
     {
-        LMS_SCOPED_TRACE_DETAILED("Database", "Analyze");
+        LMS_SCOPED_TRACE_OVERVIEW("Database", "Analyze");
-        LMS_LOG(DB, INFO, "Analyzing database...");
+        LMS_LOG(DB, INFO, "Performing database analyze... This may take a while...");
+
+        // first select all the tables and indexes, and then analyze one by one in order to not have a big lock
+        std::vector<std::string> entries;
+        retrieveEntriesToAnalyze(entries);
+
+        for (const std::string& entry : entries)
+            analyzeEntry(entry);
+
+        LMS_LOG(DB, INFO, "Analyze complete!");
+    }
+
+    void Session::retrieveEntriesToAnalyze(std::vector<std::string>& entryList)
+    {
+        auto transaction{ createReadTransaction() };
+        entryList = utils::fetchQueryResults(_session.query<std::string>("SELECT name FROM sqlite_master WHERE type='table' OR type ='index'"));
+    }
+
+    void Session::analyzeEntry(const std::string& entry)
+    {
+        LMS_SCOPED_TRACE_DETAILED_WITH_ARG("Database", "AnalyzeEntry", "Entry", entry);
+        LMS_LOG(DB, DEBUG, "Analyzing " << entry);
         {
             auto transaction{ createWriteTransaction() };
-            _session.execute("ANALYZE");
+            _session.execute("ANALYZE " + entry);
         }
-        LMS_LOG(DB, INFO, "Database Analyze complete");
+        LMS_LOG(DB, DEBUG, "Analyzing " << entry << ": done!");
     }
-
-    void Session::optimize()
-    {
-        LMS_SCOPED_TRACE_DETAILED("Database", "Optimize");
-        LMS_LOG(DB, INFO, "Optimizing database...");
-        {
-            auto transaction{ createWriteTransaction() };
-            _session.execute("PRAGMA optimize");
-        }
-        LMS_LOG(DB, INFO, "Database optimizing complete");
-    }
-
 } // namespace lms::db

src/libs/database/include/database/Session.hpp

@@ -22,6 +22,8 @@
 #include <Wt/Dbo/Dbo.h>
 #include <Wt/Dbo/SqlConnectionPool.h>
 
+#include <string>
+#include <vector>
 #include "core/ITraceLogger.hpp"
 #include "core/RecursiveSharedMutex.hpp"
 #include "database/Object.hpp"
@@ -84,10 +86,15 @@ namespace lms::db
 #endif
         }
 
-        void analyze();
+        // All these methods will acquire transactions
-        void optimize();
+        void fullAnalyze(); // helper for retrieveEntriesToAnalyze + analyzeEntry
+        void retrieveEntriesToAnalyze(std::vector<std::string>& entryList);
+        void analyzeEntry(const std::string& entry);
 
-        void prepareTables(); // need to run only once at startup
+        void prepareTablesIfNeeded(); // need to run only once at startup
+        void migrateIfNeeded();
+        void createIndexesIfNeeded();
+        void vacuumIfNeeded();
         void refreshTracingLoggerStats();
 
         // returning a ptr here to ease further wrapping using operator->

src/libs/database/test/Common.cpp

@@ -60,8 +60,8 @@ namespace lms::db::tests
         _tmpDb = std::make_unique<TmpDatabase>();
         {
             db::Session s{ _tmpDb->getDb() };
-            s.prepareTables();
+            s.prepareTablesIfNeeded();
-            s.analyze();
+            s.createIndexesIfNeeded();
         }
     }
 

src/libs/services/scanner/CMakeLists.txt

@@ -2,6 +2,7 @@
 add_library(lmsscanner SHARED
 	impl/ScannerService.cpp
 	impl/ScannerStats.cpp
+	impl/ScanStepAnalyze.cpp
 	impl/ScanStepCheckDuplicatedDbFiles.cpp
 	impl/ScanStepComputeClusterStats.cpp
 	impl/ScanStepDiscoverFiles.cpp

src/libs/services/scanner/impl/IScanStep.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
-#include <string_view>
 #include <vector>
 
+#include "core/LiteralString.hpp"
 #include "services/scanner/ScannerStats.hpp"
 
 namespace lms::scanner
@@ -32,7 +32,7 @@ namespace lms::scanner
         virtual ~IScanStep() = default;
 
         virtual ScanStep getStep() const = 0;
-        virtual std::string_view getStepName() const = 0;
+        virtual core::LiteralString getStepName() const = 0;
 
         struct ScanContext
         {

src/libs/services/scanner/impl/ScanStepAnalyze.cpp

@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepAnalyze.hpp"
+
+#include "core/ILogger.hpp"
+#include "database/Db.hpp"
+#include "database/Session.hpp"
+
+namespace lms::scanner
+{
+    void ScanStepAnalyze::process(ScanContext& context)
+    {
+        ScanStats& stats{ context.stats };
+
+        if (stats.nbChanges() > (stats.nbFiles() / 5))
+        {
+            LMS_LOG(DBUPDATER, INFO, "Database changed substantially: triggering full analyze");
+
+            auto& session{ _db.getTLSSession() };
+
+            std::vector<std::string> entries;
+            session.retrieveEntriesToAnalyze(entries);
+            context.currentStepStats.totalElems = entries.size();
+
+            for (const std::string& entry : entries)
+            {
+                if (_abortScan)
+                    break;
+
+                _db.getTLSSession().analyzeEntry(entry);
+                context.currentStepStats.processedElems++;
+                _progressCallback(context.currentStepStats);
+            }
+        }
+    }
+}

src/libs/services/scanner/impl/ScanStepAnalyze.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+	class ScanStepAnalyze : public ScanStepBase
+	{
+		public:
+			using ScanStepBase::ScanStepBase;
+
+		private:
+			ScanStep getStep() const override { return ScanStep::Analyze; }
+			core::LiteralString getStepName() const override { return "Analyze"; }
+			void process(ScanContext& context) override;
+	};
+}

src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp

@@ -29,7 +29,7 @@ namespace lms::scanner
 			using ScanStepBase::ScanStepBase;
 
 		private:
-			std::string_view getStepName() const override { return "Checking for duplicated files"; }
+			core::LiteralString getStepName() const override { return "Checking for duplicated files"; }
 			ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; }
 			void process(ScanContext& context) override;
 	};

src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp

@@ -79,6 +79,7 @@ namespace lms::scanner
                     }
 
                     context.currentStepStats.processedElems++;
+                    _progressCallback(context.currentStepStats);
                 }
 
                 return true;

src/libs/services/scanner/impl/ScanStepComputeClusterStats.hpp

@@ -30,7 +30,7 @@ namespace lms::scanner
 
     private:
         ScanStep getStep() const override { return ScanStep::ComputeClusterStats; }
-        std::string_view getStepName() const override { return "Compute cluster stats"; }
+        core::LiteralString getStepName() const override { return "Compute cluster stats"; }
         void process(ScanContext& context) override;
     };
 }

src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp

@@ -30,7 +30,7 @@ namespace lms::scanner
 
 		private:
 			ScanStep getStep() const override { return ScanStep::DiscoveringFiles; }
-			std::string_view getStepName() const override { return "DiscoveringFiles"; }
+			core::LiteralString getStepName() const override { return "Discovering files"; }
 			void process(ScanContext& context) override;
 	};
 }

src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp

@@ -31,7 +31,7 @@ namespace lms::scanner
 			using ScanStepBase::ScanStepBase;
 
 		private:
-			std::string_view getStepName() const override { return "Checking orphaned entries"; }
+			core::LiteralString getStepName() const override { return "Checking orphaned entries"; }
 			ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; }
 			void process(ScanContext& context) override;
 

src/libs/services/scanner/impl/ScanStepScanFiles.cpp

@@ -518,10 +518,6 @@ namespace lms::scanner
                 context.stats.scans++;
 
                 processFileMetaData(context, scanResult.path, *scanResult.trackMetaData, libraryInfo);
-
-                // optimize the database during scan (if we import a very large database, it may be too late to do it once at end)
-                if ((context.stats.scans % 1'000) == 0)
-                    _db.getTLSSession().optimize();
             }
             else
             {

src/libs/services/scanner/impl/ScanStepScanFiles.hpp

@@ -40,7 +40,7 @@ namespace lms::scanner
 
     private:
         ScanStep getStep() const override { return ScanStep::ScanningFiles; }
-        std::string_view getStepName() const override { return "Scanning files"; }
+        core::LiteralString getStepName() const override { return "Scanning files"; }
         void process(ScanContext& context) override;
 
         bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);

src/libs/services/scanner/impl/ScannerService.cpp

@@ -25,15 +25,17 @@
 #include "database/TrackFeatures.hpp"
 #include "database/ScanSettings.hpp"
 #include "core/Exception.hpp"
+#include "core/Path.hpp"
 #include "core/IConfig.hpp"
 #include "core/ILogger.hpp"
-#include "core/Path.hpp"
+#include "core/ITraceLogger.hpp"
 
+#include "ScanStepAnalyze.hpp"
 #include "ScanStepCheckDuplicatedDbFiles.hpp"
+#include "ScanStepComputeClusterStats.hpp"
 #include "ScanStepDiscoverFiles.hpp"
 #include "ScanStepRemoveOrphanDbFiles.hpp"
 #include "ScanStepScanFiles.hpp"
-#include "ScanStepComputeClusterStats.hpp"
 
 namespace lms::scanner
 {
@@ -259,6 +261,8 @@ namespace lms::scanner
 
     void ScannerService::scan(bool forceScan)
     {
+        LMS_SCOPED_TRACE_OVERVIEW("Scanner", "Scan");
+
         _events.scanStarted.emit();
 
         {
@@ -278,6 +282,8 @@ namespace lms::scanner
 
         for (auto& scanStep : _scanSteps)
         {
+            LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
+
             LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
             scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() };
 
@@ -289,8 +295,6 @@ namespace lms::scanner
 
         LMS_LOG(DBUPDATER, INFO, "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ",  duplicates = " << stats.duplicates.size());
 
-        _db.getTLSSession().analyze();
-
         if (!_abortScan)
         {
             stats.stopTime = Wt::WDateTime::currentDateTime();
@@ -348,6 +352,7 @@ namespace lms::scanner
         _scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
         _scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
         _scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
+        _scanSteps.push_back(std::make_unique<ScanStepAnalyze>(params));
     }
 
     ScannerSettings ScannerService::readSettings()

src/libs/services/scanner/include/services/scanner/ScannerStats.hpp

@@ -66,8 +66,9 @@ namespace lms::scanner
         FetchingTrackFeatures,
         ReloadingSimilarityEngine,
         ComputeClusterStats,
+        Analyze,
     };
-    static inline constexpr unsigned ScanProgressStepCount{ 7 };
+    static inline constexpr unsigned ScanProgressStepCount{ 8 };
 
     // reduced scan stats
     struct ScanStepStats

src/lms/CMakeLists.txt

@@ -4,6 +4,7 @@ add_executable(lms
 	ui/Auth.cpp
 	ui/LmsApplication.cpp
 	ui/LmsApplicationManager.cpp
+	ui/LmsInitApplication.cpp
 	ui/LmsTheme.cpp
 	ui/MediaPlayer.cpp
 	ui/ModalManager.cpp

src/lms/main.cpp

@@ -40,6 +40,7 @@
 #include "subsonic/SubsonicResource.hpp"
 #include "ui/LmsApplication.hpp"
 #include "ui/LmsApplicationManager.hpp"
+#include "ui/LmsInitApplication.hpp"
 #include "core/IChildProcessManager.hpp"
 #include "core/IConfig.hpp"
 #include "core/IOContextRunner.hpp"
@@ -275,20 +276,34 @@ namespace lms
             Wt::WServer server{ argv[0] };
             server.setServerConfiguration(wtServerArgs.size(), const_cast<char**>(&wtArgv[0]));
 
+            // As initialization can take a while (db migration, analyze, etc.), we bind a temporary init entry point to warn the user
+            server.addEntryPoint(Wt::EntryPointType::Application,
+                [&](const Wt::WEnvironment& env)
+                {
+                    return ui::LmsInitApplication::create(env);
+                });
+
+            LMS_LOG(MAIN, INFO, "Starting init web server...");
+            server.start();
+
             core::IOContextRunner ioContextRunner{ ioContext, getThreadCount(), "Misc" };
 
             // Connection pool size must be twice the number of threads: we have at least 2 io pools with getThreadCount() each and they all may access the database
             db::Db database{ config->getPath("working-dir") / "lms.db", getThreadCount() * 2 };
             {
                 db::Session session{ database };
-                session.prepareTables();
+                session.prepareTablesIfNeeded();
+                session.createIndexesIfNeeded();
+
+                // As this may be quite long, we only do it during startup
+                session.vacuumIfNeeded();
 
                 // force optimize in case scanner aborted during a large import:
-                // queries may be too slow to even be able to relaunch a scan sing the web interface
+                // queries may be too slow to even be able to relaunch a scan using the web interface
-                session.analyze();
+                session.fullAnalyze();
                 database.getTLSSession().refreshTracingLoggerStats();
             }
-
+ 
             ui::LmsApplicationManager appManager;
 
             // Service initialization order is important (reverse-order for deinit)
@@ -327,8 +342,12 @@ namespace lms
             core::Service<feedback::IFeedbackService> feedbackService{ feedback::createFeedbackService(ioContext, database) };
             core::Service<scrobbling::IScrobblingService> scrobblingService{ scrobbling::createScrobblingService(ioContext, database) };
 
-            std::unique_ptr<Wt::WResource> subsonicResource;
+            LMS_LOG(MAIN, INFO, "Stopping init web server...");
+            server.stop();
 
+            server.removeEntryPoint("");
+
+            std::unique_ptr<Wt::WResource> subsonicResource;
             // bind API resources
             if (config->getBool("api-subsonic", true))
             {
@@ -345,7 +364,7 @@ namespace lms
 
             proxyScannerEventsToApplication(*scannerService, server);
 
-            LMS_LOG(MAIN, INFO, "Starting server...");
+            LMS_LOG(MAIN, INFO, "Starting init web server...");
             server.start();
 
             LMS_LOG(MAIN, INFO, "Now running...");

src/lms/ui/LmsInitApplication.cpp

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <Wt/WContainerWidget.h>
+#include <Wt/WText.h>
+
+#include "LmsInitApplication.hpp"
+
+namespace lms::ui
+{
+    LmsInitApplication::LmsInitApplication(const Wt::WEnvironment& env)
+        : Wt::WApplication{ env }
+    {
+        enableUpdates(true);
+        root()->addNew<Wt::WText>("LMS is initializing. This may take a while, please wait...");
+    }
+
+    std::unique_ptr<Wt::WApplication> LmsInitApplication::create(const Wt::WEnvironment& env)
+    {
+        return std::make_unique<LmsInitApplication>(env);
+    }
+} // namespace lms::ui

src/lms/ui/LmsInitApplication.hpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/WApplication.h>
+
+namespace lms::ui
+{
+    class LmsInitApplication : public Wt::WApplication
+    {
+    public:
+        LmsInitApplication(const Wt::WEnvironment& env);
+
+        static std::unique_ptr<Wt::WApplication> create(const Wt::WEnvironment& env);
+    };
+} // namespace lms::ui

src/lms/ui/admin/ScannerController.cpp

@@ -274,6 +274,13 @@ namespace lms::ui
             case scanner::ScanStep::ComputeClusterStats:
                 _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats")
                     .arg(status.currentScanStepStats->progress()));
+
+            case scanner::ScanStep::        Analyze:
+                _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-analyze")
+                    .arg(status.currentScanStepStats->processedElems)
+                    .arg(status.currentScanStepStats->totalElems)
+                    .arg(status.currentScanStepStats->progress()));
+                break;
             }
             break;
         }