diff --git a/approot/messages.xml b/approot/messages.xml index e78a91e9..c494f326 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -105,6 +105,7 @@ Not scheduled Scheduled on {1} Scanning: step {1}/{2} +Analyzing database... {1}/{2} entries ({3}%)... Checking for duplicate files... {1} files Checking files... {1}% Computing stats... {1}% diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index b0627408..797355a1 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -105,6 +105,7 @@ Non planifié Planifié le {1} En cours de scan : étape {1}/{2} +Analyze de la base de données... {1}/{2} entrées ({3}%)... Vérification des fichiers dupliqués... {1} fichiers Vérification des fichiers... {1}% Calcul des statistiques... {1}% diff --git a/approot/messages_it.xml b/approot/messages_it.xml index aa675ec5..640b1091 100644 --- a/approot/messages_it.xml +++ b/approot/messages_it.xml @@ -105,6 +105,7 @@ Non pianificato Pianificato il {1} Scansione: passo {1}/{2} +Analisi del database... {1}/{2} voci ({3}%)... Controllo duplicati... {1} files Controllo file... {1}% Calcolo statistiche... {1}% diff --git a/approot/messages_zh.xml b/approot/messages_zh.xml index 3a89f689..6a8ac377 100644 --- a/approot/messages_zh.xml +++ b/approot/messages_zh.xml @@ -106,6 +106,7 @@ 计划于 {1} 扫描中: 阶段 {1}/{2} + 检查文件中... {1}% 检索文件中: {1} 文件 diff --git a/src/libs/core/include/core/LiteralString.hpp b/src/libs/core/include/core/LiteralString.hpp index 7c0394bd..11b7f668 100644 --- a/src/libs/core/include/core/LiteralString.hpp +++ b/src/libs/core/include/core/LiteralString.hpp @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include @@ -41,6 +42,12 @@ namespace lms::core private: std::string_view _str; }; + + inline std::ostream& operator<<(std::ostream& os, const LiteralString& str) + { + os << str.str(); + return os; + } } namespace std diff --git a/src/libs/database/impl/Db.cpp b/src/libs/database/impl/Db.cpp index d756d1f5..23afed43 100644 --- a/src/libs/database/impl/Db.cpp +++ b/src/libs/database/impl/Db.cpp @@ -25,8 +25,9 @@ #include "database/Session.hpp" #include "database/User.hpp" #include "core/IConfig.hpp" -#include "core/Service.hpp" #include "core/ILogger.hpp" +#include "core/ITraceLogger.hpp" +#include "core/Service.hpp" namespace lms::db { @@ -49,7 +50,7 @@ namespace lms::db prepare(); } - ~Connection() + ~Connection() override { // make use of per-connection usage stats to optimize optimize(); @@ -66,16 +67,15 @@ namespace lms::db void prepare() { LMS_LOG(DB, DEBUG, "Setting per-connection settings..."); - executeSql("pragma journal_mode=WAL"); - executeSql("pragma synchronous=normal"); - executeSql("pragma analysis_limit=2000"); // to help make analyze command faster, 1000 does not seem to be enough to speed up all queries + executeSql("PRAGMA journal_mode=WAL"); + executeSql("PRAGMA synchronous=normal"); LMS_LOG(DB, DEBUG, "Setting per-connection settings done!"); } void optimize() { LMS_LOG(DB, DEBUG, "connection close: Running pragma optimize..."); - executeSql("pragma optimize"); + executeSql("PRAGMA optimize"); LMS_LOG(DB, DEBUG, "connection close: pragma optimize complete"); } diff --git a/src/libs/database/impl/Session.cpp b/src/libs/database/impl/Session.cpp index 757bdcd3..ded20802 100644 --- a/src/libs/database/impl/Session.cpp +++ b/src/libs/database/impl/Session.cpp @@ -46,6 +46,7 @@ #include "EnumSetTraits.hpp" #include "PathTraits.hpp" #include "Migration.hpp" +#include "Utils.hpp" namespace lms::db { @@ -120,7 +121,7 @@ namespace lms::db return ReadTransaction{ _session }; } - void Session::prepareTables() + void Session::prepareTablesIfNeeded() { LMS_LOG(DB, INFO, "Preparing tables..."); @@ -140,127 +141,165 @@ namespace lms::db throw e; } } + } + void Session::migrateIfNeeded() + { Migration::doDbMigration(*this); - // Indexes - { - LMS_SCOPED_TRACE_OVERVIEW("Database", "IndexCreation"); - - auto transaction{ createWriteTransaction() }; - _session.execute("CREATE INDEX IF NOT EXISTS artist_id_idx ON artist(id)"); - _session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)"); - _session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)"); - _session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)"); - - _session.execute("CREATE INDEX IF NOT EXISTS auth_token_user_idx ON auth_token(user_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS auth_token_expiry_idx ON auth_token(expiry)"); - _session.execute("CREATE INDEX IF NOT EXISTS auth_token_value_idx ON auth_token(value)"); - - _session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)"); - _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)"); - - _session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)"); - _session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)"); - _session.execute("CREATE INDEX IF NOT EXISTS listen_user_backend_idx ON listen(user_id,backend)"); - _session.execute("CREATE INDEX IF NOT EXISTS listen_track_user_backend_idx ON listen(track_id,user_id,backend)"); - _session.execute("CREATE INDEX IF NOT EXISTS listen_user_track_backend_date_time_idx ON listen(user_id,track_id,backend,date_time)"); - - _session.execute("CREATE INDEX IF NOT EXISTS release_id_idx ON release(id)"); - _session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)"); - _session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)"); - _session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)"); - _session.execute("CREATE INDEX IF NOT EXISTS release_type_name_idx ON release_type(name)"); - - _session.execute("CREATE INDEX IF NOT EXISTS track_id_idx ON track(id)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_absolute_path_idx ON track(absolute_file_path)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_recording_mbid_idx ON track(recording_mbid)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_file_last_write_idx ON track(file_last_write)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_date_idx ON track(date)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_original_date_idx ON track(original_date)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)"); - - _session.execute("CREATE INDEX IF NOT EXISTS track_media_library_idx ON track(media_library_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_media_library_release_idx ON track(media_library_id, release_id)"); - - _session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)"); - _session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)"); - - _session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)"); - - _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_idx ON track_artist_link(artist_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_type_idx ON track_artist_link(artist_id,type)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_idx ON track_artist_link(track_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_type_idx ON track_artist_link(track_id,type)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_type_idx ON track_artist_link(type)"); - - _session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_idx ON track_bookmark(user_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_track_idx ON track_bookmark(user_id,track_id)"); - - _session.execute("CREATE INDEX IF NOT EXISTS starred_artist_user_backend_idx ON starred_artist(user_id,backend)"); - _session.execute("CREATE INDEX IF NOT EXISTS starred_artist_artist_user_backend_idx ON starred_artist(artist_id,user_id,backend)"); - - _session.execute("CREATE INDEX IF NOT EXISTS starred_release_user_backend_idx ON starred_release(user_id,backend)"); - _session.execute("CREATE INDEX IF NOT EXISTS starred_release_release_user_backend_idx ON starred_release(release_id,user_id,backend)"); - - _session.execute("CREATE INDEX IF NOT EXISTS starred_track_user_backend_idx ON starred_track(user_id,backend)"); - _session.execute("CREATE INDEX IF NOT EXISTS starred_track_track_user_backend_idx ON starred_track(track_id,user_id,backend)"); - } - - // Singletons + // TODO: move this elsewhere { auto uniqueTransaction{ createWriteTransaction() }; ScanSettings::init(*this); } } + void Session::createIndexesIfNeeded() + { + LMS_SCOPED_TRACE_OVERVIEW("Database", "IndexCreation"); + + auto transaction{ createWriteTransaction() }; + _session.execute("CREATE INDEX IF NOT EXISTS artist_id_idx ON artist(id)"); + _session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)"); + _session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)"); + + _session.execute("CREATE INDEX IF NOT EXISTS auth_token_user_idx ON auth_token(user_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS auth_token_expiry_idx ON auth_token(expiry)"); + _session.execute("CREATE INDEX IF NOT EXISTS auth_token_value_idx ON auth_token(value)"); + + _session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)"); + + _session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)"); + _session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)"); + _session.execute("CREATE INDEX IF NOT EXISTS listen_user_backend_idx ON listen(user_id,backend)"); + _session.execute("CREATE INDEX IF NOT EXISTS listen_track_user_backend_idx ON listen(track_id,user_id,backend)"); + _session.execute("CREATE INDEX IF NOT EXISTS listen_user_track_backend_date_time_idx ON listen(user_id,track_id,backend,date_time)"); + + _session.execute("CREATE INDEX IF NOT EXISTS release_id_idx ON release(id)"); + _session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)"); + _session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)"); + _session.execute("CREATE INDEX IF NOT EXISTS release_type_name_idx ON release_type(name)"); + + _session.execute("CREATE INDEX IF NOT EXISTS track_id_idx ON track(id)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_absolute_path_idx ON track(absolute_file_path)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_recording_mbid_idx ON track(recording_mbid)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_file_last_write_idx ON track(file_last_write)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_date_idx ON track(date)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_original_date_idx ON track(original_date)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)"); + + _session.execute("CREATE INDEX IF NOT EXISTS track_media_library_idx ON track(media_library_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_media_library_release_idx ON track(media_library_id, release_id)"); + + _session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)"); + + _session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)"); + + _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_idx ON track_artist_link(artist_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_type_idx ON track_artist_link(artist_id,type)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_idx ON track_artist_link(track_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_type_idx ON track_artist_link(track_id,type)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_type_idx ON track_artist_link(type)"); + + _session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_idx ON track_bookmark(user_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_track_idx ON track_bookmark(user_id,track_id)"); + + _session.execute("CREATE INDEX IF NOT EXISTS starred_artist_user_backend_idx ON starred_artist(user_id,backend)"); + _session.execute("CREATE INDEX IF NOT EXISTS starred_artist_artist_user_backend_idx ON starred_artist(artist_id,user_id,backend)"); + + _session.execute("CREATE INDEX IF NOT EXISTS starred_release_user_backend_idx ON starred_release(user_id,backend)"); + _session.execute("CREATE INDEX IF NOT EXISTS starred_release_release_user_backend_idx ON starred_release(release_id,user_id,backend)"); + + _session.execute("CREATE INDEX IF NOT EXISTS starred_track_user_backend_idx ON starred_track(user_id,backend)"); + _session.execute("CREATE INDEX IF NOT EXISTS starred_track_track_user_backend_idx ON starred_track(track_id,user_id,backend)"); + } + + void Session::vacuumIfNeeded() + { + long pageCount{}; + long freeListCount{}; + + { + auto transaction{ createReadTransaction() }; + pageCount = utils::fetchQuerySingleResult(_session.query("SELECT page_count FROM pragma_page_count")); + freeListCount = utils::fetchQuerySingleResult(_session.query("SELECT freelist_count FROM pragma_freelist_count")); + } + + LMS_LOG(DB, INFO, "page stats: page_count = " << pageCount << ", freelist_count = " << freeListCount); + if (freeListCount >= (pageCount / 10)) + { + LMS_SCOPED_TRACE_OVERVIEW("Database", "Vacuum"); + LMS_LOG(DB, INFO, "Performing vacuum... This may take a while..."); + + // We manually take a lock here since vacuum cannot be inside a transaction + { + std::unique_lock lock{ _db.getMutex() }; + _db.executeSql("VACUUM"); + } + + LMS_LOG(DB, INFO, "Vacuum complete!"); + } + } + void Session::refreshTracingLoggerStats() { auto* traceLogger{ core::Service::get() }; if (!traceLogger) return; - auto& dbSession{ _db.getTLSSession() }; - auto transaction{ dbSession.createReadTransaction() }; + auto transaction{ createReadTransaction() }; - traceLogger->setMetadata("db_artist_count", std::to_string(db::Artist::getCount(dbSession))); - traceLogger->setMetadata("db_cluster_count", std::to_string(db::Cluster::getCount(dbSession))); - traceLogger->setMetadata("db_cluster_type_count", std::to_string(db::ClusterType::getCount(dbSession))); - traceLogger->setMetadata("db_starred_artist_count", std::to_string(db::StarredArtist::getCount(dbSession))); - traceLogger->setMetadata("db_starred_release_count", std::to_string(db::StarredRelease::getCount(dbSession))); - traceLogger->setMetadata("db_starred_track_count", std::to_string(db::StarredTrack::getCount(dbSession))); - traceLogger->setMetadata("db_track_bookmark_count", std::to_string(db::TrackBookmark::getCount(dbSession))); - traceLogger->setMetadata("db_listen_count", std::to_string(db::Listen::getCount(dbSession))); - traceLogger->setMetadata("db_release_count", std::to_string(db::Release::getCount(dbSession))); - traceLogger->setMetadata("db_track_count", std::to_string(db::Track::getCount(dbSession))); + traceLogger->setMetadata("db_artist_count", std::to_string(db::Artist::getCount(*this))); + traceLogger->setMetadata("db_cluster_count", std::to_string(db::Cluster::getCount(*this))); + traceLogger->setMetadata("db_cluster_type_count", std::to_string(db::ClusterType::getCount(*this))); + traceLogger->setMetadata("db_starred_artist_count", std::to_string(db::StarredArtist::getCount(*this))); + traceLogger->setMetadata("db_starred_release_count", std::to_string(db::StarredRelease::getCount(*this))); + traceLogger->setMetadata("db_starred_track_count", std::to_string(db::StarredTrack::getCount(*this))); + traceLogger->setMetadata("db_track_bookmark_count", std::to_string(db::TrackBookmark::getCount(*this))); + traceLogger->setMetadata("db_listen_count", std::to_string(db::Listen::getCount(*this))); + traceLogger->setMetadata("db_release_count", std::to_string(db::Release::getCount(*this))); + traceLogger->setMetadata("db_track_count", std::to_string(db::Track::getCount(*this))); } - void Session::analyze() + void Session::fullAnalyze() { - LMS_SCOPED_TRACE_DETAILED("Database", "Analyze"); - LMS_LOG(DB, INFO, "Analyzing database..."); + LMS_SCOPED_TRACE_OVERVIEW("Database", "Analyze"); + LMS_LOG(DB, INFO, "Performing database analyze... This may take a while..."); + + // first select all the tables and indexes, and then analyze one by one in order to not have a big lock + std::vector entries; + retrieveEntriesToAnalyze(entries); + + for (const std::string& entry : entries) + analyzeEntry(entry); + + LMS_LOG(DB, INFO, "Analyze complete!"); + } + + void Session::retrieveEntriesToAnalyze(std::vector& entryList) + { + auto transaction{ createReadTransaction() }; + entryList = utils::fetchQueryResults(_session.query("SELECT name FROM sqlite_master WHERE type='table' OR type ='index'")); + } + + void Session::analyzeEntry(const std::string& entry) + { + LMS_SCOPED_TRACE_DETAILED_WITH_ARG("Database", "AnalyzeEntry", "Entry", entry); + LMS_LOG(DB, DEBUG, "Analyzing " << entry); { auto transaction{ createWriteTransaction() }; - _session.execute("ANALYZE"); + _session.execute("ANALYZE " + entry); } - LMS_LOG(DB, INFO, "Database Analyze complete"); + LMS_LOG(DB, DEBUG, "Analyzing " << entry << ": done!"); } - - void Session::optimize() - { - LMS_SCOPED_TRACE_DETAILED("Database", "Optimize"); - LMS_LOG(DB, INFO, "Optimizing database..."); - { - auto transaction{ createWriteTransaction() }; - _session.execute("PRAGMA optimize"); - } - LMS_LOG(DB, INFO, "Database optimizing complete"); - } - } // namespace lms::db diff --git a/src/libs/database/include/database/Session.hpp b/src/libs/database/include/database/Session.hpp index d2d10083..2c8352af 100644 --- a/src/libs/database/include/database/Session.hpp +++ b/src/libs/database/include/database/Session.hpp @@ -22,6 +22,8 @@ #include #include +#include +#include #include "core/ITraceLogger.hpp" #include "core/RecursiveSharedMutex.hpp" #include "database/Object.hpp" @@ -84,10 +86,15 @@ namespace lms::db #endif } - void analyze(); - void optimize(); + // All these methods will acquire transactions + void fullAnalyze(); // helper for retrieveEntriesToAnalyze + analyzeEntry + void retrieveEntriesToAnalyze(std::vector& entryList); + void analyzeEntry(const std::string& entry); - void prepareTables(); // need to run only once at startup + void prepareTablesIfNeeded(); // need to run only once at startup + void migrateIfNeeded(); + void createIndexesIfNeeded(); + void vacuumIfNeeded(); void refreshTracingLoggerStats(); // returning a ptr here to ease further wrapping using operator-> diff --git a/src/libs/database/test/Common.cpp b/src/libs/database/test/Common.cpp index ac3089ab..d78a8557 100644 --- a/src/libs/database/test/Common.cpp +++ b/src/libs/database/test/Common.cpp @@ -60,8 +60,8 @@ namespace lms::db::tests _tmpDb = std::make_unique(); { db::Session s{ _tmpDb->getDb() }; - s.prepareTables(); - s.analyze(); + s.prepareTablesIfNeeded(); + s.createIndexesIfNeeded(); } } diff --git a/src/libs/services/scanner/CMakeLists.txt b/src/libs/services/scanner/CMakeLists.txt index 2993415a..4da8e7b8 100644 --- a/src/libs/services/scanner/CMakeLists.txt +++ b/src/libs/services/scanner/CMakeLists.txt @@ -2,6 +2,7 @@ add_library(lmsscanner SHARED impl/ScannerService.cpp impl/ScannerStats.cpp + impl/ScanStepAnalyze.cpp impl/ScanStepCheckDuplicatedDbFiles.cpp impl/ScanStepComputeClusterStats.cpp impl/ScanStepDiscoverFiles.cpp diff --git a/src/libs/services/scanner/impl/IScanStep.hpp b/src/libs/services/scanner/impl/IScanStep.hpp index e131da4b..553a8203 100644 --- a/src/libs/services/scanner/impl/IScanStep.hpp +++ b/src/libs/services/scanner/impl/IScanStep.hpp @@ -19,9 +19,9 @@ #pragma once -#include #include +#include "core/LiteralString.hpp" #include "services/scanner/ScannerStats.hpp" namespace lms::scanner @@ -32,7 +32,7 @@ namespace lms::scanner virtual ~IScanStep() = default; virtual ScanStep getStep() const = 0; - virtual std::string_view getStepName() const = 0; + virtual core::LiteralString getStepName() const = 0; struct ScanContext { diff --git a/src/libs/services/scanner/impl/ScanStepAnalyze.cpp b/src/libs/services/scanner/impl/ScanStepAnalyze.cpp new file mode 100644 index 00000000..91731641 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepAnalyze.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepAnalyze.hpp" + +#include "core/ILogger.hpp" +#include "database/Db.hpp" +#include "database/Session.hpp" + +namespace lms::scanner +{ + void ScanStepAnalyze::process(ScanContext& context) + { + ScanStats& stats{ context.stats }; + + if (stats.nbChanges() > (stats.nbFiles() / 5)) + { + LMS_LOG(DBUPDATER, INFO, "Database changed substantially: triggering full analyze"); + + auto& session{ _db.getTLSSession() }; + + std::vector entries; + session.retrieveEntriesToAnalyze(entries); + context.currentStepStats.totalElems = entries.size(); + + for (const std::string& entry : entries) + { + if (_abortScan) + break; + + _db.getTLSSession().analyzeEntry(entry); + context.currentStepStats.processedElems++; + _progressCallback(context.currentStepStats); + } + } + } +} diff --git a/src/libs/services/scanner/impl/ScanStepAnalyze.hpp b/src/libs/services/scanner/impl/ScanStepAnalyze.hpp new file mode 100644 index 00000000..9a1763cb --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepAnalyze.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "ScanStepBase.hpp" + +namespace lms::scanner +{ + class ScanStepAnalyze : public ScanStepBase + { + public: + using ScanStepBase::ScanStepBase; + + private: + ScanStep getStep() const override { return ScanStep::Analyze; } + core::LiteralString getStepName() const override { return "Analyze"; } + void process(ScanContext& context) override; + }; +} diff --git a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp index 3599791d..ba02d7ac 100644 --- a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp @@ -29,7 +29,7 @@ namespace lms::scanner using ScanStepBase::ScanStepBase; private: - std::string_view getStepName() const override { return "Checking for duplicated files"; } + core::LiteralString getStepName() const override { return "Checking for duplicated files"; } ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; } void process(ScanContext& context) override; }; diff --git a/src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp b/src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp index a62d354e..331973fc 100644 --- a/src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp +++ b/src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp @@ -79,6 +79,7 @@ namespace lms::scanner } context.currentStepStats.processedElems++; + _progressCallback(context.currentStepStats); } return true; diff --git a/src/libs/services/scanner/impl/ScanStepComputeClusterStats.hpp b/src/libs/services/scanner/impl/ScanStepComputeClusterStats.hpp index 8000f454..43aeaf36 100644 --- a/src/libs/services/scanner/impl/ScanStepComputeClusterStats.hpp +++ b/src/libs/services/scanner/impl/ScanStepComputeClusterStats.hpp @@ -30,7 +30,7 @@ namespace lms::scanner private: ScanStep getStep() const override { return ScanStep::ComputeClusterStats; } - std::string_view getStepName() const override { return "Compute cluster stats"; } + core::LiteralString getStepName() const override { return "Compute cluster stats"; } void process(ScanContext& context) override; }; } diff --git a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp index 60f3e240..2a56a3b9 100644 --- a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp @@ -30,7 +30,7 @@ namespace lms::scanner private: ScanStep getStep() const override { return ScanStep::DiscoveringFiles; } - std::string_view getStepName() const override { return "DiscoveringFiles"; } + core::LiteralString getStepName() const override { return "Discovering files"; } void process(ScanContext& context) override; }; } diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp index 5b56fd90..92c467ad 100644 --- a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp @@ -31,7 +31,7 @@ namespace lms::scanner using ScanStepBase::ScanStepBase; private: - std::string_view getStepName() const override { return "Checking orphaned entries"; } + core::LiteralString getStepName() const override { return "Checking orphaned entries"; } ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; } void process(ScanContext& context) override; diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp index cd2606a7..28bd7bc7 100644 --- a/src/libs/services/scanner/impl/ScanStepScanFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp @@ -518,10 +518,6 @@ namespace lms::scanner context.stats.scans++; processFileMetaData(context, scanResult.path, *scanResult.trackMetaData, libraryInfo); - - // optimize the database during scan (if we import a very large database, it may be too late to do it once at end) - if ((context.stats.scans % 1'000) == 0) - _db.getTLSSession().optimize(); } else { diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp index d656ea99..05398a39 100644 --- a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp @@ -40,7 +40,7 @@ namespace lms::scanner private: ScanStep getStep() const override { return ScanStep::ScanningFiles; } - std::string_view getStepName() const override { return "Scanning files"; } + core::LiteralString getStepName() const override { return "Scanning files"; } void process(ScanContext& context) override; bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo); diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index 356cbcbb..fc82af62 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -25,15 +25,17 @@ #include "database/TrackFeatures.hpp" #include "database/ScanSettings.hpp" #include "core/Exception.hpp" +#include "core/Path.hpp" #include "core/IConfig.hpp" #include "core/ILogger.hpp" -#include "core/Path.hpp" +#include "core/ITraceLogger.hpp" +#include "ScanStepAnalyze.hpp" #include "ScanStepCheckDuplicatedDbFiles.hpp" +#include "ScanStepComputeClusterStats.hpp" #include "ScanStepDiscoverFiles.hpp" #include "ScanStepRemoveOrphanDbFiles.hpp" #include "ScanStepScanFiles.hpp" -#include "ScanStepComputeClusterStats.hpp" namespace lms::scanner { @@ -259,6 +261,8 @@ namespace lms::scanner void ScannerService::scan(bool forceScan) { + LMS_SCOPED_TRACE_OVERVIEW("Scanner", "Scan"); + _events.scanStarted.emit(); { @@ -278,6 +282,8 @@ namespace lms::scanner for (auto& scanStep : _scanSteps) { + LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName()); + LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'"); scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() }; @@ -289,8 +295,6 @@ namespace lms::scanner LMS_LOG(DBUPDATER, INFO, "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size()); - _db.getTLSSession().analyze(); - if (!_abortScan) { stats.stopTime = Wt::WDateTime::currentDateTime(); @@ -348,6 +352,7 @@ namespace lms::scanner _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); } ScannerSettings ScannerService::readSettings() diff --git a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp index 02094980..ed5dffe8 100644 --- a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp +++ b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp @@ -66,8 +66,9 @@ namespace lms::scanner FetchingTrackFeatures, ReloadingSimilarityEngine, ComputeClusterStats, + Analyze, }; - static inline constexpr unsigned ScanProgressStepCount{ 7 }; + static inline constexpr unsigned ScanProgressStepCount{ 8 }; // reduced scan stats struct ScanStepStats diff --git a/src/lms/CMakeLists.txt b/src/lms/CMakeLists.txt index a21e8098..2a877826 100644 --- a/src/lms/CMakeLists.txt +++ b/src/lms/CMakeLists.txt @@ -4,6 +4,7 @@ add_executable(lms ui/Auth.cpp ui/LmsApplication.cpp ui/LmsApplicationManager.cpp + ui/LmsInitApplication.cpp ui/LmsTheme.cpp ui/MediaPlayer.cpp ui/ModalManager.cpp diff --git a/src/lms/main.cpp b/src/lms/main.cpp index dd6c684c..dfe1f0b8 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -40,6 +40,7 @@ #include "subsonic/SubsonicResource.hpp" #include "ui/LmsApplication.hpp" #include "ui/LmsApplicationManager.hpp" +#include "ui/LmsInitApplication.hpp" #include "core/IChildProcessManager.hpp" #include "core/IConfig.hpp" #include "core/IOContextRunner.hpp" @@ -275,20 +276,34 @@ namespace lms Wt::WServer server{ argv[0] }; server.setServerConfiguration(wtServerArgs.size(), const_cast(&wtArgv[0])); + // As initialization can take a while (db migration, analyze, etc.), we bind a temporary init entry point to warn the user + server.addEntryPoint(Wt::EntryPointType::Application, + [&](const Wt::WEnvironment& env) + { + return ui::LmsInitApplication::create(env); + }); + + LMS_LOG(MAIN, INFO, "Starting init web server..."); + server.start(); + core::IOContextRunner ioContextRunner{ ioContext, getThreadCount(), "Misc" }; // Connection pool size must be twice the number of threads: we have at least 2 io pools with getThreadCount() each and they all may access the database db::Db database{ config->getPath("working-dir") / "lms.db", getThreadCount() * 2 }; { db::Session session{ database }; - session.prepareTables(); + session.prepareTablesIfNeeded(); + session.createIndexesIfNeeded(); + + // As this may be quite long, we only do it during startup + session.vacuumIfNeeded(); // force optimize in case scanner aborted during a large import: - // queries may be too slow to even be able to relaunch a scan sing the web interface - session.analyze(); + // queries may be too slow to even be able to relaunch a scan using the web interface + session.fullAnalyze(); database.getTLSSession().refreshTracingLoggerStats(); } - + ui::LmsApplicationManager appManager; // Service initialization order is important (reverse-order for deinit) @@ -327,8 +342,12 @@ namespace lms core::Service feedbackService{ feedback::createFeedbackService(ioContext, database) }; core::Service scrobblingService{ scrobbling::createScrobblingService(ioContext, database) }; - std::unique_ptr subsonicResource; + LMS_LOG(MAIN, INFO, "Stopping init web server..."); + server.stop(); + server.removeEntryPoint(""); + + std::unique_ptr subsonicResource; // bind API resources if (config->getBool("api-subsonic", true)) { @@ -345,7 +364,7 @@ namespace lms proxyScannerEventsToApplication(*scannerService, server); - LMS_LOG(MAIN, INFO, "Starting server..."); + LMS_LOG(MAIN, INFO, "Starting init web server..."); server.start(); LMS_LOG(MAIN, INFO, "Now running..."); diff --git a/src/lms/ui/LmsInitApplication.cpp b/src/lms/ui/LmsInitApplication.cpp new file mode 100644 index 00000000..4870b6ee --- /dev/null +++ b/src/lms/ui/LmsInitApplication.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include +#include + +#include "LmsInitApplication.hpp" + +namespace lms::ui +{ + LmsInitApplication::LmsInitApplication(const Wt::WEnvironment& env) + : Wt::WApplication{ env } + { + enableUpdates(true); + root()->addNew("LMS is initializing. This may take a while, please wait..."); + } + + std::unique_ptr LmsInitApplication::create(const Wt::WEnvironment& env) + { + return std::make_unique(env); + } +} // namespace lms::ui \ No newline at end of file diff --git a/src/lms/ui/LmsInitApplication.hpp b/src/lms/ui/LmsInitApplication.hpp new file mode 100644 index 00000000..2e6d4a8d --- /dev/null +++ b/src/lms/ui/LmsInitApplication.hpp @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +namespace lms::ui +{ + class LmsInitApplication : public Wt::WApplication + { + public: + LmsInitApplication(const Wt::WEnvironment& env); + + static std::unique_ptr create(const Wt::WEnvironment& env); + }; +} // namespace lms::ui \ No newline at end of file diff --git a/src/lms/ui/admin/ScannerController.cpp b/src/lms/ui/admin/ScannerController.cpp index 65f08fd6..d7eb29a7 100644 --- a/src/lms/ui/admin/ScannerController.cpp +++ b/src/lms/ui/admin/ScannerController.cpp @@ -274,6 +274,13 @@ namespace lms::ui case scanner::ScanStep::ComputeClusterStats: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats") .arg(status.currentScanStepStats->progress())); + + case scanner::ScanStep:: Analyze: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-analyze") + .arg(status.currentScanStepStats->processedElems) + .arg(status.currentScanStepStats->totalElems) + .arg(status.currentScanStepStats->progress())); + break; } break; }