From 0780d1ab21c85fe32be9c5d58c063f3955dfb581 Mon Sep 17 00:00:00 2001 From: emeric Date: Sat, 20 Apr 2024 23:57:01 +0200 Subject: [PATCH] Added a way to manually compact and/or optimize the db --- approot/admin-scannercontroller.xml | 85 +++++++----- approot/messages.xml | 8 +- approot/messages_fr.xml | 8 +- approot/messages_it.xml | 7 +- approot/messages_zh.xml | 8 +- src/libs/services/scanner/CMakeLists.txt | 3 +- src/libs/services/scanner/impl/IScanStep.hpp | 3 +- .../impl/ScanStepCheckDuplicatedDbFiles.hpp | 4 +- .../services/scanner/impl/ScanStepCompact.cpp | 33 +++++ ...canStepAnalyze.hpp => ScanStepCompact.hpp} | 6 +- .../scanner/impl/ScanStepDiscoverFiles.hpp | 4 +- ...anStepAnalyze.cpp => ScanStepOptimize.cpp} | 11 +- .../scanner/impl/ScanStepOptimize.hpp | 36 +++++ .../impl/ScanStepRemoveOrphanDbFiles.hpp | 4 +- .../scanner/impl/ScanStepScanFiles.cpp | 2 +- .../scanner/impl/ScanStepScanFiles.hpp | 4 +- .../services/scanner/impl/ScannerService.cpp | 27 ++-- .../services/scanner/impl/ScannerService.hpp | 9 +- .../services/scanner/IScannerService.hpp | 55 ++++---- .../services/scanner/ScannerOptions.hpp | 30 ++++ .../include/services/scanner/ScannerStats.hpp | 18 +-- .../impl/entrypoints/MediaLibraryScanning.cpp | 4 +- src/lms/ui/admin/ScannerController.cpp | 129 +++++++++++------- src/lms/ui/admin/ScannerController.hpp | 5 + 24 files changed, 341 insertions(+), 162 deletions(-) create mode 100644 src/libs/services/scanner/impl/ScanStepCompact.cpp rename src/libs/services/scanner/impl/{ScanStepAnalyze.hpp => ScanStepCompact.hpp} (82%) rename src/libs/services/scanner/impl/{ScanStepAnalyze.cpp => ScanStepOptimize.cpp} (79%) create mode 100644 src/libs/services/scanner/impl/ScanStepOptimize.hpp create mode 100644 src/libs/services/scanner/include/services/scanner/ScannerOptions.hpp diff --git a/approot/admin-scannercontroller.xml b/approot/admin-scannercontroller.xml index d2660d0e..cbe01b13 100644 --- a/approot/admin-scannercontroller.xml +++ b/approot/admin-scannercontroller.xml @@ -2,42 +2,63 @@ -
- ${tr:Lms.Admin.ScannerController.scanner} -
-
- - ${status class="form-control"} + ${tr:Lms.Admin.ScannerController.scanner} +
+
+ + ${status class="form-control"} +
+
+ + ${step-status class="form-control"} +
+
+ +
+ ${last-scan class="form-control"} + ${report-btn class="btn btn-outline-info"}
-
- - ${step-status class="form-control"} -
-
- -
- ${last-scan class="form-control"} - ${report-btn class="btn btn-outline-info"} -
-
-
-
- ${scan-btn class="btn btn-primary"} - -
    -
  • ${full-scan-btn class="dropdown-item"}
    • -
+
+
+
+
+
+ ${tr:Lms.Admin.ScannerController.scan-options} +
+
+
+ ${full-scan class="form-check-input"} + +
+
+ ${force-optimize class="form-check-input"} + +
+
+ ${compact class="form-check-input"} + +
+
- +
+ ${scan-btn class="btn btn-primary me-2"} + +
+
diff --git a/approot/messages.xml b/approot/messages.xml index c494f326..058d29e0 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -89,9 +89,11 @@ Cannot get track duration Cannot parse file Cannot read file +Compact the database. Caution: this may take a while and will block the whole application during the compact step! {1} duplicate files: {1} errors: -Force full rescan now +Force database optimization +Rescan all files Get report Last scan Not available @@ -100,17 +102,19 @@ Duplicated file hash Duplicated track MBID Scan now +Scan options Scanner Status Not scheduled Scheduled on {1} Scanning: step {1}/{2} -Analyzing database... {1}/{2} entries ({3}%)... Checking for duplicate files... {1} files Checking files... {1}% +Compacting database... Computing stats... {1}% Discovering files: {1} files Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)... +Optimizing database... {1}/{2} entries ({3}%)... Reloading similarity engine: {1}%... Scanning files: {1}/{2} files ({3}%)... Step status diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 797355a1..77da8540 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -89,9 +89,11 @@ Impossible de récupérer la durée de la piste Impossible d'analyser le fichier Impossible de lire le fichier +Compacter la base de données. Attention : cette opération peut prendre du temps et va vérouiller l'application pendant toute l'étape de compactage! {1} fichiers dupliqués : {1} erreurs : -Forcer un rescan complet +Forcer l'optimisation de la base de données +Rescanner tous les fichiers Rapport Dernier scan Non disponible @@ -100,17 +102,19 @@ Hash dupliqué Track MBID dupliqué Lancer un scan +Options de scan Scanner Statut Non planifié Planifié le {1} En cours de scan : étape {1}/{2} -Analyze de la base de données... {1}/{2} entrées ({3}%)... Vérification des fichiers dupliqués... {1} fichiers Vérification des fichiers... {1}% +Compactage de la base de données... Calcul des statistiques... {1}% Découverte des fichiers : {1} fichiers Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)... +Optimisation de la base de données... {1}/{2} entrées ({3}%)... Rechargement du moteur de recommandation : {1}%... Scan des fichiers : {1}/{2} fichiers ({3}%)... Statut de l'étape diff --git a/approot/messages_it.xml b/approot/messages_it.xml index 640b1091..8a2da532 100644 --- a/approot/messages_it.xml +++ b/approot/messages_it.xml @@ -89,9 +89,11 @@ Non sono stato in grado di determinare la durata della traccia Non in grado di analizzare il file Non in grado di leggere il file +Compatta il database. Attenzione: ciò potrebbe richiedere del tempo e bloccherà l'intera applicazione durante il passaggio di compattazione! {1} file duplicati: {1} errori: -Forza una nuova scansione completa +Forza l'ottimizzazione del database +Forza la scansione di tutti i file Scarica un resoconto Ultima scansione Non disponibile @@ -100,17 +102,18 @@ Hash doppio Track MBID doppio Scansiona ora +Opzioni di scansione Scanner Stato Non pianificato Pianificato il {1} Scansione: passo {1}/{2} -Analisi del database... {1}/{2} voci ({3}%)... Controllo duplicati... {1} files Controllo file... {1}% Calcolo statistiche... {1}% File trovati: {1} files Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)... +Ottimizzazione del database... {1}/{2} voci ({3}%)... Ricarica motore di tracce simili: {1}%... Scansione files: {1}/{2} files ({3}%)... Stato passo diff --git a/approot/messages_zh.xml b/approot/messages_zh.xml index 6a8ac377..05774126 100644 --- a/approot/messages_zh.xml +++ b/approot/messages_zh.xml @@ -89,9 +89,11 @@ 无法获得音轨时间 无法解析文件 无法读取文件 + {1} 个重复文件： {1} 个错误： -立即强制全盘扫描 + + 生成报告 最后扫描 不可用 @@ -100,17 +102,19 @@ 相同文件哈希值 相同 MBID 立即扫描 + 扫描器 状态 无计划 计划于 {1} 扫描中: 阶段 {1}/{2} - 检查文件中... {1}% + 检索文件中: {1} 文件 从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)... + 重载相似引擎中 {1}%... 扫描文件中: {1}/{2} 个文件 ({3}%)... 当前步骤状态 diff --git a/src/libs/services/scanner/CMakeLists.txt b/src/libs/services/scanner/CMakeLists.txt index 4da8e7b8..dbe27c9c 100644 --- a/src/libs/services/scanner/CMakeLists.txt +++ b/src/libs/services/scanner/CMakeLists.txt @@ -2,10 +2,11 @@ add_library(lmsscanner SHARED impl/ScannerService.cpp impl/ScannerStats.cpp - impl/ScanStepAnalyze.cpp impl/ScanStepCheckDuplicatedDbFiles.cpp + impl/ScanStepCompact.cpp impl/ScanStepComputeClusterStats.cpp impl/ScanStepDiscoverFiles.cpp + impl/ScanStepOptimize.cpp impl/ScanStepRemoveOrphanDbFiles.cpp impl/ScanStepScanFiles.cpp ) diff --git a/src/libs/services/scanner/impl/IScanStep.hpp b/src/libs/services/scanner/impl/IScanStep.hpp index 553a8203..0d00af85 100644 --- a/src/libs/services/scanner/impl/IScanStep.hpp +++ b/src/libs/services/scanner/impl/IScanStep.hpp @@ -22,6 +22,7 @@ #include #include "core/LiteralString.hpp" +#include "services/scanner/ScannerOptions.hpp" #include "services/scanner/ScannerStats.hpp" namespace lms::scanner @@ -36,7 +37,7 @@ namespace lms::scanner struct ScanContext { - const bool forceScan; + ScanOptions scanOptions; ScanStats stats; ScanStepStats currentStepStats; }; diff --git a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp index ba02d7ac..d6762599 100644 --- a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp @@ -29,8 +29,8 @@ namespace lms::scanner using ScanStepBase::ScanStepBase; private: - core::LiteralString getStepName() const override { return "Checking for duplicated files"; } - ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; } + core::LiteralString getStepName() const override { return "Check for duplicated files"; } + ScanStep getStep() const override { return ScanStep::CheckForDuplicateFiles; } void process(ScanContext& context) override; }; } diff --git a/src/libs/services/scanner/impl/ScanStepCompact.cpp b/src/libs/services/scanner/impl/ScanStepCompact.cpp new file mode 100644 index 00000000..b6078055 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepCompact.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepCompact.hpp" + +#include "database/Db.hpp" +#include "database/Session.hpp" + +namespace lms::scanner +{ + void ScanStepCompact::process(ScanContext& context) + { + // Don't auto compact as it may be too annoying to block the whole application + if (context.scanOptions.compact) + _db.getTLSSession().vacuum(); + } +} diff --git a/src/libs/services/scanner/impl/ScanStepAnalyze.hpp b/src/libs/services/scanner/impl/ScanStepCompact.hpp similarity index 82% rename from src/libs/services/scanner/impl/ScanStepAnalyze.hpp rename to src/libs/services/scanner/impl/ScanStepCompact.hpp index 9a1763cb..f56b4f24 100644 --- a/src/libs/services/scanner/impl/ScanStepAnalyze.hpp +++ b/src/libs/services/scanner/impl/ScanStepCompact.hpp @@ -23,14 +23,14 @@ namespace lms::scanner { - class ScanStepAnalyze : public ScanStepBase + class ScanStepCompact : public ScanStepBase { public: using ScanStepBase::ScanStepBase; private: - ScanStep getStep() const override { return ScanStep::Analyze; } - core::LiteralString getStepName() const override { return "Analyze"; } + ScanStep getStep() const override { return ScanStep::Compact; } + core::LiteralString getStepName() const override { return "Compact"; } void process(ScanContext& context) override; }; } diff --git a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp index 2a56a3b9..3a350e4f 100644 --- a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp @@ -29,8 +29,8 @@ namespace lms::scanner using ScanStepBase::ScanStepBase; private: - ScanStep getStep() const override { return ScanStep::DiscoveringFiles; } - core::LiteralString getStepName() const override { return "Discovering files"; } + ScanStep getStep() const override { return ScanStep::DiscoverFiles; } + core::LiteralString getStepName() const override { return "Discover files"; } void process(ScanContext& context) override; }; } diff --git a/src/libs/services/scanner/impl/ScanStepAnalyze.cpp b/src/libs/services/scanner/impl/ScanStepOptimize.cpp similarity index 79% rename from src/libs/services/scanner/impl/ScanStepAnalyze.cpp rename to src/libs/services/scanner/impl/ScanStepOptimize.cpp index f7101db9..0bcfc7df 100644 --- a/src/libs/services/scanner/impl/ScanStepAnalyze.cpp +++ b/src/libs/services/scanner/impl/ScanStepOptimize.cpp @@ -17,7 +17,7 @@ * along with LMS. If not, see . */ -#include "ScanStepAnalyze.hpp" +#include "ScanStepOptimize.hpp" #include "core/ILogger.hpp" #include "database/Db.hpp" @@ -25,19 +25,20 @@ namespace lms::scanner { - void ScanStepAnalyze::process(ScanContext& context) + void ScanStepOptimize::process(ScanContext& context) { ScanStats& stats{ context.stats }; - if (stats.nbChanges() > (stats.nbFiles() / 5)) + if (context.scanOptions.forceOptimize || (stats.nbChanges() > (stats.nbFiles() / 5))) { - LMS_LOG(DBUPDATER, INFO, "Database changed substantially: triggering full analyze"); + LMS_LOG(DBUPDATER, INFO, "Database analyze started"); auto& session{ _db.getTLSSession() }; std::vector entries; session.retrieveEntriesToAnalyze(entries); context.currentStepStats.totalElems = entries.size(); + _progressCallback(context.currentStepStats); for (const std::string& entry : entries) { @@ -48,6 +49,8 @@ namespace lms::scanner context.currentStepStats.processedElems++; _progressCallback(context.currentStepStats); } + + LMS_LOG(DBUPDATER, INFO, "Database analyze complete"); } } } diff --git a/src/libs/services/scanner/impl/ScanStepOptimize.hpp b/src/libs/services/scanner/impl/ScanStepOptimize.hpp new file mode 100644 index 00000000..a0fac852 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepOptimize.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "ScanStepBase.hpp" + +namespace lms::scanner +{ + class ScanStepOptimize : public ScanStepBase + { + public: + using ScanStepBase::ScanStepBase; + + private: + ScanStep getStep() const override { return ScanStep::Optimize; } + core::LiteralString getStepName() const override { return "Optimize"; } + void process(ScanContext& context) override; + }; +} diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp index 92c467ad..fbc6391b 100644 --- a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp @@ -31,8 +31,8 @@ namespace lms::scanner using ScanStepBase::ScanStepBase; private: - core::LiteralString getStepName() const override { return "Checking orphaned entries"; } - ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; } + core::LiteralString getStepName() const override { return "Check orphaned entries"; } + ScanStep getStep() const override { return ScanStep::CheckForMissingFiles; } void process(ScanContext& context) override; void removeOrphanTracks(ScanContext& context); diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp index 28bd7bc7..4534918f 100644 --- a/src/libs/services/scanner/impl/ScanStepScanFiles.cpp +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp @@ -459,7 +459,7 @@ namespace lms::scanner } bool needUpdateLibrary{}; - if (!context.forceScan) + if (!context.scanOptions.fullScan) { // Skip file if last write is the same db::Session& dbSession{ _db.getTLSSession() }; diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp index 05398a39..d176857e 100644 --- a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp @@ -39,8 +39,8 @@ namespace lms::scanner ScanStepScanFiles(InitParams& initParams); private: - ScanStep getStep() const override { return ScanStep::ScanningFiles; } - core::LiteralString getStepName() const override { return "Scanning files"; } + ScanStep getStep() const override { return ScanStep::ScanFiles; } + core::LiteralString getStepName() const override { return "Scan files"; } void process(ScanContext& context) override; bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo); diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index fc82af62..b4ff7db2 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -30,10 +30,11 @@ #include "core/ILogger.hpp" #include "core/ITraceLogger.hpp" -#include "ScanStepAnalyze.hpp" #include "ScanStepCheckDuplicatedDbFiles.hpp" +#include "ScanStepCompact.hpp" #include "ScanStepComputeClusterStats.hpp" #include "ScanStepDiscoverFiles.hpp" +#include "ScanStepOptimize.hpp" #include "ScanStepRemoveOrphanDbFiles.hpp" #include "ScanStepScanFiles.hpp" @@ -135,15 +136,15 @@ namespace lms::scanner _events.scanAborted.emit(); } - void ScannerService::requestImmediateScan(bool force) + void ScannerService::requestImmediateScan(const ScanOptions& scanOptions) { abortScan(); - _ioService.post([this, force] + _ioService.post([this, scanOptions] { if (_abortScan) return; - scheduleScan(force); + scheduleScan(scanOptions); }); } @@ -220,7 +221,7 @@ namespace lms::scanner } if (nextScanDateTime.isValid()) - scheduleScan(false, nextScanDateTime); + scheduleScan(ScanOptions{}, nextScanDateTime); { std::unique_lock lock{ _statusMutex }; @@ -231,14 +232,14 @@ namespace lms::scanner _events.scanScheduled.emit(_nextScheduledScan); } - void ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime) + void ScannerService::scheduleScan(const ScanOptions& scanOptions, const Wt::WDateTime& dateTime) { - auto cb{ [this, force](boost::system::error_code ec) + auto cb{ [this, scanOptions](boost::system::error_code ec) { if (ec) return; - scan(force); + scan(scanOptions); } }; if (dateTime.isNull()) @@ -259,7 +260,7 @@ namespace lms::scanner } } - void ScannerService::scan(bool forceScan) + void ScannerService::scan(const ScanOptions& scanOptions) { LMS_SCOPED_TRACE_OVERVIEW("Scanner", "Scan"); @@ -276,16 +277,17 @@ namespace lms::scanner refreshScanSettings(); - IScanStep::ScanContext scanContext{ forceScan, ScanStats {}, ScanStepStats {} }; + IScanStep::ScanContext scanContext{ scanOptions, ScanStats {}, ScanStepStats {} }; ScanStats& stats{ scanContext.stats }; stats.startTime = Wt::WDateTime::currentDateTime(); + std::size_t stepIndex{}; for (auto& scanStep : _scanSteps) { LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName()); LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'"); - scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() }; + scanContext.currentStepStats = ScanStepStats{ .startTime = Wt::WDateTime::currentDateTime(), .stepIndex = stepIndex++, .currentStep = scanStep->getStep() }; notifyInProgress(scanContext.currentStepStats); scanStep->process(scanContext); @@ -350,9 +352,10 @@ namespace lms::scanner _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); _scanSteps.push_back(std::make_unique(params)); - _scanSteps.push_back(std::make_unique(params)); } ScannerSettings ScannerService::readSettings() diff --git a/src/libs/services/scanner/impl/ScannerService.hpp b/src/libs/services/scanner/impl/ScannerService.hpp index d0beb160..ea52aa1a 100644 --- a/src/libs/services/scanner/impl/ScannerService.hpp +++ b/src/libs/services/scanner/impl/ScannerService.hpp @@ -46,12 +46,13 @@ namespace lms::scanner ScannerService(db::Db& db); ~ScannerService(); + private: ScannerService(const ScannerService&) = delete; ScannerService& operator=(const ScannerService&) = delete; - private: + void requestStop() override; void requestReload() override; - void requestImmediateScan(bool force) override; + void requestImmediateScan(const ScanOptions& scanOptions) override; Status getStatus() const override; Events& getEvents() override { return _events; } @@ -62,12 +63,12 @@ namespace lms::scanner // Job handling void scheduleNextScan(); - void scheduleScan(bool force, const Wt::WDateTime& dateTime = {}); + void scheduleScan(const ScanOptions& scanOptions, const Wt::WDateTime& dateTime = {}); void abortScan(); // Update database (scheduled callback) - void scan(bool force); + void scan(const ScanOptions& scanOptions); void scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats); diff --git a/src/libs/services/scanner/include/services/scanner/IScannerService.hpp b/src/libs/services/scanner/include/services/scanner/IScannerService.hpp index f6906835..c66d864c 100644 --- a/src/libs/services/scanner/include/services/scanner/IScannerService.hpp +++ b/src/libs/services/scanner/include/services/scanner/IScannerService.hpp @@ -22,47 +22,46 @@ #include #include "ScannerEvents.hpp" +#include "ScannerOptions.hpp" #include "ScannerStats.hpp" namespace lms::db { - class Db; + class Db; } namespace lms::scanner { + class IScannerService + { + public: + virtual ~IScannerService() = default; - class IScannerService - { - public: - virtual ~IScannerService() = default; + // Async requests + virtual void requestStop() = 0; + virtual void requestReload() = 0; - // Async requests - virtual void requestStop() = 0; - virtual void requestReload() = 0; - virtual void requestImmediateScan(bool force) = 0; + virtual void requestImmediateScan(const ScanOptions& options = {}) = 0; - enum class State - { - NotScheduled, - Scheduled, - InProgress, - }; + enum class State + { + NotScheduled, + Scheduled, + InProgress, + }; - struct Status - { - State currentState {State::NotScheduled}; - Wt::WDateTime nextScheduledScan; - std::optional lastCompleteScanStats; - std::optional currentScanStepStats; - }; + struct Status + { + State currentState{ State::NotScheduled }; + Wt::WDateTime nextScheduledScan; + std::optional lastCompleteScanStats; + std::optional currentScanStepStats; + }; - virtual Status getStatus() const = 0; + virtual Status getStatus() const = 0; - virtual Events& getEvents() = 0; - }; - - std::unique_ptr createScannerService(db::Db& db); + virtual Events& getEvents() = 0; + }; + std::unique_ptr createScannerService(db::Db& db); } // Scanner - diff --git a/src/libs/services/scanner/include/services/scanner/ScannerOptions.hpp b/src/libs/services/scanner/include/services/scanner/ScannerOptions.hpp new file mode 100644 index 00000000..33ac40b2 --- /dev/null +++ b/src/libs/services/scanner/include/services/scanner/ScannerOptions.hpp @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2024 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +namespace lms::scanner +{ + struct ScanOptions + { + bool fullScan{}; // scan files even if not changed + bool forceOptimize{}; // force optimize database + bool compact{}; // compact the database + }; +} \ No newline at end of file diff --git a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp index ed5dffe8..9c8d9621 100644 --- a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp +++ b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp @@ -59,22 +59,24 @@ namespace lms::scanner enum class ScanStep { - DiscoveringFiles, - ScanningFiles, - ChekingForMissingFiles, - CheckingForDuplicateFiles, - FetchingTrackFeatures, - ReloadingSimilarityEngine, + CheckForMissingFiles, + CheckForDuplicateFiles, ComputeClusterStats, - Analyze, + Compact, + DiscoverFiles, + FetchTrackFeatures, + Optimize, + ReloadSimilarityEngine, + ScanFiles, }; - static inline constexpr unsigned ScanProgressStepCount{ 8 }; + static inline constexpr unsigned ScanProgressStepCount{ 9 }; // reduced scan stats struct ScanStepStats { Wt::WDateTime startTime; + std::size_t stepIndex{}; ScanStep currentStep; std::size_t totalElems{}; diff --git a/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp b/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp index 38beb278..a692b728 100644 --- a/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp +++ b/src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp @@ -39,7 +39,7 @@ namespace lms::api::subsonic::Scan { std::size_t count{}; - if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanningFiles) + if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanFiles) count = scanStatus.currentScanStepStats->processedElems; statusResponse.setAttribute("count", count); @@ -59,7 +59,7 @@ namespace lms::api::subsonic::Scan Response handleStartScan(RequestContext& context) { - core::Service::get()->requestImmediateScan(false); + core::Service::get()->requestImmediateScan(); Response response{ Response::createOkResponse(context.serverProtocolVersion) }; response.addNode("scanStatus", createStatusResponseNode()); diff --git a/src/lms/ui/admin/ScannerController.cpp b/src/lms/ui/admin/ScannerController.cpp index c7c7d305..8fcb8501 100644 --- a/src/lms/ui/admin/ScannerController.cpp +++ b/src/lms/ui/admin/ScannerController.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -156,16 +157,19 @@ namespace lms::ui _reportBtn->setLink(link); } + Wt::WCheckBox* fullScan{ bindNew("full-scan") }; + Wt::WCheckBox* forceOptimize{ bindNew("force-optimize") }; + Wt::WCheckBox* compact{ bindNew("compact") }; Wt::WPushButton* scanBtn{ bindNew("scan-btn", Wt::WString::tr("Lms.Admin.ScannerController.scan-now")) }; - scanBtn->clicked().connect([] + scanBtn->clicked().connect([=] { - core::Service::get()->requestImmediateScan(false); - }); - - Wt::WPushButton* fullScanBtn{ bindNew("full-scan-btn", Wt::WString::tr("Lms.Admin.ScannerController.force-scan-now")) }; - fullScanBtn->clicked().connect([] - { - core::Service::get()->requestImmediateScan(true); + const scanner::ScanOptions scanOptions + { + .fullScan = fullScan->isChecked(), + .forceOptimize = forceOptimize->isChecked(), + .compact = compact->isChecked(), + }; + core::Service::get()->requestImmediateScan(scanOptions); }); _lastScanStatus = bindNew("last-scan"); @@ -199,6 +203,13 @@ namespace lms::ui using namespace scanner; const IScannerService::Status status{ core::Service::get()->getStatus() }; + + refreshLastScanStatus(status); + refreshStatus(status); + } + + void ScannerController::refreshLastScanStatus(const scanner::IScannerService::Status& status) + { if (status.lastCompleteScanStats) { _lastScanStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.last-scan-status") @@ -218,6 +229,11 @@ namespace lms::ui _lastScanStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.last-scan-not-available")); _reportBtn->setEnabled(false); } + } + + void ScannerController::refreshStatus(const scanner::IScannerService::Status& status) + { + using namespace scanner; switch (status.currentState) { @@ -225,64 +241,77 @@ namespace lms::ui _status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-not-scheduled")); _stepStatus->setText(""); break; + case IScannerService::State::Scheduled: _status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-scheduled") .arg(status.nextScheduledScan.toString())); _stepStatus->setText(""); break; + case IScannerService::State::InProgress: _status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress") - .arg(static_cast(status.currentScanStepStats->currentStep) + 1) + .arg(status.currentScanStepStats->stepIndex + 1) .arg(scanner::ScanProgressStepCount)); - switch (status.currentScanStepStats->currentStep) - { - case scanner::ScanStep::CheckingForDuplicateFiles: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files") - .arg(status.currentScanStepStats->processedElems)); - break; + refreshCurrentStep(*status.currentScanStepStats); + break; + } + } - case scanner::ScanStep::ChekingForMissingFiles: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files") - .arg(status.currentScanStepStats->progress())); - break; + void ScannerController::refreshCurrentStep(const scanner::ScanStepStats& stepStats) + { + using namespace scanner; - case scanner::ScanStep::DiscoveringFiles: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files") - .arg(status.currentScanStepStats->processedElems)); - break; + switch (stepStats.currentStep) + { + case ScanStep::CheckForDuplicateFiles: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files") + .arg(stepStats.processedElems)); + break; - case scanner::ScanStep::ScanningFiles: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files") - .arg(status.currentScanStepStats->processedElems) - .arg(status.currentScanStepStats->totalElems) - .arg(status.currentScanStepStats->progress())); - break; + case scanner::ScanStep::CheckForMissingFiles: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files") + .arg(stepStats.progress())); + break; - case scanner::ScanStep::FetchingTrackFeatures: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features") - .arg(status.currentScanStepStats->processedElems) - .arg(status.currentScanStepStats->totalElems) - .arg(status.currentScanStepStats->progress())); - break; + case scanner::ScanStep::Compact: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compact")); + break; - case scanner::ScanStep::ReloadingSimilarityEngine: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine") - .arg(status.currentScanStepStats->progress())); - break; + case scanner::ScanStep::ComputeClusterStats: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats") + .arg(stepStats.progress())); + break; - case scanner::ScanStep::ComputeClusterStats: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats") - .arg(status.currentScanStepStats->progress())); - break; + case scanner::ScanStep::DiscoverFiles: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files") + .arg(stepStats.processedElems)); + break; - case scanner::ScanStep:: Analyze: - _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-analyze") - .arg(status.currentScanStepStats->processedElems) - .arg(status.currentScanStepStats->totalElems) - .arg(status.currentScanStepStats->progress())); - break; - } + case scanner::ScanStep::FetchTrackFeatures: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features") + .arg(stepStats.processedElems) + .arg(stepStats.totalElems) + .arg(stepStats.progress())); + break; + + case scanner::ScanStep::Optimize: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-optimize") + .arg(stepStats.processedElems) + .arg(stepStats.totalElems) + .arg(stepStats.progress())); + break; + + case scanner::ScanStep::ReloadSimilarityEngine: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine") + .arg(stepStats.progress())); + break; + + case scanner::ScanStep::ScanFiles: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files") + .arg(stepStats.processedElems) + .arg(stepStats.totalElems) + .arg(stepStats.progress())); break; } } diff --git a/src/lms/ui/admin/ScannerController.hpp b/src/lms/ui/admin/ScannerController.hpp index b10fdf05..e8091c7b 100644 --- a/src/lms/ui/admin/ScannerController.hpp +++ b/src/lms/ui/admin/ScannerController.hpp @@ -23,6 +23,8 @@ #include #include +#include "services/scanner/IScannerService.hpp" + namespace lms::ui { class ScannerController : public Wt::WTemplate @@ -32,6 +34,9 @@ namespace lms::ui private: void refreshContents(); + void refreshLastScanStatus(const scanner::IScannerService::Status& status); + void refreshStatus(const scanner::IScannerService::Status& status); + void refreshCurrentStep(const scanner::ScanStepStats& stepStats); Wt::WPushButton* _reportBtn; Wt::WLineEdit* _lastScanStatus;