owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a way to manually compact and/or optimize the db

Browse Source
This commit is contained in:
emeric
2024-04-20 23:57:01 +02:00
parent 8cde7ddf43
commit 0780d1ab21
24 changed files with 341 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
approot/admin-scannercontroller.xml
+53 -32
View File
@@ -2,42 +2,63 @@
<messages xmlns:if="Wt.WTemplate.conditions">
<message id="Lms.Admin.ScannerController.template">
<form>
<legend>${tr:Lms.Admin.ScannerController.scanner}</legend>
<div class="row g-3">
<div class="col-lg-6">
<label class="form-label" for="${id:status}">
${tr:Lms.Admin.ScannerController.status}
</label>
${status class="form-control"}
<legend>${tr:Lms.Admin.ScannerController.scanner}</legend>
<div class="row g-3">
<div class="col-lg-6">
<label class="form-label" for="${id:status}">
${tr:Lms.Admin.ScannerController.status}
</label>
${status class="form-control"}
</div>
<div class="col-lg-6">
<label class="form-label" for="${id:step-status}">
${tr:Lms.Admin.ScannerController.step-status}
</label>
${step-status class="form-control"}
</div>
<div class="col-12">
<label class="form-label" for="${id:last-scan}">
${tr:Lms.Admin.ScannerController.last-scan}
</label>
<div class="input-group">
${last-scan class="form-control"}
${report-btn class="btn btn-outline-info"}
</div>
<div class="col-lg-6">
<label class="form-label" for="${id:step-status}">
${tr:Lms.Admin.ScannerController.step-status}
</label>
${step-status class="form-control"}
</div>
<div class="col-12">
<label class="form-label" for="${id:last-scan}">
${tr:Lms.Admin.ScannerController.last-scan}
</label>
<div class="input-group">
${last-scan class="form-control"}
${report-btn class="btn btn-outline-info"}
</div>
</div>
<div class="col-12">
<div class="btn-group">
${scan-btn class="btn btn-primary"}
<button type="button" class="btn btn-primary dropdown-toggle dropdown-toggle-split" data-bs-toggle="dropdown" aria-expanded="false">
</button>
<ul class="dropdown-menu">
<li>${full-scan-btn class="dropdown-item"}</li>
</ul>
</div>
<div class="col-12">
<div class="collapse" id="scanOptions">
<div class="card bg-dark">
<div class="card-header">
${tr:Lms.Admin.ScannerController.scan-options}
</div>
<div class="card-body">
<div class="form-check mb-1">
${full-scan class="form-check-input"}
<label class="form-check-label" for="${id:full-scan}">
${tr:Lms.Admin.ScannerController.full-scan}
</label>
</div>
<div class="form-check mb-1">
${force-optimize class="form-check-input"}
<label class="form-check-label" for="${id:force-optimize}">
${tr:Lms.Admin.ScannerController.force-optimize}
</label>
</div>
<div class="form-check">
${compact class="form-check-input"}
<label class="form-check-label" for="${id:compact}">
${tr:Lms.Admin.ScannerController.compact}
</label>
</div>
</div>
</div>
</div>
</div>
</form>
<div class="col-12">
${scan-btn class="btn btn-primary me-2"}
<button class="btn btn-secondary" type="button" data-bs-toggle="collapse" data-bs-target="#scanOptions" aria-expanded="false" aria-controls="scanOptions">${tr:Lms.Admin.ScannerController.scan-options}</button>
</div>
</div>
</message>
</messages>
approot/messages.xml
+6 -2
View File
@@ -89,9 +89,11 @@
<message id="Lms.Admin.ScannerController.bad-duration">Cannot get track duration</message>
<message id="Lms.Admin.ScannerController.cannot-parse-file">Cannot parse file</message>
<message id="Lms.Admin.ScannerController.cannot-read-file">Cannot read file</message>
<message id="Lms.Admin.ScannerController.compact">Compact the database. <strong>Caution:</strong> this may take a while and will block the whole application during the compact step!</message>
<message id="Lms.Admin.ScannerController.duplicates-header">{1} duplicate files:</message>
<message id="Lms.Admin.ScannerController.errors-header">{1} errors:</message>
<message id="Lms.Admin.ScannerController.force-scan-now">Force full rescan now</message>
<message id="Lms.Admin.ScannerController.force-optimize">Force database optimization</message>
<message id="Lms.Admin.ScannerController.full-scan">Rescan all files</message>
<message id="Lms.Admin.ScannerController.get-report">Get report</message>
<message id="Lms.Admin.ScannerController.last-scan">Last scan</message>
<message id="Lms.Admin.ScannerController.last-scan-not-available">Not available</message>
@@ -100,17 +102,19 @@
<message id="Lms.Admin.ScannerController.same-hash">Duplicated file hash</message>
<message id="Lms.Admin.ScannerController.same-mbid">Duplicated track MBID</message>
<message id="Lms.Admin.ScannerController.scan-now">Scan now</message>
<message id="Lms.Admin.ScannerController.scan-options">Scan options</message>
<message id="Lms.Admin.ScannerController.scanner">Scanner</message>
<message id="Lms.Admin.ScannerController.status">Status</message>
<message id="Lms.Admin.ScannerController.status-not-scheduled">Not scheduled</message>
<message id="Lms.Admin.ScannerController.status-scheduled">Scheduled on {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">Scanning: step {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-analyze">Analyzing database... {1}/{2} entries ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Checking for duplicate files... {1} files</message>
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Checking files... {1}%</message>
<message id="Lms.Admin.ScannerController.step-compact">Compacting database...</message>
<message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Computing stats... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">Discovering files: {1} files</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-optimize">Optimizing database... {1}/{2} entries ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Reloading similarity engine: {1}%...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Scanning files: {1}/{2} files ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-status">Step status</message>
approot/messages_fr.xml
+6 -2
View File
@@ -89,9 +89,11 @@
<message id="Lms.Admin.ScannerController.bad-duration">Impossible de récupérer la durée de la piste</message>
<message id="Lms.Admin.ScannerController.cannot-parse-file">Impossible d'analyser le fichier</message>
<message id="Lms.Admin.ScannerController.cannot-read-file">Impossible de lire le fichier</message>
<message id="Lms.Admin.ScannerController.compact">Compacter la base de données. <strong>Attention :</strong> cette opération peut prendre du temps et va vérouiller l'application pendant toute l'étape de compactage!</message>
<message id="Lms.Admin.ScannerController.duplicates-header">{1} fichiers dupliqués :</message>
<message id="Lms.Admin.ScannerController.errors-header">{1} erreurs :</message>
<message id="Lms.Admin.ScannerController.force-scan-now">Forcer un rescan complet</message>
<message id="Lms.Admin.ScannerController.force-optimize">Forcer l'optimisation de la base de données</message>
<message id="Lms.Admin.ScannerController.full-scan">Rescanner tous les fichiers</message>
<message id="Lms.Admin.ScannerController.get-report">Rapport</message>
<message id="Lms.Admin.ScannerController.last-scan">Dernier scan</message>
<message id="Lms.Admin.ScannerController.last-scan-not-available">Non disponible</message>
@@ -100,17 +102,19 @@
<message id="Lms.Admin.ScannerController.same-hash">Hash dupliqué</message>
<message id="Lms.Admin.ScannerController.same-mbid">Track MBID dupliqué</message>
<message id="Lms.Admin.ScannerController.scan-now">Lancer un scan</message>
<message id="Lms.Admin.ScannerController.scan-options">Options de scan</message>
<message id="Lms.Admin.ScannerController.scanner">Scanner</message>
<message id="Lms.Admin.ScannerController.status">Statut</message>
<message id="Lms.Admin.ScannerController.status-not-scheduled">Non planifié</message>
<message id="Lms.Admin.ScannerController.status-scheduled">Planifié le {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">En cours de scan : étape {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-analyze">Analyze de la base de données... {1}/{2} entrées ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Vérification des fichiers dupliqués... {1} fichiers</message>
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Vérification des fichiers... {1}%</message>
<message id="Lms.Admin.ScannerController.step-compact">Compactage de la base de données...</message>
<message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calcul des statistiques... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers : {1} fichiers</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-optimize">Optimisation de la base de données... {1}/{2} entrées ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Rechargement du moteur de recommandation : {1}%...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Scan des fichiers : {1}/{2} fichiers ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-status">Statut de l'étape</message>
approot/messages_it.xml
+5 -2
View File
@@ -89,9 +89,11 @@
<message id="Lms.Admin.ScannerController.bad-duration">Non sono stato in grado di determinare la durata della traccia</message>
<message id="Lms.Admin.ScannerController.cannot-parse-file">Non in grado di analizzare il file</message>
<message id="Lms.Admin.ScannerController.cannot-read-file">Non in grado di leggere il file</message>
<message id="Lms.Admin.ScannerController.compact">Compatta il database. <strong>Attenzione:</strong> ciò potrebbe richiedere del tempo e bloccherà l'intera applicazione durante il passaggio di compattazione!</message>
<message id="Lms.Admin.ScannerController.duplicates-header">{1} file duplicati:</message>
<message id="Lms.Admin.ScannerController.errors-header">{1} errori:</message>
<message id="Lms.Admin.ScannerController.force-scan-now">Forza una nuova scansione completa</message>
<message id="Lms.Admin.ScannerController.force-optimize">Forza l'ottimizzazione del database</message>
<message id="Lms.Admin.ScannerController.full-scan">Forza la scansione di tutti i file</message>
<message id="Lms.Admin.ScannerController.get-report">Scarica un resoconto</message>
<message id="Lms.Admin.ScannerController.last-scan">Ultima scansione</message>
<message id="Lms.Admin.ScannerController.last-scan-not-available">Non disponibile</message>
@@ -100,17 +102,18 @@
<message id="Lms.Admin.ScannerController.same-hash">Hash doppio</message>
<message id="Lms.Admin.ScannerController.same-mbid">Track MBID doppio</message>
<message id="Lms.Admin.ScannerController.scan-now">Scansiona ora</message>
<message id="Lms.Admin.ScannerController.scan-options">Opzioni di scansione</message>
<message id="Lms.Admin.ScannerController.scanner">Scanner</message>
<message id="Lms.Admin.ScannerController.status">Stato</message>
<message id="Lms.Admin.ScannerController.status-not-scheduled">Non pianificato</message>
<message id="Lms.Admin.ScannerController.status-scheduled">Pianificato il {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">Scansione: passo {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-analyze">Analisi del database... {1}/{2} voci ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Controllo duplicati... {1} files</message>
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Controllo file... {1}%</message>
<message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calcolo statistiche... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">File trovati: {1} files</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-optimize">Ottimizzazione del database... {1}/{2} voci ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Ricarica motore di tracce simili: {1}%...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Scansione files: {1}/{2} files ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-status">Stato passo</message>
approot/messages_zh.xml
+6 -2
View File
@@ -89,9 +89,11 @@
<message id="Lms.Admin.ScannerController.bad-duration">无法获得音轨时间</message>
<message id="Lms.Admin.ScannerController.cannot-parse-file">无法解析文件</message>
<message id="Lms.Admin.ScannerController.cannot-read-file">无法读取文件</message>
<message id="Lms.Admin.ScannerController.duplicates-header">{1} 个重复文件：</message>
<message id="Lms.Admin.ScannerController.errors-header">{1} 个错误：</message>
<message id="Lms.Admin.ScannerController.force-scan-now">立即强制全盘扫描</message>
<message id="Lms.Admin.ScannerController.get-report">生成报告</message>
<message id="Lms.Admin.ScannerController.last-scan">最后扫描</message>
<message id="Lms.Admin.ScannerController.last-scan-not-available">不可用</message>
@@ -100,17 +102,19 @@
<message id="Lms.Admin.ScannerController.same-hash">相同文件哈希值</message>
<message id="Lms.Admin.ScannerController.same-mbid">相同 MBID</message>
<message id="Lms.Admin.ScannerController.scan-now">立即扫描</message>
<message id="Lms.Admin.ScannerController.scanner">扫描器</message>
<message id="Lms.Admin.ScannerController.status">状态</message>
<message id="Lms.Admin.ScannerController.status-not-scheduled">无计划</message>
<message id="Lms.Admin.ScannerController.status-scheduled">计划于 {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">扫描中: 阶段 {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">检查文件中... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">检索文件中: {1} 文件</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">重载相似引擎中 {1}%...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">扫描文件中: {1}/{2} 个文件 ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-status">当前步骤状态</message>
src/libs/services/scanner/CMakeLists.txt
+2 -1
View File
@@ -2,10 +2,11 @@
add_library(lmsscanner SHARED
impl/ScannerService.cpp
impl/ScannerStats.cpp
impl/ScanStepAnalyze.cpp
impl/ScanStepCheckDuplicatedDbFiles.cpp
impl/ScanStepCompact.cpp
impl/ScanStepComputeClusterStats.cpp
impl/ScanStepDiscoverFiles.cpp
impl/ScanStepOptimize.cpp
impl/ScanStepRemoveOrphanDbFiles.cpp
impl/ScanStepScanFiles.cpp
)
src/libs/services/scanner/impl/IScanStep.hpp
+2 -1
View File
@@ -22,6 +22,7 @@
#include <vector>
#include "core/LiteralString.hpp"
#include "services/scanner/ScannerOptions.hpp"
#include "services/scanner/ScannerStats.hpp"
namespace lms::scanner
@@ -36,7 +37,7 @@ namespace lms::scanner
struct ScanContext
{
const bool forceScan;
ScanOptions scanOptions;
ScanStats stats;
ScanStepStats currentStepStats;
};
src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp
+2 -2
View File
@@ -29,8 +29,8 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
core::LiteralString getStepName() const override { return "Checking for duplicated files"; }
ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; }
core::LiteralString getStepName() const override { return "Check for duplicated files"; }
ScanStep getStep() const override { return ScanStep::CheckForDuplicateFiles; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepCompact.cpp
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepCompact.hpp"
#include "database/Db.hpp"
#include "database/Session.hpp"
namespace lms::scanner
{
void ScanStepCompact::process(ScanContext& context)
{
// Don't auto compact as it may be too annoying to block the whole application
if (context.scanOptions.compact)
_db.getTLSSession().vacuum();
}
}
src/libs/services/scanner/impl/ScanStepAnalyze.hpp → src/libs/services/scanner/impl/ScanStepCompact.hpp
+3 -3
View File
@@ -23,14 +23,14 @@
namespace lms::scanner
{
class ScanStepAnalyze : public ScanStepBase
class ScanStepCompact : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::Analyze; }
core::LiteralString getStepName() const override { return "Analyze"; }
ScanStep getStep() const override { return ScanStep::Compact; }
core::LiteralString getStepName() const override { return "Compact"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp
+2 -2
View File
@@ -29,8 +29,8 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::DiscoveringFiles; }
core::LiteralString getStepName() const override { return "Discovering files"; }
ScanStep getStep() const override { return ScanStep::DiscoverFiles; }
core::LiteralString getStepName() const override { return "Discover files"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepAnalyze.cpp → src/libs/services/scanner/impl/ScanStepOptimize.cpp
+7 -4
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepAnalyze.hpp"
#include "ScanStepOptimize.hpp"
#include "core/ILogger.hpp"
#include "database/Db.hpp"
@@ -25,19 +25,20 @@
namespace lms::scanner
{
void ScanStepAnalyze::process(ScanContext& context)
void ScanStepOptimize::process(ScanContext& context)
{
ScanStats& stats{ context.stats };
if (stats.nbChanges() > (stats.nbFiles() / 5))
if (context.scanOptions.forceOptimize || (stats.nbChanges() > (stats.nbFiles() / 5)))
{
LMS_LOG(DBUPDATER, INFO, "Database changed substantially: triggering full analyze");
LMS_LOG(DBUPDATER, INFO, "Database analyze started");
auto& session{ _db.getTLSSession() };
std::vector<std::string> entries;
session.retrieveEntriesToAnalyze(entries);
context.currentStepStats.totalElems = entries.size();
_progressCallback(context.currentStepStats);
for (const std::string& entry : entries)
{
@@ -48,6 +49,8 @@ namespace lms::scanner
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
LMS_LOG(DBUPDATER, INFO, "Database analyze complete");
}
}
}
src/libs/services/scanner/impl/ScanStepOptimize.hpp
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ScanStepBase.hpp"
namespace lms::scanner
{
class ScanStepOptimize : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::Optimize; }
core::LiteralString getStepName() const override { return "Optimize"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp
+2 -2
View File
@@ -31,8 +31,8 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
core::LiteralString getStepName() const override { return "Checking orphaned entries"; }
ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; }
core::LiteralString getStepName() const override { return "Check orphaned entries"; }
ScanStep getStep() const override { return ScanStep::CheckForMissingFiles; }
void process(ScanContext& context) override;
void removeOrphanTracks(ScanContext& context);
src/libs/services/scanner/impl/ScanStepScanFiles.cpp
+1 -1
View File
@@ -459,7 +459,7 @@ namespace lms::scanner
}
bool needUpdateLibrary{};
if (!context.forceScan)
if (!context.scanOptions.fullScan)
{
// Skip file if last write is the same
db::Session& dbSession{ _db.getTLSSession() };
src/libs/services/scanner/impl/ScanStepScanFiles.hpp
+2 -2
View File
@@ -39,8 +39,8 @@ namespace lms::scanner
ScanStepScanFiles(InitParams& initParams);
private:
ScanStep getStep() const override { return ScanStep::ScanningFiles; }
core::LiteralString getStepName() const override { return "Scanning files"; }
ScanStep getStep() const override { return ScanStep::ScanFiles; }
core::LiteralString getStepName() const override { return "Scan files"; }
void process(ScanContext& context) override;
bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);
src/libs/services/scanner/impl/ScannerService.cpp
+15 -12
View File
@@ -30,10 +30,11 @@
#include "core/ILogger.hpp"
#include "core/ITraceLogger.hpp"
#include "ScanStepAnalyze.hpp"
#include "ScanStepCheckDuplicatedDbFiles.hpp"
#include "ScanStepCompact.hpp"
#include "ScanStepComputeClusterStats.hpp"
#include "ScanStepDiscoverFiles.hpp"
#include "ScanStepOptimize.hpp"
#include "ScanStepRemoveOrphanDbFiles.hpp"
#include "ScanStepScanFiles.hpp"
@@ -135,15 +136,15 @@ namespace lms::scanner
_events.scanAborted.emit();
}
void ScannerService::requestImmediateScan(bool force)
void ScannerService::requestImmediateScan(const ScanOptions& scanOptions)
{
abortScan();
_ioService.post([this, force]
_ioService.post([this, scanOptions]
{
if (_abortScan)
return;
scheduleScan(force);
scheduleScan(scanOptions);
});
}
@@ -220,7 +221,7 @@ namespace lms::scanner
}
if (nextScanDateTime.isValid())
scheduleScan(false, nextScanDateTime);
scheduleScan(ScanOptions{}, nextScanDateTime);
{
std::unique_lock lock{ _statusMutex };
@@ -231,14 +232,14 @@ namespace lms::scanner
_events.scanScheduled.emit(_nextScheduledScan);
}
void ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime)
void ScannerService::scheduleScan(const ScanOptions& scanOptions, const Wt::WDateTime& dateTime)
{
auto cb{ [this, force](boost::system::error_code ec)
auto cb{ [this, scanOptions](boost::system::error_code ec)
{
if (ec)
return;
scan(force);
scan(scanOptions);
} };
if (dateTime.isNull())
@@ -259,7 +260,7 @@ namespace lms::scanner
}
}
void ScannerService::scan(bool forceScan)
void ScannerService::scan(const ScanOptions& scanOptions)
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "Scan");
@@ -276,16 +277,17 @@ namespace lms::scanner
refreshScanSettings();
IScanStep::ScanContext scanContext{ forceScan, ScanStats {}, ScanStepStats {} };
IScanStep::ScanContext scanContext{ scanOptions, ScanStats {}, ScanStepStats {} };
ScanStats& stats{ scanContext.stats };
stats.startTime = Wt::WDateTime::currentDateTime();
std::size_t stepIndex{};
for (auto& scanStep : _scanSteps)
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() };
scanContext.currentStepStats = ScanStepStats{ .startTime = Wt::WDateTime::currentDateTime(), .stepIndex = stepIndex++, .currentStep = scanStep->getStep() };
notifyInProgress(scanContext.currentStepStats);
scanStep->process(scanContext);
@@ -350,9 +352,10 @@ namespace lms::scanner
_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepCompact>(params));
_scanSteps.push_back(std::make_unique<ScanStepOptimize>(params));
_scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepAnalyze>(params));
}
ScannerSettings ScannerService::readSettings()
src/libs/services/scanner/impl/ScannerService.hpp
+5 -4
View File
@@ -46,12 +46,13 @@ namespace lms::scanner
ScannerService(db::Db& db);
~ScannerService();
private:
ScannerService(const ScannerService&) = delete;
ScannerService& operator=(const ScannerService&) = delete;
private:
void requestStop() override;
void requestReload() override;
void requestImmediateScan(bool force) override;
void requestImmediateScan(const ScanOptions& scanOptions) override;
Status getStatus() const override;
Events& getEvents() override { return _events; }
@@ -62,12 +63,12 @@ namespace lms::scanner
// Job handling
void scheduleNextScan();
void scheduleScan(bool force, const Wt::WDateTime& dateTime = {});
void scheduleScan(const ScanOptions& scanOptions, const Wt::WDateTime& dateTime = {});
void abortScan();
// Update database (scheduled callback)
void scan(bool force);
void scan(const ScanOptions& scanOptions);
void scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
src/libs/services/scanner/include/services/scanner/IScannerService.hpp
+27 -28
View File
@@ -22,47 +22,46 @@
#include <optional>
#include "ScannerEvents.hpp"
#include "ScannerOptions.hpp"
#include "ScannerStats.hpp"
namespace lms::db
{
class Db;
class Db;
}
namespace lms::scanner
{
class IScannerService
{
public:
virtual ~IScannerService() = default;
class IScannerService
{
public:
virtual ~IScannerService() = default;
// Async requests
virtual void requestStop() = 0;
virtual void requestReload() = 0;
// Async requests
virtual void requestStop() = 0;
virtual void requestReload() = 0;
virtual void requestImmediateScan(bool force) = 0;
virtual void requestImmediateScan(const ScanOptions& options = {}) = 0;
enum class State
{
NotScheduled,
Scheduled,
InProgress,
};
enum class State
{
NotScheduled,
Scheduled,
InProgress,
};
struct Status
{
State currentState {State::NotScheduled};
Wt::WDateTime nextScheduledScan;
std::optional<ScanStats> lastCompleteScanStats;
std::optional<ScanStepStats> currentScanStepStats;
};
struct Status
{
State currentState{ State::NotScheduled };
Wt::WDateTime nextScheduledScan;
std::optional<ScanStats> lastCompleteScanStats;
std::optional<ScanStepStats> currentScanStepStats;
};
virtual Status getStatus() const = 0;
virtual Status getStatus() const = 0;
virtual Events& getEvents() = 0;
};
std::unique_ptr<IScannerService> createScannerService(db::Db& db);
virtual Events& getEvents() = 0;
};
std::unique_ptr<IScannerService> createScannerService(db::Db& db);
} // Scanner
src/libs/services/scanner/include/services/scanner/ScannerOptions.hpp
+30
View File
@@ -0,0 +1,30 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
namespace lms::scanner
{
struct ScanOptions
{
bool fullScan{}; // scan files even if not changed
bool forceOptimize{}; // force optimize database
bool compact{}; // compact the database
};
}
src/libs/services/scanner/include/services/scanner/ScannerStats.hpp
+10 -8
View File
@@ -59,22 +59,24 @@ namespace lms::scanner
enum class ScanStep
{
DiscoveringFiles,
ScanningFiles,
ChekingForMissingFiles,
CheckingForDuplicateFiles,
FetchingTrackFeatures,
ReloadingSimilarityEngine,
CheckForMissingFiles,
CheckForDuplicateFiles,
ComputeClusterStats,
Analyze,
Compact,
DiscoverFiles,
FetchTrackFeatures,
Optimize,
ReloadSimilarityEngine,
ScanFiles,
};
static inline constexpr unsigned ScanProgressStepCount{ 8 };
static inline constexpr unsigned ScanProgressStepCount{ 9 };
// reduced scan stats
struct ScanStepStats
{
Wt::WDateTime startTime;
std::size_t stepIndex{};
ScanStep currentStep;
std::size_t totalElems{};
src/libs/subsonic/impl/entrypoints/MediaLibraryScanning.cpp
+2 -2
View File
@@ -39,7 +39,7 @@ namespace lms::api::subsonic::Scan
{
std::size_t count{};
if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanningFiles)
if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanFiles)
count = scanStatus.currentScanStepStats->processedElems;
statusResponse.setAttribute("count", count);
@@ -59,7 +59,7 @@ namespace lms::api::subsonic::Scan
Response handleStartScan(RequestContext& context)
{
core::Service<IScannerService>::get()->requestImmediateScan(false);
core::Service<IScannerService>::get()->requestImmediateScan();
Response response{ Response::createOkResponse(context.serverProtocolVersion) };
response.addNode("scanStatus", createStatusResponseNode());
src/lms/ui/admin/ScannerController.cpp
+79 -50
View File
@@ -22,6 +22,7 @@
#include <iomanip>
#include <Wt/Http/Response.h>
#include <Wt/WCheckBox.h>
#include <Wt/WDateTime.h>
#include <Wt/WPushButton.h>
#include <Wt/WResource.h>
@@ -156,16 +157,19 @@ namespace lms::ui
_reportBtn->setLink(link);
}
Wt::WCheckBox* fullScan{ bindNew<Wt::WCheckBox>("full-scan") };
Wt::WCheckBox* forceOptimize{ bindNew<Wt::WCheckBox>("force-optimize") };
Wt::WCheckBox* compact{ bindNew<Wt::WCheckBox>("compact") };
Wt::WPushButton* scanBtn{ bindNew<Wt::WPushButton>("scan-btn", Wt::WString::tr("Lms.Admin.ScannerController.scan-now")) };
scanBtn->clicked().connect([]
scanBtn->clicked().connect([=]
{
core::Service<scanner::IScannerService>::get()->requestImmediateScan(false);
});
Wt::WPushButton* fullScanBtn{ bindNew<Wt::WPushButton>("full-scan-btn", Wt::WString::tr("Lms.Admin.ScannerController.force-scan-now")) };
fullScanBtn->clicked().connect([]
{
core::Service<scanner::IScannerService>::get()->requestImmediateScan(true);
const scanner::ScanOptions scanOptions
{
.fullScan = fullScan->isChecked(),
.forceOptimize = forceOptimize->isChecked(),
.compact = compact->isChecked(),
};
core::Service<scanner::IScannerService>::get()->requestImmediateScan(scanOptions);
});
_lastScanStatus = bindNew<Wt::WLineEdit>("last-scan");
@@ -199,6 +203,13 @@ namespace lms::ui
using namespace scanner;
const IScannerService::Status status{ core::Service<IScannerService>::get()->getStatus() };
refreshLastScanStatus(status);
refreshStatus(status);
}
void ScannerController::refreshLastScanStatus(const scanner::IScannerService::Status& status)
{
if (status.lastCompleteScanStats)
{
_lastScanStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.last-scan-status")
@@ -218,6 +229,11 @@ namespace lms::ui
_lastScanStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.last-scan-not-available"));
_reportBtn->setEnabled(false);
}
}
void ScannerController::refreshStatus(const scanner::IScannerService::Status& status)
{
using namespace scanner;
switch (status.currentState)
{
@@ -225,64 +241,77 @@ namespace lms::ui
_status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-not-scheduled"));
_stepStatus->setText("");
break;
case IScannerService::State::Scheduled:
_status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-scheduled")
.arg(status.nextScheduledScan.toString()));
_stepStatus->setText("");
break;
case IScannerService::State::InProgress:
_status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress")
.arg(static_cast<int>(status.currentScanStepStats->currentStep) + 1)
.arg(status.currentScanStepStats->stepIndex + 1)
.arg(scanner::ScanProgressStepCount));
switch (status.currentScanStepStats->currentStep)
{
case scanner::ScanStep::CheckingForDuplicateFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files")
.arg(status.currentScanStepStats->processedElems));
break;
refreshCurrentStep(*status.currentScanStepStats);
break;
}
}
case scanner::ScanStep::ChekingForMissingFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files")
.arg(status.currentScanStepStats->progress()));
break;
void ScannerController::refreshCurrentStep(const scanner::ScanStepStats& stepStats)
{
using namespace scanner;
case scanner::ScanStep::DiscoveringFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
.arg(status.currentScanStepStats->processedElems));
break;
switch (stepStats.currentStep)
{
case ScanStep::CheckForDuplicateFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files")
.arg(stepStats.processedElems));
break;
case scanner::ScanStep::ScanningFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
.arg(status.currentScanStepStats->processedElems)
.arg(status.currentScanStepStats->totalElems)
.arg(status.currentScanStepStats->progress()));
break;
case scanner::ScanStep::CheckForMissingFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files")
.arg(stepStats.progress()));
break;
case scanner::ScanStep::FetchingTrackFeatures:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
.arg(status.currentScanStepStats->processedElems)
.arg(status.currentScanStepStats->totalElems)
.arg(status.currentScanStepStats->progress()));
break;
case scanner::ScanStep::Compact:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compact"));
break;
case scanner::ScanStep::ReloadingSimilarityEngine:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine")
.arg(status.currentScanStepStats->progress()));
break;
case scanner::ScanStep::ComputeClusterStats:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats")
.arg(stepStats.progress()));
break;
case scanner::ScanStep::ComputeClusterStats:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats")
.arg(status.currentScanStepStats->progress()));
break;
case scanner::ScanStep::DiscoverFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
.arg(stepStats.processedElems));
break;
case scanner::ScanStep:: Analyze:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-analyze")
.arg(status.currentScanStepStats->processedElems)
.arg(status.currentScanStepStats->totalElems)
.arg(status.currentScanStepStats->progress()));
break;
}
case scanner::ScanStep::FetchTrackFeatures:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
.arg(stepStats.processedElems)
.arg(stepStats.totalElems)
.arg(stepStats.progress()));
break;
case scanner::ScanStep::Optimize:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-optimize")
.arg(stepStats.processedElems)
.arg(stepStats.totalElems)
.arg(stepStats.progress()));
break;
case scanner::ScanStep::ReloadSimilarityEngine:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine")
.arg(stepStats.progress()));
break;
case scanner::ScanStep::ScanFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
.arg(stepStats.processedElems)
.arg(stepStats.totalElems)
.arg(stepStats.progress()));
break;
}
}
src/lms/ui/admin/ScannerController.hpp
+5
View File
@@ -23,6 +23,8 @@
#include <Wt/WTemplate.h>
#include <Wt/WLineEdit.h>
#include "services/scanner/IScannerService.hpp"
namespace lms::ui
{
class ScannerController : public Wt::WTemplate
@@ -32,6 +34,9 @@ namespace lms::ui
private:
void refreshContents();
void refreshLastScanStatus(const scanner::IScannerService::Status& status);
void refreshStatus(const scanner::IScannerService::Status& status);
void refreshCurrentStep(const scanner::ScanStepStats& stepStats);
Wt::WPushButton* _reportBtn;
Wt::WLineEdit* _lastScanStatus;