owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scanner: removed the file discover step, as it brings an important time penalty for remote file systems, for big databases (up to several minutes)

Browse Source
This commit is contained in:
emeric
2025-07-07 00:11:12 +02:00
parent 385625b6f7
commit 747b5ef6e8
13 changed files with 14 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
approot/messages.xml
+1 -1
View File
@@ -145,7 +145,7 @@
<message id="Lms.Admin.ScannerController.step-reconciliate-artists">Reconciliating artists: {1} entries...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Reloading similarity engine: {1}%...</message>
<message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Removing orphaned entries: {1} entries...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Scanning files: {1}/{2} ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Library analysis: {1} files...</message>
<message id="Lms.Admin.ScannerController.step-status">Step status</message>
<message id="Lms.Admin.ScannerController.step-updating-library-fields">Updating library fields: {1} entries</message>
approot/messages_es.xml
+1 -2
View File
@@ -139,13 +139,12 @@
<message id="Lms.Admin.ScannerController.step-checking-for-removed-files">Comprobando ficheros eliminados... {1}%</message>
<message id="Lms.Admin.ScannerController.step-compact">Compactando la base de datos...</message>
<message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calculando estadísticas... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">Descubriendo ficheros: {1} ficheros</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Accediendo a las características de la pista desde AcousticBrainz : {1}/{2} ficheros ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-optimize">Optimizando la base de datos... {1}%...</message>
<message id="Lms.Admin.ScannerController.step-reconciliate-artists">Reconciliando artistas: {1} entradas...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Recargando el motor de similitud: {1}%...</message>
<message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Borrando entradas huérfanas: {1} entradas...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Escaneando ficheros: {1} / {2} ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Análisis de bibliotecas: {1} ficheros...</message>
<message id="Lms.Admin.ScannerController.step-status">Estado de las etapas</message>
<message id="Lms.Admin.ScannerController.step-updating-library-fields">Actualizando campos de la biblioteca: {1} entradas</message>
approot/messages_fr.xml
+1 -2
View File
@@ -139,13 +139,12 @@
<message id="Lms.Admin.ScannerController.step-checking-for-removed-files">Vérification des fichiers supprimés... {1}%</message>
<message id="Lms.Admin.ScannerController.step-compact">Compactage de la base de données...</message>
<message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calcul des statistiques... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers : {1} fichiers</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-optimize">Optimisation de la base de données... {1}%...</message>
<message id="Lms.Admin.ScannerController.step-reconciliate-artists">Reconciliation des artistes: {1} entrées...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Rechargement du moteur de recommandation : {1}%...</message>
<message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Retrait des entrées orphelines: {1} entrées...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Scan des fichiers : {1}/{2} ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Analyse des bibliothèques : {1} fichiers...</message>
<message id="Lms.Admin.ScannerController.step-status">Statut de l'étape</message>
<message id="Lms.Admin.ScannerController.step-updating-library-fields">Mise à jour des champs des bibliothèques: {1} entrées</message>
approot/messages_it.xml
+1 -2
View File
@@ -139,13 +139,12 @@
<message id="Lms.Admin.ScannerController.step-checking-for-removed-files">Controllo file... {1}%</message>
<message id="Lms.Admin.ScannerController.step-compact">Compattazione del database...</message>
<message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calcolo statistiche... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">File trovati: {1} files</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-optimize">Ottimizzazione del database... {1}%...</message>
<message id="Lms.Admin.ScannerController.step-reconciliate-artists">Riconciliazione artisti: {1} voci...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Ricarica motore di tracce simili: {1}%...</message>
<message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Rimozione voci orfane: {1} voci...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Scansione dei file: {1}/{2} ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Analisi delle librerie: {1} file...</message>
<message id="Lms.Admin.ScannerController.step-status">Stato passo</message>
<message id="Lms.Admin.ScannerController.step-updating-library-fields">Aggiornamento dei campi della libreria: {1} voci</message>
approot/messages_pl.xml
+5 -6
View File
@@ -152,17 +152,16 @@
<message id="Lms.Admin.ScannerController.step-checking-for-removed-files">Sprawdzanie plików... {1}%</message>
<message id="Lms.Admin.ScannerController.step-compact">Prasowanie bazy danych...</message>
<message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Obliczanie statystyk... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">
<plural case="0">Odkrywanie plików: {1} plik</plural>
<plural case="1">Odkrywanie plików: {1} pliki</plural>
<plural case="2">Odkrywanie plików: {1} plików</plural>
</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Pobieranie danych o ścieżce z AcousticBrainz: {1}/{2} ścieżek ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-optimize">Optymalizowanie bazy danych... {1}%...</message>
<message id="Lms.Admin.ScannerController.step-reconciliate-artists">Uzgodnianie artystów: {1} wpisów...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Przeładowywanie silnika podobieństw: {1}%...</message>
<message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Usuwanie osieroconych wpisów: {1} wpisów...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">Skanowanie plików: {1}/{2} ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">
<plural case="0">Analiza bibliotek: {1} plik</plural>
<plural case="1">Analiza bibliotek: {1} pliki</plural>
<plural case="2">Analiza bibliotek: {1} plików</plural>
</message>
<message id="Lms.Admin.ScannerController.step-status">Obecny krok</message>
<message id="Lms.Admin.ScannerController.step-updating-library-fields">Aktualizowanie pól biblioteki: {1} wpisów</message>
approot/messages_zh.xml
+1 -2
View File
@@ -137,13 +137,12 @@
<message id="Lms.Admin.ScannerController.step-checking-for-removed-files">检查文件中... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">检索文件中: {1} 文件</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">重载相似引擎中 {1}%...</message>
<message id="Lms.Admin.ScannerController.step-scanning-files">扫描文件中: {1}/{2} 个文件 ({3}%)...</message>
<message id="Lms.Admin.ScannerController.step-status">当前步骤状态</message>
src/libs/services/scanner/CMakeLists.txt
-1
View File
@@ -21,7 +21,6 @@ add_library(lmsscanner STATIC
impl/steps/ScanStepCheckForRemovedFiles.cpp
impl/steps/ScanStepCompact.cpp
impl/steps/ScanStepComputeClusterStats.cpp
impl/steps/ScanStepDiscoverFiles.cpp
impl/steps/ScanStepOptimize.cpp
impl/steps/ScanStepRemoveOrphanedDbEntries.cpp
impl/steps/ScanStepScanFiles.cpp
src/libs/services/scanner/impl/ScannerService.cpp
-2
View File
@@ -47,7 +47,6 @@
#include "steps/ScanStepCheckForRemovedFiles.hpp"
#include "steps/ScanStepCompact.hpp"
#include "steps/ScanStepComputeClusterStats.hpp"
#include "steps/ScanStepDiscoverFiles.hpp"
#include "steps/ScanStepOptimize.hpp"
#include "steps/ScanStepRemoveOrphanedDbEntries.hpp"
#include "steps/ScanStepScanFiles.hpp"
@@ -474,7 +473,6 @@ namespace lms::scanner
// Order is important: steps are sequential
_scanSteps.clear();
_scanSteps.emplace_back(std::make_unique<ScanStepDiscoverFiles>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepScanFiles>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepCheckForRemovedFiles>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepArtistReconciliation>(params));
src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.cpp
-69
View File
@@ -1,69 +0,0 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepDiscoverFiles.hpp"
#include "core/ILogger.hpp"
#include "core/Path.hpp"
#include "MediaLibraryInfo.hpp"
#include "ScannerSettings.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
bool ScanStepDiscoverFiles::needProcess([[maybe_unused]] const ScanContext& context) const
{
// always discover files
return true;
}
void ScanStepDiscoverFiles::process(ScanContext& context)
{
context.stats.totalFileCount = 0;
for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
{
std::size_t currentDirectoryProcessElemsCount{};
core::pathUtils::exploreFilesRecursive(
mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) {
if (_abortScan)
return false;
// we don't report errors here (done in the actual scan step)
if (!ec && selectFileScanner(path))
{
context.currentStepStats.processedElems++;
currentDirectoryProcessElemsCount++;
_progressCallback(context.currentStepStats);
}
return true;
},
&excludeDirFileName);
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in " << mediaLibrary.rootDirectory);
}
context.stats.totalFileCount = context.currentStepStats.processedElems;
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.totalFileCount << " files in all directories");
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.hpp
-37
View File
@@ -1,37 +0,0 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ScanStepBase.hpp"
namespace lms::scanner
{
class ScanStepDiscoverFiles : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::DiscoverFiles; }
core::LiteralString getStepName() const override { return "Discover files"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp
+2 -2
View File
@@ -107,8 +107,6 @@ namespace lms::scanner
void ScanStepScanFiles::process(ScanContext& context)
{
context.currentStepStats.totalElems = context.stats.totalFileCount;
for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
process(context, mediaLibrary);
}
@@ -165,6 +163,8 @@ namespace lms::scanner
return true;
},
&excludeDirFileName);
context.stats.totalFileCount = context.currentStepStats.totalElems;
}
void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<std::unique_ptr<core::IJob>> scanJobs)
src/libs/services/scanner/include/services/scanner/ScannerStats.hpp
+1 -2
View File
@@ -54,7 +54,6 @@ namespace lms::scanner
CheckForRemovedFiles,
ComputeClusterStats,
Compact,
DiscoverFiles,
FetchTrackFeatures,
Optimize,
ReconciliateArtists,
@@ -84,7 +83,7 @@ namespace lms::scanner
Wt::WDateTime startTime;
Wt::WDateTime stopTime;
std::size_t totalFileCount{}; // Total number of files (estimated)
std::size_t totalFileCount{}; // Total number of files (only valid after the file scan step)
std::size_t skips{}; // no change since last scan
std::size_t scans{}; // count of scanned files
src/lms/ui/admin/ScannerController.cpp
+1 -8
View File
@@ -212,11 +212,6 @@ namespace lms::ui
.arg(stepStats.progress()));
break;
case ScanStep::DiscoverFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
.arg(stepStats.processedElems));
break;
case ScanStep::FetchTrackFeatures:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
.arg(stepStats.processedElems)
@@ -246,9 +241,7 @@ namespace lms::ui
case ScanStep::ScanFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
.arg(stepStats.processedElems)
.arg(stepStats.totalElems)
.arg(stepStats.progress()));
.arg(stepStats.processedElems));
break;
case ScanStep::UpdateLibraryFields: