Scanner: removed the file discover step, as it brings an important time penalty for remote file systems, for big databases (up to several minutes)

approot/messages.xml

@@ -145,7 +145,7 @@
 <message id="Lms.Admin.ScannerController.step-reconciliate-artists">Reconciliating artists: {1} entries...</message>
 <message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Reloading similarity engine: {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Removing orphaned entries: {1} entries...</message>
-<message id="Lms.Admin.ScannerController.step-scanning-files">Scanning files: {1}/{2} ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-scanning-files">Library analysis: {1} files...</message>
 <message id="Lms.Admin.ScannerController.step-status">Step status</message>
 <message id="Lms.Admin.ScannerController.step-updating-library-fields">Updating library fields: {1} entries</message>
 

approot/messages_es.xml

@@ -139,13 +139,12 @@
 <message id="Lms.Admin.ScannerController.step-checking-for-removed-files">Comprobando ficheros eliminados... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-compact">Compactando la base de datos...</message>
 <message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calculando estadísticas... {1}%</message>
-<message id="Lms.Admin.ScannerController.step-discovering-files">Descubriendo ficheros: {1} ficheros</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">Accediendo a las características de la pista desde AcousticBrainz : {1}/{2} ficheros ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-optimize">Optimizando la base de datos... {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-reconciliate-artists">Reconciliando artistas: {1} entradas...</message>
 <message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Recargando el motor de similitud: {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Borrando entradas huérfanas: {1} entradas...</message>
-<message id="Lms.Admin.ScannerController.step-scanning-files">Escaneando ficheros: {1} / {2} ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-scanning-files">Análisis de bibliotecas: {1} ficheros...</message>
 <message id="Lms.Admin.ScannerController.step-status">Estado de las etapas</message>
 <message id="Lms.Admin.ScannerController.step-updating-library-fields">Actualizando campos de la biblioteca: {1} entradas</message>
 

approot/messages_fr.xml

@@ -139,13 +139,12 @@
 <message id="Lms.Admin.ScannerController.step-checking-for-removed-files">Vérification des fichiers supprimés... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-compact">Compactage de la base de données...</message>
 <message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calcul des statistiques... {1}%</message>
-<message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers : {1} fichiers</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-optimize">Optimisation de la base de données... {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-reconciliate-artists">Reconciliation des artistes: {1} entrées...</message>
 <message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Rechargement du moteur de recommandation : {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Retrait des entrées orphelines: {1} entrées...</message>
-<message id="Lms.Admin.ScannerController.step-scanning-files">Scan des fichiers : {1}/{2} ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-scanning-files">Analyse des bibliothèques : {1} fichiers...</message>
 <message id="Lms.Admin.ScannerController.step-status">Statut de l'étape</message>
 <message id="Lms.Admin.ScannerController.step-updating-library-fields">Mise à jour des champs des bibliothèques: {1} entrées</message>
 

approot/messages_it.xml

@@ -139,13 +139,12 @@
 <message id="Lms.Admin.ScannerController.step-checking-for-removed-files">Controllo file... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-compact">Compattazione del database...</message>
 <message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Calcolo statistiche... {1}%</message>
-<message id="Lms.Admin.ScannerController.step-discovering-files">File trovati: {1} files</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-optimize">Ottimizzazione del database... {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-reconciliate-artists">Riconciliazione artisti: {1} voci...</message>
 <message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Ricarica motore di tracce simili: {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Rimozione voci orfane: {1} voci...</message>
-<message id="Lms.Admin.ScannerController.step-scanning-files">Scansione dei file: {1}/{2} ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-scanning-files">Analisi delle librerie: {1} file...</message>
 <message id="Lms.Admin.ScannerController.step-status">Stato passo</message>
 <message id="Lms.Admin.ScannerController.step-updating-library-fields">Aggiornamento dei campi della libreria: {1} voci</message>
 

approot/messages_pl.xml

@@ -152,17 +152,16 @@
 <message id="Lms.Admin.ScannerController.step-checking-for-removed-files">Sprawdzanie plików... {1}%</message>
 <message id="Lms.Admin.ScannerController.step-compact">Prasowanie bazy danych...</message>
 <message id="Lms.Admin.ScannerController.step-compute-cluster-stats">Obliczanie statystyk... {1}%</message>
-<message id="Lms.Admin.ScannerController.step-discovering-files">
-	<plural case="0">Odkrywanie plików: {1} plik</plural>
-	<plural case="1">Odkrywanie plików: {1} pliki</plural>
-	<plural case="2">Odkrywanie plików: {1} plików</plural>
-</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">Pobieranie danych o ścieżce z AcousticBrainz: {1}/{2} ścieżek ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-optimize">Optymalizowanie bazy danych... {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-reconciliate-artists">Uzgodnianie artystów: {1} wpisów...</message>
 <message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">Przeładowywanie silnika podobieństw: {1}%...</message>
 <message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Usuwanie osieroconych wpisów: {1} wpisów...</message>
-<message id="Lms.Admin.ScannerController.step-scanning-files">Skanowanie plików: {1}/{2} ({3}%)...</message>
+<message id="Lms.Admin.ScannerController.step-scanning-files">
+	<plural case="0">Analiza bibliotek: {1} plik</plural>
+	<plural case="1">Analiza bibliotek: {1} pliki</plural>
+	<plural case="2">Analiza bibliotek: {1} plików</plural>
+</message>
 <message id="Lms.Admin.ScannerController.step-status">Obecny krok</message>
 <message id="Lms.Admin.ScannerController.step-updating-library-fields">Aktualizowanie pól biblioteki: {1} wpisów</message>
 

approot/messages_zh.xml

@@ -137,13 +137,12 @@
 <message id="Lms.Admin.ScannerController.step-checking-for-removed-files">检查文件中... {1}%</message>
 
 
-<message id="Lms.Admin.ScannerController.step-discovering-files">检索文件中: {1} 文件</message>
 <message id="Lms.Admin.ScannerController.step-fetching-track-features">从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)...</message>
 
 
 <message id="Lms.Admin.ScannerController.step-reloading-similarity-engine">重载相似引擎中 {1}%...</message>
 
-<message id="Lms.Admin.ScannerController.step-scanning-files">扫描文件中: {1}/{2} 个文件 ({3}%)...</message>
+
 <message id="Lms.Admin.ScannerController.step-status">当前步骤状态</message>
 
 

src/libs/services/scanner/CMakeLists.txt

@@ -21,7 +21,6 @@ add_library(lmsscanner STATIC
 	impl/steps/ScanStepCheckForRemovedFiles.cpp
 	impl/steps/ScanStepCompact.cpp
 	impl/steps/ScanStepComputeClusterStats.cpp
-	impl/steps/ScanStepDiscoverFiles.cpp
 	impl/steps/ScanStepOptimize.cpp
 	impl/steps/ScanStepRemoveOrphanedDbEntries.cpp
 	impl/steps/ScanStepScanFiles.cpp

src/libs/services/scanner/impl/ScannerService.cpp

@@ -47,7 +47,6 @@
 #include "steps/ScanStepCheckForRemovedFiles.hpp"
 #include "steps/ScanStepCompact.hpp"
 #include "steps/ScanStepComputeClusterStats.hpp"
-#include "steps/ScanStepDiscoverFiles.hpp"
 #include "steps/ScanStepOptimize.hpp"
 #include "steps/ScanStepRemoveOrphanedDbEntries.hpp"
 #include "steps/ScanStepScanFiles.hpp"
@@ -474,7 +473,6 @@ namespace lms::scanner
 
         // Order is important: steps are sequential
         _scanSteps.clear();
-        _scanSteps.emplace_back(std::make_unique<ScanStepDiscoverFiles>(params));
         _scanSteps.emplace_back(std::make_unique<ScanStepScanFiles>(params));
         _scanSteps.emplace_back(std::make_unique<ScanStepCheckForRemovedFiles>(params));
         _scanSteps.emplace_back(std::make_unique<ScanStepArtistReconciliation>(params));

src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.cpp

@@ -1,69 +0,0 @@
-/*
- * Copyright (C) 2023 Emeric Poupon
- *
- * This file is part of LMS.
- *
- * LMS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LMS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "ScanStepDiscoverFiles.hpp"
-
-#include "core/ILogger.hpp"
-#include "core/Path.hpp"
-
-#include "MediaLibraryInfo.hpp"
-#include "ScannerSettings.hpp"
-
-#include "ScanContext.hpp"
-
-namespace lms::scanner
-{
-    bool ScanStepDiscoverFiles::needProcess([[maybe_unused]] const ScanContext& context) const
-    {
-        // always discover files
-        return true;
-    }
-
-    void ScanStepDiscoverFiles::process(ScanContext& context)
-    {
-        context.stats.totalFileCount = 0;
-
-        for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
-        {
-            std::size_t currentDirectoryProcessElemsCount{};
-            core::pathUtils::exploreFilesRecursive(
-                mediaLibrary.rootDirectory, [&](std::error_code ec, const std::filesystem::path& path) {
-                    if (_abortScan)
-                        return false;
-
-                    // we don't report errors here (done in the actual scan step)
-                    if (!ec && selectFileScanner(path))
-                    {
-                        context.currentStepStats.processedElems++;
-                        currentDirectoryProcessElemsCount++;
-                        _progressCallback(context.currentStepStats);
-                    }
-
-                    return true;
-                },
-                &excludeDirFileName);
-
-            LMS_LOG(DBUPDATER, DEBUG, "Discovered " << currentDirectoryProcessElemsCount << " files in " << mediaLibrary.rootDirectory);
-        }
-
-        context.stats.totalFileCount = context.currentStepStats.processedElems;
-
-        LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.totalFileCount << " files in all directories");
-    }
-} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.hpp

@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2023 Emeric Poupon
- *
- * This file is part of LMS.
- *
- * LMS is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LMS is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#pragma once
-
-#include "ScanStepBase.hpp"
-
-namespace lms::scanner
-{
-    class ScanStepDiscoverFiles : public ScanStepBase
-    {
-    public:
-        using ScanStepBase::ScanStepBase;
-
-    private:
-        ScanStep getStep() const override { return ScanStep::DiscoverFiles; }
-        core::LiteralString getStepName() const override { return "Discover files"; }
-        bool needProcess(const ScanContext& context) const override;
-        void process(ScanContext& context) override;
-    };
-} // namespace lms::scanner

src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp

@@ -107,8 +107,6 @@ namespace lms::scanner
 
     void ScanStepScanFiles::process(ScanContext& context)
     {
-        context.currentStepStats.totalElems = context.stats.totalFileCount;
-
         for (const MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
             process(context, mediaLibrary);
     }
@@ -165,6 +163,8 @@ namespace lms::scanner
                 return true;
             },
             &excludeDirFileName);
+
+        context.stats.totalFileCount = context.currentStepStats.totalElems;
     }
 
     void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<std::unique_ptr<core::IJob>> scanJobs)

src/libs/services/scanner/include/services/scanner/ScannerStats.hpp

@@ -54,7 +54,6 @@ namespace lms::scanner
         CheckForRemovedFiles,
         ComputeClusterStats,
         Compact,
-        DiscoverFiles,
         FetchTrackFeatures,
         Optimize,
         ReconciliateArtists,
@@ -84,7 +83,7 @@ namespace lms::scanner
         Wt::WDateTime startTime;
         Wt::WDateTime stopTime;
 
-        std::size_t totalFileCount{}; // Total number of files (estimated)
+        std::size_t totalFileCount{}; // Total number of files (only valid after the file scan step)
 
         std::size_t skips{}; // no change since last scan
         std::size_t scans{}; // count of scanned files

src/lms/ui/admin/ScannerController.cpp

@@ -212,11 +212,6 @@ namespace lms::ui
                                      .arg(stepStats.progress()));
             break;
 
-        case ScanStep::DiscoverFiles:
-            _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
-                                     .arg(stepStats.processedElems));
-            break;
-
         case ScanStep::FetchTrackFeatures:
             _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
                                      .arg(stepStats.processedElems)
@@ -246,9 +241,7 @@ namespace lms::ui
 
         case ScanStep::ScanFiles:
             _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
-                                     .arg(stepStats.processedElems)
+                                     .arg(stepStats.processedElems));
-                                     .arg(stepStats.totalElems)
-                                     .arg(stepStats.progress()));
             break;
 
         case ScanStep::UpdateLibraryFields: