Optimized library assignation to directories

SUBSONIC.md

@@ -1,9 +1,12 @@
 # Subsonic API
 The API version implemented is 1.16.0 and has been tested on _Android_ using _Subsonic Player_, _Ultrasonic_, _Symfonium_, and _DSub_.
-Since _LMS_ uses metadata tags to organize music, a compatibility mode is used to browse the collection when using the directory browsing commands.
+
+Folder navigation commands are supported. However, since _LMS_ does not store information for each folder, it is not possible to star/unstar folders considered as artists.
+Given the API limitations of folder navigation commands, it is recommended to place all tracks of an album in the same folder and not to mix multiple albums in the same folder.
+
 The Subsonic API is enabled by default.
 
-__Note__: since _LMS_ may store hashed and salted passwords or may forward authentication requests to external services, it cannot handle the __token authentication__ method. You may need to check your client to make sure to use the __password__ authentication method.
+__Note__: since _LMS_ may store hashed and salted passwords or may forward authentication requests to external services, it cannot handle the __token authentication__ method. You may need to check your client to make sure to use the __password__ authentication method. Since logins/passwords are passed in plain text through URLs, it is highly recommended to use a unique password when using the Subsonic API. Note that this may affect the use of authentication via PAM. In any case, ensure that read access to the web server logs (and to the proxy, if applicable) is well protected.
 
 # OpenSubsonic API
 OpenSubsonic is an initiative to patch and extend the legacy Subsonic API. You'll find more details in the [official documentation](https://opensubsonic.netlify.app/)

approot/messages.xml

@@ -121,6 +121,7 @@
 <message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Removing orphaned entries: {1} entries...</message>
 <message id="Lms.Admin.ScannerController.step-scanning-files">Scanning files: {1}/{2} ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-status">Step status</message>
+<message id="Lms.Admin.ScannerController.step-updating-library-fields">Updating library fields: {1} entries</message>
 
 <!--Tracing-->
 <message id="Lms.Admin.Tracing.export-current-buffer">Export traces</message>

approot/messages_fr.xml

@@ -121,6 +121,7 @@
 <message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Retrait des entrées orphelines: {1} entrées...</message>
 <message id="Lms.Admin.ScannerController.step-scanning-files">Scan des fichiers : {1}/{2} ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-status">Statut de l'étape</message>
+<message id="Lms.Admin.ScannerController.step-updating-library-fields">Mise à jour des champs des bibliothèques: {1} entrées</message>
 
 <!--Tracing-->
 <message id="Lms.Admin.Tracing.export-current-buffer">Exporter les traces</message>

approot/messages_it.xml

@@ -121,6 +121,7 @@
 <message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Rimozione voci orfane: {1} voci...</message>
 <message id="Lms.Admin.ScannerController.step-scanning-files">Scansione dei file: {1}/{2} ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-status">Stato passo</message>
+<message id="Lms.Admin.ScannerController.step-updating-library-fields">Aggiornamento dei campi della libreria: {1} voci</message>
 
 <!--Tracing-->
 <message id="Lms.Admin.Tracing.export-current-buffer">Esporta tracce</message>

approot/messages_pl.xml

@@ -138,6 +138,7 @@
 <message id="Lms.Admin.ScannerController.step-removing-orphaned-entries">Usuwanie osieroconych wpisów: {1} wpisów...</message>
 <message id="Lms.Admin.ScannerController.step-scanning-files">Skanowanie plików: {1}/{2} ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-status">Obecny krok</message>
+<message id="Lms.Admin.ScannerController.step-updating-library-fields">Aktualizowanie pól biblioteki: {1} wpisów</message>
 
 <!--Tracing-->
 <message id="Lms.Admin.Tracing.export-current-buffer">Eksportuj ślady</message>

approot/messages_zh.xml

@@ -122,6 +122,7 @@
 <message id="Lms.Admin.ScannerController.step-scanning-files">扫描文件中: {1}/{2} 个文件 ({3}%)...</message>
 <message id="Lms.Admin.ScannerController.step-status">当前步骤状态</message>
 
+
 <!--Tracing-->
 
 

src/libs/database/impl/Directory.cpp

@@ -83,6 +83,25 @@ namespace lms::db
 
             return query;
         }
+
+        std::filesystem::path getPathWithTrailingSeparator(const std::filesystem::path& path)
+        {
+            if (path.empty())
+                return path;
+
+            // Convert the path to string
+            std::string pathStr{ path.string() };
+
+            // Check if the last character is a directory separator
+            if (pathStr.back() != std::filesystem::path::preferred_separator)
+            {
+                // If not, add the preferred separator
+                pathStr += std::filesystem::path::preferred_separator;
+            }
+
+            // Return the new path
+            return std::filesystem::path{ pathStr };
+        }
     } // namespace
 
     Directory::Directory(const std::filesystem::path& p)
@@ -157,6 +176,17 @@ namespace lms::db
         return utils::execRangeQuery<DirectoryId>(query, range);
     }
 
+    RangeResults<DirectoryId> Directory::findMismatchedLibrary(Session& session, std::optional<Range> range, const std::filesystem::path& rootPath, MediaLibraryId expectedLibraryId)
+    {
+        session.checkReadTransaction();
+
+        auto query{ session.getDboSession()->query<DirectoryId>("SELECT d.id FROM directory d") };
+        query.where("d.absolute_path = ? OR d.absolute_path LIKE ?").bind(rootPath).bind(getPathWithTrailingSeparator(rootPath).string() + "%");
+        query.where("d.media_library_id <> ? OR d.media_library_id IS NULL").bind(expectedLibraryId);
+
+        return utils::execRangeQuery<DirectoryId>(query, range);
+    }
+
     RangeResults<Directory::pointer> Directory::findRootDirectories(Session& session, std::optional<Range> range)
     {
         auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Directory>>("SELECT d from directory d").where("d.parent_directory_id IS NULL") };

src/libs/database/include/database/Directory.hpp

@@ -102,6 +102,7 @@ namespace lms::db
         static RangeResults<Directory::pointer> find(Session& session, const FindParameters& params);
         static void find(Session& session, const FindParameters& parameters, const std::function<void(const Directory::pointer&)>& func);
         static RangeResults<DirectoryId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt);
+        static RangeResults<DirectoryId> findMismatchedLibrary(Session& session, std::optional<Range> range, const std::filesystem::path& rootPath, MediaLibraryId expectedLibraryId);
         static RangeResults<pointer> findRootDirectories(Session& session, std::optional<Range> range = std::nullopt);
 
         // getters

src/libs/database/test/Directory.cpp

@@ -252,4 +252,37 @@ namespace lms::db::tests
             EXPECT_EQ(res[1]->getId(), child2.getId());
         }
     }
+
+    TEST_F(DatabaseFixture, Directory_findMismatchedLibrary)
+    {
+        ScopedDirectory parent1{ session, "/root" };
+        ScopedDirectory child1{ session, "/root/foo" };
+        ScopedDirectory parent2{ session, "/root_1" };
+        ScopedDirectory child2{ session, "/root_1/foo" };
+
+        ScopedMediaLibrary library{ session, "/root" };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            const auto res{ Directory::findMismatchedLibrary(session, std::nullopt, library->getPath(), library->getId()).results };
+            ASSERT_EQ(res.size(), 2);
+            EXPECT_EQ(res[0], parent1.getId());
+            EXPECT_EQ(res[1], child1.getId());
+        }
+
+        {
+            auto transaction{ session.createWriteTransaction() };
+
+            parent1.get().modify()->setMediaLibrary(library.get());
+            child1.get().modify()->setMediaLibrary(library.get());
+        }
+
+        {
+            auto transaction{ session.createReadTransaction() };
+
+            const auto res{ Directory::findMismatchedLibrary(session, std::nullopt, library->getPath(), library->getId()).results };
+            EXPECT_EQ(res.size(), 0);
+        }
+    }
 } // namespace lms::db::tests

src/libs/services/scanner/CMakeLists.txt

@@ -12,6 +12,7 @@ add_library(lmsscanner SHARED
 	impl/ScanStepOptimize.cpp
 	impl/ScanStepRemoveOrphanedDbEntries.cpp
 	impl/ScanStepScanFiles.cpp
+	impl/ScanStepUpdateLibraryFields.cpp
 	)
 
 target_include_directories(lmsscanner INTERFACE

src/libs/services/scanner/impl/ScanStepScanFiles.cpp

@@ -383,17 +383,6 @@ namespace lms::scanner
 
                     if (fileMatched)
                     {
-                        // Not very efficient way to update media_library for directories
-                        if (path.has_parent_path())
-                        {
-                            const std::filesystem::path directory{ path.parent_path() };
-                            if (directory != currentDirectory)
-                            {
-                                updateDirectoryIfNeeded(currentDirectory, mediaLibrary);
-                                currentDirectory = directory;
-                            }
-                        }
-
                         context.currentStepStats.processedElems++;
                         _progressCallback(context.currentStepStats);
                     }
@@ -792,34 +781,4 @@ namespace lms::scanner
             stats.updates++;
         }
     }
-
-    void ScanStepScanFiles::updateDirectoryIfNeeded(const std::filesystem::path& dirPath, const ScannerSettings::MediaLibraryInfo& libraryInfo)
-    {
-        db::Session& dbSession{ _db.getTLSSession() };
-
-        const bool needUpdateLibrary{ [&] {
-            auto transaction{ dbSession.createReadTransaction() };
-
-            Directory::pointer directory{ Directory::find(dbSession, dirPath) };
-            if (!directory)
-                return false; // we create directories only of we find images or tracks inside
-
-            MediaLibrary::pointer currentLibrary{ directory->getMediaLibrary() };
-            return !currentLibrary || currentLibrary->getId() != libraryInfo.id;
-        }() };
-
-        if (needUpdateLibrary)
-        {
-            auto transaction{ dbSession.createWriteTransaction() };
-
-            Directory::pointer directory{ Directory::find(dbSession, dirPath) };
-            assert(directory);
-            directory.modify()->setMediaLibrary(MediaLibrary::find(dbSession, libraryInfo.id)); // may be null if settings are updated in // => next scan will correct this
-        }
-
-        if (dirPath != libraryInfo.rootDirectory && dirPath.has_parent_path())
-        {
-            updateDirectoryIfNeeded(dirPath.parent_path(), libraryInfo);
-        }
-    }
 } // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepScanFiles.hpp

@@ -49,8 +49,6 @@ namespace lms::scanner
         void processAudioFileScanData(ScanContext& context, const std::filesystem::path& path, const metadata::Track* trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo);
         void processImageFileScanData(ScanContext& context, const std::filesystem::path& path, const ImageInfo* imageInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo);
 
-        void updateDirectoryIfNeeded(const std::filesystem::path& directory, const ScannerSettings::MediaLibraryInfo& libraryInfo);
-
         std::unique_ptr<metadata::IParser> _metadataParser;
         const std::vector<std::string> _extraTagsToParse;
 

src/libs/services/scanner/impl/ScanStepUpdateLibraryFields.cpp

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepUpdateLibraryFields.hpp"
+
+#include "core/ILogger.hpp"
+#include "core/Path.hpp"
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/MediaLibrary.hpp"
+#include "database/Session.hpp"
+
+namespace lms::scanner
+{
+
+    void ScanStepUpdateLibraryFields::process(ScanContext& context)
+    {
+        processDirectories(context);
+    }
+
+    void ScanStepUpdateLibraryFields::processDirectories(ScanContext& context)
+    {
+        for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
+        {
+            if (_abortScan)
+                break;
+
+            processDirectory(context, mediaLibrary);
+        }
+    }
+
+    void ScanStepUpdateLibraryFields::processDirectory(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary)
+    {
+        db::Session& session{ _db.getTLSSession() };
+
+        constexpr std::size_t batchSize = 100;
+
+        db::RangeResults<db::DirectoryId> entries;
+        while (!_abortScan)
+        {
+            {
+                auto transaction{ session.createReadTransaction() };
+
+                entries = db::Directory::findMismatchedLibrary(session, db::Range{ 0, batchSize }, mediaLibrary.rootDirectory, mediaLibrary.id);
+            };
+
+            if (entries.results.empty())
+                break;
+
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                db::MediaLibrary::pointer library{ db::MediaLibrary::find(session, mediaLibrary.id) };
+                if (!library) // may be legit
+                    break;
+
+                for (const db::DirectoryId directoryId : entries.results)
+                {
+                    if (_abortScan)
+                        break;
+
+                    db::Directory::pointer directory{ db::Directory::find(session, directoryId) };
+                    directory.modify()->setMediaLibrary(library);
+                }
+            }
+
+            context.currentStepStats.processedElems += entries.results.size();
+            _progressCallback(context.currentStepStats);
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepUpdateLibraryFields.hpp

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2023 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "database/DirectoryId.hpp"
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepUpdateLibraryFields : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        core::LiteralString getStepName() const override { return "Update Library fields"; }
+        ScanStep getStep() const override { return ScanStep::UpdateLibraryFields; }
+        void process(ScanContext& context) override;
+
+        void processDirectories(ScanContext& context);
+        void processDirectory(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary);
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScannerService.cpp

@@ -40,6 +40,7 @@
 #include "ScanStepOptimize.hpp"
 #include "ScanStepRemoveOrphanedDbEntries.hpp"
 #include "ScanStepScanFiles.hpp"
+#include "ScanStepUpdateLibraryFields.hpp"
 
 namespace lms::scanner
 {
@@ -279,7 +280,7 @@ namespace lms::scanner
             LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
 
             LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
-            scanContext.currentStepStats = ScanStepStats{ .startTime = Wt::WDateTime::currentDateTime(), .stepIndex = stepIndex++, .currentStep = scanStep->getStep() };
+            scanContext.currentStepStats = ScanStepStats{ .startTime = Wt::WDateTime::currentDateTime(), .stepCount = _scanSteps.size(), .stepIndex = stepIndex++, .currentStep = scanStep->getStep() };
 
             notifyInProgress(scanContext.currentStepStats);
             scanStep->process(scanContext);
@@ -339,11 +340,12 @@ namespace lms::scanner
             _db
         };
 
-        // Order is important
+        // Order is important, steps are sequential
         _scanSteps.clear();
         _scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
         _scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
         _scanSteps.push_back(std::make_unique<ScanStepCheckForRemovedFiles>(params));
+        _scanSteps.push_back(std::make_unique<ScanStepUpdateLibraryFields>(params));
         _scanSteps.push_back(std::make_unique<ScanStepAssociateArtistImages>(params));
         _scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanedDbEntries>(params));
         _scanSteps.push_back(std::make_unique<ScanStepCompact>(params));

src/libs/services/scanner/include/services/scanner/ScannerStats.hpp

@@ -72,14 +72,15 @@ namespace lms::scanner
         ReloadSimilarityEngine,
         RemoveOrphanedDbEntries,
         ScanFiles,
+        UpdateLibraryFields,
     };
-    static inline constexpr unsigned ScanProgressStepCount{ 11 };
 
     // reduced scan stats
     struct ScanStepStats
     {
         Wt::WDateTime startTime;
 
+        std::size_t stepCount{};
         std::size_t stepIndex{};
         ScanStep currentStep;
 

src/lms/ui/admin/ScannerController.cpp

@@ -256,7 +256,7 @@ namespace lms::ui
 
             _status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress")
                                  .arg(status.currentScanStepStats->stepIndex + 1)
-                                 .arg(scanner::ScanProgressStepCount));
+                                 .arg(status.currentScanStepStats->stepCount));
 
             refreshCurrentStep(*status.currentScanStepStats);
             break;
@@ -326,6 +326,11 @@ namespace lms::ui
                                      .arg(stepStats.totalElems)
                                      .arg(stepStats.progress()));
             break;
+
+        case ScanStep::UpdateLibraryFields:
+            _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-updating-library-fields")
+                                     .arg(stepStats.processedElems));
+            break;
         }
     }
 } // namespace lms::ui