Added embedded/external lyrics support: parsing + indexing, ref #379

src/libs/services/scanner/CMakeLists.txt

@@ -4,6 +4,7 @@ add_library(lmsscanner SHARED
 	impl/ScannerService.cpp
 	impl/ScannerStats.cpp
 	impl/ScanStepAssociateArtistImages.cpp
+	impl/ScanStepAssociateExternalLyrics.cpp
 	impl/ScanStepAssociateReleaseImages.cpp
 	impl/ScanStepCheckForDuplicatedFiles.cpp
 	impl/ScanStepCheckForRemovedFiles.cpp

src/libs/services/scanner/impl/FileScanQueue.cpp

@@ -19,6 +19,8 @@
 
 #include "FileScanQueue.hpp"
 
+#include <fstream>
+
 #include "core/Exception.hpp"
 #include "core/IConfig.hpp"
 #include "core/ILogger.hpp"
@@ -62,6 +64,10 @@ namespace lms::scanner
                     break;
                 case ScanRequestType::ImageFile:
                     result.scanData = scanImageFile(path);
+                    break;
+                case ScanRequestType::LyricsFile:
+                    result.scanData = scanLyricsFile(path);
+                    break;
                 }
 
                 {
@@ -114,6 +120,28 @@ namespace lms::scanner
         return optInfo;
     }
 
+    LyricsFileScanData FileScanQueue::scanLyricsFile(const std::filesystem::path& path)
+    {
+        LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ScanLyricsFile");
+
+        LyricsFileScanData lyrics;
+
+        try
+        {
+            std::ifstream ifs{ path.string() };
+            if (!ifs)
+                LMS_LOG(DBUPDATER, ERROR, "Cannot open file '" << path.string() << "'");
+            else
+                lyrics = metadata::parseLyrics(ifs);
+        }
+        catch (const std::exception& e)
+        {
+            LMS_LOG(DBUPDATER, ERROR, "Cannot read lyrics in file '" << path.string() << "': " << e.what());
+        }
+
+        return lyrics;
+    }
+
     std::size_t FileScanQueue::getResultsCount() const
     {
         std::scoped_lock lock{ _mutex };

src/libs/services/scanner/impl/FileScanQueue.hpp

@@ -29,6 +29,7 @@
 
 #include "core/IOContextRunner.hpp"
 #include "metadata/IParser.hpp"
+#include "metadata/Lyrics.hpp"
 
 namespace lms::scanner
 {
@@ -40,10 +41,11 @@ namespace lms::scanner
 
     using AudioFileScanData = std::unique_ptr<metadata::Track>;
     using ImageFileScanData = std::optional<ImageInfo>;
+    using LyricsFileScanData = std::optional<metadata::Lyrics>;
     struct FileScanResult
     {
         std::filesystem::path path;
-        std::variant<std::monostate, AudioFileScanData, ImageFileScanData> scanData;
+        std::variant<std::monostate, AudioFileScanData, ImageFileScanData, LyricsFileScanData> scanData;
     };
 
     class FileScanQueue
@@ -57,6 +59,7 @@ namespace lms::scanner
         {
             AudioFile,
             ImageFile,
+            LyricsFile,
         };
         void pushScanRequest(const std::filesystem::path& path, ScanRequestType type);
 
@@ -68,6 +71,7 @@ namespace lms::scanner
     private:
         AudioFileScanData scanAudioFile(const std::filesystem::path& path);
         ImageFileScanData scanImageFile(const std::filesystem::path& path);
+        LyricsFileScanData scanLyricsFile(const std::filesystem::path& path);
 
         metadata::IParser& _metadataParser;
         boost::asio::io_context _scanContext;

src/libs/services/scanner/impl/IScanStep.hpp

@@ -19,8 +19,6 @@
 
 #pragma once
 
-#include <vector>
-
 #include "core/LiteralString.hpp"
 #include "services/scanner/ScannerOptions.hpp"
 #include "services/scanner/ScannerStats.hpp"

src/libs/services/scanner/impl/ScanStepAssociateArtistImages.cpp

@@ -54,6 +54,7 @@ namespace lms::scanner
         {
             db::Session& session;
             db::ArtistId lastRetrievedArtistId;
+            std::size_t processedArtistCount{};
             const std::vector<std::string>& artistFileNames;
         };
 
@@ -150,6 +151,7 @@ namespace lms::scanner
                         LMS_LOG(DBUPDATER, DEBUG, "Updating artist image for artist '" << artist->getName() << "', using '" << (image ? image->getAbsoluteFilePath().c_str() : "<none>") << "'");
                         artistImageAssociations.push_back(ArtistImageAssociation{ artist->getId(), image ? image->getId() : db::ImageId{} });
                     }
+                    searchContext.processedArtistCount++;
                 });
             }
 
@@ -231,7 +233,7 @@ namespace lms::scanner
                 return;
 
             updateArtistImages(session, artistImageAssociations);
-            context.currentStepStats.processedElems += readBatchSize;
+            context.currentStepStats.processedElems = searchContext.processedArtistCount;
             _progressCallback(context.currentStepStats);
         }
     }

src/libs/services/scanner/impl/ScanStepAssociateExternalLyrics.cpp

@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "ScanStepAssociateExternalLyrics.hpp"
+
+#include <deque>
+
+#include "core/ILogger.hpp"
+#include "database/Db.hpp"
+#include "database/Directory.hpp"
+#include "database/Session.hpp"
+#include "database/Track.hpp"
+#include "database/TrackLyrics.hpp"
+
+namespace lms::scanner
+{
+    namespace
+    {
+        constexpr std::size_t readBatchSize{ 100 };
+        constexpr std::size_t writeBatchSize{ 20 };
+
+        struct TrackLyricsAssociation
+        {
+            db::TrackLyricsId trackLyricsId;
+            db::TrackId trackId;
+        };
+        using TrackLyricsAssociationContainer = std::deque<TrackLyricsAssociation>;
+
+        struct SearchTrackLyricsContext
+        {
+            db::Session& session;
+            db::TrackLyricsId lastRetrievedTrackLyricsId;
+            std::size_t processedLyricsCount{};
+        };
+
+        db::Track::pointer getMatchingTrack(db::Session& session, const db::TrackLyrics::pointer& lyrics)
+        {
+            db::Track::pointer matchingTrack;
+
+            auto tryMatch = [&](std::string_view stem) {
+                db::Track::FindParameters params;
+                assert(lyrics->getDirectory()->getId().isValid());
+                assert(!lyrics->getFileStem().empty());
+
+                params.setDirectory(lyrics->getDirectory()->getId());
+                params.setStem(stem);
+
+                db::Track::find(session, params, [&](const db::Track::pointer track) {
+                    if (matchingTrack)
+                        LMS_LOG(DBUPDATER, DEBUG, "External lyrics '" << lyrics->getAbsoluteFilePath() << "' already matched with '" << matchingTrack->getAbsoluteFilePath() << "', replaced by '" << track->getAbsoluteFilePath() << "'");
+
+                    matchingTrack = track;
+                });
+            };
+
+            // First try with the stem. If it does not match, try again with the parent steam, if it exists, to handle the file.laguagecode.lrc case
+            tryMatch(lyrics->getFileStem());
+            if (!matchingTrack)
+            {
+                std::filesystem::path stem{ lyrics->getFileStem() };
+                if (stem.has_extension())
+                    tryMatch(stem.stem().string());
+            }
+
+            return matchingTrack;
+        }
+
+        bool fetchNextTrackLyricsToUpdate(SearchTrackLyricsContext& searchContext, TrackLyricsAssociationContainer& trackLyricsAssociations)
+        {
+            const db::TrackLyricsId trackLyricsId{ searchContext.lastRetrievedTrackLyricsId };
+
+            {
+                auto transaction{ searchContext.session.createReadTransaction() };
+
+                db::TrackLyrics::find(searchContext.session, searchContext.lastRetrievedTrackLyricsId, readBatchSize, [&](const db::TrackLyrics::pointer& trackLyrics) {
+                    // Only iterate over external lyrics
+                    if (trackLyrics->getAbsoluteFilePath().empty())
+                        return;
+
+                    db::Track::pointer track{ getMatchingTrack(searchContext.session, trackLyrics) };
+                    if (track != trackLyrics->getTrack())
+                    {
+                        LMS_LOG(DBUPDATER, DEBUG, "Updating track for external lyrics '" << trackLyrics->getAbsoluteFilePath() << "', using '" << (track ? track->getAbsoluteFilePath().c_str() : "<none>") << "'");
+                        trackLyricsAssociations.push_back(TrackLyricsAssociation{ .trackLyricsId = trackLyrics->getId(), .trackId = (track ? track->getId() : db::TrackId{}) });
+                    }
+                    searchContext.processedLyricsCount++;
+                });
+            }
+
+            return trackLyricsId != searchContext.lastRetrievedTrackLyricsId;
+        }
+
+        void updateTrackLyrics(db::Session& session, const TrackLyricsAssociation& trackLyricsAssociation)
+        {
+            db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(session, trackLyricsAssociation.trackLyricsId) };
+            assert(lyrics);
+
+            db::Track::pointer track;
+            if (trackLyricsAssociation.trackId.isValid())
+                track = db::Track::find(session, trackLyricsAssociation.trackId);
+
+            lyrics.modify()->setTrack(track);
+        }
+
+        void updateTrackLyrics(db::Session& session, TrackLyricsAssociationContainer& lyricsAssociations)
+        {
+            while (!lyricsAssociations.empty())
+            {
+                auto transaction{ session.createWriteTransaction() };
+
+                for (std::size_t i{}; !lyricsAssociations.empty() && i < writeBatchSize; ++i)
+                {
+                    updateTrackLyrics(session, lyricsAssociations.front());
+                    lyricsAssociations.pop_front();
+                }
+            }
+        }
+    } // namespace
+
+    void ScanStepAssociateExternalLyrics::process(ScanContext& context)
+    {
+        if (_abortScan)
+            return;
+
+        if (context.stats.nbChanges() == 0)
+            return;
+
+        auto& session{ _db.getTLSSession() };
+
+        {
+            auto transaction{ session.createReadTransaction() };
+            context.currentStepStats.totalElems = db::TrackLyrics::getExternalLyricsCount(session);
+        }
+
+        SearchTrackLyricsContext searchContext{
+            .session = session,
+            .lastRetrievedTrackLyricsId = {},
+        };
+
+        TrackLyricsAssociationContainer trackLyricsAssociations;
+        while (fetchNextTrackLyricsToUpdate(searchContext, trackLyricsAssociations))
+        {
+            if (_abortScan)
+                return;
+
+            updateTrackLyrics(session, trackLyricsAssociations);
+            context.currentStepStats.processedElems = searchContext.processedLyricsCount;
+            _progressCallback(context.currentStepStats);
+        }
+    }
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepAssociateExternalLyrics.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "ScanStepBase.hpp"
+
+namespace lms::scanner
+{
+    class ScanStepAssociateExternalLyrics : public ScanStepBase
+    {
+    public:
+        using ScanStepBase::ScanStepBase;
+
+    private:
+        ScanStep getStep() const override { return ScanStep::AssociateExternalLyrics; }
+        core::LiteralString getStepName() const override { return "Associate external lyrics"; }
+        void process(ScanContext& context) override;
+    };
+} // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepAssociateReleaseImages.cpp

@@ -54,6 +54,7 @@ namespace lms::scanner
         {
             db::Session& session;
             db::ReleaseId lastRetrievedReleaseId;
+            std::size_t processedReleaseCount{};
             const std::vector<std::string>& releaseFileNames;
         };
 
@@ -146,6 +147,7 @@ namespace lms::scanner
                         LMS_LOG(DBUPDATER, DEBUG, "Updating release image for release '" << release->getName() << "', using '" << (image ? image->getAbsoluteFilePath().c_str() : "<none>") << "'");
                         releaseImageAssociations.push_back(ReleaseImageAssociation{ release->getId(), image ? image->getId() : db::ImageId{} });
                     }
+                    searchContext.processedReleaseCount++;
                 });
             }
 
@@ -227,7 +229,7 @@ namespace lms::scanner
                 return;
 
             updateReleaseImages(session, releaseImageAssociations);
-            context.currentStepStats.processedElems += readBatchSize;
+            context.currentStepStats.processedElems = searchContext.processedReleaseCount;
             _progressCallback(context.currentStepStats);
         }
     }

src/libs/services/scanner/impl/ScanStepAssociateReleaseImages.hpp

@@ -32,7 +32,7 @@ namespace lms::scanner
         ScanStepAssociateReleaseImages(InitParams& initParams);
 
     private:
-        ScanStep getStep() const override { return ScanStep::AssociateArtistImages; }
+        ScanStep getStep() const override { return ScanStep::AssociateReleaseImages; }
         core::LiteralString getStepName() const override { return "Associate release images"; }
         void process(ScanContext& context) override;
 

src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.cpp

@@ -25,6 +25,7 @@
 #include "database/Image.hpp"
 #include "database/Session.hpp"
 #include "database/Track.hpp"
+#include "database/TrackLyrics.hpp"
 
 namespace lms::scanner
 {
@@ -45,15 +46,17 @@ namespace lms::scanner
             context.currentStepStats.totalElems = 0;
             context.currentStepStats.totalElems += db::Track::getCount(session);
             context.currentStepStats.totalElems += db::Image::getCount(session);
+            context.currentStepStats.totalElems += db::TrackLyrics::getExternalLyricsCount(session);
         }
         LMS_LOG(DBUPDATER, DEBUG, context.currentStepStats.totalElems << " files to be checked...");
 
         checkForRemovedFiles<db::Track>(context, _settings.supportedAudioFileExtensions);
         checkForRemovedFiles<db::Image>(context, _settings.supportedImageFileExtensions);
+        checkForRemovedFiles<db::TrackLyrics>(context, _settings.supportedLyricsFileExtensions);
     }
 
     template<typename Object>
-    void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context, const std::vector<std::filesystem::path>& supportedFileExtensions)
+    void ScanStepCheckForRemovedFiles::checkForRemovedFiles(ScanContext& context, std::span<const std::filesystem::path> supportedFileExtensions)
     {
         using namespace db;
 
@@ -79,6 +82,13 @@ namespace lms::scanner
                 Object::find(session, lastCheckedId, batchSize, [&](const typename Object::pointer& object) {
                     endReached = false;
 
+                    // special case for track lyrics, only check external lyrics
+                    if constexpr (std::is_same_v<Object, TrackLyrics>)
+                    {
+                        if (object->getAbsoluteFilePath().empty())
+                            return;
+                    }
+
                     if (!checkFile(object->getAbsoluteFilePath(), supportedFileExtensions))
                         objectsToRemove.push_back(object);
 
@@ -101,7 +111,7 @@ namespace lms::scanner
         }
     }
 
-    bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p, const std::vector<std::filesystem::path>& allowedExtensions)
+    bool ScanStepCheckForRemovedFiles::checkFile(const std::filesystem::path& p, std::span<const std::filesystem::path> allowedExtensions)
     {
         try
         {

src/libs/services/scanner/impl/ScanStepCheckForRemovedFiles.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include <filesystem>
+#include <span>
 
 #include "ScanStepBase.hpp"
 
@@ -36,8 +37,8 @@ namespace lms::scanner
         void process(ScanContext& context) override;
 
         template<typename Object>
-        void checkForRemovedFiles(ScanContext& context, const std::vector<std::filesystem::path>& supportedFileExtensions);
+        void checkForRemovedFiles(ScanContext& context, std::span<const std::filesystem::path> supportedFileExtensions);
 
-        bool checkFile(const std::filesystem::path& p, const std::vector<std::filesystem::path>& allowedExtensions);
+        bool checkFile(const std::filesystem::path& p, std::span<const std::filesystem::path> allowedExtensions);
     };
 } // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepCompact.cpp

@@ -26,7 +26,7 @@ namespace lms::scanner
 {
     void ScanStepCompact::process(ScanContext& context)
     {
-        // Don't auto compact as it may be too annoying to block the whole application
+        // Don't auto compact as it may be too annoying to block the whole application for very large databases
         if (context.scanOptions.compact)
             _db.getTLSSession().vacuum();
     }

src/libs/services/scanner/impl/ScanStepScanFiles.cpp

@@ -35,6 +35,7 @@
 #include "database/Track.hpp"
 #include "database/TrackArtistLink.hpp"
 #include "database/TrackFeatures.hpp"
+#include "database/TrackLyrics.hpp"
 #include "metadata/Exception.hpp"
 #include "metadata/IParser.hpp"
 
@@ -122,6 +123,22 @@ namespace lms::scanner
             return directory;
         }
 
+        db::TrackLyrics::pointer createLyrics(Session& session, const metadata::Lyrics& lyricsInfo)
+        {
+            db::TrackLyrics::pointer lyrics{ session.create<db::TrackLyrics>() };
+
+            lyrics.modify()->setLanguage(lyricsInfo.language);
+            lyrics.modify()->setOffset(lyricsInfo.offset);
+            lyrics.modify()->setDisplayArtist(lyricsInfo.displayArtist);
+            lyrics.modify()->setDisplayTitle(lyricsInfo.displayTitle);
+            if (!lyricsInfo.synchronizedLines.empty())
+                lyrics.modify()->setSynchronizedLines(lyricsInfo.synchronizedLines);
+            else
+                lyrics.modify()->setUnsynchronizedLines(lyricsInfo.unsynchronizedLines);
+
+            return lyrics;
+        }
+
         Artist::pointer createArtist(Session& session, const metadata::Artist& artistInfo)
         {
             Artist::pointer artist{ session.create<Artist>(artistInfo.name) };
@@ -448,6 +465,12 @@ namespace lms::scanner
                         if (checkImageFileNeedScan(context, path))
                             _fileScanQueue.pushScanRequest(path, FileScanQueue::ScanRequestType::ImageFile);
                     }
+                    else if (core::pathUtils::hasFileAnyExtension(path, _settings.supportedLyricsFileExtensions))
+                    {
+                        fileMatched = true;
+                        if (checkLyricsFileNeedScan(context, path))
+                            _fileScanQueue.pushScanRequest(path, FileScanQueue::ScanRequestType::LyricsFile);
+                    }
 
                     if (fileMatched)
                     {
@@ -555,6 +578,34 @@ namespace lms::scanner
         return true; // need to scan
     }
 
+    bool ScanStepScanFiles::checkLyricsFileNeedScan(ScanContext& context, const std::filesystem::path& file)
+    {
+        ScanStats& stats{ context.stats };
+
+        const Wt::WDateTime lastWriteTime{ retrieveFileGetLastWrite(file) };
+        // Should rarely fail as we are currently iterating it
+        if (!lastWriteTime.isValid())
+        {
+            stats.skips++;
+            return false;
+        }
+
+        if (!context.scanOptions.fullScan)
+        {
+            db::Session& dbSession{ _db.getTLSSession() };
+            auto transaction{ _db.getTLSSession().createReadTransaction() };
+
+            const db::TrackLyrics::pointer lyrics{ db::TrackLyrics::find(dbSession, file) };
+            if (lyrics && lyrics->getLastWriteTime() == lastWriteTime)
+            {
+                stats.skips++;
+                return false;
+            }
+        }
+
+        return true; // need to scan
+    }
+
     void ScanStepScanFiles::processFileScanResults(ScanContext& context, std::span<const FileScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo)
     {
         LMS_SCOPED_TRACE_OVERVIEW("Scanner", "ProcessScanResults");
@@ -577,6 +628,11 @@ namespace lms::scanner
                 context.stats.scans++;
                 processImageFileScanData(context, scanResult.path, scanData->has_value() ? &scanData->value() : nullptr, libraryInfo);
             }
+            else if (const LyricsFileScanData * scanData{ std::get_if<LyricsFileScanData>(&scanResult.scanData) })
+            {
+                context.stats.scans++;
+                processLyricsFileScanData(context, scanResult.path, scanData->has_value() ? &scanData->value() : nullptr, libraryInfo);
+            }
         }
     }
 
@@ -783,6 +839,13 @@ namespace lms::scanner
         track.modify()->setTrackReplayGain(trackMetadata->replayGain);
         track.modify()->setArtistDisplayName(trackMetadata->artistDisplayName);
 
+        track.modify()->clearEmbeddedLyrics();
+        for (const metadata::Lyrics& lyricsInfo : trackMetadata->lyrics)
+        {
+            db::TrackLyrics::pointer lyrics{ createLyrics(dbSession, lyricsInfo) };
+            track.modify()->addLyrics(lyrics);
+        }
+
         if (added)
         {
             LMS_LOG(DBUPDATER, DEBUG, "Added audio file '" << file.string() << "'");
@@ -851,4 +914,70 @@ namespace lms::scanner
             stats.updates++;
         }
     }
+
+    void ScanStepScanFiles::processLyricsFileScanData(ScanContext& context, const std::filesystem::path& file, const metadata::Lyrics* lyricsInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo)
+    {
+        LMS_SCOPED_TRACE_DETAILED("Scanner", "ProcessImageScanData");
+
+        ScanStats& stats{ context.stats };
+
+        const std::optional<FileInfo> fileInfo{ retrieveFileInfo(file, libraryInfo.rootDirectory) };
+        if (!fileInfo)
+        {
+            stats.skips++;
+            return;
+        }
+
+        db::Session& dbSession{ _db.getTLSSession() };
+        db::TrackLyrics::pointer trackLyrics{ db::TrackLyrics::find(dbSession, file) };
+
+        if (!lyricsInfo)
+        {
+            if (trackLyrics)
+            {
+                trackLyrics.remove();
+                stats.deletions++;
+            }
+            context.stats.errors.emplace_back(file, ScanErrorType::CannotReadLyricsFile);
+            return;
+        }
+
+        bool added;
+        if (!trackLyrics)
+        {
+            trackLyrics = dbSession.create<db::TrackLyrics>();
+            trackLyrics.modify()->setAbsoluteFilePath(file);
+            added = true;
+        }
+        else
+        {
+            added = false;
+        }
+
+        trackLyrics.modify()->setLastWriteTime(fileInfo->lastWriteTime);
+        trackLyrics.modify()->setFileSize(fileInfo->fileSize);
+        trackLyrics.modify()->setLanguage(lyricsInfo->language);
+        trackLyrics.modify()->setOffset(lyricsInfo->offset);
+        trackLyrics.modify()->setDisplayTitle(lyricsInfo->displayTitle);
+        trackLyrics.modify()->setDisplayArtist(lyricsInfo->displayArtist);
+        if (!lyricsInfo->synchronizedLines.empty())
+            trackLyrics.modify()->setSynchronizedLines(lyricsInfo->synchronizedLines);
+        else
+            trackLyrics.modify()->setUnsynchronizedLines(lyricsInfo->unsynchronizedLines);
+
+        MediaLibrary::pointer mediaLibrary{ MediaLibrary::find(dbSession, libraryInfo.id) }; // may be null if settings are updated in // => next scan will correct this
+        trackLyrics.modify()->setDirectory(getOrCreateDirectory(dbSession, file.parent_path(), mediaLibrary));
+
+        if (added)
+        {
+            LMS_LOG(DBUPDATER, DEBUG, "Added external lyrics '" << file.string() << "'");
+            stats.additions++;
+        }
+        else
+        {
+            LMS_LOG(DBUPDATER, DEBUG, "Updated external lyrics '" << file.string() << "'");
+            stats.updates++;
+        }
+    }
+
 } // namespace lms::scanner

src/libs/services/scanner/impl/ScanStepScanFiles.hpp

@@ -44,10 +44,12 @@ namespace lms::scanner
 
         bool checkAudioFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);
         bool checkImageFileNeedScan(ScanContext& context, const std::filesystem::path& file);
+        bool checkLyricsFileNeedScan(ScanContext& context, const std::filesystem::path& file);
 
         void processFileScanResults(ScanContext& context, std::span<const FileScanResult> scanResults, const ScannerSettings::MediaLibraryInfo& libraryInfo);
         void processAudioFileScanData(ScanContext& context, const std::filesystem::path& path, const metadata::Track* trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo);
         void processImageFileScanData(ScanContext& context, const std::filesystem::path& path, const ImageInfo* imageInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo);
+        void processLyricsFileScanData(ScanContext& context, const std::filesystem::path& path, const metadata::Lyrics* lyrics, const ScannerSettings::MediaLibraryInfo& libraryInfo);
 
         std::unique_ptr<metadata::IParser> _metadataParser;
         const std::vector<std::string> _extraTagsToParse;

src/libs/services/scanner/impl/ScannerService.cpp

@@ -32,6 +32,7 @@
 #include "image/Image.hpp"
 
 #include "ScanStepAssociateArtistImages.hpp"
+#include "ScanStepAssociateExternalLyrics.hpp"
 #include "ScanStepAssociateReleaseImages.hpp"
 #include "ScanStepCheckForDuplicatedFiles.hpp"
 #include "ScanStepCheckForRemovedFiles.hpp"
@@ -271,7 +272,8 @@ namespace lms::scanner
 
         refreshScanSettings();
 
-        IScanStep::ScanContext scanContext{ scanOptions, ScanStats{}, ScanStepStats{} };
+        IScanStep::ScanContext scanContext;
+        scanContext.scanOptions = scanOptions;
         ScanStats& stats{ scanContext.stats };
         stats.startTime = Wt::WDateTime::currentDateTime();
 
@@ -281,7 +283,14 @@ namespace lms::scanner
             LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
 
             LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
-            scanContext.currentStepStats = ScanStepStats{ .startTime = Wt::WDateTime::currentDateTime(), .stepCount = _scanSteps.size(), .stepIndex = stepIndex++, .currentStep = scanStep->getStep() };
+            scanContext.currentStepStats = ScanStepStats{
+                .startTime = Wt::WDateTime::currentDateTime(),
+                .stepCount = _scanSteps.size(),
+                .stepIndex = stepIndex++,
+                .currentStep = scanStep->getStep(),
+                .totalElems = 0,
+                .processedElems = 0
+                };
 
             notifyInProgress(scanContext.currentStepStats);
             scanStep->process(scanContext);
@@ -341,7 +350,7 @@ namespace lms::scanner
             _db
         };
 
-        // Order is important, steps are sequential
+        // Order is important: steps are sequential
         _scanSteps.clear();
         _scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
         _scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
@@ -349,6 +358,7 @@ namespace lms::scanner
         _scanSteps.push_back(std::make_unique<ScanStepUpdateLibraryFields>(params));
         _scanSteps.push_back(std::make_unique<ScanStepAssociateArtistImages>(params));
         _scanSteps.push_back(std::make_unique<ScanStepAssociateReleaseImages>(params));
+        _scanSteps.push_back(std::make_unique<ScanStepAssociateExternalLyrics>(params));
         _scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanedDbEntries>(params));
         _scanSteps.push_back(std::make_unique<ScanStepCompact>(params));
         _scanSteps.push_back(std::make_unique<ScanStepOptimize>(params));
@@ -373,14 +383,21 @@ namespace lms::scanner
             {
                 const auto audioFileExtensions{ scanSettings->getAudioFileExtensions() };
                 newSettings.supportedAudioFileExtensions.reserve(audioFileExtensions.size());
-                std::transform(std::cbegin(audioFileExtensions), std::end(audioFileExtensions), std::back_inserter(newSettings.supportedAudioFileExtensions),
+                std::transform(std::cbegin(audioFileExtensions), std::cend(audioFileExtensions), std::back_inserter(newSettings.supportedAudioFileExtensions),
                     [](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; });
             }
 
             {
                 const auto imageFileExtensions{ image::getSupportedFileExtensions() };
                 newSettings.supportedImageFileExtensions.reserve(imageFileExtensions.size());
-                std::transform(std::cbegin(imageFileExtensions), std::end(imageFileExtensions), std::back_inserter(newSettings.supportedImageFileExtensions),
+                std::transform(std::cbegin(imageFileExtensions), std::cend(imageFileExtensions), std::back_inserter(newSettings.supportedImageFileExtensions),
+                    [](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; });
+            }
+
+            {
+                const auto lyricsFileExtensions{ metadata::getSupportedLyricsFileExtensions() };
+                newSettings.supportedLyricsFileExtensions.reserve(lyricsFileExtensions.size());
+                std::transform(std::cbegin(lyricsFileExtensions), std::cend(lyricsFileExtensions), std::back_inserter(newSettings.supportedLyricsFileExtensions),
                     [](const std::filesystem::path& extension) { return std::filesystem::path{ core::stringUtils::stringToLower(extension.string()) }; });
             }
 
@@ -410,7 +427,7 @@ namespace lms::scanner
         }
 
         const std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
-        _events.scanInProgress(stepStats);
+        _events.scanInProgress.emit(stepStats);
         _lastScanInProgressEmit = now;
     }
 
@@ -418,7 +435,7 @@ namespace lms::scanner
     {
         std::chrono::system_clock::time_point now{ std::chrono::system_clock::now() };
 
-        if (std::chrono::duration_cast<std::chrono::seconds>(now - _lastScanInProgressEmit).count() > 1)
+        if (now - _lastScanInProgressEmit >= std::chrono::seconds{ 1 })
             notifyInProgress(stepStats);
     }
 } // namespace lms::scanner

src/libs/services/scanner/impl/ScannerSettings.hpp

@@ -37,6 +37,7 @@ namespace lms::scanner
         db::ScanSettings::UpdatePeriod updatePeriod{ db::ScanSettings::UpdatePeriod::Never };
         std::vector<std::filesystem::path> supportedAudioFileExtensions;
         std::vector<std::filesystem::path> supportedImageFileExtensions;
+        std::vector<std::filesystem::path> supportedLyricsFileExtensions;
         bool skipDuplicateMBID{};
         std::vector<std::string> extraTags;
         std::vector<std::string> artistTagDelimiters;

src/libs/services/scanner/impl/ScannerStats.cpp

@@ -40,6 +40,8 @@ namespace lms::scanner
 
     unsigned ScanStepStats::progress() const
     {
-        return (processedElems / static_cast<float>(totalElems ? totalElems : 1)) * 100;
+        const unsigned res{ static_cast<unsigned>((processedElems / static_cast<float>(totalElems ? totalElems : 1)) * 100) };
+        // can technically be above 100% since we may add files while iterating the filesystem
+        return res;
     }
 } // namespace lms::scanner

src/libs/services/scanner/include/services/scanner/ScannerStats.hpp

@@ -30,11 +30,12 @@ namespace lms::scanner
 {
     enum class ScanErrorType
     {
-        CannotReadFile,      // cannot read file
-        CannotReadAudioFile, // cannot parse audio file
-        CannotReadImageFile, // cannot parse image file
-        NoAudioTrack,        // no audio track found
-        BadDuration,         // bad duration
+        CannotReadFile,       // cannot read file
+        CannotReadAudioFile,  // cannot parse audio file
+        CannotReadImageFile,  // cannot parse image file
+        CannotReadLyricsFile, // cannot parse lyrics file
+        NoAudioTrack,         // no audio track found
+        BadDuration,          // bad duration
     };
 
     enum class DuplicateReason
@@ -62,6 +63,7 @@ namespace lms::scanner
     enum class ScanStep
     {
         AssociateArtistImages,
+        AssociateExternalLyrics,
         AssociateReleaseImages,
         CheckForDuplicatedFiles,
         CheckForRemovedFiles,