diff --git a/approot/messages.xml b/approot/messages.xml index 1f6add56..a91e9681 100644 --- a/approot/messages.xml +++ b/approot/messages.xml @@ -83,6 +83,7 @@ Not scheduled Scheduled on {1} Scanning: step {1}/{2} +Checking for duplicate files... {1} files Checking files... {1}% Discovering files: {1} files Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)... diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 95afa35c..eef64859 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -83,6 +83,7 @@ Non planifié Planifié le {1} En cours de scan : étape {1}/{2} +Vérification des fichiers dupliqués... {1} fichiers Vérification des fichiers... {1}% Découverte des fichiers : {1} fichiers Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)... diff --git a/approot/messages_it.xml b/approot/messages_it.xml index 5c21c420..aa6a37b0 100644 --- a/approot/messages_it.xml +++ b/approot/messages_it.xml @@ -83,6 +83,7 @@ Non pianificato Pianificato il {1} Scansione: passo {1}/{2} + Controllo file... {1}% File trovati: {1} files Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)... diff --git a/approot/messages_zh.xml b/approot/messages_zh.xml index e685897f..ea1c2b4c 100644 --- a/approot/messages_zh.xml +++ b/approot/messages_zh.xml @@ -83,6 +83,7 @@ 无计划 计划于 {1} 扫描中: 阶段 {1}/{2} + 检查文件中... {1}% 检索文件中: {1} 文件 从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)... diff --git a/src/libs/services/scanner/CMakeLists.txt b/src/libs/services/scanner/CMakeLists.txt index 53f65df3..e913996f 100644 --- a/src/libs/services/scanner/CMakeLists.txt +++ b/src/libs/services/scanner/CMakeLists.txt @@ -2,6 +2,10 @@ add_library(lmsscanner SHARED impl/ScannerService.cpp impl/ScannerStats.cpp + impl/ScanStepCheckDuplicatedDbFiles.cpp + impl/ScanStepDiscoverFiles.cpp + impl/ScanStepRemoveOrphanDbFiles.cpp + impl/ScanStepScanFiles.cpp ) target_include_directories(lmsscanner INTERFACE diff --git a/src/libs/services/scanner/impl/IScanStep.hpp b/src/libs/services/scanner/impl/IScanStep.hpp new file mode 100644 index 00000000..2eeb7b98 --- /dev/null +++ b/src/libs/services/scanner/impl/IScanStep.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include "services/scanner/ScannerStats.hpp" + +namespace Scanner +{ + class IScanStep + { + public: + virtual ~IScanStep() = default; + + virtual ScanStep getStep() const = 0; + virtual std::string_view getStepName() const = 0; + + struct ScanContext + { + const std::filesystem::path directory; + const bool forceScan; + ScanStats stats; + ScanStepStats currentStepStats; + }; + virtual void process(ScanContext& context) = 0; + }; +} diff --git a/src/libs/services/scanner/impl/ScanStepBase.hpp b/src/libs/services/scanner/impl/ScanStepBase.hpp new file mode 100644 index 00000000..6e774e60 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepBase.hpp @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include "services/scanner/ScannerStats.hpp" +#include "IScanStep.hpp" +#include "ScannerSettings.hpp" + +namespace Database +{ + class Db; +} + +namespace Scanner +{ + class ScanStepBase : public IScanStep + { + public: + static inline const std::filesystem::path excludeDirFileName {".lmsignore"}; + using ProgressCallback = std::function; + + struct InitParams + { + const ScannerSettings& settings; + ProgressCallback progressCallback; + bool& abortScan; + Database::Db& db; + }; + ScanStepBase(InitParams& initParams) + : _settings {initParams.settings} + , _progressCallback {initParams.progressCallback} + , _abortScan {initParams.abortScan} + , _db {initParams.db} + {} + + protected: + const ScannerSettings& _settings; + ProgressCallback _progressCallback; + bool& _abortScan; + Database::Db& _db; + }; +} diff --git a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp new file mode 100644 index 00000000..8d1a0773 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepCheckDuplicatedDbFiles.hpp" + +#include "services/database/Db.hpp" +#include "services/database/Session.hpp" +#include "services/database/Track.hpp" +#include "utils/Logger.hpp" + +namespace Scanner +{ + void + ScanStepCheckDuplicatedDbFiles::process(ScanContext& context) + { + using namespace Database; + + if (_abortScan) + return; + + Session& session {_db.getTLSSession()}; + auto transaction {session.createSharedTransaction()}; + + const RangeResults tracks = Track::findTrackMBIDDuplicates(session, Range {}); + for (const TrackId trackId : tracks.results) + { + const Track::pointer track {Track::find(session, trackId)}; + if (auto trackMBID {track->getTrackMBID()}) + { + LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName(); + context.stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID}); + context.currentStepStats.processedElems++; + _progressCallback(context.currentStepStats); + } + } + + LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files"; + } +} diff --git a/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp new file mode 100644 index 00000000..655ed863 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "ScanStepBase.hpp" + +namespace Scanner +{ + class ScanStepCheckDuplicatedDbFiles : public ScanStepBase + { + public: + using ScanStepBase::ScanStepBase; + + private: + std::string_view getStepName() const override { return "Checking for duplicated files"; } + ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; } + void process(ScanContext& context) override; + }; +} diff --git a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp new file mode 100644 index 00000000..02f3dcfa --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepDiscoverFiles.hpp" +#include "utils/Logger.hpp" +#include "utils/Path.hpp" + +namespace Scanner +{ + void + ScanStepDiscoverFiles::process(ScanContext& context) + { + context.stats.filesScanned = 0; + PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path) + { + if (_abortScan) + return false; + + if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions)) + { + context.currentStepStats.processedElems++; + _progressCallback(context.currentStepStats); + } + + return true; + }, &excludeDirFileName); + + context.stats.filesScanned = context.currentStepStats.processedElems; + + LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'"; + } +} diff --git a/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp new file mode 100644 index 00000000..1e17ca34 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "ScanStepBase.hpp" + +namespace Scanner +{ + class ScanStepDiscoverFiles : public ScanStepBase + { + public: + using ScanStepBase::ScanStepBase; + + private: + ScanStep getStep() const override { return ScanStep::DiscoveringFiles; } + std::string_view getStepName() const override { return "DiscoveringFiles"; } + void process(ScanContext& context) override; + }; +} diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp new file mode 100644 index 00000000..7e9eebc1 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepRemoveOrphanDbFiles.hpp" + +#include "services/database/Artist.hpp" +#include "services/database/Cluster.hpp" +#include "services/database/Db.hpp" +#include "services/database/Release.hpp" +#include "services/database/Session.hpp" +#include "services/database/Track.hpp" +#include "utils/Logger.hpp" +#include "utils/Path.hpp" + +namespace Scanner +{ + void + ScanStepRemoveOrphanDbFiles::process(ScanContext& context) + { + removeOrphanTracks(context); + removeOrphanClusters(); + removeOrphanArtists(); + removeOrphanReleases(); + } + + void ScanStepRemoveOrphanDbFiles::removeOrphanTracks(ScanContext& context) + { + using namespace Database; + + if (_abortScan) + return; + + static constexpr std::size_t batchSize {50}; + Session& session {_db.getTLSSession()}; + + LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed..."; + std::size_t trackCount {}; + + { + auto transaction {session.createSharedTransaction()}; + trackCount = Track::getCount(session); + } + LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked..."; + + context.currentStepStats.totalElems = trackCount; + + RangeResults trackPaths; + std::vector tracksToRemove; + + // TODO handle only files in context.directory + for (std::size_t i {trackCount < batchSize ? 0 : trackCount - batchSize}; ; i -= (i > batchSize ? batchSize : i)) + { + tracksToRemove.clear(); + + { + auto transaction {session.createSharedTransaction()}; + trackPaths = Track::findPaths(session, Range {i, batchSize}); + } + + for (const Track::PathResult& trackPath : trackPaths.results) + { + if (_abortScan) + return; + + if (!checkFile(trackPath.path, _settings.mediaDirectory)) + tracksToRemove.push_back(trackPath.trackId); + + context.currentStepStats.processedElems++; + } + + if (!tracksToRemove.empty()) + { + auto transaction {session.createSharedTransaction()}; + + for (const TrackId trackId : tracksToRemove) + { + Track::pointer track {Track::find(session, trackId)}; + if (track) + { + track.remove(); + context.stats.deletions++; + } + } + } + + _progressCallback(context.currentStepStats); + + if (i == 0) + break; + } + + LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks checked!"; + } + + void + ScanStepRemoveOrphanDbFiles::removeOrphanClusters() + { + using namespace Database; + + LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters..."; + Session& session {_db.getTLSSession()}; + auto transaction {session.createUniqueTransaction()}; + + // Now process orphan Cluster (no track) + auto clusterIds {Cluster::findOrphans(session, Range {})}; + for (ClusterId clusterId : clusterIds.results) + { + Cluster::pointer cluster {Cluster::find(session, clusterId)}; + LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'"; + cluster.remove(); + } + } + + void + ScanStepRemoveOrphanDbFiles::removeOrphanArtists() + { + using namespace Database; + + LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists..."; + + Session& session {_db.getTLSSession()}; + auto transaction {session.createUniqueTransaction()}; + + auto artistIds {Artist::findAllOrphans(session, Range {})}; + for (const ArtistId artistId : artistIds.results) + { + Artist::pointer artist {Artist::find(session, artistId)}; + LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'"; + artist.remove(); + } + } + + void + ScanStepRemoveOrphanDbFiles::removeOrphanReleases() + { + using namespace Database; + + LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases..."; + + Session& session {_db.getTLSSession()}; + auto transaction {session.createUniqueTransaction()}; + + auto releases {Release::findOrphans(session, Range {})}; + for (const ReleaseId releaseId : releases.results) + { + Release::pointer release {Release::find(session, releaseId)}; + LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'"; + release.remove(); + } + } + + bool + ScanStepRemoveOrphanDbFiles::checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory) + { + try + { + // For each track, make sure the the file still exists + // and still belongs to a media directory + if (!std::filesystem::exists( p ) + || !std::filesystem::is_regular_file( p ) ) + { + LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing"; + return false; + } + + if (!PathUtils::isPathInRootPath(p, mediaDirectory, &excludeDirFileName)) + { + LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory"; + return false; + } + + if (!PathUtils::hasFileAnyExtension(p, _settings.supportedExtensions)) + { + LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled"; + return false; + } + + return true; + } + catch (std::filesystem::filesystem_error& e) + { + LMS_LOG(DBUPDATER, ERROR) << "Caught exception while checking file '" << p.string() << "': " << e.what(); + return false; + } + } +} diff --git a/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp new file mode 100644 index 00000000..460cdb84 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include "ScanStepBase.hpp" + +namespace Scanner +{ + class ScanStepRemoveOrphanDbFiles : public ScanStepBase + { + public: + using ScanStepBase::ScanStepBase; + + private: + std::string_view getStepName() const override { return "Checking orphaned entries"; } + ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; } + void process(ScanContext& context) override; + + void removeOrphanTracks(ScanContext& context); + void removeOrphanClusters(); + void removeOrphanArtists(); + void removeOrphanReleases(); + bool checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory); + }; +} diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.cpp b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp new file mode 100644 index 00000000..82cca76e --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.cpp @@ -0,0 +1,430 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ScanStepScanFiles.hpp" + +#include "metadata/IParser.hpp" +#include "services/database/Artist.hpp" +#include "services/database/Cluster.hpp" +#include "services/database/Db.hpp" +#include "services/database/Release.hpp" +#include "services/database/Session.hpp" +#include "services/database/Track.hpp" +#include "services/database/TrackFeatures.hpp" +#include "services/database/TrackArtistLink.hpp" +#include "utils/Exception.hpp" +#include "utils/IConfig.hpp" +#include "utils/Logger.hpp" +#include "utils/Path.hpp" + +using namespace Database; + +namespace +{ + Artist::pointer + createArtist(Session& session, const MetaData::Artist& artistInfo) + { + Artist::pointer artist {session.create(artistInfo.name)}; + + if (artistInfo.musicBrainzArtistID) + artist.modify()->setMBID(*artistInfo.musicBrainzArtistID); + if (artistInfo.sortName) + artist.modify()->setSortName(*artistInfo.sortName); + + return artist; + } + + void + updateArtistIfNeeded(Artist::pointer artist, const MetaData::Artist& artistInfo) + { + // Name may have been updated + if (artist->getName() != artistInfo.name) + { + artist.modify()->setName(artistInfo.name); + } + + // Sortname may have been updated + if (artistInfo.sortName && *artistInfo.sortName != artist->getSortName() ) + { + artist.modify()->setSortName(*artistInfo.sortName); + } + } + + std::vector + getOrCreateArtists(Session& session, const std::vector& artistsInfo, bool allowFallbackOnMBIDEntries) + { + std::vector artists; + + for (const MetaData::Artist& artistInfo : artistsInfo) + { + Artist::pointer artist; + + // First try to get by MBID + if (artistInfo.musicBrainzArtistID) + { + artist = Artist::find(session, *artistInfo.musicBrainzArtistID); + if (!artist) + artist = createArtist(session, artistInfo); + else + updateArtistIfNeeded(artist, artistInfo); + + artists.emplace_back(std::move(artist)); + continue; + } + + // Fall back on artist name (collisions may occur) + if (!artistInfo.name.empty()) + { + for (const Artist::pointer& sameNamedArtist : Artist::find(session, artistInfo.name)) + { + // Do not fallback on artist that is correctly tagged + if (!allowFallbackOnMBIDEntries && sameNamedArtist->getMBID()) + continue; + + artist = sameNamedArtist; + break; + } + + // No Artist found with the same name and without MBID -> creating + if (!artist) + artist = createArtist(session, artistInfo); + else + updateArtistIfNeeded(artist, artistInfo); + + artists.emplace_back(std::move(artist)); + continue; + } + } + + return artists; + } + + Release::pointer + getOrCreateRelease(Session& session, const MetaData::Album& album) + { + Release::pointer release; + + // First try to get by MBID + if (album.musicBrainzAlbumID) + { + release = Release::find(session, *album.musicBrainzAlbumID); + if (!release) + { + release = session.create(album.name, album.musicBrainzAlbumID); + } + else if (release->getName() != album.name) + { + // Name may have been updated + release.modify()->setName(album.name); + } + + return release; + } + + // Fall back on release name (collisions may occur) + if (!album.name.empty()) + { + for (const Release::pointer& sameNamedRelease : Release::find(session, album.name)) + { + // do not fallback on properly tagged releases + if (!sameNamedRelease->getMBID()) + { + release = sameNamedRelease; + break; + } + } + + // No release found with the same name and without MBID -> creating + if (!release) + release = session.create(album.name); + + return release; + } + + return Release::pointer{}; + } + + std::vector + getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames) + { + std::vector< Cluster::pointer > clusters; + + for (auto clusterNames : clustersNames) + { + auto clusterType = ClusterType::find(session, clusterNames.first); + if (!clusterType) + continue; + + for (auto clusterName : clusterNames.second) + { + auto cluster = clusterType->getCluster(clusterName); + if (!cluster) + cluster = session.create(clusterType, clusterName); + + clusters.push_back(cluster); + } + } + + return clusters; + } + + MetaData::ParserReadStyle + getParserReadStyle() + { + std::string_view readStyle {Service::get()->getString("scanner-parser-read-style", "accurate")}; + + if (readStyle == "fast") + return MetaData::ParserReadStyle::Fast; + else if (readStyle == "average") + return MetaData::ParserReadStyle::Average; + else if (readStyle == "accurate") + return MetaData::ParserReadStyle::Accurate; + + throw LmsException {"Invalid value for 'scanner-parser-read-style'"}; + } +} // namespace + +namespace Scanner +{ + ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams) + : ScanStepBase {initParams} + , _metadataParser {MetaData::createParser(MetaData::ParserType::TagLib, getParserReadStyle())} // For now, always use TagLib + { + } + + void + ScanStepScanFiles::process(ScanContext& context) + { + _metadataParser->setClusterTypeNames(_settings.clusterTypeNames); + + context.currentStepStats.totalElems = context.stats.filesScanned; + + PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path) + { + if (_abortScan) + return false; + + if (ec) + { + LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message(); + context.stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()}); + } + else if (PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions)) + { + scanAudioFile(path, context); + + context.currentStepStats.processedElems++; + _progressCallback(context.currentStepStats); + } + + return true; + }, &excludeDirFileName); + } + + void + ScanStepScanFiles::scanAudioFile(const std::filesystem::path& file, ScanContext& context) + { + ScanStats& stats {context.stats}; + Wt::WDateTime lastWriteTime; + try + { + lastWriteTime = PathUtils::getLastWriteTime(file); + } + catch (LmsException& e) + { + LMS_LOG(DBUPDATER, ERROR) << e.what(); + stats.skips++; + return; + } + + if (!context.forceScan) + { + // Skip file if last write is the same + Database::Session& dbSession {_db.getTLSSession()}; + auto transaction {_db.getTLSSession().createSharedTransaction()}; + + const Track::pointer track {Track::findByPath(dbSession, file)}; + + if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t() + && track->getScanVersion() == _settings.scanVersion) + { + stats.skips++; + return; + } + } + + std::optional trackInfo {_metadataParser->parse(file)}; + if (!trackInfo) + { + context.stats.errors.emplace_back(file, ScanErrorType::CannotParseFile); + return; + } + + stats.scans++; + + Database::Session& dbSession {_db.getTLSSession()}; + auto uniqueTransaction {dbSession.createUniqueTransaction()}; + + Track::pointer track {Track::findByPath(dbSession, file) }; + + // Skip duplicate recording MBID + if (trackInfo->recordingMBID && _settings.skipDuplicateRecordingMBID) + { + for (Track::pointer otherTrack : Track::findByRecordingMBID(dbSession, *trackInfo->recordingMBID)) + { + if (track && track->getId() == otherTrack->getId()) + continue; + + LMS_LOG(DBUPDATER, DEBUG) << "Skipped '" << file.string() << "' (similar recording MBID in '" << otherTrack->getPath().string() << "')"; + // This recording MBID already exists, just remove what we just scanned + if (track) + { + track.remove(); + stats.deletions++; + } + return; + } + } + + // We estimate this is an audio file if: + // - we found a least one audio stream + // - the duration is not null + if (trackInfo->audioStreams.empty()) + { + LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (no audio stream found)"; + + // If Track exists here, delete it! + if (track) + { + track.remove(); + stats.deletions++; + } + stats.errors.emplace_back(ScanError {file, ScanErrorType::NoAudioTrack}); + return; + } + if (trackInfo->duration == std::chrono::milliseconds::zero()) + { + LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (duration is 0)"; + + // If Track exists here, delete it! + if (track) + { + track.remove(); + stats.deletions++; + } + stats.errors.emplace_back(ScanError {file, ScanErrorType::BadDuration}); + return; + } + + // ***** Title + std::string title; + if (!trackInfo->title.empty()) + title = trackInfo->title; + else + { + // TODO parse file name guess track etc. + // For now juste use file name as title + title = file.filename().string(); + } + + // If file already exists, update its data + // Otherwise, create it + if (!track) + { + track = dbSession.create(file); + LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'"; + stats.additions++; + } + else + { + LMS_LOG(DBUPDATER, INFO) << "Updating '" << file.string() << "'"; + + stats.updates++; + } + + // Track related data + assert(track); + + track.modify()->clearArtistLinks(); + // Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files + for (const Artist::pointer& artist : getOrCreateArtists(dbSession, trackInfo->artists, false)) + track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, artist, TrackArtistLinkType::Artist)); + + for (const Artist::pointer& releaseArtist : getOrCreateArtists(dbSession, trackInfo->albumArtists, false)) + track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, releaseArtist, TrackArtistLinkType::ReleaseArtist)); + + // Allow fallbacks on artists with the same name even if they have MBID, since there is no tag to indicate the MBID of these artists + // We could ask MusicBrainz to get all the information, but that would heavily slow down the import process + for (const Artist::pointer& conductor : getOrCreateArtists(dbSession, trackInfo->conductorArtists, true)) + track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, conductor, TrackArtistLinkType::Conductor)); + + for (const Artist::pointer& composer : getOrCreateArtists(dbSession, trackInfo->composerArtists, true)) + track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, composer, TrackArtistLinkType::Composer)); + + for (const Artist::pointer& lyricist : getOrCreateArtists(dbSession, trackInfo->lyricistArtists, true)) + track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, lyricist, TrackArtistLinkType::Lyricist)); + + for (const Artist::pointer& mixer : getOrCreateArtists(dbSession, trackInfo->mixerArtists, true)) + track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, mixer, TrackArtistLinkType::Mixer)); + + for (const auto& [role, performers] : trackInfo->performerArtists) + { + for (const Artist::pointer& performer : getOrCreateArtists(dbSession, performers, true)) + track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, performer, TrackArtistLinkType::Performer, role)); + } + + for (const Artist::pointer& producer : getOrCreateArtists(dbSession, trackInfo->producerArtists, true)) + track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, producer, TrackArtistLinkType::Producer)); + + for (const Artist::pointer& remixer : getOrCreateArtists(dbSession, trackInfo->remixerArtists, true)) + track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, remixer, TrackArtistLinkType::Remixer)); + + track.modify()->setScanVersion(_settings.scanVersion); + if (trackInfo->album) + track.modify()->setRelease(getOrCreateRelease(dbSession, *trackInfo->album)); + else + track.modify()->setRelease({}); + track.modify()->setClusters(getOrCreateClusters(dbSession, trackInfo->clusters)); + track.modify()->setLastWriteTime(lastWriteTime); + track.modify()->setName(title); + track.modify()->setDuration(trackInfo->duration); + track.modify()->setAddedTime(Wt::WDateTime::currentDateTime()); + track.modify()->setTrackNumber(trackInfo->trackNumber ? *trackInfo->trackNumber : 0); + track.modify()->setDiscNumber(trackInfo->discNumber ? *trackInfo->discNumber : 0); + track.modify()->setTotalTrack(trackInfo->totalTrack); + track.modify()->setTotalDisc(trackInfo->totalDisc); + track.modify()->setDiscSubtitle(trackInfo->discSubtitle); + track.modify()->setDate(trackInfo->date); + track.modify()->setOriginalDate(trackInfo->originalDate); + + // If a file has an OriginalYear but no Year, set it to ease filtering + if (!trackInfo->date.isValid() && trackInfo->originalDate.isValid()) + track.modify()->setDate(trackInfo->originalDate); + + track.modify()->setRecordingMBID(trackInfo->recordingMBID); + track.modify()->setTrackMBID(trackInfo->trackMBID); + if (auto trackFeatures {TrackFeatures::find(dbSession, track->getId())}) + trackFeatures.remove(); // TODO: only if MBID changed? + track.modify()->setHasCover(trackInfo->hasCover); + track.modify()->setCopyright(trackInfo->copyright); + track.modify()->setCopyrightURL(trackInfo->copyrightURL); + track.modify()->setTrackReplayGain(trackInfo->trackReplayGain); + track.modify()->setReleaseReplayGain(trackInfo->albumReplayGain); + } +} diff --git a/src/libs/services/scanner/impl/ScanStepScanFiles.hpp b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp new file mode 100644 index 00000000..d74d7c79 --- /dev/null +++ b/src/libs/services/scanner/impl/ScanStepScanFiles.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include "metadata/IParser.hpp" +#include "ScanStepBase.hpp" + +namespace Scanner +{ + class ScanStepScanFiles : public ScanStepBase + { + public: + ScanStepScanFiles(InitParams& initParams); + + private: + ScanStep getStep() const override { return ScanStep::ScanningFiles; } + std::string_view getStepName() const override { return "Scanning files"; } + void process(ScanContext& context) override; + + void scanAudioFile(const std::filesystem::path& file, ScanContext& context); + + std::unique_ptr _metadataParser; + }; +} diff --git a/src/libs/services/scanner/impl/ScannerService.cpp b/src/libs/services/scanner/impl/ScannerService.cpp index 80b6fddd..9c16a426 100644 --- a/src/libs/services/scanner/impl/ScannerService.cpp +++ b/src/libs/services/scanner/impl/ScannerService.cpp @@ -22,27 +22,25 @@ #include #include -#include "services/database/Artist.hpp" #include "services/database/Cluster.hpp" -#include "services/database/Release.hpp" -#include "services/database/ScanSettings.hpp" -#include "services/database/Track.hpp" -#include "services/database/TrackArtistLink.hpp" #include "services/database/TrackFeatures.hpp" -#include "metadata/IParser.hpp" +#include "services/database/ScanSettings.hpp" #include "services/recommendation/IRecommendationService.hpp" #include "utils/Exception.hpp" #include "utils/IConfig.hpp" #include "utils/Logger.hpp" #include "utils/Path.hpp" -#include "utils/UUID.hpp" +#include "utils/Tuple.hpp" + +#include "ScanStepCheckDuplicatedDbFiles.hpp" +#include "ScanStepDiscoverFiles.hpp" +#include "ScanStepRemoveOrphanDbFiles.hpp" +#include "ScanStepScanFiles.hpp" using namespace Database; namespace { -const std::filesystem::path excludeDirFileName {".lmsignore"}; - Wt::WDate getNextMonday(Wt::WDate current) { @@ -65,183 +63,6 @@ getNextFirstOfMonth(Wt::WDate current) return current; } -bool -isFileSupported(const std::filesystem::path& file, const std::vector& extensions) -{ - const std::filesystem::path extension {StringUtils::stringToLower(file.extension().string())}; - - return (std::find(std::cbegin(extensions), std::cend(extensions), extension) != std::cend(extensions)); -} - -bool -isPathInMediaDirectory(const std::filesystem::path& path, const std::filesystem::path& rootPath) -{ - std::filesystem::path curPath = path; - - while (curPath.parent_path() != curPath) - { - curPath = curPath.parent_path(); - - std::error_code ec; - if (std::filesystem::exists(curPath / excludeDirFileName, ec)) - return false; - - if (curPath == rootPath) - return true; - } - - return false; -} - -static -Artist::pointer -createArtist(Session& session, const MetaData::Artist& artistInfo) -{ - Artist::pointer artist {session.create(artistInfo.name)}; - - if (artistInfo.musicBrainzArtistID) - artist.modify()->setMBID(*artistInfo.musicBrainzArtistID); - if (artistInfo.sortName) - artist.modify()->setSortName(*artistInfo.sortName); - - return artist; -} - -static -void -updateArtistIfNeeded(Artist::pointer artist, const MetaData::Artist& artistInfo) -{ - // Name may have been updated - if (artist->getName() != artistInfo.name) - { - artist.modify()->setName(artistInfo.name); - } - - // Sortname may have been updated - if (artistInfo.sortName && *artistInfo.sortName != artist->getSortName() ) - { - artist.modify()->setSortName(*artistInfo.sortName); - } -} - -std::vector -getOrCreateArtists(Session& session, const std::vector& artistsInfo, bool allowFallbackOnMBIDEntries) -{ - std::vector artists; - - for (const MetaData::Artist& artistInfo : artistsInfo) - { - Artist::pointer artist; - - // First try to get by MBID - if (artistInfo.musicBrainzArtistID) - { - artist = Artist::find(session, *artistInfo.musicBrainzArtistID); - if (!artist) - artist = createArtist(session, artistInfo); - else - updateArtistIfNeeded(artist, artistInfo); - - artists.emplace_back(std::move(artist)); - continue; - } - - // Fall back on artist name (collisions may occur) - if (!artistInfo.name.empty()) - { - for (const Artist::pointer& sameNamedArtist : Artist::find(session, artistInfo.name)) - { - // Do not fallback on artist that is correctly tagged - if (!allowFallbackOnMBIDEntries && sameNamedArtist->getMBID()) - continue; - - artist = sameNamedArtist; - break; - } - - // No Artist found with the same name and without MBID -> creating - if (!artist) - artist = createArtist(session, artistInfo); - else - updateArtistIfNeeded(artist, artistInfo); - - artists.emplace_back(std::move(artist)); - continue; - } - } - - return artists; -} - -Release::pointer -getOrCreateRelease(Session& session, const MetaData::Album& album) -{ - Release::pointer release; - - // First try to get by MBID - if (album.musicBrainzAlbumID) - { - release = Release::find(session, *album.musicBrainzAlbumID); - if (!release) - { - release = session.create(album.name, album.musicBrainzAlbumID); - } - else if (release->getName() != album.name) - { - // Name may have been updated - release.modify()->setName(album.name); - } - - return release; - } - - // Fall back on release name (collisions may occur) - if (!album.name.empty()) - { - for (const Release::pointer& sameNamedRelease : Release::find(session, album.name)) - { - // do not fallback on properly tagged releases - if (!sameNamedRelease->getMBID()) - { - release = sameNamedRelease; - break; - } - } - - // No release found with the same name and without MBID -> creating - if (!release) - release = session.create(album.name); - - return release; - } - - return Release::pointer{}; -} - -std::vector -getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames) -{ - std::vector< Cluster::pointer > clusters; - - for (auto clusterNames : clustersNames) - { - auto clusterType = ClusterType::find(session, clusterNames.first); - if (!clusterType) - continue; - - for (auto clusterName : clusterNames.second) - { - auto cluster = clusterType->getCluster(clusterName); - if (!cluster) - cluster = session.create(clusterType, clusterName); - - clusters.push_back(cluster); - } - } - - return clusters; -} - } // namespace namespace Scanner { @@ -252,29 +73,11 @@ createScannerService(Db& db, Recommendation::IRecommendationService& recommendat return std::make_unique(db, recommendationService); } -MetaData::ParserReadStyle -getParserReadStyle() -{ - std::string_view readStyle {Service::get()->getString("scanner-parser-read-style", "accurate")}; - - if (readStyle == "fast") - return MetaData::ParserReadStyle::Fast; - else if (readStyle == "average") - return MetaData::ParserReadStyle::Average; - else if (readStyle == "accurate") - return MetaData::ParserReadStyle::Accurate; - - throw LmsException {"Invalid value for 'scanner-parser-read-style'"}; -} - ScannerService::ScannerService(Db& db, Recommendation::IRecommendationService& recommendationService) : _recommendationService {recommendationService} -, _skipDuplicateRecordingMBID {Service::get()->getBool("scanner-skip-duplicate-recording-mbid", false)} +, _db {db} , _dbSession {db} -, _metadataParser {MetaData::createParser(MetaData::ParserType::TagLib, getParserReadStyle())} // For now, always use TagLib { - LMS_LOG(DBUPDATER, INFO) << "skipDuplicateRecordingMBID = " << _skipDuplicateRecordingMBID; - _ioService.setThreadCount(1); refreshScanSettings(); @@ -383,34 +186,34 @@ ScannerService::getStatus() const void ScannerService::scheduleNextScan() { - LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan"; + LMS_LOG(DBUPDATER, DEBUG) << "Scheduling next scan"; refreshScanSettings(); const Wt::WDateTime now {Wt::WDateTime::currentDateTime()}; Wt::WDateTime nextScanDateTime; - switch (_updatePeriod) + switch (_settings.updatePeriod) { case ScanSettings::UpdatePeriod::Daily: - if (now.time() < _startTime) - nextScanDateTime = {now.date(), _startTime}; + if (now.time() < _settings.startTime) + nextScanDateTime = {now.date(), _settings.startTime}; else - nextScanDateTime = {now.date().addDays(1), _startTime}; + nextScanDateTime = {now.date().addDays(1), _settings.startTime}; break; case ScanSettings::UpdatePeriod::Weekly: - if (now.time() < _startTime && now.date().dayOfWeek() == 1) - nextScanDateTime = {now.date(), _startTime}; + if (now.time() < _settings.startTime && now.date().dayOfWeek() == 1) + nextScanDateTime = {now.date(), _settings.startTime}; else - nextScanDateTime = {getNextMonday(now.date()), _startTime}; + nextScanDateTime = {getNextMonday(now.date()), _settings.startTime}; break; case ScanSettings::UpdatePeriod::Monthly: - if (now.time() < _startTime && now.date().day() == 1) - nextScanDateTime = {now.date(), _startTime}; + if (now.time() < _settings.startTime && now.date().day() == 1) + nextScanDateTime = {now.date(), _settings.startTime}; else - nextScanDateTime = {getNextFirstOfMonth(now.date()), _startTime}; + nextScanDateTime = {getNextFirstOfMonth(now.date()), _settings.startTime}; break; case ScanSettings::UpdatePeriod::Hourly: @@ -434,31 +237,6 @@ ScannerService::scheduleNextScan() _events.scanScheduled.emit(_nextScheduledScan); } -void -ScannerService::countAllFiles(ScanStats& stats) -{ - ScanStepStats stepStats{stats.startTime, ScanProgressStep::DiscoveringFiles}; - - stats.filesScanned = 0; - notifyInProgress(stepStats); - - exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) - { - if (_abortScan) - return false; - - if (!ec && isFileSupported(path, _fileExtensions)) - { - stats.filesScanned++; - stepStats.processedElems++; - notifyInProgressIfNeeded(stepStats); - } - - return true; - }, excludeDirFileName); - notifyInProgress(stepStats); -} - void ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime) { @@ -499,35 +277,29 @@ ScannerService::scan(bool forceScan) _nextScheduledScan = {}; } - ScanStats stats; - stats.startTime = Wt::WDateTime::currentDateTime(); LMS_LOG(UI, INFO) << "New scan started!"; refreshScanSettings(); - removeMissingTracks(stats); + IScanStep::ScanContext scanContext {_settings.mediaDirectory, forceScan, ScanStats {}, ScanStepStats {}}; + ScanStats& stats {scanContext.stats}; + stats.startTime = Wt::WDateTime::currentDateTime(); - LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'..."; - countAllFiles(stats); - LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesScanned; - - LMS_LOG(UI, INFO) << "Checks complete, force scan = " << forceScan; - - LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "'..."; - scanMediaDirectory(_mediaDirectory, forceScan, stats); - LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "' DONE"; - - removeOrphanEntries(); - - if (!_abortScan) + for (auto& scanStep : _scanSteps) { - checkDuplicatedAudioFiles(stats); - reloadSimilarityEngine(stats); + LMS_LOG(DBUPDATER, DEBUG) << "Starting scan step '" << scanStep->getStepName() << "'"; + scanContext.currentStepStats = ScanStepStats {Wt::WDateTime::currentDateTime(), scanStep->getStep()}; + + notifyInProgress(scanContext.currentStepStats); + scanStep->process(scanContext); + notifyInProgress(scanContext.currentStepStats); + LMS_LOG(DBUPDATER, DEBUG) << "Completed scan step '" << scanStep->getStepName() << "'"; } LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size(); + // TODO make it a scan step _dbSession.optimize(); if (!_abortScan) @@ -559,33 +331,71 @@ ScannerService::scan(bool forceScan) void ScannerService::refreshScanSettings() { - auto transaction {_dbSession.createSharedTransaction()}; + ScannerSettings newSettings {readSettings()}; + if (_settings == newSettings) + return; - const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; + LMS_LOG(DBUPDATER, DEBUG) << "Scanner settings updated"; + LMS_LOG(DBUPDATER, DEBUG) << "skipDuplicateRecordingMBID = " << newSettings.skipDuplicateRecordingMBID; + LMS_LOG(DBUPDATER, DEBUG) << "Using scan settings version " << newSettings.scanVersion; - LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion(); + _settings = std::move(newSettings); - _scanVersion = scanSettings->getScanVersion(); - _startTime = scanSettings->getUpdateStartTime(); - _updatePeriod = scanSettings->getUpdatePeriod(); + auto cbFunc {[this](const ScanStepStats& stats) + { + notifyInProgressIfNeeded(stats); + }}; + ScanStepBase::InitParams params { - const auto fileExtensions {scanSettings->getAudioFileExtensions()}; - _fileExtensions.clear(); - std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(_fileExtensions), + _settings, + cbFunc, + _abortScan, + _db + }; + + _scanSteps.clear(); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); + _scanSteps.push_back(std::make_unique(params)); +} + +ScannerSettings +ScannerService::readSettings() +{ + ScannerSettings newSettings; + + newSettings.skipDuplicateRecordingMBID = Service::get()->getBool("scanner-skip-duplicate-recording-mbid", false); + { + auto transaction {_dbSession.createSharedTransaction()}; + + const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; + + newSettings.scanVersion = scanSettings->getScanVersion(); + newSettings.startTime = scanSettings->getUpdateStartTime(); + newSettings.updatePeriod = scanSettings->getUpdatePeriod(); + + { + const auto fileExtensions {scanSettings->getAudioFileExtensions()}; + newSettings.supportedExtensions.reserve(fileExtensions.size()); + std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions), [](const std::filesystem::path& extension) { return std::filesystem::path{ StringUtils::stringToLower(extension.string()) }; }); + } + newSettings.recommendationServiceType = scanSettings->getRecommendationEngineType(); + newSettings.mediaDirectory = scanSettings->getMediaDirectory(); + + const auto clusterTypes = scanSettings->getClusterTypes(); + std::set clusterTypeNames; + + std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes), + std::inserter(clusterTypeNames, clusterTypeNames.begin()), + [](ClusterType::pointer clusterType) { return clusterType->getName(); }); + + newSettings.clusterTypeNames = std::move(clusterTypeNames); } - _mediaDirectory = scanSettings->getMediaDirectory(); - _recommendationServiceType = scanSettings->getRecommendationEngineType(); - const auto clusterTypes = scanSettings->getClusterTypes(); - std::set clusterTypeNames; - - std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes), - std::inserter(clusterTypeNames, clusterTypeNames.begin()), - [](ClusterType::pointer clusterType) { return clusterType->getName(); }); - - _metadataParser->setClusterTypeNames(clusterTypeNames); + return newSettings; } void @@ -610,410 +420,4 @@ ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats) notifyInProgress(stepStats); } -void -ScannerService::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats) -{ - Wt::WDateTime lastWriteTime; - try - { - lastWriteTime = getLastWriteTime(file); - } - catch (LmsException& e) - { - LMS_LOG(DBUPDATER, ERROR) << e.what(); - stats.skips++; - return; - } - - if (!forceScan) - { - // Skip file if last write is the same - auto transaction {_dbSession.createSharedTransaction()}; - - const Track::pointer track {Track::findByPath(_dbSession, file)}; - - if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t() - && track->getScanVersion() == _scanVersion) - { - stats.skips++; - return; - } - } - - std::optional trackInfo {_metadataParser->parse(file)}; - if (!trackInfo) - { - stats.errors.emplace_back(file, ScanErrorType::CannotParseFile); - return; - } - - stats.scans++; - - auto uniqueTransaction {_dbSession.createUniqueTransaction()}; - - Track::pointer track {Track::findByPath(_dbSession, file) }; - - // Skip duplicate recording MBID - if (trackInfo->recordingMBID && _skipDuplicateRecordingMBID) - { - for (Track::pointer otherTrack : Track::findByRecordingMBID(_dbSession, *trackInfo->recordingMBID)) - { - if (track && track->getId() == otherTrack->getId()) - continue; - - LMS_LOG(DBUPDATER, DEBUG) << "Skipped '" << file.string() << "' (similar recording MBID in '" << otherTrack->getPath().string() << "')"; - // This recording MBID already exists, just remove what we just scanned - if (track) - { - track.remove(); - stats.deletions++; - } - return; - } - } - - // We estimate this is an audio file if: - // - we found a least one audio stream - // - the duration is not null - if (trackInfo->audioStreams.empty()) - { - LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (no audio stream found)"; - - // If Track exists here, delete it! - if (track) - { - track.remove(); - stats.deletions++; - } - stats.errors.emplace_back(ScanError {file, ScanErrorType::NoAudioTrack}); - return; - } - if (trackInfo->duration == std::chrono::milliseconds::zero()) - { - LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (duration is 0)"; - - // If Track exists here, delete it! - if (track) - { - track.remove(); - stats.deletions++; - } - stats.errors.emplace_back(ScanError {file, ScanErrorType::BadDuration}); - return; - } - - // ***** Title - std::string title; - if (!trackInfo->title.empty()) - title = trackInfo->title; - else - { - // TODO parse file name guess track etc. - // For now juste use file name as title - title = file.filename().string(); - } - - // If file already exist, update data - // Otherwise, create it - if (!track) - { - // Create a new song - track = _dbSession.create(file); - LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'"; - stats.additions++; - } - else - { - LMS_LOG(DBUPDATER, INFO) << "Updating '" << file.string() << "'"; - - stats.updates++; - } - - // Track related data - assert(track); - - track.modify()->clearArtistLinks(); - // Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files - for (const Artist::pointer& artist : getOrCreateArtists(_dbSession, trackInfo->artists, false)) - track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, artist, TrackArtistLinkType::Artist)); - - for (const Artist::pointer& releaseArtist : getOrCreateArtists(_dbSession, trackInfo->albumArtists, false)) - track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, releaseArtist, TrackArtistLinkType::ReleaseArtist)); - - // Allow fallbacks on artists with the same name even if they have MBID, since there is no tag to indicate the MBID of these artists - // We could ask MusicBrainz to get all the information, but that would heavily slow down the import process - for (const Artist::pointer& conductor : getOrCreateArtists(_dbSession, trackInfo->conductorArtists, true)) - track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, conductor, TrackArtistLinkType::Conductor)); - - for (const Artist::pointer& composer : getOrCreateArtists(_dbSession, trackInfo->composerArtists, true)) - track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, composer, TrackArtistLinkType::Composer)); - - for (const Artist::pointer& lyricist : getOrCreateArtists(_dbSession, trackInfo->lyricistArtists, true)) - track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, lyricist, TrackArtistLinkType::Lyricist)); - - for (const Artist::pointer& mixer : getOrCreateArtists(_dbSession, trackInfo->mixerArtists, true)) - track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, mixer, TrackArtistLinkType::Mixer)); - - for (const auto& [role, performers] : trackInfo->performerArtists) - { - for (const Artist::pointer& performer : getOrCreateArtists(_dbSession, performers, true)) - track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, performer, TrackArtistLinkType::Performer, role)); - } - - for (const Artist::pointer& producer : getOrCreateArtists(_dbSession, trackInfo->producerArtists, true)) - track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, producer, TrackArtistLinkType::Producer)); - - for (const Artist::pointer& remixer : getOrCreateArtists(_dbSession, trackInfo->remixerArtists, true)) - track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, remixer, TrackArtistLinkType::Remixer)); - - track.modify()->setScanVersion(_scanVersion); - if (trackInfo->album) - track.modify()->setRelease(getOrCreateRelease(_dbSession, *trackInfo->album)); - else - track.modify()->setRelease({}); - track.modify()->setClusters(getOrCreateClusters(_dbSession, trackInfo->clusters)); - track.modify()->setLastWriteTime(lastWriteTime); - track.modify()->setName(title); - track.modify()->setDuration(trackInfo->duration); - track.modify()->setAddedTime(Wt::WDateTime::currentDateTime()); - track.modify()->setTrackNumber(trackInfo->trackNumber ? *trackInfo->trackNumber : 0); - track.modify()->setDiscNumber(trackInfo->discNumber ? *trackInfo->discNumber : 0); - track.modify()->setTotalTrack(trackInfo->totalTrack); - track.modify()->setTotalDisc(trackInfo->totalDisc); - track.modify()->setDiscSubtitle(trackInfo->discSubtitle); - track.modify()->setDate(trackInfo->date); - track.modify()->setOriginalDate(trackInfo->originalDate); - - // If a file has an OriginalYear but no Year, set it to ease filtering - if (!trackInfo->date.isValid() && trackInfo->originalDate.isValid()) - track.modify()->setDate(trackInfo->originalDate); - - track.modify()->setRecordingMBID(trackInfo->recordingMBID); - track.modify()->setTrackMBID(trackInfo->trackMBID); - if (auto trackFeatures {TrackFeatures::find(_dbSession, track->getId())}) - trackFeatures.remove(); // TODO: only if MBID changed? - track.modify()->setHasCover(trackInfo->hasCover); - track.modify()->setCopyright(trackInfo->copyright); - track.modify()->setCopyrightURL(trackInfo->copyrightURL); - track.modify()->setTrackReplayGain(trackInfo->trackReplayGain); - track.modify()->setReleaseReplayGain(trackInfo->albumReplayGain); -} - -void -ScannerService::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats) -{ - ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles}; - stepStats.totalElems = stats.filesScanned; - notifyInProgress(stepStats); - - exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path) - { - if (_abortScan) - return false; - - if (ec) - { - LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message(); - stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()}); - } - else if (isFileSupported(path, _fileExtensions)) - { - scanAudioFile(path, forceScan, stats ); - - stepStats.processedElems++; - notifyInProgressIfNeeded(stepStats); - } - - return true; - }, excludeDirFileName); - - notifyInProgress(stepStats); -} - -// Check if a file exists and is still in a media directory -static bool -checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::vector& extensions) -{ - try - { - // For each track, make sure the the file still exists - // and still belongs to a media directory - if (!std::filesystem::exists( p ) - || !std::filesystem::is_regular_file( p ) ) - { - LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing"; - return false; - } - - if (!isPathInMediaDirectory(p, mediaDirectory)) - { - LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory"; - return false; - } - - if (!isFileSupported(p, extensions)) - { - LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled"; - return false; - } - - return true; - } - catch (std::filesystem::filesystem_error& e) - { - LMS_LOG(DBUPDATER, ERROR) << "Caught exception while checking file '" << p.string() << "': " << e.what(); - return false; - } -} - -void -ScannerService::removeMissingTracks(ScanStats& stats) -{ - static constexpr std::size_t batchSize {50}; - - ScanStepStats stepStats{stats.startTime, ScanProgressStep::ChekingForMissingFiles}; - - LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed..."; - std::size_t trackCount {}; - - { - auto transaction {_dbSession.createSharedTransaction()}; - trackCount = Track::getCount(_dbSession); - } - LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked..."; - - stepStats.totalElems = trackCount; - notifyInProgress(stepStats); - - RangeResults trackPaths; - std::vector tracksToRemove; - - for (std::size_t i {trackCount < batchSize ? 0 : trackCount - batchSize}; ; i -= (i > batchSize ? batchSize : i)) - { - tracksToRemove.clear(); - - { - auto transaction {_dbSession.createSharedTransaction()}; - trackPaths = Track::findPaths(_dbSession, Range {i, batchSize}); - } - - for (const Track::PathResult& trackPath : trackPaths.results) - { - if (_abortScan) - return; - - if (!checkFile(trackPath.path, _mediaDirectory, _fileExtensions)) - tracksToRemove.push_back(trackPath.trackId); - - stepStats.processedElems++; - } - - if (!tracksToRemove.empty()) - { - auto transaction {_dbSession.createUniqueTransaction()}; - - for (const TrackId trackId : tracksToRemove) - { - Track::pointer track {Track::find(_dbSession, trackId)}; - if (track) - { - track.remove(); - stats.deletions++; - } - } - } - - notifyInProgressIfNeeded(stepStats); - - if (i == 0) - break; - } - - LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks checked!"; -} - -void -ScannerService::removeOrphanEntries() -{ - LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters..."; - { - auto transaction {_dbSession.createUniqueTransaction()}; - - // Now process orphan Cluster (no track) - auto clusterIds {Cluster::findOrphans(_dbSession, Range {})}; - for (ClusterId clusterId : clusterIds.results) - { - Cluster::pointer cluster {Cluster::find(_dbSession, clusterId)}; - LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'"; - cluster.remove(); - } - } - - LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists..."; - { - auto transaction {_dbSession.createUniqueTransaction()}; - - auto artistIds {Artist::findAllOrphans(_dbSession, Range {})}; - for (const ArtistId artistId : artistIds.results) - { - Artist::pointer artist {Artist::find(_dbSession, artistId)}; - LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'"; - artist.remove(); - } - } - - LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases..."; - { - auto transaction {_dbSession.createUniqueTransaction()}; - - auto releases {Release::findOrphans(_dbSession, Range {})}; - for (const ReleaseId releaseId : releases.results) - { - Release::pointer release {Release::find(_dbSession, releaseId)}; - LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'"; - release.remove(); - } - } - - LMS_LOG(DBUPDATER, INFO) << "Check audio files done!"; -} - -void -ScannerService::checkDuplicatedAudioFiles(ScanStats& stats) -{ - LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files"; - - auto transaction {_dbSession.createSharedTransaction()}; - - const RangeResults tracks = Track::findTrackMBIDDuplicates(_dbSession, Range {}); - for (const TrackId trackId : tracks.results) - { - const Track::pointer track {Track::find(_dbSession, trackId)}; - if (auto trackMBID {track->getTrackMBID()}) - { - LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName(); - stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID}); - } - } - - LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!"; -} - -void -ScannerService::reloadSimilarityEngine(ScanStats& stats) -{ - ScanStepStats stepStats {stats.startTime, ScanProgressStep::ReloadingSimilarityEngine}; - - auto progressCallback {[&](const Recommendation::Progress& progress) - { - stepStats.totalElems = progress.totalElems; - stepStats.processedElems = progress.processedElems; - notifyInProgressIfNeeded(stepStats); - }}; - - notifyInProgress(stepStats); - _recommendationService.load(stats.nbChanges() > 0, progressCallback); - notifyInProgress(stepStats); -} - } // namespace Scanner diff --git a/src/libs/services/scanner/impl/ScannerService.hpp b/src/libs/services/scanner/impl/ScannerService.hpp index 9e7ec5ea..133dfef2 100644 --- a/src/libs/services/scanner/impl/ScannerService.hpp +++ b/src/libs/services/scanner/impl/ScannerService.hpp @@ -30,14 +30,13 @@ #include -#include "services/database/Types.hpp" -#include "services/database/ScanSettings.hpp" +#include "services/database/Db.hpp" #include "services/database/Session.hpp" -#include "metadata/IParser.hpp" +#include "services/database/Types.hpp" #include "services/scanner/IScannerService.hpp" #include "utils/Path.hpp" - -class UUID; +#include "IScanStep.hpp" +#include "ScannerSettings.hpp" namespace Recommendation { @@ -80,27 +79,24 @@ namespace Scanner // Helpers void refreshScanSettings(); + ScannerSettings readSettings(); - void countAllFiles(ScanStats& stats); - void removeMissingTracks(ScanStats& stats); - void removeOrphanEntries(); - void checkDuplicatedAudioFiles(ScanStats& stats); - void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats); void notifyInProgressIfNeeded(const ScanStepStats& stats); void notifyInProgress(const ScanStepStats& stats); void reloadSimilarityEngine(ScanStats& stats); Recommendation::IRecommendationService& _recommendationService; + std::vector> _scanSteps; + std::mutex _controlMutex; - std::atomic _abortScan {}; + bool _abortScan {}; Wt::WIOService _ioService; boost::asio::system_timer _scheduleTimer {_ioService}; - const bool _skipDuplicateRecordingMBID {}; Events _events; std::chrono::system_clock::time_point _lastScanInProgressEmit {}; + Database::Db& _db; Database::Session _dbSession; - std::unique_ptr _metadataParser; mutable std::shared_mutex _statusMutex; State _curState {State::NotScheduled}; @@ -108,13 +104,7 @@ namespace Scanner std::optional _currentScanStepStats; Wt::WDateTime _nextScheduledScan; - // Current scan settings - std::size_t _scanVersion {}; - Wt::WTime _startTime; - Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never}; - std::vector _fileExtensions; - std::filesystem::path _mediaDirectory; - Database::ScanSettings::RecommendationEngineType _recommendationServiceType; + ScannerSettings _settings; }; } // Scanner diff --git a/src/libs/services/scanner/impl/ScannerSettings.hpp b/src/libs/services/scanner/impl/ScannerSettings.hpp new file mode 100644 index 00000000..36b6d7d7 --- /dev/null +++ b/src/libs/services/scanner/impl/ScannerSettings.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2023 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include +#include "services/database/ScanSettings.hpp" + +namespace Scanner +{ + struct ScannerSettings + { + std::size_t scanVersion {}; + Wt::WTime startTime; + Database::ScanSettings::UpdatePeriod updatePeriod {Database::ScanSettings::UpdatePeriod::Never}; + std::vector supportedExtensions; + Database::ScanSettings::RecommendationEngineType recommendationServiceType; + std::filesystem::path mediaDirectory; + bool skipDuplicateRecordingMBID {}; + std::set clusterTypeNames; + + bool operator==(const ScannerSettings& rhs) const + { + return scanVersion == rhs.scanVersion + && startTime == rhs.startTime + && updatePeriod == rhs.updatePeriod + && supportedExtensions == rhs.supportedExtensions + && recommendationServiceType == rhs.recommendationServiceType + && mediaDirectory == rhs.mediaDirectory + && skipDuplicateRecordingMBID == rhs.skipDuplicateRecordingMBID + && clusterTypeNames == rhs.clusterTypeNames; + } + }; +} diff --git a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp index b6b0a436..c0a8bba8 100644 --- a/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp +++ b/src/libs/services/scanner/include/services/scanner/ScannerStats.hpp @@ -57,11 +57,12 @@ namespace Scanner DuplicateReason reason; }; - enum class ScanProgressStep : unsigned + enum class ScanStep { - ChekingForMissingFiles = 0, DiscoveringFiles, ScanningFiles, + ChekingForMissingFiles, + CheckingForDuplicateFiles, FetchingTrackFeatures, ReloadingSimilarityEngine, }; @@ -72,7 +73,7 @@ namespace Scanner { Wt::WDateTime startTime; - ScanProgressStep currentStep; + ScanStep currentStep; std::size_t totalElems {}; std::size_t processedElems {}; diff --git a/src/libs/subsonic/impl/Scan.cpp b/src/libs/subsonic/impl/Scan.cpp index d9a48d67..b006e6b1 100644 --- a/src/libs/subsonic/impl/Scan.cpp +++ b/src/libs/subsonic/impl/Scan.cpp @@ -39,7 +39,7 @@ namespace API::Subsonic::Scan { std::size_t count{}; - if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanProgressStep::ScanningFiles) + if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanningFiles) count = scanStatus.currentScanStepStats->processedElems; statusResponse.setAttribute("count", count); diff --git a/src/libs/utils/impl/Path.cpp b/src/libs/utils/impl/Path.cpp index 029173a0..089472d5 100644 --- a/src/libs/utils/impl/Path.cpp +++ b/src/libs/utils/impl/Path.cpp @@ -31,106 +31,145 @@ #include "utils/Crc32Calculator.hpp" #include "utils/Exception.hpp" #include "utils/Logger.hpp" +#include "utils/String.hpp" -std::uint32_t -computeCrc32(const std::filesystem::path& p) +namespace PathUtils { - Utils::Crc32Calculator crc32; - std::ifstream ifs {p.string().c_str(), std::ios_base::binary}; - if (ifs) + std::uint32_t + computeCrc32(const std::filesystem::path& p) { - do + Utils::Crc32Calculator crc32; + + std::ifstream ifs {p.string().c_str(), std::ios_base::binary}; + if (ifs) { - std::array buffer; + do + { + std::array buffer; - ifs.read( buffer.data(), buffer.size() ); - crc32.processBytes( reinterpret_cast(buffer.data()), ifs.gcount() ); - } - while (ifs); - } - else - { - LMS_LOG(DBUPDATER, ERROR) << "Failed to open file '" << p.string() << "'"; - throw LmsException("Failed to open file '" + p.string() + "'" ); - } - - return crc32.getResult(); -} - -bool -ensureDirectory(const std::filesystem::path& dir) -{ - if (std::filesystem::exists(dir)) - return std::filesystem::is_directory(dir); - else - return std::filesystem::create_directory(dir); -} - -Wt::WDateTime -getLastWriteTime(const std::filesystem::path& file) -{ - struct stat sb {}; - - if (stat(file.string().c_str(), &sb) == -1) - throw LmsException("Failed to get stats on file '" + file.string() + "'" ); - - return Wt::WDateTime::fromTime_t(sb.st_mtime); -} - -bool -exploreFilesRecursive(const std::filesystem::path& directory, std::function cb, const std::filesystem::path& excludeDirFileName) -{ - std::error_code ec; - std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec}; - - if (ec) - { - cb(ec, directory); - return true; // try to continue exploring anyway - } - - if (!excludeDirFileName.empty()) - { - const std::filesystem::path excludePath {directory / excludeDirFileName}; - - if (std::filesystem::exists(excludePath, ec)) - { - LMS_LOG(DBUPDATER, DEBUG) << "Found '" << excludePath.string() << "': skipping directory"; - return true; - } - } - - std::filesystem::directory_iterator itEnd; - while (itPath != itEnd) - { - bool continueExploring {true}; - - if (ec) - { - continueExploring = cb(ec, *itPath); + ifs.read( buffer.data(), buffer.size() ); + crc32.processBytes( reinterpret_cast(buffer.data()), ifs.gcount() ); + } + while (ifs); } else { - if (std::filesystem::is_regular_file(*itPath, ec)) + LMS_LOG(DBUPDATER, ERROR) << "Failed to open file '" << p.string() << "'"; + throw LmsException("Failed to open file '" + p.string() + "'" ); + } + + return crc32.getResult(); + } + + bool + ensureDirectory(const std::filesystem::path& dir) + { + if (std::filesystem::exists(dir)) + return std::filesystem::is_directory(dir); + else + return std::filesystem::create_directory(dir); + } + + Wt::WDateTime + getLastWriteTime(const std::filesystem::path& file) + { + struct stat sb {}; + + if (stat(file.string().c_str(), &sb) == -1) + throw LmsException("Failed to get stats on file '" + file.string() + "'" ); + + return Wt::WDateTime::fromTime_t(sb.st_mtime); + } + + bool + exploreFilesRecursive(const std::filesystem::path& directory, std::function cb, const std::filesystem::path* excludeDirFileName) + { + std::error_code ec; + std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec}; + + if (ec) + { + cb(ec, directory); + return true; // try to continue exploring anyway + } + + if (excludeDirFileName && !excludeDirFileName->empty()) + { + const std::filesystem::path excludePath {directory / *excludeDirFileName}; + + if (std::filesystem::exists(excludePath, ec)) { - continueExploring = cb(ec, *itPath); - } - else if (std::filesystem::is_directory(*itPath, ec)) - { - if (!ec) - continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName); - else - continueExploring = cb(ec, *itPath); + LMS_LOG(DBUPDATER, DEBUG) << "Found '" << excludePath.string() << "': skipping directory"; + return true; } } - if (!continueExploring) - return false; + std::filesystem::directory_iterator itEnd; + while (itPath != itEnd) + { + bool continueExploring {true}; - itPath.increment(ec); + if (ec) + { + continueExploring = cb(ec, *itPath); + } + else + { + if (std::filesystem::is_regular_file(*itPath, ec)) + { + continueExploring = cb(ec, *itPath); + } + else if (std::filesystem::is_directory(*itPath, ec)) + { + if (!ec) + continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName); + else + continueExploring = cb(ec, *itPath); + } + } + + if (!continueExploring) + return false; + + itPath.increment(ec); + } + + return true; } - return true; -} + bool + hasFileAnyExtension(const std::filesystem::path& file, const std::vector& supportedExtensions) + { + const std::filesystem::path extension {StringUtils::stringToLower(file.extension().string())}; + return (std::find(std::cbegin(supportedExtensions), std::cend(supportedExtensions), extension) != std::cend(supportedExtensions)); + } + + bool + isPathInRootPath(const std::filesystem::path& path, const std::filesystem::path& rootPath, const std::filesystem::path* excludeDirFileName) + { + std::filesystem::path curPath = path; + + while (curPath.parent_path() != curPath) + { + curPath = curPath.parent_path(); + + if (excludeDirFileName && !excludeDirFileName->empty()) + { + assert(!excludeDirFileName->has_parent_path()); + + std::error_code ec; + if (std::filesystem::exists(curPath / *excludeDirFileName, ec)) + return false; + } + + if (curPath == rootPath) + return true; + } + + return false; + } + + +} // ns PathUtils diff --git a/src/libs/utils/impl/Zipper.cpp b/src/libs/utils/impl/Zipper.cpp index 33933378..3e11d0ed 100644 --- a/src/libs/utils/impl/Zipper.cpp +++ b/src/libs/utils/impl/Zipper.cpp @@ -266,7 +266,7 @@ namespace Zip if (lastModifiedTime.isValid()) fileContext.lastModifiedTime = lastModifiedTime; else - fileContext.lastModifiedTime = getLastWriteTime(filePath); + fileContext.lastModifiedTime = PathUtils::getLastWriteTime(filePath); _files[filename] = std::move(fileContext); diff --git a/src/libs/utils/include/utils/Path.hpp b/src/libs/utils/include/utils/Path.hpp index e6361a5f..52f62136 100644 --- a/src/libs/utils/include/utils/Path.hpp +++ b/src/libs/utils/include/utils/Path.hpp @@ -26,15 +26,25 @@ #include -std::uint32_t computeCrc32(const std::filesystem::path& p); +namespace PathUtils +{ + std::uint32_t computeCrc32(const std::filesystem::path& p); -// Make sure the given path is a directory -// Create it if needed -bool ensureDirectory(const std::filesystem::path& dir); + // Make sure the given path is a directory + // Create it if needed + bool ensureDirectory(const std::filesystem::path& dir); -// Get the last write time since Epoch -Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir); + // Get the last write time since Epoch + Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir); -// returns false if aborted by user -bool exploreFilesRecursive(const std::filesystem::path& directory, std::function cb, const std::filesystem::path& excludeDirFileName = {}); + // returns false if aborted by user + bool exploreFilesRecursive(const std::filesystem::path& directory, std::function cb, const std::filesystem::path* excludeDirFileName = {}); + // Check if file's extension is one of provided extensions + bool hasFileAnyExtension(const std::filesystem::path& file, const std::vector& extensions); + + // Check if a path is within a directory (excludeDirFileName is a relative can be used to exclude a whole directory and its subdirectory, must not have parent_path) + bool isPathInRootPath(const std::filesystem::path& path, const std::filesystem::path& rootPath, const std::filesystem::path* excludeDirFileName = {}); + + +} diff --git a/src/libs/utils/include/utils/Tuple.hpp b/src/libs/utils/include/utils/Tuple.hpp new file mode 100644 index 00000000..f8de7387 --- /dev/null +++ b/src/libs/utils/include/utils/Tuple.hpp @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2015 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +namespace Utils +{ + namespace Details + { + template + struct Seq { }; + + template + struct GenSeq : GenSeq { }; + + template + struct GenSeq<0, Is...> : Seq { }; + + template + void forEachTypeInTuple(T&& t, Func f, Seq) + { + auto l = { (f(std::get(t)), 0)... }; + } + } + + template + void forEachTypeInTuple(std::tuple const& t, Func f) + { + Details::forEachTypeInTuple(t, f, Details::GenSeq()); + } +} + + diff --git a/src/lms/ui/admin/ScannerController.cpp b/src/lms/ui/admin/ScannerController.cpp index c8bd0a39..4965eac4 100644 --- a/src/lms/ui/admin/ScannerController.cpp +++ b/src/lms/ui/admin/ScannerController.cpp @@ -237,30 +237,35 @@ ScannerController::refreshContents() switch (status.currentScanStepStats->currentStep) { - case Scanner::ScanProgressStep::ChekingForMissingFiles: + case Scanner::ScanStep::CheckingForDuplicateFiles: + _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files") + .arg(status.currentScanStepStats->processedElems)); + break; + + case Scanner::ScanStep::ChekingForMissingFiles: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files") .arg(status.currentScanStepStats->progress())); break; - case Scanner::ScanProgressStep::DiscoveringFiles: + case Scanner::ScanStep::DiscoveringFiles: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files") .arg(status.currentScanStepStats->processedElems)); break; - case Scanner::ScanProgressStep::ScanningFiles: + case Scanner::ScanStep::ScanningFiles: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files") .arg(status.currentScanStepStats->processedElems) .arg(status.currentScanStepStats->totalElems) .arg(status.currentScanStepStats->progress())); break; - case Scanner::ScanProgressStep::FetchingTrackFeatures: + case Scanner::ScanStep::FetchingTrackFeatures: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features") .arg(status.currentScanStepStats->processedElems) .arg(status.currentScanStepStats->totalElems) .arg(status.currentScanStepStats->progress())); break; - case Scanner::ScanProgressStep::ReloadingSimilarityEngine: + case Scanner::ScanStep::ReloadingSimilarityEngine: _stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine") .arg(status.currentScanStepStats->progress())); break;