owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored scanner to ease further improvements

Browse Source
This commit is contained in:
emeric
2023-02-19 22:58:55 +01:00
parent 962775e920
commit 12d967fe4f
25 changed files with 1370 additions and 808 deletions
Download Patch File Download Diff File Expand all files Collapse all files
approot/messages.xml
+1
View File
@@ -83,6 +83,7 @@
<message id="Lms.Admin.ScannerController.status-not-scheduled">Not scheduled</message>
<message id="Lms.Admin.ScannerController.status-scheduled">Scheduled on {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">Scanning: step {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Checking for duplicate files... {1} files</message>
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Checking files... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">Discovering files: {1} files</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Fetching track features from AcousticBrainz: {1}/{2} tracks ({3}%)...</message>
approot/messages_fr.xml
+1
View File
@@ -83,6 +83,7 @@
<message id="Lms.Admin.ScannerController.status-not-scheduled">Non planifié</message>
<message id="Lms.Admin.ScannerController.status-scheduled">Planifié le {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">En cours de scan : étape {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-checking-for-duplicate-files">Vérification des fichiers dupliqués... {1} fichiers</message>
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Vérification des fichiers... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">Découverte des fichiers : {1} fichiers</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Récupération des métadonnées AcousticBrainz : {1}/{2} fichiers ({3}%)...</message>
approot/messages_it.xml
+1
View File
@@ -83,6 +83,7 @@
<message id="Lms.Admin.ScannerController.status-not-scheduled">Non pianificato</message>
<message id="Lms.Admin.ScannerController.status-scheduled">Pianificato il {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">Scansione: passo {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">Controllo file... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">File trovati: {1} files</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">Recupero metadati da AcousticBrainz: {1}/{2} tracce ({3}%)...</message>
approot/messages_zh.xml
+1
View File
@@ -83,6 +83,7 @@
<message id="Lms.Admin.ScannerController.status-not-scheduled">无计划</message>
<message id="Lms.Admin.ScannerController.status-scheduled">计划于 {1}</message>
<message id="Lms.Admin.ScannerController.status-in-progress">扫描中: 阶段 {1}/{2}</message>
<message id="Lms.Admin.ScannerController.step-checking-for-missing-files">检查文件中... {1}%</message>
<message id="Lms.Admin.ScannerController.step-discovering-files">检索文件中: {1} 文件</message>
<message id="Lms.Admin.ScannerController.step-fetching-track-features">从 AcousticBrainz 获取音轨特征: {1}/{2} 音轨 ({3}%)...</message>
src/libs/services/scanner/CMakeLists.txt
+4
View File
@@ -2,6 +2,10 @@
add_library(lmsscanner SHARED
impl/ScannerService.cpp
impl/ScannerStats.cpp
impl/ScanStepCheckDuplicatedDbFiles.cpp
impl/ScanStepDiscoverFiles.cpp
impl/ScanStepRemoveOrphanDbFiles.cpp
impl/ScanStepScanFiles.cpp
)
target_include_directories(lmsscanner INTERFACE
src/libs/services/scanner/impl/IScanStep.hpp
+45
View File
@@ -0,0 +1,45 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <string_view>
#include "services/scanner/ScannerStats.hpp"
namespace Scanner
{
class IScanStep
{
public:
virtual ~IScanStep() = default;
virtual ScanStep getStep() const = 0;
virtual std::string_view getStepName() const = 0;
struct ScanContext
{
const std::filesystem::path directory;
const bool forceScan;
ScanStats stats;
ScanStepStats currentStepStats;
};
virtual void process(ScanContext& context) = 0;
};
}
src/libs/services/scanner/impl/ScanStepBase.hpp
+61
View File
@@ -0,0 +1,61 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <functional>
#include "services/scanner/ScannerStats.hpp"
#include "IScanStep.hpp"
#include "ScannerSettings.hpp"
namespace Database
{
class Db;
}
namespace Scanner
{
class ScanStepBase : public IScanStep
{
public:
static inline const std::filesystem::path excludeDirFileName {".lmsignore"};
using ProgressCallback = std::function<void(const ScanStepStats& stats)>;
struct InitParams
{
const ScannerSettings& settings;
ProgressCallback progressCallback;
bool& abortScan;
Database::Db& db;
};
ScanStepBase(InitParams& initParams)
: _settings {initParams.settings}
, _progressCallback {initParams.progressCallback}
, _abortScan {initParams.abortScan}
, _db {initParams.db}
{}
protected:
const ScannerSettings& _settings;
ProgressCallback _progressCallback;
bool& _abortScan;
Database::Db& _db;
};
}
src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepCheckDuplicatedDbFiles.hpp"
#include "services/database/Db.hpp"
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "utils/Logger.hpp"
namespace Scanner
{
void
ScanStepCheckDuplicatedDbFiles::process(ScanContext& context)
{
using namespace Database;
if (_abortScan)
return;
Session& session {_db.getTLSSession()};
auto transaction {session.createSharedTransaction()};
const RangeResults<TrackId> tracks = Track::findTrackMBIDDuplicates(session, Range {});
for (const TrackId trackId : tracks.results)
{
const Track::pointer track {Track::find(session, trackId)};
if (auto trackMBID {track->getTrackMBID()})
{
LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
context.stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID});
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
}
LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files";
}
}
src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ScanStepBase.hpp"
namespace Scanner
{
class ScanStepCheckDuplicatedDbFiles : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
std::string_view getStepName() const override { return "Checking for duplicated files"; }
ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp
+48
View File
@@ -0,0 +1,48 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepDiscoverFiles.hpp"
#include "utils/Logger.hpp"
#include "utils/Path.hpp"
namespace Scanner
{
void
ScanStepDiscoverFiles::process(ScanContext& context)
{
context.stats.filesScanned = 0;
PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path)
{
if (_abortScan)
return false;
if (!ec && PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
{
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
return true;
}, &excludeDirFileName);
context.stats.filesScanned = context.currentStepStats.processedElems;
LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'";
}
}
src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ScanStepBase.hpp"
namespace Scanner
{
class ScanStepDiscoverFiles : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::DiscoveringFiles; }
std::string_view getStepName() const override { return "DiscoveringFiles"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp
+202
View File
@@ -0,0 +1,202 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepRemoveOrphanDbFiles.hpp"
#include "services/database/Artist.hpp"
#include "services/database/Cluster.hpp"
#include "services/database/Db.hpp"
#include "services/database/Release.hpp"
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "utils/Logger.hpp"
#include "utils/Path.hpp"
namespace Scanner
{
void
ScanStepRemoveOrphanDbFiles::process(ScanContext& context)
{
removeOrphanTracks(context);
removeOrphanClusters();
removeOrphanArtists();
removeOrphanReleases();
}
void ScanStepRemoveOrphanDbFiles::removeOrphanTracks(ScanContext& context)
{
using namespace Database;
if (_abortScan)
return;
static constexpr std::size_t batchSize {50};
Session& session {_db.getTLSSession()};
LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
std::size_t trackCount {};
{
auto transaction {session.createSharedTransaction()};
trackCount = Track::getCount(session);
}
LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked...";
context.currentStepStats.totalElems = trackCount;
RangeResults<Track::PathResult> trackPaths;
std::vector<TrackId> tracksToRemove;
// TODO handle only files in context.directory
for (std::size_t i {trackCount < batchSize ? 0 : trackCount - batchSize}; ; i -= (i > batchSize ? batchSize : i))
{
tracksToRemove.clear();
{
auto transaction {session.createSharedTransaction()};
trackPaths = Track::findPaths(session, Range {i, batchSize});
}
for (const Track::PathResult& trackPath : trackPaths.results)
{
if (_abortScan)
return;
if (!checkFile(trackPath.path, _settings.mediaDirectory))
tracksToRemove.push_back(trackPath.trackId);
context.currentStepStats.processedElems++;
}
if (!tracksToRemove.empty())
{
auto transaction {session.createSharedTransaction()};
for (const TrackId trackId : tracksToRemove)
{
Track::pointer track {Track::find(session, trackId)};
if (track)
{
track.remove();
context.stats.deletions++;
}
}
}
_progressCallback(context.currentStepStats);
if (i == 0)
break;
}
LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks checked!";
}
void
ScanStepRemoveOrphanDbFiles::removeOrphanClusters()
{
using namespace Database;
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
Session& session {_db.getTLSSession()};
auto transaction {session.createUniqueTransaction()};
// Now process orphan Cluster (no track)
auto clusterIds {Cluster::findOrphans(session, Range {})};
for (ClusterId clusterId : clusterIds.results)
{
Cluster::pointer cluster {Cluster::find(session, clusterId)};
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'";
cluster.remove();
}
}
void
ScanStepRemoveOrphanDbFiles::removeOrphanArtists()
{
using namespace Database;
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists...";
Session& session {_db.getTLSSession()};
auto transaction {session.createUniqueTransaction()};
auto artistIds {Artist::findAllOrphans(session, Range {})};
for (const ArtistId artistId : artistIds.results)
{
Artist::pointer artist {Artist::find(session, artistId)};
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'";
artist.remove();
}
}
void
ScanStepRemoveOrphanDbFiles::removeOrphanReleases()
{
using namespace Database;
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases...";
Session& session {_db.getTLSSession()};
auto transaction {session.createUniqueTransaction()};
auto releases {Release::findOrphans(session, Range {})};
for (const ReleaseId releaseId : releases.results)
{
Release::pointer release {Release::find(session, releaseId)};
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'";
release.remove();
}
}
bool
ScanStepRemoveOrphanDbFiles::checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory)
{
try
{
// For each track, make sure the the file still exists
// and still belongs to a media directory
if (!std::filesystem::exists( p )
|| !std::filesystem::is_regular_file( p ) )
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing";
return false;
}
if (!PathUtils::isPathInRootPath(p, mediaDirectory, &excludeDirFileName))
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory";
return false;
}
if (!PathUtils::hasFileAnyExtension(p, _settings.supportedExtensions))
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled";
return false;
}
return true;
}
catch (std::filesystem::filesystem_error& e)
{
LMS_LOG(DBUPDATER, ERROR) << "Caught exception while checking file '" << p.string() << "': " << e.what();
return false;
}
}
}
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp
+44
View File
@@ -0,0 +1,44 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <filesystem>
#include "ScanStepBase.hpp"
namespace Scanner
{
class ScanStepRemoveOrphanDbFiles : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
std::string_view getStepName() const override { return "Checking orphaned entries"; }
ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; }
void process(ScanContext& context) override;
void removeOrphanTracks(ScanContext& context);
void removeOrphanClusters();
void removeOrphanArtists();
void removeOrphanReleases();
bool checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory);
};
}
src/libs/services/scanner/impl/ScanStepScanFiles.cpp
+430
View File
@@ -0,0 +1,430 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepScanFiles.hpp"
#include "metadata/IParser.hpp"
#include "services/database/Artist.hpp"
#include "services/database/Cluster.hpp"
#include "services/database/Db.hpp"
#include "services/database/Release.hpp"
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "services/database/TrackFeatures.hpp"
#include "services/database/TrackArtistLink.hpp"
#include "utils/Exception.hpp"
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/Path.hpp"
using namespace Database;
namespace
{
Artist::pointer
createArtist(Session& session, const MetaData::Artist& artistInfo)
{
Artist::pointer artist {session.create<Artist>(artistInfo.name)};
if (artistInfo.musicBrainzArtistID)
artist.modify()->setMBID(*artistInfo.musicBrainzArtistID);
if (artistInfo.sortName)
artist.modify()->setSortName(*artistInfo.sortName);
return artist;
}
void
updateArtistIfNeeded(Artist::pointer artist, const MetaData::Artist& artistInfo)
{
// Name may have been updated
if (artist->getName() != artistInfo.name)
{
artist.modify()->setName(artistInfo.name);
}
// Sortname may have been updated
if (artistInfo.sortName && *artistInfo.sortName != artist->getSortName() )
{
artist.modify()->setSortName(*artistInfo.sortName);
}
}
std::vector<Artist::pointer>
getOrCreateArtists(Session& session, const std::vector<MetaData::Artist>& artistsInfo, bool allowFallbackOnMBIDEntries)
{
std::vector<Artist::pointer> artists;
for (const MetaData::Artist& artistInfo : artistsInfo)
{
Artist::pointer artist;
// First try to get by MBID
if (artistInfo.musicBrainzArtistID)
{
artist = Artist::find(session, *artistInfo.musicBrainzArtistID);
if (!artist)
artist = createArtist(session, artistInfo);
else
updateArtistIfNeeded(artist, artistInfo);
artists.emplace_back(std::move(artist));
continue;
}
// Fall back on artist name (collisions may occur)
if (!artistInfo.name.empty())
{
for (const Artist::pointer& sameNamedArtist : Artist::find(session, artistInfo.name))
{
// Do not fallback on artist that is correctly tagged
if (!allowFallbackOnMBIDEntries && sameNamedArtist->getMBID())
continue;
artist = sameNamedArtist;
break;
}
// No Artist found with the same name and without MBID -> creating
if (!artist)
artist = createArtist(session, artistInfo);
else
updateArtistIfNeeded(artist, artistInfo);
artists.emplace_back(std::move(artist));
continue;
}
}
return artists;
}
Release::pointer
getOrCreateRelease(Session& session, const MetaData::Album& album)
{
Release::pointer release;
// First try to get by MBID
if (album.musicBrainzAlbumID)
{
release = Release::find(session, *album.musicBrainzAlbumID);
if (!release)
{
release = session.create<Release>(album.name, album.musicBrainzAlbumID);
}
else if (release->getName() != album.name)
{
// Name may have been updated
release.modify()->setName(album.name);
}
return release;
}
// Fall back on release name (collisions may occur)
if (!album.name.empty())
{
for (const Release::pointer& sameNamedRelease : Release::find(session, album.name))
{
// do not fallback on properly tagged releases
if (!sameNamedRelease->getMBID())
{
release = sameNamedRelease;
break;
}
}
// No release found with the same name and without MBID -> creating
if (!release)
release = session.create<Release>(album.name);
return release;
}
return Release::pointer{};
}
std::vector<Cluster::pointer>
getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames)
{
std::vector< Cluster::pointer > clusters;
for (auto clusterNames : clustersNames)
{
auto clusterType = ClusterType::find(session, clusterNames.first);
if (!clusterType)
continue;
for (auto clusterName : clusterNames.second)
{
auto cluster = clusterType->getCluster(clusterName);
if (!cluster)
cluster = session.create<Cluster>(clusterType, clusterName);
clusters.push_back(cluster);
}
}
return clusters;
}
MetaData::ParserReadStyle
getParserReadStyle()
{
std::string_view readStyle {Service<IConfig>::get()->getString("scanner-parser-read-style", "accurate")};
if (readStyle == "fast")
return MetaData::ParserReadStyle::Fast;
else if (readStyle == "average")
return MetaData::ParserReadStyle::Average;
else if (readStyle == "accurate")
return MetaData::ParserReadStyle::Accurate;
throw LmsException {"Invalid value for 'scanner-parser-read-style'"};
}
} // namespace
namespace Scanner
{
ScanStepScanFiles::ScanStepScanFiles(InitParams& initParams)
: ScanStepBase {initParams}
, _metadataParser {MetaData::createParser(MetaData::ParserType::TagLib, getParserReadStyle())} // For now, always use TagLib
{
}
void
ScanStepScanFiles::process(ScanContext& context)
{
_metadataParser->setClusterTypeNames(_settings.clusterTypeNames);
context.currentStepStats.totalElems = context.stats.filesScanned;
PathUtils::exploreFilesRecursive(context.directory, [&](std::error_code ec, const std::filesystem::path& path)
{
if (_abortScan)
return false;
if (ec)
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message();
context.stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()});
}
else if (PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
{
scanAudioFile(path, context);
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
return true;
}, &excludeDirFileName);
}
void
ScanStepScanFiles::scanAudioFile(const std::filesystem::path& file, ScanContext& context)
{
ScanStats& stats {context.stats};
Wt::WDateTime lastWriteTime;
try
{
lastWriteTime = PathUtils::getLastWriteTime(file);
}
catch (LmsException& e)
{
LMS_LOG(DBUPDATER, ERROR) << e.what();
stats.skips++;
return;
}
if (!context.forceScan)
{
// Skip file if last write is the same
Database::Session& dbSession {_db.getTLSSession()};
auto transaction {_db.getTLSSession().createSharedTransaction()};
const Track::pointer track {Track::findByPath(dbSession, file)};
if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t()
&& track->getScanVersion() == _settings.scanVersion)
{
stats.skips++;
return;
}
}
std::optional<MetaData::Track> trackInfo {_metadataParser->parse(file)};
if (!trackInfo)
{
context.stats.errors.emplace_back(file, ScanErrorType::CannotParseFile);
return;
}
stats.scans++;
Database::Session& dbSession {_db.getTLSSession()};
auto uniqueTransaction {dbSession.createUniqueTransaction()};
Track::pointer track {Track::findByPath(dbSession, file) };
// Skip duplicate recording MBID
if (trackInfo->recordingMBID && _settings.skipDuplicateRecordingMBID)
{
for (Track::pointer otherTrack : Track::findByRecordingMBID(dbSession, *trackInfo->recordingMBID))
{
if (track && track->getId() == otherTrack->getId())
continue;
LMS_LOG(DBUPDATER, DEBUG) << "Skipped '" << file.string() << "' (similar recording MBID in '" << otherTrack->getPath().string() << "')";
// This recording MBID already exists, just remove what we just scanned
if (track)
{
track.remove();
stats.deletions++;
}
return;
}
}
// We estimate this is an audio file if:
// - we found a least one audio stream
// - the duration is not null
if (trackInfo->audioStreams.empty())
{
LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (no audio stream found)";
// If Track exists here, delete it!
if (track)
{
track.remove();
stats.deletions++;
}
stats.errors.emplace_back(ScanError {file, ScanErrorType::NoAudioTrack});
return;
}
if (trackInfo->duration == std::chrono::milliseconds::zero())
{
LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (duration is 0)";
// If Track exists here, delete it!
if (track)
{
track.remove();
stats.deletions++;
}
stats.errors.emplace_back(ScanError {file, ScanErrorType::BadDuration});
return;
}
// ***** Title
std::string title;
if (!trackInfo->title.empty())
title = trackInfo->title;
else
{
// TODO parse file name guess track etc.
// For now juste use file name as title
title = file.filename().string();
}
// If file already exists, update its data
// Otherwise, create it
if (!track)
{
track = dbSession.create<Track>(file);
LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'";
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, INFO) << "Updating '" << file.string() << "'";
stats.updates++;
}
// Track related data
assert(track);
track.modify()->clearArtistLinks();
// Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files
for (const Artist::pointer& artist : getOrCreateArtists(dbSession, trackInfo->artists, false))
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, artist, TrackArtistLinkType::Artist));
for (const Artist::pointer& releaseArtist : getOrCreateArtists(dbSession, trackInfo->albumArtists, false))
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, releaseArtist, TrackArtistLinkType::ReleaseArtist));
// Allow fallbacks on artists with the same name even if they have MBID, since there is no tag to indicate the MBID of these artists
// We could ask MusicBrainz to get all the information, but that would heavily slow down the import process
for (const Artist::pointer& conductor : getOrCreateArtists(dbSession, trackInfo->conductorArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, conductor, TrackArtistLinkType::Conductor));
for (const Artist::pointer& composer : getOrCreateArtists(dbSession, trackInfo->composerArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, composer, TrackArtistLinkType::Composer));
for (const Artist::pointer& lyricist : getOrCreateArtists(dbSession, trackInfo->lyricistArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, lyricist, TrackArtistLinkType::Lyricist));
for (const Artist::pointer& mixer : getOrCreateArtists(dbSession, trackInfo->mixerArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, mixer, TrackArtistLinkType::Mixer));
for (const auto& [role, performers] : trackInfo->performerArtists)
{
for (const Artist::pointer& performer : getOrCreateArtists(dbSession, performers, true))
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, performer, TrackArtistLinkType::Performer, role));
}
for (const Artist::pointer& producer : getOrCreateArtists(dbSession, trackInfo->producerArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, producer, TrackArtistLinkType::Producer));
for (const Artist::pointer& remixer : getOrCreateArtists(dbSession, trackInfo->remixerArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(dbSession, track, remixer, TrackArtistLinkType::Remixer));
track.modify()->setScanVersion(_settings.scanVersion);
if (trackInfo->album)
track.modify()->setRelease(getOrCreateRelease(dbSession, *trackInfo->album));
else
track.modify()->setRelease({});
track.modify()->setClusters(getOrCreateClusters(dbSession, trackInfo->clusters));
track.modify()->setLastWriteTime(lastWriteTime);
track.modify()->setName(title);
track.modify()->setDuration(trackInfo->duration);
track.modify()->setAddedTime(Wt::WDateTime::currentDateTime());
track.modify()->setTrackNumber(trackInfo->trackNumber ? *trackInfo->trackNumber : 0);
track.modify()->setDiscNumber(trackInfo->discNumber ? *trackInfo->discNumber : 0);
track.modify()->setTotalTrack(trackInfo->totalTrack);
track.modify()->setTotalDisc(trackInfo->totalDisc);
track.modify()->setDiscSubtitle(trackInfo->discSubtitle);
track.modify()->setDate(trackInfo->date);
track.modify()->setOriginalDate(trackInfo->originalDate);
// If a file has an OriginalYear but no Year, set it to ease filtering
if (!trackInfo->date.isValid() && trackInfo->originalDate.isValid())
track.modify()->setDate(trackInfo->originalDate);
track.modify()->setRecordingMBID(trackInfo->recordingMBID);
track.modify()->setTrackMBID(trackInfo->trackMBID);
if (auto trackFeatures {TrackFeatures::find(dbSession, track->getId())})
trackFeatures.remove(); // TODO: only if MBID changed?
track.modify()->setHasCover(trackInfo->hasCover);
track.modify()->setCopyright(trackInfo->copyright);
track.modify()->setCopyrightURL(trackInfo->copyrightURL);
track.modify()->setTrackReplayGain(trackInfo->trackReplayGain);
track.modify()->setReleaseReplayGain(trackInfo->albumReplayGain);
}
}
src/libs/services/scanner/impl/ScanStepScanFiles.hpp
+43
View File
@@ -0,0 +1,43 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <filesystem>
#include "metadata/IParser.hpp"
#include "ScanStepBase.hpp"
namespace Scanner
{
class ScanStepScanFiles : public ScanStepBase
{
public:
ScanStepScanFiles(InitParams& initParams);
private:
ScanStep getStep() const override { return ScanStep::ScanningFiles; }
std::string_view getStepName() const override { return "Scanning files"; }
void process(ScanContext& context) override;
void scanAudioFile(const std::filesystem::path& file, ScanContext& context);
std::unique_ptr<MetaData::IParser> _metadataParser;
};
}
src/libs/services/scanner/impl/ScannerService.cpp
+88 -684
View File
@@ -22,27 +22,25 @@
#include <ctime>
#include <boost/asio/placeholders.hpp>
#include "services/database/Artist.hpp"
#include "services/database/Cluster.hpp"
#include "services/database/Release.hpp"
#include "services/database/ScanSettings.hpp"
#include "services/database/Track.hpp"
#include "services/database/TrackArtistLink.hpp"
#include "services/database/TrackFeatures.hpp"
#include "metadata/IParser.hpp"
#include "services/database/ScanSettings.hpp"
#include "services/recommendation/IRecommendationService.hpp"
#include "utils/Exception.hpp"
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/Path.hpp"
#include "utils/UUID.hpp"
#include "utils/Tuple.hpp"
#include "ScanStepCheckDuplicatedDbFiles.hpp"
#include "ScanStepDiscoverFiles.hpp"
#include "ScanStepRemoveOrphanDbFiles.hpp"
#include "ScanStepScanFiles.hpp"
using namespace Database;
namespace {
const std::filesystem::path excludeDirFileName {".lmsignore"};
Wt::WDate
getNextMonday(Wt::WDate current)
{
@@ -65,183 +63,6 @@ getNextFirstOfMonth(Wt::WDate current)
return current;
}
bool
isFileSupported(const std::filesystem::path& file, const std::vector<std::filesystem::path>& extensions)
{
const std::filesystem::path extension {StringUtils::stringToLower(file.extension().string())};
return (std::find(std::cbegin(extensions), std::cend(extensions), extension) != std::cend(extensions));
}
bool
isPathInMediaDirectory(const std::filesystem::path& path, const std::filesystem::path& rootPath)
{
std::filesystem::path curPath = path;
while (curPath.parent_path() != curPath)
{
curPath = curPath.parent_path();
std::error_code ec;
if (std::filesystem::exists(curPath / excludeDirFileName, ec))
return false;
if (curPath == rootPath)
return true;
}
return false;
}
static
Artist::pointer
createArtist(Session& session, const MetaData::Artist& artistInfo)
{
Artist::pointer artist {session.create<Artist>(artistInfo.name)};
if (artistInfo.musicBrainzArtistID)
artist.modify()->setMBID(*artistInfo.musicBrainzArtistID);
if (artistInfo.sortName)
artist.modify()->setSortName(*artistInfo.sortName);
return artist;
}
static
void
updateArtistIfNeeded(Artist::pointer artist, const MetaData::Artist& artistInfo)
{
// Name may have been updated
if (artist->getName() != artistInfo.name)
{
artist.modify()->setName(artistInfo.name);
}
// Sortname may have been updated
if (artistInfo.sortName && *artistInfo.sortName != artist->getSortName() )
{
artist.modify()->setSortName(*artistInfo.sortName);
}
}
std::vector<Artist::pointer>
getOrCreateArtists(Session& session, const std::vector<MetaData::Artist>& artistsInfo, bool allowFallbackOnMBIDEntries)
{
std::vector<Artist::pointer> artists;
for (const MetaData::Artist& artistInfo : artistsInfo)
{
Artist::pointer artist;
// First try to get by MBID
if (artistInfo.musicBrainzArtistID)
{
artist = Artist::find(session, *artistInfo.musicBrainzArtistID);
if (!artist)
artist = createArtist(session, artistInfo);
else
updateArtistIfNeeded(artist, artistInfo);
artists.emplace_back(std::move(artist));
continue;
}
// Fall back on artist name (collisions may occur)
if (!artistInfo.name.empty())
{
for (const Artist::pointer& sameNamedArtist : Artist::find(session, artistInfo.name))
{
// Do not fallback on artist that is correctly tagged
if (!allowFallbackOnMBIDEntries && sameNamedArtist->getMBID())
continue;
artist = sameNamedArtist;
break;
}
// No Artist found with the same name and without MBID -> creating
if (!artist)
artist = createArtist(session, artistInfo);
else
updateArtistIfNeeded(artist, artistInfo);
artists.emplace_back(std::move(artist));
continue;
}
}
return artists;
}
Release::pointer
getOrCreateRelease(Session& session, const MetaData::Album& album)
{
Release::pointer release;
// First try to get by MBID
if (album.musicBrainzAlbumID)
{
release = Release::find(session, *album.musicBrainzAlbumID);
if (!release)
{
release = session.create<Release>(album.name, album.musicBrainzAlbumID);
}
else if (release->getName() != album.name)
{
// Name may have been updated
release.modify()->setName(album.name);
}
return release;
}
// Fall back on release name (collisions may occur)
if (!album.name.empty())
{
for (const Release::pointer& sameNamedRelease : Release::find(session, album.name))
{
// do not fallback on properly tagged releases
if (!sameNamedRelease->getMBID())
{
release = sameNamedRelease;
break;
}
}
// No release found with the same name and without MBID -> creating
if (!release)
release = session.create<Release>(album.name);
return release;
}
return Release::pointer{};
}
std::vector<Cluster::pointer>
getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames)
{
std::vector< Cluster::pointer > clusters;
for (auto clusterNames : clustersNames)
{
auto clusterType = ClusterType::find(session, clusterNames.first);
if (!clusterType)
continue;
for (auto clusterName : clusterNames.second)
{
auto cluster = clusterType->getCluster(clusterName);
if (!cluster)
cluster = session.create<Cluster>(clusterType, clusterName);
clusters.push_back(cluster);
}
}
return clusters;
}
} // namespace
namespace Scanner {
@@ -252,29 +73,11 @@ createScannerService(Db& db, Recommendation::IRecommendationService& recommendat
return std::make_unique<ScannerService>(db, recommendationService);
}
MetaData::ParserReadStyle
getParserReadStyle()
{
std::string_view readStyle {Service<IConfig>::get()->getString("scanner-parser-read-style", "accurate")};
if (readStyle == "fast")
return MetaData::ParserReadStyle::Fast;
else if (readStyle == "average")
return MetaData::ParserReadStyle::Average;
else if (readStyle == "accurate")
return MetaData::ParserReadStyle::Accurate;
throw LmsException {"Invalid value for 'scanner-parser-read-style'"};
}
ScannerService::ScannerService(Db& db, Recommendation::IRecommendationService& recommendationService)
: _recommendationService {recommendationService}
, _skipDuplicateRecordingMBID {Service<IConfig>::get()->getBool("scanner-skip-duplicate-recording-mbid", false)}
, _db {db}
, _dbSession {db}
, _metadataParser {MetaData::createParser(MetaData::ParserType::TagLib, getParserReadStyle())} // For now, always use TagLib
{
LMS_LOG(DBUPDATER, INFO) << "skipDuplicateRecordingMBID = " << _skipDuplicateRecordingMBID;
_ioService.setThreadCount(1);
refreshScanSettings();
@@ -383,34 +186,34 @@ ScannerService::getStatus() const
void
ScannerService::scheduleNextScan()
{
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan";
LMS_LOG(DBUPDATER, DEBUG) << "Scheduling next scan";
refreshScanSettings();
const Wt::WDateTime now {Wt::WDateTime::currentDateTime()};
Wt::WDateTime nextScanDateTime;
switch (_updatePeriod)
switch (_settings.updatePeriod)
{
case ScanSettings::UpdatePeriod::Daily:
if (now.time() < _startTime)
nextScanDateTime = {now.date(), _startTime};
if (now.time() < _settings.startTime)
nextScanDateTime = {now.date(), _settings.startTime};
else
nextScanDateTime = {now.date().addDays(1), _startTime};
nextScanDateTime = {now.date().addDays(1), _settings.startTime};
break;
case ScanSettings::UpdatePeriod::Weekly:
if (now.time() < _startTime && now.date().dayOfWeek() == 1)
nextScanDateTime = {now.date(), _startTime};
if (now.time() < _settings.startTime && now.date().dayOfWeek() == 1)
nextScanDateTime = {now.date(), _settings.startTime};
else
nextScanDateTime = {getNextMonday(now.date()), _startTime};
nextScanDateTime = {getNextMonday(now.date()), _settings.startTime};
break;
case ScanSettings::UpdatePeriod::Monthly:
if (now.time() < _startTime && now.date().day() == 1)
nextScanDateTime = {now.date(), _startTime};
if (now.time() < _settings.startTime && now.date().day() == 1)
nextScanDateTime = {now.date(), _settings.startTime};
else
nextScanDateTime = {getNextFirstOfMonth(now.date()), _startTime};
nextScanDateTime = {getNextFirstOfMonth(now.date()), _settings.startTime};
break;
case ScanSettings::UpdatePeriod::Hourly:
@@ -434,31 +237,6 @@ ScannerService::scheduleNextScan()
_events.scanScheduled.emit(_nextScheduledScan);
}
void
ScannerService::countAllFiles(ScanStats& stats)
{
ScanStepStats stepStats{stats.startTime, ScanProgressStep::DiscoveringFiles};
stats.filesScanned = 0;
notifyInProgress(stepStats);
exploreFilesRecursive(_mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
{
if (_abortScan)
return false;
if (!ec && isFileSupported(path, _fileExtensions))
{
stats.filesScanned++;
stepStats.processedElems++;
notifyInProgressIfNeeded(stepStats);
}
return true;
}, excludeDirFileName);
notifyInProgress(stepStats);
}
void
ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime)
{
@@ -499,35 +277,29 @@ ScannerService::scan(bool forceScan)
_nextScheduledScan = {};
}
ScanStats stats;
stats.startTime = Wt::WDateTime::currentDateTime();
LMS_LOG(UI, INFO) << "New scan started!";
refreshScanSettings();
removeMissingTracks(stats);
IScanStep::ScanContext scanContext {_settings.mediaDirectory, forceScan, ScanStats {}, ScanStepStats {}};
ScanStats& stats {scanContext.stats};
stats.startTime = Wt::WDateTime::currentDateTime();
LMS_LOG(DBUPDATER, DEBUG) << "Counting files in media directory '" << _mediaDirectory.string() << "'...";
countAllFiles(stats);
LMS_LOG(DBUPDATER, DEBUG) << "-> Nb files = " << stats.filesScanned;
LMS_LOG(UI, INFO) << "Checks complete, force scan = " << forceScan;
LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "'...";
scanMediaDirectory(_mediaDirectory, forceScan, stats);
LMS_LOG(DBUPDATER, INFO) << "scaning media directory '" << _mediaDirectory.string() << "' DONE";
removeOrphanEntries();
if (!_abortScan)
for (auto& scanStep : _scanSteps)
{
checkDuplicatedAudioFiles(stats);
reloadSimilarityEngine(stats);
LMS_LOG(DBUPDATER, DEBUG) << "Starting scan step '" << scanStep->getStepName() << "'";
scanContext.currentStepStats = ScanStepStats {Wt::WDateTime::currentDateTime(), scanStep->getStep()};
notifyInProgress(scanContext.currentStepStats);
scanStep->process(scanContext);
notifyInProgress(scanContext.currentStepStats);
LMS_LOG(DBUPDATER, DEBUG) << "Completed scan step '" << scanStep->getStepName() << "'";
}
LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size();
// TODO make it a scan step
_dbSession.optimize();
if (!_abortScan)
@@ -559,33 +331,71 @@ ScannerService::scan(bool forceScan)
void
ScannerService::refreshScanSettings()
{
auto transaction {_dbSession.createSharedTransaction()};
ScannerSettings newSettings {readSettings()};
if (_settings == newSettings)
return;
const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
LMS_LOG(DBUPDATER, DEBUG) << "Scanner settings updated";
LMS_LOG(DBUPDATER, DEBUG) << "skipDuplicateRecordingMBID = " << newSettings.skipDuplicateRecordingMBID;
LMS_LOG(DBUPDATER, DEBUG) << "Using scan settings version " << newSettings.scanVersion;
LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion();
_settings = std::move(newSettings);
_scanVersion = scanSettings->getScanVersion();
_startTime = scanSettings->getUpdateStartTime();
_updatePeriod = scanSettings->getUpdatePeriod();
auto cbFunc {[this](const ScanStepStats& stats)
{
notifyInProgressIfNeeded(stats);
}};
ScanStepBase::InitParams params
{
const auto fileExtensions {scanSettings->getAudioFileExtensions()};
_fileExtensions.clear();
std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(_fileExtensions),
_settings,
cbFunc,
_abortScan,
_db
};
_scanSteps.clear();
_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
}
ScannerSettings
ScannerService::readSettings()
{
ScannerSettings newSettings;
newSettings.skipDuplicateRecordingMBID = Service<IConfig>::get()->getBool("scanner-skip-duplicate-recording-mbid", false);
{
auto transaction {_dbSession.createSharedTransaction()};
const ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)};
newSettings.scanVersion = scanSettings->getScanVersion();
newSettings.startTime = scanSettings->getUpdateStartTime();
newSettings.updatePeriod = scanSettings->getUpdatePeriod();
{
const auto fileExtensions {scanSettings->getAudioFileExtensions()};
newSettings.supportedExtensions.reserve(fileExtensions.size());
std::transform(std::cbegin(fileExtensions), std::end(fileExtensions), std::back_inserter(newSettings.supportedExtensions),
[](const std::filesystem::path& extension) { return std::filesystem::path{ StringUtils::stringToLower(extension.string()) }; });
}
newSettings.recommendationServiceType = scanSettings->getRecommendationEngineType();
newSettings.mediaDirectory = scanSettings->getMediaDirectory();
const auto clusterTypes = scanSettings->getClusterTypes();
std::set<std::string> clusterTypeNames;
std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
std::inserter(clusterTypeNames, clusterTypeNames.begin()),
[](ClusterType::pointer clusterType) { return clusterType->getName(); });
newSettings.clusterTypeNames = std::move(clusterTypeNames);
}
_mediaDirectory = scanSettings->getMediaDirectory();
_recommendationServiceType = scanSettings->getRecommendationEngineType();
const auto clusterTypes = scanSettings->getClusterTypes();
std::set<std::string> clusterTypeNames;
std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
std::inserter(clusterTypeNames, clusterTypeNames.begin()),
[](ClusterType::pointer clusterType) { return clusterType->getName(); });
_metadataParser->setClusterTypeNames(clusterTypeNames);
return newSettings;
}
void
@@ -610,410 +420,4 @@ ScannerService::notifyInProgressIfNeeded(const ScanStepStats& stepStats)
notifyInProgress(stepStats);
}
void
ScannerService::scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats)
{
Wt::WDateTime lastWriteTime;
try
{
lastWriteTime = getLastWriteTime(file);
}
catch (LmsException& e)
{
LMS_LOG(DBUPDATER, ERROR) << e.what();
stats.skips++;
return;
}
if (!forceScan)
{
// Skip file if last write is the same
auto transaction {_dbSession.createSharedTransaction()};
const Track::pointer track {Track::findByPath(_dbSession, file)};
if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t()
&& track->getScanVersion() == _scanVersion)
{
stats.skips++;
return;
}
}
std::optional<MetaData::Track> trackInfo {_metadataParser->parse(file)};
if (!trackInfo)
{
stats.errors.emplace_back(file, ScanErrorType::CannotParseFile);
return;
}
stats.scans++;
auto uniqueTransaction {_dbSession.createUniqueTransaction()};
Track::pointer track {Track::findByPath(_dbSession, file) };
// Skip duplicate recording MBID
if (trackInfo->recordingMBID && _skipDuplicateRecordingMBID)
{
for (Track::pointer otherTrack : Track::findByRecordingMBID(_dbSession, *trackInfo->recordingMBID))
{
if (track && track->getId() == otherTrack->getId())
continue;
LMS_LOG(DBUPDATER, DEBUG) << "Skipped '" << file.string() << "' (similar recording MBID in '" << otherTrack->getPath().string() << "')";
// This recording MBID already exists, just remove what we just scanned
if (track)
{
track.remove();
stats.deletions++;
}
return;
}
}
// We estimate this is an audio file if:
// - we found a least one audio stream
// - the duration is not null
if (trackInfo->audioStreams.empty())
{
LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (no audio stream found)";
// If Track exists here, delete it!
if (track)
{
track.remove();
stats.deletions++;
}
stats.errors.emplace_back(ScanError {file, ScanErrorType::NoAudioTrack});
return;
}
if (trackInfo->duration == std::chrono::milliseconds::zero())
{
LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file.string() << "' (duration is 0)";
// If Track exists here, delete it!
if (track)
{
track.remove();
stats.deletions++;
}
stats.errors.emplace_back(ScanError {file, ScanErrorType::BadDuration});
return;
}
// ***** Title
std::string title;
if (!trackInfo->title.empty())
title = trackInfo->title;
else
{
// TODO parse file name guess track etc.
// For now juste use file name as title
title = file.filename().string();
}
// If file already exist, update data
// Otherwise, create it
if (!track)
{
// Create a new song
track = _dbSession.create<Track>(file);
LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'";
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, INFO) << "Updating '" << file.string() << "'";
stats.updates++;
}
// Track related data
assert(track);
track.modify()->clearArtistLinks();
// Do not fallback on artists with the same name but having a MBID for artist and releaseArtists, as it may be corrected by properly tagging files
for (const Artist::pointer& artist : getOrCreateArtists(_dbSession, trackInfo->artists, false))
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, artist, TrackArtistLinkType::Artist));
for (const Artist::pointer& releaseArtist : getOrCreateArtists(_dbSession, trackInfo->albumArtists, false))
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, releaseArtist, TrackArtistLinkType::ReleaseArtist));
// Allow fallbacks on artists with the same name even if they have MBID, since there is no tag to indicate the MBID of these artists
// We could ask MusicBrainz to get all the information, but that would heavily slow down the import process
for (const Artist::pointer& conductor : getOrCreateArtists(_dbSession, trackInfo->conductorArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, conductor, TrackArtistLinkType::Conductor));
for (const Artist::pointer& composer : getOrCreateArtists(_dbSession, trackInfo->composerArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, composer, TrackArtistLinkType::Composer));
for (const Artist::pointer& lyricist : getOrCreateArtists(_dbSession, trackInfo->lyricistArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, lyricist, TrackArtistLinkType::Lyricist));
for (const Artist::pointer& mixer : getOrCreateArtists(_dbSession, trackInfo->mixerArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, mixer, TrackArtistLinkType::Mixer));
for (const auto& [role, performers] : trackInfo->performerArtists)
{
for (const Artist::pointer& performer : getOrCreateArtists(_dbSession, performers, true))
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, performer, TrackArtistLinkType::Performer, role));
}
for (const Artist::pointer& producer : getOrCreateArtists(_dbSession, trackInfo->producerArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, producer, TrackArtistLinkType::Producer));
for (const Artist::pointer& remixer : getOrCreateArtists(_dbSession, trackInfo->remixerArtists, true))
track.modify()->addArtistLink(TrackArtistLink::create(_dbSession, track, remixer, TrackArtistLinkType::Remixer));
track.modify()->setScanVersion(_scanVersion);
if (trackInfo->album)
track.modify()->setRelease(getOrCreateRelease(_dbSession, *trackInfo->album));
else
track.modify()->setRelease({});
track.modify()->setClusters(getOrCreateClusters(_dbSession, trackInfo->clusters));
track.modify()->setLastWriteTime(lastWriteTime);
track.modify()->setName(title);
track.modify()->setDuration(trackInfo->duration);
track.modify()->setAddedTime(Wt::WDateTime::currentDateTime());
track.modify()->setTrackNumber(trackInfo->trackNumber ? *trackInfo->trackNumber : 0);
track.modify()->setDiscNumber(trackInfo->discNumber ? *trackInfo->discNumber : 0);
track.modify()->setTotalTrack(trackInfo->totalTrack);
track.modify()->setTotalDisc(trackInfo->totalDisc);
track.modify()->setDiscSubtitle(trackInfo->discSubtitle);
track.modify()->setDate(trackInfo->date);
track.modify()->setOriginalDate(trackInfo->originalDate);
// If a file has an OriginalYear but no Year, set it to ease filtering
if (!trackInfo->date.isValid() && trackInfo->originalDate.isValid())
track.modify()->setDate(trackInfo->originalDate);
track.modify()->setRecordingMBID(trackInfo->recordingMBID);
track.modify()->setTrackMBID(trackInfo->trackMBID);
if (auto trackFeatures {TrackFeatures::find(_dbSession, track->getId())})
trackFeatures.remove(); // TODO: only if MBID changed?
track.modify()->setHasCover(trackInfo->hasCover);
track.modify()->setCopyright(trackInfo->copyright);
track.modify()->setCopyrightURL(trackInfo->copyrightURL);
track.modify()->setTrackReplayGain(trackInfo->trackReplayGain);
track.modify()->setReleaseReplayGain(trackInfo->albumReplayGain);
}
void
ScannerService::scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats)
{
ScanStepStats stepStats{stats.startTime, ScanProgressStep::ScanningFiles};
stepStats.totalElems = stats.filesScanned;
notifyInProgress(stepStats);
exploreFilesRecursive(mediaDirectory, [&](std::error_code ec, const std::filesystem::path& path)
{
if (_abortScan)
return false;
if (ec)
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message();
stats.errors.emplace_back(ScanError {path, ScanErrorType::CannotReadFile, ec.message()});
}
else if (isFileSupported(path, _fileExtensions))
{
scanAudioFile(path, forceScan, stats );
stepStats.processedElems++;
notifyInProgressIfNeeded(stepStats);
}
return true;
}, excludeDirFileName);
notifyInProgress(stepStats);
}
// Check if a file exists and is still in a media directory
static bool
checkFile(const std::filesystem::path& p, const std::filesystem::path& mediaDirectory, const std::vector<std::filesystem::path>& extensions)
{
try
{
// For each track, make sure the the file still exists
// and still belongs to a media directory
if (!std::filesystem::exists( p )
|| !std::filesystem::is_regular_file( p ) )
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing";
return false;
}
if (!isPathInMediaDirectory(p, mediaDirectory))
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory";
return false;
}
if (!isFileSupported(p, extensions))
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled";
return false;
}
return true;
}
catch (std::filesystem::filesystem_error& e)
{
LMS_LOG(DBUPDATER, ERROR) << "Caught exception while checking file '" << p.string() << "': " << e.what();
return false;
}
}
void
ScannerService::removeMissingTracks(ScanStats& stats)
{
static constexpr std::size_t batchSize {50};
ScanStepStats stepStats{stats.startTime, ScanProgressStep::ChekingForMissingFiles};
LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
std::size_t trackCount {};
{
auto transaction {_dbSession.createSharedTransaction()};
trackCount = Track::getCount(_dbSession);
}
LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked...";
stepStats.totalElems = trackCount;
notifyInProgress(stepStats);
RangeResults<Track::PathResult> trackPaths;
std::vector<TrackId> tracksToRemove;
for (std::size_t i {trackCount < batchSize ? 0 : trackCount - batchSize}; ; i -= (i > batchSize ? batchSize : i))
{
tracksToRemove.clear();
{
auto transaction {_dbSession.createSharedTransaction()};
trackPaths = Track::findPaths(_dbSession, Range {i, batchSize});
}
for (const Track::PathResult& trackPath : trackPaths.results)
{
if (_abortScan)
return;
if (!checkFile(trackPath.path, _mediaDirectory, _fileExtensions))
tracksToRemove.push_back(trackPath.trackId);
stepStats.processedElems++;
}
if (!tracksToRemove.empty())
{
auto transaction {_dbSession.createUniqueTransaction()};
for (const TrackId trackId : tracksToRemove)
{
Track::pointer track {Track::find(_dbSession, trackId)};
if (track)
{
track.remove();
stats.deletions++;
}
}
}
notifyInProgressIfNeeded(stepStats);
if (i == 0)
break;
}
LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks checked!";
}
void
ScannerService::removeOrphanEntries()
{
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
{
auto transaction {_dbSession.createUniqueTransaction()};
// Now process orphan Cluster (no track)
auto clusterIds {Cluster::findOrphans(_dbSession, Range {})};
for (ClusterId clusterId : clusterIds.results)
{
Cluster::pointer cluster {Cluster::find(_dbSession, clusterId)};
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'";
cluster.remove();
}
}
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists...";
{
auto transaction {_dbSession.createUniqueTransaction()};
auto artistIds {Artist::findAllOrphans(_dbSession, Range {})};
for (const ArtistId artistId : artistIds.results)
{
Artist::pointer artist {Artist::find(_dbSession, artistId)};
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'";
artist.remove();
}
}
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases...";
{
auto transaction {_dbSession.createUniqueTransaction()};
auto releases {Release::findOrphans(_dbSession, Range {})};
for (const ReleaseId releaseId : releases.results)
{
Release::pointer release {Release::find(_dbSession, releaseId)};
LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'";
release.remove();
}
}
LMS_LOG(DBUPDATER, INFO) << "Check audio files done!";
}
void
ScannerService::checkDuplicatedAudioFiles(ScanStats& stats)
{
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files";
auto transaction {_dbSession.createSharedTransaction()};
const RangeResults<TrackId> tracks = Track::findTrackMBIDDuplicates(_dbSession, Range {});
for (const TrackId trackId : tracks.results)
{
const Track::pointer track {Track::find(_dbSession, trackId)};
if (auto trackMBID {track->getTrackMBID()})
{
LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
stats.duplicates.emplace_back(ScanDuplicate {track->getId(), DuplicateReason::SameTrackMBID});
}
}
LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!";
}
void
ScannerService::reloadSimilarityEngine(ScanStats& stats)
{
ScanStepStats stepStats {stats.startTime, ScanProgressStep::ReloadingSimilarityEngine};
auto progressCallback {[&](const Recommendation::Progress& progress)
{
stepStats.totalElems = progress.totalElems;
stepStats.processedElems = progress.processedElems;
notifyInProgressIfNeeded(stepStats);
}};
notifyInProgress(stepStats);
_recommendationService.load(stats.nbChanges() > 0, progressCallback);
notifyInProgress(stepStats);
}
} // namespace Scanner
src/libs/services/scanner/impl/ScannerService.hpp
+10 -20
View File
@@ -30,14 +30,13 @@
#include <boost/asio/system_timer.hpp>
#include "services/database/Types.hpp"
#include "services/database/ScanSettings.hpp"
#include "services/database/Db.hpp"
#include "services/database/Session.hpp"
#include "metadata/IParser.hpp"
#include "services/database/Types.hpp"
#include "services/scanner/IScannerService.hpp"
#include "utils/Path.hpp"
class UUID;
#include "IScanStep.hpp"
#include "ScannerSettings.hpp"
namespace Recommendation
{
@@ -80,27 +79,24 @@ namespace Scanner
// Helpers
void refreshScanSettings();
ScannerSettings readSettings();
void countAllFiles(ScanStats& stats);
void removeMissingTracks(ScanStats& stats);
void removeOrphanEntries();
void checkDuplicatedAudioFiles(ScanStats& stats);
void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats);
void notifyInProgressIfNeeded(const ScanStepStats& stats);
void notifyInProgress(const ScanStepStats& stats);
void reloadSimilarityEngine(ScanStats& stats);
Recommendation::IRecommendationService& _recommendationService;
std::vector<std::unique_ptr<IScanStep>> _scanSteps;
std::mutex _controlMutex;
std::atomic<bool> _abortScan {};
bool _abortScan {};
Wt::WIOService _ioService;
boost::asio::system_timer _scheduleTimer {_ioService};
const bool _skipDuplicateRecordingMBID {};
Events _events;
std::chrono::system_clock::time_point _lastScanInProgressEmit {};
Database::Db& _db;
Database::Session _dbSession;
std::unique_ptr<MetaData::IParser> _metadataParser;
mutable std::shared_mutex _statusMutex;
State _curState {State::NotScheduled};
@@ -108,13 +104,7 @@ namespace Scanner
std::optional<ScanStepStats> _currentScanStepStats;
Wt::WDateTime _nextScheduledScan;
// Current scan settings
std::size_t _scanVersion {};
Wt::WTime _startTime;
Database::ScanSettings::UpdatePeriod _updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
std::vector<std::filesystem::path> _fileExtensions;
std::filesystem::path _mediaDirectory;
Database::ScanSettings::RecommendationEngineType _recommendationServiceType;
ScannerSettings _settings;
};
} // Scanner
src/libs/services/scanner/impl/ScannerSettings.hpp
+54
View File
@@ -0,0 +1,54 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <filesystem>
#include <set>
#include <string>
#include <vector>
#include <Wt/WDateTime.h>
#include "services/database/ScanSettings.hpp"
namespace Scanner
{
struct ScannerSettings
{
std::size_t scanVersion {};
Wt::WTime startTime;
Database::ScanSettings::UpdatePeriod updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
std::vector<std::filesystem::path> supportedExtensions;
Database::ScanSettings::RecommendationEngineType recommendationServiceType;
std::filesystem::path mediaDirectory;
bool skipDuplicateRecordingMBID {};
std::set<std::string> clusterTypeNames;
bool operator==(const ScannerSettings& rhs) const
{
return scanVersion == rhs.scanVersion
&& startTime == rhs.startTime
&& updatePeriod == rhs.updatePeriod
&& supportedExtensions == rhs.supportedExtensions
&& recommendationServiceType == rhs.recommendationServiceType
&& mediaDirectory == rhs.mediaDirectory
&& skipDuplicateRecordingMBID == rhs.skipDuplicateRecordingMBID
&& clusterTypeNames == rhs.clusterTypeNames;
}
};
}
src/libs/services/scanner/include/services/scanner/ScannerStats.hpp
+4 -3
View File
@@ -57,11 +57,12 @@ namespace Scanner
DuplicateReason reason;
};
enum class ScanProgressStep : unsigned
enum class ScanStep
{
ChekingForMissingFiles = 0,
DiscoveringFiles,
ScanningFiles,
ChekingForMissingFiles,
CheckingForDuplicateFiles,
FetchingTrackFeatures,
ReloadingSimilarityEngine,
};
@@ -72,7 +73,7 @@ namespace Scanner
{
Wt::WDateTime startTime;
ScanProgressStep currentStep;
ScanStep currentStep;
std::size_t totalElems {};
std::size_t processedElems {};
src/libs/subsonic/impl/Scan.cpp
+1 -1
View File
@@ -39,7 +39,7 @@ namespace API::Subsonic::Scan
{
std::size_t count{};
if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanProgressStep::ScanningFiles)
if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanningFiles)
count = scanStatus.currentScanStepStats->processedElems;
statusResponse.setAttribute("count", count);
src/libs/utils/impl/Path.cpp
+125 -86
View File
@@ -31,106 +31,145 @@
#include "utils/Crc32Calculator.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/String.hpp"
std::uint32_t
computeCrc32(const std::filesystem::path& p)
namespace PathUtils
{
Utils::Crc32Calculator crc32;
std::ifstream ifs {p.string().c_str(), std::ios_base::binary};
if (ifs)
std::uint32_t
computeCrc32(const std::filesystem::path& p)
{
do
Utils::Crc32Calculator crc32;
std::ifstream ifs {p.string().c_str(), std::ios_base::binary};
if (ifs)
{
std::array<char,1024> buffer;
do
{
std::array<char,1024> buffer;
ifs.read( buffer.data(), buffer.size() );
crc32.processBytes( reinterpret_cast<const std::byte*>(buffer.data()), ifs.gcount() );
}
while (ifs);
}
else
{
LMS_LOG(DBUPDATER, ERROR) << "Failed to open file '" << p.string() << "'";
throw LmsException("Failed to open file '" + p.string() + "'" );
}
return crc32.getResult();
}
bool
ensureDirectory(const std::filesystem::path& dir)
{
if (std::filesystem::exists(dir))
return std::filesystem::is_directory(dir);
else
return std::filesystem::create_directory(dir);
}
Wt::WDateTime
getLastWriteTime(const std::filesystem::path& file)
{
struct stat sb {};
if (stat(file.string().c_str(), &sb) == -1)
throw LmsException("Failed to get stats on file '" + file.string() + "'" );
return Wt::WDateTime::fromTime_t(sb.st_mtime);
}
bool
exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path& excludeDirFileName)
{
std::error_code ec;
std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec};
if (ec)
{
cb(ec, directory);
return true; // try to continue exploring anyway
}
if (!excludeDirFileName.empty())
{
const std::filesystem::path excludePath {directory / excludeDirFileName};
if (std::filesystem::exists(excludePath, ec))
{
LMS_LOG(DBUPDATER, DEBUG) << "Found '" << excludePath.string() << "': skipping directory";
return true;
}
}
std::filesystem::directory_iterator itEnd;
while (itPath != itEnd)
{
bool continueExploring {true};
if (ec)
{
continueExploring = cb(ec, *itPath);
ifs.read( buffer.data(), buffer.size() );
crc32.processBytes( reinterpret_cast<const std::byte*>(buffer.data()), ifs.gcount() );
}
while (ifs);
}
else
{
if (std::filesystem::is_regular_file(*itPath, ec))
LMS_LOG(DBUPDATER, ERROR) << "Failed to open file '" << p.string() << "'";
throw LmsException("Failed to open file '" + p.string() + "'" );
}
return crc32.getResult();
}
bool
ensureDirectory(const std::filesystem::path& dir)
{
if (std::filesystem::exists(dir))
return std::filesystem::is_directory(dir);
else
return std::filesystem::create_directory(dir);
}
Wt::WDateTime
getLastWriteTime(const std::filesystem::path& file)
{
struct stat sb {};
if (stat(file.string().c_str(), &sb) == -1)
throw LmsException("Failed to get stats on file '" + file.string() + "'" );
return Wt::WDateTime::fromTime_t(sb.st_mtime);
}
bool
exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path* excludeDirFileName)
{
std::error_code ec;
std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec};
if (ec)
{
cb(ec, directory);
return true; // try to continue exploring anyway
}
if (excludeDirFileName && !excludeDirFileName->empty())
{
const std::filesystem::path excludePath {directory / *excludeDirFileName};
if (std::filesystem::exists(excludePath, ec))
{
continueExploring = cb(ec, *itPath);
}
else if (std::filesystem::is_directory(*itPath, ec))
{
if (!ec)
continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName);
else
continueExploring = cb(ec, *itPath);
LMS_LOG(DBUPDATER, DEBUG) << "Found '" << excludePath.string() << "': skipping directory";
return true;
}
}
if (!continueExploring)
return false;
std::filesystem::directory_iterator itEnd;
while (itPath != itEnd)
{
bool continueExploring {true};
itPath.increment(ec);
if (ec)
{
continueExploring = cb(ec, *itPath);
}
else
{
if (std::filesystem::is_regular_file(*itPath, ec))
{
continueExploring = cb(ec, *itPath);
}
else if (std::filesystem::is_directory(*itPath, ec))
{
if (!ec)
continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName);
else
continueExploring = cb(ec, *itPath);
}
}
if (!continueExploring)
return false;
itPath.increment(ec);
}
return true;
}
return true;
}
bool
hasFileAnyExtension(const std::filesystem::path& file, const std::vector<std::filesystem::path>& supportedExtensions)
{
const std::filesystem::path extension {StringUtils::stringToLower(file.extension().string())};
return (std::find(std::cbegin(supportedExtensions), std::cend(supportedExtensions), extension) != std::cend(supportedExtensions));
}
bool
isPathInRootPath(const std::filesystem::path& path, const std::filesystem::path& rootPath, const std::filesystem::path* excludeDirFileName)
{
std::filesystem::path curPath = path;
while (curPath.parent_path() != curPath)
{
curPath = curPath.parent_path();
if (excludeDirFileName && !excludeDirFileName->empty())
{
assert(!excludeDirFileName->has_parent_path());
std::error_code ec;
if (std::filesystem::exists(curPath / *excludeDirFileName, ec))
return false;
}
if (curPath == rootPath)
return true;
}
return false;
}
} // ns PathUtils
src/libs/utils/impl/Zipper.cpp
+1 -1
View File
@@ -266,7 +266,7 @@ namespace Zip
if (lastModifiedTime.isValid())
fileContext.lastModifiedTime = lastModifiedTime;
else
fileContext.lastModifiedTime = getLastWriteTime(filePath);
fileContext.lastModifiedTime = PathUtils::getLastWriteTime(filePath);
_files[filename] = std::move(fileContext);
src/libs/utils/include/utils/Path.hpp
+18 -8
View File
@@ -26,15 +26,25 @@
#include <Wt/WDateTime.h>
std::uint32_t computeCrc32(const std::filesystem::path& p);
namespace PathUtils
{
std::uint32_t computeCrc32(const std::filesystem::path& p);
// Make sure the given path is a directory
// Create it if needed
bool ensureDirectory(const std::filesystem::path& dir);
// Make sure the given path is a directory
// Create it if needed
bool ensureDirectory(const std::filesystem::path& dir);
// Get the last write time since Epoch
Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir);
// Get the last write time since Epoch
Wt::WDateTime getLastWriteTime(const std::filesystem::path& dir);
// returns false if aborted by user
bool exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path& excludeDirFileName = {});
// returns false if aborted by user
bool exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path* excludeDirFileName = {});
// Check if file's extension is one of provided extensions
bool hasFileAnyExtension(const std::filesystem::path& file, const std::vector<std::filesystem::path>& extensions);
// Check if a path is within a directory (excludeDirFileName is a relative can be used to exclude a whole directory and its subdirectory, must not have parent_path)
bool isPathInRootPath(const std::filesystem::path& path, const std::filesystem::path& rootPath, const std::filesystem::path* excludeDirFileName = {});
}
src/libs/utils/include/utils/Tuple.hpp
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright (C) 2015 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <tuple>
namespace Utils
{
namespace Details
{
template<int... Is>
struct Seq { };
template<int N, int... Is>
struct GenSeq : GenSeq<N - 1, N - 1, Is...> { };
template<int... Is>
struct GenSeq<0, Is...> : Seq<Is...> { };
template<typename T, typename Func, int... Is>
void forEachTypeInTuple(T&& t, Func f, Seq<Is...>)
{
auto l = { (f(std::get<Is>(t)), 0)... };
}
}
template<typename... Ts, typename Func>
void forEachTypeInTuple(std::tuple<Ts...> const& t, Func f)
{
Details::forEachTypeInTuple(t, f, Details::GenSeq<sizeof...(Ts)>());
}
}
src/lms/ui/admin/ScannerController.cpp
+10 -5
View File
@@ -237,30 +237,35 @@ ScannerController::refreshContents()
switch (status.currentScanStepStats->currentStep)
{
case Scanner::ScanProgressStep::ChekingForMissingFiles:
case Scanner::ScanStep::CheckingForDuplicateFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files")
.arg(status.currentScanStepStats->processedElems));
break;
case Scanner::ScanStep::ChekingForMissingFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files")
.arg(status.currentScanStepStats->progress()));
break;
case Scanner::ScanProgressStep::DiscoveringFiles:
case Scanner::ScanStep::DiscoveringFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
.arg(status.currentScanStepStats->processedElems));
break;
case Scanner::ScanProgressStep::ScanningFiles:
case Scanner::ScanStep::ScanningFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
.arg(status.currentScanStepStats->processedElems)
.arg(status.currentScanStepStats->totalElems)
.arg(status.currentScanStepStats->progress()));
break;
case Scanner::ScanProgressStep::FetchingTrackFeatures:
case Scanner::ScanStep::FetchingTrackFeatures:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
.arg(status.currentScanStepStats->processedElems)
.arg(status.currentScanStepStats->totalElems)
.arg(status.currentScanStepStats->progress()));
break;
case Scanner::ScanProgressStep::ReloadingSimilarityEngine:
case Scanner::ScanStep::ReloadingSimilarityEngine:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine")
.arg(status.currentScanStepStats->progress()));
break;