owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

No need to full rescan when changing some settings

Browse Source
This commit is contained in:
emeric
2025-04-07 22:48:10 +02:00
parent 15c5f0b89b
commit 75cbbde598
41 changed files with 331 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/database/impl/Migration.cpp
+8 -1
View File
@@ -35,7 +35,7 @@ namespace lms::db
{
namespace
{
static constexpr Version LMS_DATABASE_VERSION{ 86 };
static constexpr Version LMS_DATABASE_VERSION{ 87 };
}
VersionInfo::VersionInfo()
@@ -1178,6 +1178,12 @@ FROM tracklist)");
utils::executeCommand(*session.getDboSession(), "UPDATE scan_settings SET scan_version = scan_version + 1");
}
void migrateFromV86(Session& session)
{
utils::executeCommand(*session.getDboSession(), "ALTER TABLE scan_settings ADD COLUMN name TEXT NON NULL DEFAULT('')");
utils::executeCommand(*session.getDboSession(), "ALTER TABLE scan_settings RENAME COLUMN scan_version TO audio_scan_version");
}
bool doDbMigration(Session& session)
{
constexpr std::string_view outdatedMsg{ "Outdated database, please rebuild it (delete the .db file and restart)" };
@@ -1240,6 +1246,7 @@ FROM tracklist)");
{ 83, migrateFromV83 },
{ 84, migrateFromV84 },
{ 85, migrateFromV85 },
{ 86, migrateFromV86 },
};
bool migrationPerformed{};
src/libs/database/impl/ScanSettings.cpp
+15 -20
View File
@@ -26,24 +26,25 @@
#include "database/Session.hpp"
#include "Utils.hpp"
#include "traits/StringViewTraits.hpp"
namespace lms::db
{
void ScanSettings::init(Session& session)
ScanSettings::ScanSettings(std::string_view name)
: _name{ name }
{
session.checkWriteTransaction();
if (pointer settings{ get(session) })
return;
session.getDboSession()->add(std::make_unique<ScanSettings>());
}
ScanSettings::pointer ScanSettings::get(Session& session)
ScanSettings::pointer ScanSettings::create(Session& session, std::string_view name)
{
return session.getDboSession()->add(std::unique_ptr<ScanSettings>(new ScanSettings{ name }));
}
ScanSettings::pointer ScanSettings::get(Session& session, std::string_view name)
{
session.checkReadTransaction();
return utils::fetchQuerySingleResult(session.getDboSession()->find<ScanSettings>());
return utils::fetchQuerySingleResult(session.getDboSession()->find<ScanSettings>().where("name = ?").bind(name));
}
std::vector<std::string_view> ScanSettings::getExtraTagsToScan() const
@@ -69,7 +70,7 @@ namespace lms::db
{
std::string newTagsToScan{ core::stringUtils::joinStrings(extraTags, ";") };
if (newTagsToScan != _extraTagsToScan)
incScanVersion();
incAudioScanVersion();
_extraTagsToScan = std::move(newTagsToScan);
}
@@ -80,7 +81,7 @@ namespace lms::db
if (tagDelimiters != _artistTagDelimiters)
{
_artistTagDelimiters.swap(tagDelimiters);
incScanVersion();
incAudioScanVersion();
}
}
@@ -90,30 +91,24 @@ namespace lms::db
if (tagDelimiters != _defaultTagDelimiters)
{
_defaultTagDelimiters.swap(tagDelimiters);
incScanVersion();
incAudioScanVersion();
}
}
void ScanSettings::setSkipSingleReleasePlayLists(bool value)
{
if (_skipSingleReleasePlayLists != value)
{
_skipSingleReleasePlayLists = value;
incScanVersion();
}
}
void ScanSettings::setAllowMBIDArtistMerge(bool value)
{
if (_allowMBIDArtistMerge != value)
{
_allowMBIDArtistMerge = value;
incScanVersion();
}
}
void ScanSettings::incScanVersion()
void ScanSettings::incAudioScanVersion()
{
_scanVersion += 1;
_audioScanVersion += 1;
}
} // namespace lms::db
src/libs/database/impl/Session.cpp
+3 -1
View File
@@ -180,7 +180,9 @@ namespace lms::db
// TODO: move this elsewhere
{
auto uniqueTransaction{ createWriteTransaction() };
ScanSettings::init(*this);
if (!ScanSettings::get(*this))
create<ScanSettings>();
}
return migrationPerformed;
src/libs/database/include/database/ScanSettings.hpp
+14 -7
View File
@@ -19,7 +19,6 @@
#pragma once
#include <filesystem>
#include <span>
#include <string>
#include <string_view>
@@ -58,12 +57,12 @@ namespace lms::db
None,
};
static void init(Session& session);
ScanSettings() = default;
static pointer get(Session& session);
static pointer get(Session& session, std::string_view name = "");
// Getters
std::size_t getScanVersion() const { return _scanVersion; }
std::size_t getAudioScanVersion() const { return _audioScanVersion; }
Wt::WTime getUpdateStartTime() const { return _startTime; }
UpdatePeriod getUpdatePeriod() const { return _updatePeriod; }
std::vector<std::string_view> getExtraTagsToScan() const;
@@ -82,12 +81,12 @@ namespace lms::db
void setDefaultTagDelimiters(std::span<const std::string_view> delimiters);
void setSkipSingleReleasePlayLists(bool value);
void setAllowMBIDArtistMerge(bool value);
void incScanVersion();
template<class Action>
void persist(Action& a)
{
Wt::Dbo::field(a, _scanVersion, "scan_version");
Wt::Dbo::field(a, _name, "name");
Wt::Dbo::field(a, _audioScanVersion, "audio_scan_version");
Wt::Dbo::field(a, _startTime, "start_time");
Wt::Dbo::field(a, _updatePeriod, "update_period");
Wt::Dbo::field(a, _similarityEngineType, "similarity_engine_type");
@@ -99,7 +98,15 @@ namespace lms::db
}
private:
int _scanVersion{};
friend class Session;
ScanSettings(std::string_view name);
static pointer create(Session& session, std::string_view name = "");
void incAudioScanVersion();
std::string _name;
int _audioScanVersion{};
Wt::WTime _startTime = Wt::WTime{ 0, 0, 0 };
UpdatePeriod _updatePeriod{ UpdatePeriod::Never };
SimilarityEngineType _similarityEngineType{ SimilarityEngineType::Clusters };
src/libs/services/scanner/CMakeLists.txt
+1
View File
@@ -13,6 +13,7 @@ add_library(lmsscanner STATIC
impl/steps/ScanStepAssociateExternalLyrics.cpp
impl/steps/ScanStepAssociatePlayListTracks.cpp
impl/steps/ScanStepAssociateReleaseImages.cpp
impl/steps/ScanStepBase.cpp
impl/steps/ScanStepCheckForDuplicatedFiles.cpp
impl/steps/ScanStepCheckForRemovedFiles.cpp
impl/steps/ScanStepCompact.cpp
src/libs/services/scanner/impl/ScannerService.cpp
+106 -61
View File
@@ -28,6 +28,7 @@
#include "core/ITraceLogger.hpp"
#include "database/MediaLibrary.hpp"
#include "database/ScanSettings.hpp"
#include "database/Session.hpp"
#include "scanners/ArtistInfoFileScanner.hpp"
#include "scanners/AudioFileScanner.hpp"
@@ -56,6 +57,9 @@ namespace lms::scanner
namespace
{
static constexpr std::string_view currentSettingsName{ "" };
static constexpr std::string_view lastScanSettingsName{ "last_scan" };
Wt::WDate getNextMonday(Wt::WDate current)
{
do
@@ -75,6 +79,60 @@ namespace lms::scanner
return current;
}
std::optional<ScannerSettings> readScannerSettings(db::Session& session, std::string_view name)
{
std::optional<ScannerSettings> settings;
auto transaction{ session.createReadTransaction() };
const ScanSettings::pointer scanSettings{ ScanSettings::get(session, name) };
if (!scanSettings)
return settings;
settings.emplace();
settings->audioScanVersion = scanSettings->getAudioScanVersion();
settings->startTime = scanSettings->getUpdateStartTime();
settings->updatePeriod = scanSettings->getUpdatePeriod();
MediaLibrary::find(session, [&](const MediaLibrary::pointer& mediaLibrary) {
MediaLibraryInfo info;
info.firstScan = mediaLibrary->isEmpty();
info.id = mediaLibrary->getId();
info.rootDirectory = mediaLibrary->getPath().lexically_normal();
settings->mediaLibraries.push_back(info);
});
{
const auto& tags{ scanSettings->getExtraTagsToScan() };
std::transform(std::cbegin(tags), std::cend(tags), std::back_inserter(settings->extraTags), [](std::string_view tag) { return std::string{ tag }; });
}
settings->artistTagDelimiters = scanSettings->getArtistTagDelimiters();
settings->defaultTagDelimiters = scanSettings->getDefaultTagDelimiters();
settings->skipSingleReleasePlayLists = scanSettings->getSkipSingleReleasePlayLists();
settings->allowArtistMBIDFallback = scanSettings->getAllowMBIDArtistMerge();
// TODO, store this in DB + expose in UI
settings->skipDuplicateTrackMBID = core::Service<core::IConfig>::get()->getBool("scanner-skip-duplicate-mbid", false);
return settings;
}
void writeScannerSettings(db::Session& session, std::string_view name, const ScannerSettings& settings)
{
auto transaction{ session.createWriteTransaction() };
ScanSettings::pointer scanSettings{ ScanSettings::get(session, name) };
if (!scanSettings)
scanSettings = session.create<ScanSettings>(name);
scanSettings.modify()->setAllowMBIDArtistMerge(settings.allowArtistMBIDFallback);
scanSettings.modify()->setSkipSingleReleasePlayLists(settings.skipSingleReleasePlayLists);
// TODO add more fields
}
} // namespace
std::unique_ptr<IScannerService> createScannerService(Db& db)
@@ -283,26 +341,7 @@ namespace lms::scanner
ScanStats& stats{ scanContext.stats };
stats.startTime = Wt::WDateTime::currentDateTime();
std::size_t stepIndex{};
for (auto& scanStep : _scanSteps)
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
scanContext.currentStepStats = ScanStepStats{
.startTime = Wt::WDateTime::currentDateTime(),
.stepCount = _scanSteps.size(),
.stepIndex = stepIndex++,
.currentStep = scanStep->getStep(),
.totalElems = 0,
.processedElems = 0
};
notifyInProgress(scanContext.currentStepStats);
scanStep->process(scanContext);
notifyInProgress(scanContext.currentStepStats);
LMS_LOG(DBUPDATER, DEBUG, "Completed scan step '" << scanStep->getStepName() << "'");
}
processScanSteps(scanContext);
{
std::unique_lock lock{ _statusMutex };
@@ -322,6 +361,10 @@ namespace lms::scanner
_lastCompleteScanStats = stats;
}
// save current settings as last scan settings to compare during next scans if something changed
writeScannerSettings(_db.getTLSSession(), lastScanSettingsName, _settings);
_lastScanSettings = _settings;
LMS_LOG(DBUPDATER, DEBUG, "Scan not aborted, scheduling next scan!");
scheduleNextScan();
@@ -333,16 +376,54 @@ namespace lms::scanner
}
}
void ScannerService::processScanSteps(ScanContext& context)
{
std::size_t stepIndex{};
for (auto& scanStep : _scanSteps)
{
context.currentStepStats = ScanStepStats{
.startTime = Wt::WDateTime::currentDateTime(),
.stepCount = _scanSteps.size(),
.stepIndex = stepIndex++,
.currentStep = scanStep->getStep(),
.totalElems = 0,
.processedElems = 0
};
if (_abortScan)
break;
if (!scanStep->needProcess(context))
{
LMS_LOG(DBUPDATER, DEBUG, "Skipping scan step '" << scanStep->getStepName() << "'");
continue;
}
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
notifyInProgress(context.currentStepStats);
scanStep->process(context);
notifyInProgress(context.currentStepStats);
LMS_LOG(DBUPDATER, DEBUG, "Completed scan step '" << scanStep->getStepName() << "'");
}
}
}
void ScannerService::refreshScanSettings()
{
ScannerSettings newSettings{ readSettings() };
if (_settings == newSettings)
std::optional<ScannerSettings> newSettings{ readScannerSettings(_db.getTLSSession(), currentSettingsName) };
assert(newSettings.has_value());
if (_settings == *newSettings)
return;
LMS_LOG(DBUPDATER, DEBUG, "Scanner settings updated");
LMS_LOG(DBUPDATER, DEBUG, "Using scan settings version " << newSettings.scanVersion);
LMS_LOG(DBUPDATER, DEBUG, "Using audio scan settings version " << newSettings->audioScanVersion);
_settings = std::move(newSettings);
_settings = std::move(*newSettings);
if (!_lastScanSettings)
_lastScanSettings = readScannerSettings(_db.getTLSSession(), lastScanSettingsName);
auto cbFunc{ [this](const ScanStepStats& stats) {
notifyInProgressIfNeeded(stats);
@@ -360,6 +441,7 @@ namespace lms::scanner
ScanStepBase::InitParams params{
.settings = _settings,
.lastScanSettings = _lastScanSettings.has_value() ? &(_lastScanSettings.value()) : nullptr,
.progressCallback = cbFunc,
.abortScan = _abortScan,
.db = _db,
@@ -384,43 +466,6 @@ namespace lms::scanner
_scanSteps.emplace_back(std::make_unique<ScanStepCheckForDuplicatedFiles>(params));
}
ScannerSettings ScannerService::readSettings()
{
ScannerSettings newSettings;
newSettings.skipDuplicateTrackMBID = core::Service<core::IConfig>::get()->getBool("scanner-skip-duplicate-mbid", false);
{
auto transaction{ _db.getTLSSession().createReadTransaction() };
const ScanSettings::pointer scanSettings{ ScanSettings::get(_db.getTLSSession()) };
newSettings.scanVersion = scanSettings->getScanVersion();
newSettings.startTime = scanSettings->getUpdateStartTime();
newSettings.updatePeriod = scanSettings->getUpdatePeriod();
MediaLibrary::find(_db.getTLSSession(), [&](const MediaLibrary::pointer& mediaLibrary) {
MediaLibraryInfo info;
info.firstScan = mediaLibrary->isEmpty();
info.id = mediaLibrary->getId();
info.rootDirectory = mediaLibrary->getPath().lexically_normal();
newSettings.mediaLibraries.push_back(info);
});
{
const auto& tags{ scanSettings->getExtraTagsToScan() };
std::transform(std::cbegin(tags), std::cend(tags), std::back_inserter(newSettings.extraTags), [](std::string_view tag) { return std::string{ tag }; });
}
newSettings.artistTagDelimiters = scanSettings->getArtistTagDelimiters();
newSettings.defaultTagDelimiters = scanSettings->getDefaultTagDelimiters();
newSettings.skipSingleReleasePlayLists = scanSettings->getSkipSingleReleasePlayLists();
}
return newSettings;
}
void ScannerService::notifyInProgress(const ScanStepStats& stepStats)
{
{
src/libs/services/scanner/impl/ScannerService.hpp
+2 -1
View File
@@ -68,12 +68,12 @@ namespace lms::scanner
// Update database (scheduled callback)
void scan(const ScanOptions& scanOptions);
void processScanSteps(ScanContext& context);
void scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
// Helpers
void refreshScanSettings();
ScannerSettings readSettings();
void notifyInProgressIfNeeded(const ScanStepStats& stats);
void notifyInProgress(const ScanStepStats& stats);
@@ -96,5 +96,6 @@ namespace lms::scanner
Wt::WDateTime _nextScheduledScan;
ScannerSettings _settings;
std::optional<ScannerSettings> _lastScanSettings;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/ScannerSettings.hpp
+1 -1
View File
@@ -35,7 +35,7 @@ namespace lms::scanner
struct ScannerSettings
{
std::size_t scanVersion{};
std::size_t audioScanVersion{};
Wt::WTime startTime;
db::ScanSettings::UpdatePeriod updatePeriod{ db::ScanSettings::UpdatePeriod::Never };
bool skipDuplicateTrackMBID{};
src/libs/services/scanner/impl/scanners/AudioFileScanOperation.cpp
+1 -1
View File
@@ -645,6 +645,7 @@ namespace lms::scanner
// Track related data
assert(track);
track.modify()->setScanVersion(_settings.audioScanVersion);
// Audio properties
track.modify()->setBitrate(_parsedTrack->audioProperties.bitrate);
@@ -692,7 +693,6 @@ namespace lms::scanner
for (const auto& [role, performers] : _parsedTrack->performerArtists)
createTrackArtistLinks(dbSession, track, db::TrackArtistLinkType::Performer, role, performers, allowFallback);
track.modify()->setScanVersion(_settings.scanVersion);
if (_parsedTrack->medium && _parsedTrack->medium->release)
track.modify()->setRelease(getOrCreateRelease(dbSession, *_parsedTrack->medium->release, directory));
else
src/libs/services/scanner/impl/scanners/AudioFileScanner.cpp
+1 -1
View File
@@ -106,7 +106,7 @@ namespace lms::scanner
const db::Track::pointer track{ db::Track::findByPath(dbSession, file.file) };
if (track
&& track->getLastWriteTime() == lastWriteTime
&& track->getScanVersion() == _settings.scanVersion)
&& track->getScanVersion() == _settings.audioScanVersion)
{
// this file may have been moved from one library to another, then we just need to update the media library id instead of a full rescan
const auto trackMediaLibrary{ track->getMediaLibrary() };
src/libs/services/scanner/impl/steps/IScanStep.hpp
+1
View File
@@ -32,6 +32,7 @@ namespace lms::scanner
virtual ScanStep getStep() const = 0;
virtual core::LiteralString getStepName() const = 0;
virtual bool needProcess(const ScanContext& context) const = 0;
virtual void process(ScanContext& context) = 0;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepArtistReconciliation.cpp
+10 -4
View File
@@ -74,6 +74,12 @@ namespace lms::scanner
}
} // namespace
bool ScanStepArtistReconciliation::needProcess([[maybe_unused]] const ScanContext& context) const
{
// Since this step is very fast in case there is nothing to do, no need to skip if nothing has changed
return true;
}
void ScanStepArtistReconciliation::process(ScanContext& context)
{
// Reconcile artist links
@@ -99,7 +105,7 @@ namespace lms::scanner
db::Session& session{ _db.getTLSSession() };
std::vector<db::ArtistInfo::pointer> artistInfo;
while (true)
while (!_abortScan)
{
artistInfo.clear();
{
@@ -133,7 +139,7 @@ namespace lms::scanner
db::Session& session{ _db.getTLSSession() };
std::vector<db::ArtistInfo::pointer> artistInfo;
while (true)
while (!_abortScan)
{
artistInfo.clear();
{
@@ -166,7 +172,7 @@ namespace lms::scanner
db::Session& session{ _db.getTLSSession() };
std::vector<db::TrackArtistLink::pointer> links;
while (true)
while (!_abortScan)
{
links.clear();
{
@@ -200,7 +206,7 @@ namespace lms::scanner
db::Session& session{ _db.getTLSSession() };
std::vector<db::TrackArtistLink::pointer> links;
while (true)
while (!_abortScan)
{
links.clear();
{
src/libs/services/scanner/impl/steps/ScanStepArtistReconciliation.hpp
+1
View File
@@ -31,6 +31,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::ReconciliateArtists; }
core::LiteralString getStepName() const override { return "Artist reconciliation"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
void updateLinksForArtistNameNoLongerMatch(ScanContext& context);
src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.cpp
+8 -6
View File
@@ -254,14 +254,16 @@ namespace lms::scanner
{
}
bool ScanStepAssociateArtistImages::needProcess(const ScanContext& context) const
{
if (context.stats.nbChanges() > 0)
return true;
return false;
}
void ScanStepAssociateArtistImages::process(ScanContext& context)
{
if (_abortScan)
return;
if (context.stats.nbChanges() == 0)
return;
auto& session{ _db.getTLSSession() };
{
src/libs/services/scanner/impl/steps/ScanStepAssociateArtistImages.hpp
+1
View File
@@ -37,6 +37,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::AssociateArtistImages; }
core::LiteralString getStepName() const override { return "Associate artist images"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
const std::vector<std::string> _artistFileNames;
src/libs/services/scanner/impl/steps/ScanStepAssociateExternalLyrics.cpp
+8 -6
View File
@@ -138,14 +138,16 @@ namespace lms::scanner
}
} // namespace
bool ScanStepAssociateExternalLyrics::needProcess(const ScanContext& context) const
{
if (context.stats.nbChanges() > 0)
return true;
return false;
}
void ScanStepAssociateExternalLyrics::process(ScanContext& context)
{
if (_abortScan)
return;
if (context.stats.nbChanges() == 0)
return;
auto& session{ _db.getTLSSession() };
{
src/libs/services/scanner/impl/steps/ScanStepAssociateExternalLyrics.hpp
+1
View File
@@ -31,6 +31,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::AssociateExternalLyrics; }
core::LiteralString getStepName() const override { return "Associate external lyrics"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.cpp
+11 -6
View File
@@ -212,14 +212,19 @@ namespace lms::scanner
}
} // namespace
bool ScanStepAssociatePlayListTracks::needProcess(const ScanContext& context) const
{
if (context.stats.nbChanges() > 0)
return true;
if (getLastScanSettings() && getLastScanSettings()->skipSingleReleasePlayLists != _settings.skipSingleReleasePlayLists)
return true;
return false;
}
void ScanStepAssociatePlayListTracks::process(ScanContext& context)
{
if (_abortScan)
return;
if (context.stats.nbChanges() == 0)
return;
auto& session{ _db.getTLSSession() };
{
src/libs/services/scanner/impl/steps/ScanStepAssociatePlayListTracks.hpp
+1
View File
@@ -31,6 +31,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::AssociatePlayListTracks; }
core::LiteralString getStepName() const override { return "Associate playlist tracks"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp
+8 -6
View File
@@ -199,14 +199,16 @@ namespace lms::scanner
{
}
bool ScanStepAssociateReleaseImages::needProcess(const ScanContext& context) const
{
if (context.stats.nbChanges() > 0)
return true;
return false;
}
void ScanStepAssociateReleaseImages::process(ScanContext& context)
{
if (_abortScan)
return;
if (context.stats.nbChanges() == 0)
return;
auto& session{ _db.getTLSSession() };
{
src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.hpp
+1
View File
@@ -37,6 +37,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::AssociateReleaseImages; }
core::LiteralString getStepName() const override { return "Associate release images"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
const std::vector<std::string> _releaseFileNames;
src/libs/services/scanner/impl/steps/ScanStepBase.cpp
+35
View File
@@ -0,0 +1,35 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepBase.hpp"
namespace lms::scanner
{
ScanStepBase::ScanStepBase(InitParams& initParams)
: _settings{ initParams.settings }
, _progressCallback{ initParams.progressCallback }
, _abortScan{ initParams.abortScan }
, _db{ initParams.db }
, _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners))
, _lastScanSettings{ initParams.lastScanSettings }
{
}
ScanStepBase::~ScanStepBase() = default;
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepBase.hpp
+8 -9
View File
@@ -44,28 +44,27 @@ namespace lms::scanner
struct InitParams
{
const ScannerSettings& settings;
const ScannerSettings* lastScanSettings{};
ProgressCallback progressCallback;
bool& abortScan;
db::Db& db;
std::span<IFileScanner*> fileScanners;
};
ScanStepBase(InitParams& initParams)
: _settings{ initParams.settings }
, _progressCallback{ initParams.progressCallback }
, _abortScan{ initParams.abortScan }
, _db{ initParams.db }
, _fileScanners(std::cbegin(initParams.fileScanners), std::cend(initParams.fileScanners))
{
}
~ScanStepBase() override = default;
ScanStepBase(InitParams& initParams);
~ScanStepBase() override;
ScanStepBase(const ScanStepBase&) = delete;
ScanStepBase& operator=(const ScanStepBase&) = delete;
protected:
const ScannerSettings* getLastScanSettings() const { return _lastScanSettings; }
const ScannerSettings& _settings;
ProgressCallback _progressCallback;
bool& _abortScan;
db::Db& _db;
std::vector<IFileScanner*> _fileScanners;
private:
const ScannerSettings* _lastScanSettings{};
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepCheckForDuplicatedFiles.cpp
+6 -3
View File
@@ -26,13 +26,16 @@
namespace lms::scanner
{
bool ScanStepCheckForDuplicatedFiles::needProcess([[maybe_unused]] const ScanContext& context) const
{
// Always check for everything
return true;
}
void ScanStepCheckForDuplicatedFiles::process(ScanContext& context)
{
using namespace db;
if (_abortScan)
return;
Session& session{ _db.getTLSSession() };
auto transaction{ session.createReadTransaction() };
src/libs/services/scanner/impl/steps/ScanStepCheckForDuplicatedFiles.hpp
+1
View File
@@ -31,6 +31,7 @@ namespace lms::scanner
private:
core::LiteralString getStepName() const override { return "Check for duplicated files"; }
ScanStep getStep() const override { return ScanStep::CheckForDuplicatedFiles; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.cpp
+6 -3
View File
@@ -41,11 +41,14 @@ namespace lms::scanner
constexpr std::size_t batchSize = 100;
}
bool ScanStepCheckForRemovedFiles::needProcess([[maybe_unused]] const ScanContext& context) const
{
// always check for removed files
return true;
}
void ScanStepCheckForRemovedFiles::process(ScanContext& context)
{
if (_abortScan)
return;
db::Session& session{ _db.getTLSSession() };
{
src/libs/services/scanner/impl/steps/ScanStepCheckForRemovedFiles.hpp
+1
View File
@@ -34,6 +34,7 @@ namespace lms::scanner
private:
core::LiteralString getStepName() const override { return "Check for removed files"; }
ScanStep getStep() const override { return ScanStep::CheckForRemovedFiles; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
template<typename Object>
src/libs/services/scanner/impl/steps/ScanStepCompact.cpp
+7 -3
View File
@@ -24,10 +24,14 @@
namespace lms::scanner
{
void ScanStepCompact::process(ScanContext& context)
bool ScanStepCompact::needProcess(const ScanContext& context) const
{
// Don't auto compact as it may be too annoying to block the whole application for very large databases
if (context.scanOptions.compact)
_db.getTLSSession().vacuum();
return context.scanOptions.compact;
}
void ScanStepCompact::process([[maybe_unused]] ScanContext& context)
{
_db.getTLSSession().vacuum();
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepCompact.hpp
+1
View File
@@ -31,6 +31,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::Compact; }
core::LiteralString getStepName() const override { return "Compact"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.cpp
+8 -3
View File
@@ -25,13 +25,18 @@
namespace lms::scanner
{
bool ScanStepComputeClusterStats::needProcess(const ScanContext& context) const
{
if (context.stats.nbChanges() > 0)
return true;
return false;
}
void ScanStepComputeClusterStats::process(ScanContext& context)
{
using namespace db;
if (context.stats.nbChanges() == 0)
return;
Session& dbSession{ _db.getTLSSession() };
const std::size_t clusterCount{ [&] {
src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.hpp
+1
View File
@@ -31,6 +31,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::ComputeClusterStats; }
core::LiteralString getStepName() const override { return "Compute cluster stats"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.cpp
+6
View File
@@ -28,6 +28,12 @@
namespace lms::scanner
{
bool ScanStepDiscoverFiles::needProcess([[maybe_unused]] const ScanContext& context) const
{
// always discover files
return true;
}
void ScanStepDiscoverFiles::process(ScanContext& context)
{
context.stats.totalFileCount = 0;
src/libs/services/scanner/impl/steps/ScanStepDiscoverFiles.hpp
+1
View File
@@ -31,6 +31,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::DiscoverFiles; }
core::LiteralString getStepName() const override { return "Discover files"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepOptimize.cpp
+26 -20
View File
@@ -25,32 +25,38 @@
namespace lms::scanner
{
bool ScanStepOptimize::needProcess(const ScanContext& context) const
{
if (context.scanOptions.forceOptimize)
return true;
if (context.stats.nbChanges() > (context.stats.nbFiles() / 10))
return true;
return false;
}
void ScanStepOptimize::process(ScanContext& context)
{
ScanStats& stats{ context.stats };
LMS_LOG(DBUPDATER, INFO, "Database analyze started");
if (context.scanOptions.forceOptimize || (stats.nbChanges() > (stats.nbFiles() / 10)))
auto& session{ _db.getTLSSession() };
std::vector<std::string> entries;
session.retrieveEntriesToAnalyze(entries);
context.currentStepStats.totalElems = entries.size();
_progressCallback(context.currentStepStats);
for (const std::string& entry : entries)
{
LMS_LOG(DBUPDATER, INFO, "Database analyze started");
if (_abortScan)
break;
auto& session{ _db.getTLSSession() };
std::vector<std::string> entries;
session.retrieveEntriesToAnalyze(entries);
context.currentStepStats.totalElems = entries.size();
_db.getTLSSession().analyzeEntry(entry);
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
for (const std::string& entry : entries)
{
if (_abortScan)
break;
_db.getTLSSession().analyzeEntry(entry);
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
LMS_LOG(DBUPDATER, INFO, "Database analyze complete");
}
LMS_LOG(DBUPDATER, INFO, "Database analyze complete");
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepOptimize.hpp
+1
View File
@@ -31,6 +31,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::Optimize; }
core::LiteralString getStepName() const override { return "Optimize"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
};
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.cpp
+6
View File
@@ -31,6 +31,12 @@
namespace lms::scanner
{
bool ScanStepRemoveOrphanedDbEntries::needProcess([[maybe_unused]] const ScanContext& context) const
{
// fast enough when there is nothing to do
return true;
}
void ScanStepRemoveOrphanedDbEntries::process(ScanContext& context)
{
removeOrphanedClusters(context);
src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.hpp
+1
View File
@@ -31,6 +31,7 @@ namespace lms::scanner
private:
core::LiteralString getStepName() const override { return "Remove orphaned DB entries"; }
ScanStep getStep() const override { return ScanStep::RemoveOrphanedDbEntries; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
void removeOrphanedClusters(ScanContext& context);
src/libs/services/scanner/impl/steps/ScanStepScanFiles.cpp
+6
View File
@@ -64,6 +64,12 @@ namespace lms::scanner
LMS_LOG(DBUPDATER, INFO, "Using " << _fileScanQueue.getThreadCount() << " thread(s) for scanning file metadata");
}
bool ScanStepScanFiles::needProcess([[maybe_unused]] const ScanContext& context) const
{
// Always need to scan files
return true;
}
void ScanStepScanFiles::process(ScanContext& context)
{
context.currentStepStats.totalElems = context.stats.totalFileCount;
src/libs/services/scanner/impl/steps/ScanStepScanFiles.hpp
+1
View File
@@ -38,6 +38,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::ScanFiles; }
core::LiteralString getStepName() const override { return "Scan files"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
void process(ScanContext& context, const MediaLibraryInfo& mediaLibrary);
void processFileScanResults(ScanContext& context, std::span<std::unique_ptr<IFileScanOperation>> scanOperations);
src/libs/services/scanner/impl/steps/ScanStepUpdateLibraryFields.cpp
+5
View File
@@ -29,6 +29,11 @@
namespace lms::scanner
{
bool ScanStepUpdateLibraryFields::needProcess([[maybe_unused]] const ScanContext& context) const
{
// Fast enough when nothing to do
return true;
}
void ScanStepUpdateLibraryFields::process(ScanContext& context)
{
src/libs/services/scanner/impl/steps/ScanStepUpdateLibraryFields.hpp
+1
View File
@@ -33,6 +33,7 @@ namespace lms::scanner
private:
core::LiteralString getStepName() const override { return "Update Library fields"; }
ScanStep getStep() const override { return ScanStep::UpdateLibraryFields; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
void processDirectories(ScanContext& context);