Added a way to manually compact and/or optimize the db

This commit is contained in:
emeric
2024-04-20 23:57:01 +02:00
parent 8cde7ddf43
commit 0780d1ab21
24 changed files with 341 additions and 162 deletions
+2 -1
View File
@@ -2,10 +2,11 @@
add_library(lmsscanner SHARED
impl/ScannerService.cpp
impl/ScannerStats.cpp
impl/ScanStepAnalyze.cpp
impl/ScanStepCheckDuplicatedDbFiles.cpp
impl/ScanStepCompact.cpp
impl/ScanStepComputeClusterStats.cpp
impl/ScanStepDiscoverFiles.cpp
impl/ScanStepOptimize.cpp
impl/ScanStepRemoveOrphanDbFiles.cpp
impl/ScanStepScanFiles.cpp
)
+2 -1
View File
@@ -22,6 +22,7 @@
#include <vector>
#include "core/LiteralString.hpp"
#include "services/scanner/ScannerOptions.hpp"
#include "services/scanner/ScannerStats.hpp"
namespace lms::scanner
@@ -36,7 +37,7 @@ namespace lms::scanner
struct ScanContext
{
const bool forceScan;
ScanOptions scanOptions;
ScanStats stats;
ScanStepStats currentStepStats;
};
@@ -29,8 +29,8 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
core::LiteralString getStepName() const override { return "Checking for duplicated files"; }
ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; }
core::LiteralString getStepName() const override { return "Check for duplicated files"; }
ScanStep getStep() const override { return ScanStep::CheckForDuplicateFiles; }
void process(ScanContext& context) override;
};
}
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepCompact.hpp"
#include "database/Db.hpp"
#include "database/Session.hpp"
namespace lms::scanner
{
void ScanStepCompact::process(ScanContext& context)
{
// Don't auto compact as it may be too annoying to block the whole application
if (context.scanOptions.compact)
_db.getTLSSession().vacuum();
}
}
@@ -23,14 +23,14 @@
namespace lms::scanner
{
class ScanStepAnalyze : public ScanStepBase
class ScanStepCompact : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::Analyze; }
core::LiteralString getStepName() const override { return "Analyze"; }
ScanStep getStep() const override { return ScanStep::Compact; }
core::LiteralString getStepName() const override { return "Compact"; }
void process(ScanContext& context) override;
};
}
@@ -29,8 +29,8 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::DiscoveringFiles; }
core::LiteralString getStepName() const override { return "Discovering files"; }
ScanStep getStep() const override { return ScanStep::DiscoverFiles; }
core::LiteralString getStepName() const override { return "Discover files"; }
void process(ScanContext& context) override;
};
}
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepAnalyze.hpp"
#include "ScanStepOptimize.hpp"
#include "core/ILogger.hpp"
#include "database/Db.hpp"
@@ -25,19 +25,20 @@
namespace lms::scanner
{
void ScanStepAnalyze::process(ScanContext& context)
void ScanStepOptimize::process(ScanContext& context)
{
ScanStats& stats{ context.stats };
if (stats.nbChanges() > (stats.nbFiles() / 5))
if (context.scanOptions.forceOptimize || (stats.nbChanges() > (stats.nbFiles() / 5)))
{
LMS_LOG(DBUPDATER, INFO, "Database changed substantially: triggering full analyze");
LMS_LOG(DBUPDATER, INFO, "Database analyze started");
auto& session{ _db.getTLSSession() };
std::vector<std::string> entries;
session.retrieveEntriesToAnalyze(entries);
context.currentStepStats.totalElems = entries.size();
_progressCallback(context.currentStepStats);
for (const std::string& entry : entries)
{
@@ -48,6 +49,8 @@ namespace lms::scanner
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
LMS_LOG(DBUPDATER, INFO, "Database analyze complete");
}
}
}
@@ -0,0 +1,36 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ScanStepBase.hpp"
namespace lms::scanner
{
class ScanStepOptimize : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::Optimize; }
core::LiteralString getStepName() const override { return "Optimize"; }
void process(ScanContext& context) override;
};
}
@@ -31,8 +31,8 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
core::LiteralString getStepName() const override { return "Checking orphaned entries"; }
ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; }
core::LiteralString getStepName() const override { return "Check orphaned entries"; }
ScanStep getStep() const override { return ScanStep::CheckForMissingFiles; }
void process(ScanContext& context) override;
void removeOrphanTracks(ScanContext& context);
@@ -459,7 +459,7 @@ namespace lms::scanner
}
bool needUpdateLibrary{};
if (!context.forceScan)
if (!context.scanOptions.fullScan)
{
// Skip file if last write is the same
db::Session& dbSession{ _db.getTLSSession() };
@@ -39,8 +39,8 @@ namespace lms::scanner
ScanStepScanFiles(InitParams& initParams);
private:
ScanStep getStep() const override { return ScanStep::ScanningFiles; }
core::LiteralString getStepName() const override { return "Scanning files"; }
ScanStep getStep() const override { return ScanStep::ScanFiles; }
core::LiteralString getStepName() const override { return "Scan files"; }
void process(ScanContext& context) override;
bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);
@@ -30,10 +30,11 @@
#include "core/ILogger.hpp"
#include "core/ITraceLogger.hpp"
#include "ScanStepAnalyze.hpp"
#include "ScanStepCheckDuplicatedDbFiles.hpp"
#include "ScanStepCompact.hpp"
#include "ScanStepComputeClusterStats.hpp"
#include "ScanStepDiscoverFiles.hpp"
#include "ScanStepOptimize.hpp"
#include "ScanStepRemoveOrphanDbFiles.hpp"
#include "ScanStepScanFiles.hpp"
@@ -135,15 +136,15 @@ namespace lms::scanner
_events.scanAborted.emit();
}
void ScannerService::requestImmediateScan(bool force)
void ScannerService::requestImmediateScan(const ScanOptions& scanOptions)
{
abortScan();
_ioService.post([this, force]
_ioService.post([this, scanOptions]
{
if (_abortScan)
return;
scheduleScan(force);
scheduleScan(scanOptions);
});
}
@@ -220,7 +221,7 @@ namespace lms::scanner
}
if (nextScanDateTime.isValid())
scheduleScan(false, nextScanDateTime);
scheduleScan(ScanOptions{}, nextScanDateTime);
{
std::unique_lock lock{ _statusMutex };
@@ -231,14 +232,14 @@ namespace lms::scanner
_events.scanScheduled.emit(_nextScheduledScan);
}
void ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime)
void ScannerService::scheduleScan(const ScanOptions& scanOptions, const Wt::WDateTime& dateTime)
{
auto cb{ [this, force](boost::system::error_code ec)
auto cb{ [this, scanOptions](boost::system::error_code ec)
{
if (ec)
return;
scan(force);
scan(scanOptions);
} };
if (dateTime.isNull())
@@ -259,7 +260,7 @@ namespace lms::scanner
}
}
void ScannerService::scan(bool forceScan)
void ScannerService::scan(const ScanOptions& scanOptions)
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "Scan");
@@ -276,16 +277,17 @@ namespace lms::scanner
refreshScanSettings();
IScanStep::ScanContext scanContext{ forceScan, ScanStats {}, ScanStepStats {} };
IScanStep::ScanContext scanContext{ scanOptions, ScanStats {}, ScanStepStats {} };
ScanStats& stats{ scanContext.stats };
stats.startTime = Wt::WDateTime::currentDateTime();
std::size_t stepIndex{};
for (auto& scanStep : _scanSteps)
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() };
scanContext.currentStepStats = ScanStepStats{ .startTime = Wt::WDateTime::currentDateTime(), .stepIndex = stepIndex++, .currentStep = scanStep->getStep() };
notifyInProgress(scanContext.currentStepStats);
scanStep->process(scanContext);
@@ -350,9 +352,10 @@ namespace lms::scanner
_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepCompact>(params));
_scanSteps.push_back(std::make_unique<ScanStepOptimize>(params));
_scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepAnalyze>(params));
}
ScannerSettings ScannerService::readSettings()
@@ -46,12 +46,13 @@ namespace lms::scanner
ScannerService(db::Db& db);
~ScannerService();
private:
ScannerService(const ScannerService&) = delete;
ScannerService& operator=(const ScannerService&) = delete;
private:
void requestStop() override;
void requestReload() override;
void requestImmediateScan(bool force) override;
void requestImmediateScan(const ScanOptions& scanOptions) override;
Status getStatus() const override;
Events& getEvents() override { return _events; }
@@ -62,12 +63,12 @@ namespace lms::scanner
// Job handling
void scheduleNextScan();
void scheduleScan(bool force, const Wt::WDateTime& dateTime = {});
void scheduleScan(const ScanOptions& scanOptions, const Wt::WDateTime& dateTime = {});
void abortScan();
// Update database (scheduled callback)
void scan(bool force);
void scan(const ScanOptions& scanOptions);
void scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
@@ -22,47 +22,46 @@
#include <optional>
#include "ScannerEvents.hpp"
#include "ScannerOptions.hpp"
#include "ScannerStats.hpp"
namespace lms::db
{
class Db;
class Db;
}
namespace lms::scanner
{
class IScannerService
{
public:
virtual ~IScannerService() = default;
class IScannerService
{
public:
virtual ~IScannerService() = default;
// Async requests
virtual void requestStop() = 0;
virtual void requestReload() = 0;
// Async requests
virtual void requestStop() = 0;
virtual void requestReload() = 0;
virtual void requestImmediateScan(bool force) = 0;
virtual void requestImmediateScan(const ScanOptions& options = {}) = 0;
enum class State
{
NotScheduled,
Scheduled,
InProgress,
};
enum class State
{
NotScheduled,
Scheduled,
InProgress,
};
struct Status
{
State currentState {State::NotScheduled};
Wt::WDateTime nextScheduledScan;
std::optional<ScanStats> lastCompleteScanStats;
std::optional<ScanStepStats> currentScanStepStats;
};
struct Status
{
State currentState{ State::NotScheduled };
Wt::WDateTime nextScheduledScan;
std::optional<ScanStats> lastCompleteScanStats;
std::optional<ScanStepStats> currentScanStepStats;
};
virtual Status getStatus() const = 0;
virtual Status getStatus() const = 0;
virtual Events& getEvents() = 0;
};
std::unique_ptr<IScannerService> createScannerService(db::Db& db);
virtual Events& getEvents() = 0;
};
std::unique_ptr<IScannerService> createScannerService(db::Db& db);
} // Scanner
@@ -0,0 +1,30 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
namespace lms::scanner
{
struct ScanOptions
{
bool fullScan{}; // scan files even if not changed
bool forceOptimize{}; // force optimize database
bool compact{}; // compact the database
};
}
@@ -59,22 +59,24 @@ namespace lms::scanner
enum class ScanStep
{
DiscoveringFiles,
ScanningFiles,
ChekingForMissingFiles,
CheckingForDuplicateFiles,
FetchingTrackFeatures,
ReloadingSimilarityEngine,
CheckForMissingFiles,
CheckForDuplicateFiles,
ComputeClusterStats,
Analyze,
Compact,
DiscoverFiles,
FetchTrackFeatures,
Optimize,
ReloadSimilarityEngine,
ScanFiles,
};
static inline constexpr unsigned ScanProgressStepCount{ 8 };
static inline constexpr unsigned ScanProgressStepCount{ 9 };
// reduced scan stats
struct ScanStepStats
{
Wt::WDateTime startTime;
std::size_t stepIndex{};
ScanStep currentStep;
std::size_t totalElems{};
@@ -39,7 +39,7 @@ namespace lms::api::subsonic::Scan
{
std::size_t count{};
if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanningFiles)
if (scanStatus.currentScanStepStats && scanStatus.currentScanStepStats->currentStep == ScanStep::ScanFiles)
count = scanStatus.currentScanStepStats->processedElems;
statusResponse.setAttribute("count", count);
@@ -59,7 +59,7 @@ namespace lms::api::subsonic::Scan
Response handleStartScan(RequestContext& context)
{
core::Service<IScannerService>::get()->requestImmediateScan(false);
core::Service<IScannerService>::get()->requestImmediateScan();
Response response{ Response::createOkResponse(context.serverProtocolVersion) };
response.addNode("scanStatus", createStatusResponseNode());
+79 -50
View File
@@ -22,6 +22,7 @@
#include <iomanip>
#include <Wt/Http/Response.h>
#include <Wt/WCheckBox.h>
#include <Wt/WDateTime.h>
#include <Wt/WPushButton.h>
#include <Wt/WResource.h>
@@ -156,16 +157,19 @@ namespace lms::ui
_reportBtn->setLink(link);
}
Wt::WCheckBox* fullScan{ bindNew<Wt::WCheckBox>("full-scan") };
Wt::WCheckBox* forceOptimize{ bindNew<Wt::WCheckBox>("force-optimize") };
Wt::WCheckBox* compact{ bindNew<Wt::WCheckBox>("compact") };
Wt::WPushButton* scanBtn{ bindNew<Wt::WPushButton>("scan-btn", Wt::WString::tr("Lms.Admin.ScannerController.scan-now")) };
scanBtn->clicked().connect([]
scanBtn->clicked().connect([=]
{
core::Service<scanner::IScannerService>::get()->requestImmediateScan(false);
});
Wt::WPushButton* fullScanBtn{ bindNew<Wt::WPushButton>("full-scan-btn", Wt::WString::tr("Lms.Admin.ScannerController.force-scan-now")) };
fullScanBtn->clicked().connect([]
{
core::Service<scanner::IScannerService>::get()->requestImmediateScan(true);
const scanner::ScanOptions scanOptions
{
.fullScan = fullScan->isChecked(),
.forceOptimize = forceOptimize->isChecked(),
.compact = compact->isChecked(),
};
core::Service<scanner::IScannerService>::get()->requestImmediateScan(scanOptions);
});
_lastScanStatus = bindNew<Wt::WLineEdit>("last-scan");
@@ -199,6 +203,13 @@ namespace lms::ui
using namespace scanner;
const IScannerService::Status status{ core::Service<IScannerService>::get()->getStatus() };
refreshLastScanStatus(status);
refreshStatus(status);
}
void ScannerController::refreshLastScanStatus(const scanner::IScannerService::Status& status)
{
if (status.lastCompleteScanStats)
{
_lastScanStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.last-scan-status")
@@ -218,6 +229,11 @@ namespace lms::ui
_lastScanStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.last-scan-not-available"));
_reportBtn->setEnabled(false);
}
}
void ScannerController::refreshStatus(const scanner::IScannerService::Status& status)
{
using namespace scanner;
switch (status.currentState)
{
@@ -225,64 +241,77 @@ namespace lms::ui
_status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-not-scheduled"));
_stepStatus->setText("");
break;
case IScannerService::State::Scheduled:
_status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-scheduled")
.arg(status.nextScheduledScan.toString()));
_stepStatus->setText("");
break;
case IScannerService::State::InProgress:
_status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress")
.arg(static_cast<int>(status.currentScanStepStats->currentStep) + 1)
.arg(status.currentScanStepStats->stepIndex + 1)
.arg(scanner::ScanProgressStepCount));
switch (status.currentScanStepStats->currentStep)
{
case scanner::ScanStep::CheckingForDuplicateFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files")
.arg(status.currentScanStepStats->processedElems));
break;
refreshCurrentStep(*status.currentScanStepStats);
break;
}
}
case scanner::ScanStep::ChekingForMissingFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files")
.arg(status.currentScanStepStats->progress()));
break;
void ScannerController::refreshCurrentStep(const scanner::ScanStepStats& stepStats)
{
using namespace scanner;
case scanner::ScanStep::DiscoveringFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
.arg(status.currentScanStepStats->processedElems));
break;
switch (stepStats.currentStep)
{
case ScanStep::CheckForDuplicateFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-duplicate-files")
.arg(stepStats.processedElems));
break;
case scanner::ScanStep::ScanningFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
.arg(status.currentScanStepStats->processedElems)
.arg(status.currentScanStepStats->totalElems)
.arg(status.currentScanStepStats->progress()));
break;
case scanner::ScanStep::CheckForMissingFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-checking-for-missing-files")
.arg(stepStats.progress()));
break;
case scanner::ScanStep::FetchingTrackFeatures:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
.arg(status.currentScanStepStats->processedElems)
.arg(status.currentScanStepStats->totalElems)
.arg(status.currentScanStepStats->progress()));
break;
case scanner::ScanStep::Compact:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compact"));
break;
case scanner::ScanStep::ReloadingSimilarityEngine:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine")
.arg(status.currentScanStepStats->progress()));
break;
case scanner::ScanStep::ComputeClusterStats:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats")
.arg(stepStats.progress()));
break;
case scanner::ScanStep::ComputeClusterStats:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats")
.arg(status.currentScanStepStats->progress()));
break;
case scanner::ScanStep::DiscoverFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-discovering-files")
.arg(stepStats.processedElems));
break;
case scanner::ScanStep:: Analyze:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-analyze")
.arg(status.currentScanStepStats->processedElems)
.arg(status.currentScanStepStats->totalElems)
.arg(status.currentScanStepStats->progress()));
break;
}
case scanner::ScanStep::FetchTrackFeatures:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-fetching-track-features")
.arg(stepStats.processedElems)
.arg(stepStats.totalElems)
.arg(stepStats.progress()));
break;
case scanner::ScanStep::Optimize:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-optimize")
.arg(stepStats.processedElems)
.arg(stepStats.totalElems)
.arg(stepStats.progress()));
break;
case scanner::ScanStep::ReloadSimilarityEngine:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-reloading-similarity-engine")
.arg(stepStats.progress()));
break;
case scanner::ScanStep::ScanFiles:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-scanning-files")
.arg(stepStats.processedElems)
.arg(stepStats.totalElems)
.arg(stepStats.progress()));
break;
}
}
+5
View File
@@ -23,6 +23,8 @@
#include <Wt/WTemplate.h>
#include <Wt/WLineEdit.h>
#include "services/scanner/IScannerService.hpp"
namespace lms::ui
{
class ScannerController : public Wt::WTemplate
@@ -32,6 +34,9 @@ namespace lms::ui
private:
void refreshContents();
void refreshLastScanStatus(const scanner::IScannerService::Status& status);
void refreshStatus(const scanner::IScannerService::Status& status);
void refreshCurrentStep(const scanner::ScanStepStats& stepStats);
Wt::WPushButton* _reportBtn;
Wt::WLineEdit* _lastScanStatus;