owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a way to manually compact and/or optimize the db

Browse Source
This commit is contained in:
emeric
2024-04-20 23:57:01 +02:00
parent 8cde7ddf43
commit 0780d1ab21
24 changed files with 341 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/scanner/impl/IScanStep.hpp
+2 -1
View File
@@ -22,6 +22,7 @@
#include <vector>
#include "core/LiteralString.hpp"
#include "services/scanner/ScannerOptions.hpp"
#include "services/scanner/ScannerStats.hpp"
namespace lms::scanner
@@ -36,7 +37,7 @@ namespace lms::scanner
struct ScanContext
{
const bool forceScan;
ScanOptions scanOptions;
ScanStats stats;
ScanStepStats currentStepStats;
};
src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp
+2 -2
View File
@@ -29,8 +29,8 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
core::LiteralString getStepName() const override { return "Checking for duplicated files"; }
ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; }
core::LiteralString getStepName() const override { return "Check for duplicated files"; }
ScanStep getStep() const override { return ScanStep::CheckForDuplicateFiles; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepCompact.cpp
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepCompact.hpp"
#include "database/Db.hpp"
#include "database/Session.hpp"
namespace lms::scanner
{
void ScanStepCompact::process(ScanContext& context)
{
// Don't auto compact as it may be too annoying to block the whole application
if (context.scanOptions.compact)
_db.getTLSSession().vacuum();
}
}
src/libs/services/scanner/impl/ScanStepAnalyze.hpp → src/libs/services/scanner/impl/ScanStepCompact.hpp
+3 -3
View File
@@ -23,14 +23,14 @@
namespace lms::scanner
{
class ScanStepAnalyze : public ScanStepBase
class ScanStepCompact : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::Analyze; }
core::LiteralString getStepName() const override { return "Analyze"; }
ScanStep getStep() const override { return ScanStep::Compact; }
core::LiteralString getStepName() const override { return "Compact"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp
+2 -2
View File
@@ -29,8 +29,8 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::DiscoveringFiles; }
core::LiteralString getStepName() const override { return "Discovering files"; }
ScanStep getStep() const override { return ScanStep::DiscoverFiles; }
core::LiteralString getStepName() const override { return "Discover files"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepAnalyze.cpp → src/libs/services/scanner/impl/ScanStepOptimize.cpp
+7 -4
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepAnalyze.hpp"
#include "ScanStepOptimize.hpp"
#include "core/ILogger.hpp"
#include "database/Db.hpp"
@@ -25,19 +25,20 @@
namespace lms::scanner
{
void ScanStepAnalyze::process(ScanContext& context)
void ScanStepOptimize::process(ScanContext& context)
{
ScanStats& stats{ context.stats };
if (stats.nbChanges() > (stats.nbFiles() / 5))
if (context.scanOptions.forceOptimize || (stats.nbChanges() > (stats.nbFiles() / 5)))
{
LMS_LOG(DBUPDATER, INFO, "Database changed substantially: triggering full analyze");
LMS_LOG(DBUPDATER, INFO, "Database analyze started");
auto& session{ _db.getTLSSession() };
std::vector<std::string> entries;
session.retrieveEntriesToAnalyze(entries);
context.currentStepStats.totalElems = entries.size();
_progressCallback(context.currentStepStats);
for (const std::string& entry : entries)
{
@@ -48,6 +49,8 @@ namespace lms::scanner
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
LMS_LOG(DBUPDATER, INFO, "Database analyze complete");
}
}
}
src/libs/services/scanner/impl/ScanStepOptimize.hpp
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ScanStepBase.hpp"
namespace lms::scanner
{
class ScanStepOptimize : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::Optimize; }
core::LiteralString getStepName() const override { return "Optimize"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp
+2 -2
View File
@@ -31,8 +31,8 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
core::LiteralString getStepName() const override { return "Checking orphaned entries"; }
ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; }
core::LiteralString getStepName() const override { return "Check orphaned entries"; }
ScanStep getStep() const override { return ScanStep::CheckForMissingFiles; }
void process(ScanContext& context) override;
void removeOrphanTracks(ScanContext& context);
src/libs/services/scanner/impl/ScanStepScanFiles.cpp
+1 -1
View File
@@ -459,7 +459,7 @@ namespace lms::scanner
}
bool needUpdateLibrary{};
if (!context.forceScan)
if (!context.scanOptions.fullScan)
{
// Skip file if last write is the same
db::Session& dbSession{ _db.getTLSSession() };
src/libs/services/scanner/impl/ScanStepScanFiles.hpp
+2 -2
View File
@@ -39,8 +39,8 @@ namespace lms::scanner
ScanStepScanFiles(InitParams& initParams);
private:
ScanStep getStep() const override { return ScanStep::ScanningFiles; }
core::LiteralString getStepName() const override { return "Scanning files"; }
ScanStep getStep() const override { return ScanStep::ScanFiles; }
core::LiteralString getStepName() const override { return "Scan files"; }
void process(ScanContext& context) override;
bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);
src/libs/services/scanner/impl/ScannerService.cpp
+15 -12
View File
@@ -30,10 +30,11 @@
#include "core/ILogger.hpp"
#include "core/ITraceLogger.hpp"
#include "ScanStepAnalyze.hpp"
#include "ScanStepCheckDuplicatedDbFiles.hpp"
#include "ScanStepCompact.hpp"
#include "ScanStepComputeClusterStats.hpp"
#include "ScanStepDiscoverFiles.hpp"
#include "ScanStepOptimize.hpp"
#include "ScanStepRemoveOrphanDbFiles.hpp"
#include "ScanStepScanFiles.hpp"
@@ -135,15 +136,15 @@ namespace lms::scanner
_events.scanAborted.emit();
}
void ScannerService::requestImmediateScan(bool force)
void ScannerService::requestImmediateScan(const ScanOptions& scanOptions)
{
abortScan();
_ioService.post([this, force]
_ioService.post([this, scanOptions]
{
if (_abortScan)
return;
scheduleScan(force);
scheduleScan(scanOptions);
});
}
@@ -220,7 +221,7 @@ namespace lms::scanner
}
if (nextScanDateTime.isValid())
scheduleScan(false, nextScanDateTime);
scheduleScan(ScanOptions{}, nextScanDateTime);
{
std::unique_lock lock{ _statusMutex };
@@ -231,14 +232,14 @@ namespace lms::scanner
_events.scanScheduled.emit(_nextScheduledScan);
}
void ScannerService::scheduleScan(bool force, const Wt::WDateTime& dateTime)
void ScannerService::scheduleScan(const ScanOptions& scanOptions, const Wt::WDateTime& dateTime)
{
auto cb{ [this, force](boost::system::error_code ec)
auto cb{ [this, scanOptions](boost::system::error_code ec)
{
if (ec)
return;
scan(force);
scan(scanOptions);
} };
if (dateTime.isNull())
@@ -259,7 +260,7 @@ namespace lms::scanner
}
}
void ScannerService::scan(bool forceScan)
void ScannerService::scan(const ScanOptions& scanOptions)
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "Scan");
@@ -276,16 +277,17 @@ namespace lms::scanner
refreshScanSettings();
IScanStep::ScanContext scanContext{ forceScan, ScanStats {}, ScanStepStats {} };
IScanStep::ScanContext scanContext{ scanOptions, ScanStats {}, ScanStepStats {} };
ScanStats& stats{ scanContext.stats };
stats.startTime = Wt::WDateTime::currentDateTime();
std::size_t stepIndex{};
for (auto& scanStep : _scanSteps)
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() };
scanContext.currentStepStats = ScanStepStats{ .startTime = Wt::WDateTime::currentDateTime(), .stepIndex = stepIndex++, .currentStep = scanStep->getStep() };
notifyInProgress(scanContext.currentStepStats);
scanStep->process(scanContext);
@@ -350,9 +352,10 @@ namespace lms::scanner
_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepCompact>(params));
_scanSteps.push_back(std::make_unique<ScanStepOptimize>(params));
_scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepAnalyze>(params));
}
ScannerSettings ScannerService::readSettings()
src/libs/services/scanner/impl/ScannerService.hpp
+5 -4
View File
@@ -46,12 +46,13 @@ namespace lms::scanner
ScannerService(db::Db& db);
~ScannerService();
private:
ScannerService(const ScannerService&) = delete;
ScannerService& operator=(const ScannerService&) = delete;
private:
void requestStop() override;
void requestReload() override;
void requestImmediateScan(bool force) override;
void requestImmediateScan(const ScanOptions& scanOptions) override;
Status getStatus() const override;
Events& getEvents() override { return _events; }
@@ -62,12 +63,12 @@ namespace lms::scanner
// Job handling
void scheduleNextScan();
void scheduleScan(bool force, const Wt::WDateTime& dateTime = {});
void scheduleScan(const ScanOptions& scanOptions, const Wt::WDateTime& dateTime = {});
void abortScan();
// Update database (scheduled callback)
void scan(bool force);
void scan(const ScanOptions& scanOptions);
void scanMediaDirectory(const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);