owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changed db optimization strategy: now perform a full analyze (only after scan if enough changes has been made, and during application startup). Also perform a vacuum if needed during startup. Added an init screen for the web interface since it can take a while to complete

Browse Source
This commit is contained in:
emeric
2024-04-18 18:46:35 +02:00
parent c63f9fa51c
commit 1862cd23c2
27 changed files with 382 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/core/include/core/LiteralString.hpp
+7
View File
@@ -20,6 +20,7 @@
#pragma once
#include <cstddef>
#include <ostream>
#include <string>
#include <string_view>
@@ -41,6 +42,12 @@ namespace lms::core
private:
std::string_view _str;
};
inline std::ostream& operator<<(std::ostream& os, const LiteralString& str)
{
os << str.str();
return os;
}
}
namespace std
src/libs/database/impl/Db.cpp
+6 -6
View File
@@ -25,8 +25,9 @@
#include "database/Session.hpp"
#include "database/User.hpp"
#include "core/IConfig.hpp"
#include "core/Service.hpp"
#include "core/ILogger.hpp"
#include "core/ITraceLogger.hpp"
#include "core/Service.hpp"
namespace lms::db
{
@@ -49,7 +50,7 @@ namespace lms::db
prepare();
}
~Connection()
~Connection() override
{
// make use of per-connection usage stats to optimize
optimize();
@@ -66,16 +67,15 @@ namespace lms::db
void prepare()
{
LMS_LOG(DB, DEBUG, "Setting per-connection settings...");
executeSql("pragma journal_mode=WAL");
executeSql("pragma synchronous=normal");
executeSql("pragma analysis_limit=2000"); // to help make analyze command faster, 1000 does not seem to be enough to speed up all queries
executeSql("PRAGMA journal_mode=WAL");
executeSql("PRAGMA synchronous=normal");
LMS_LOG(DB, DEBUG, "Setting per-connection settings done!");
}
void optimize()
{
LMS_LOG(DB, DEBUG, "connection close: Running pragma optimize...");
executeSql("pragma optimize");
executeSql("PRAGMA optimize");
LMS_LOG(DB, DEBUG, "connection close: pragma optimize complete");
}
src/libs/database/impl/Session.cpp
+140 -101
View File
@@ -46,6 +46,7 @@
#include "EnumSetTraits.hpp"
#include "PathTraits.hpp"
#include "Migration.hpp"
#include "Utils.hpp"
namespace lms::db
{
@@ -120,7 +121,7 @@ namespace lms::db
return ReadTransaction{ _session };
}
void Session::prepareTables()
void Session::prepareTablesIfNeeded()
{
LMS_LOG(DB, INFO, "Preparing tables...");
@@ -140,127 +141,165 @@ namespace lms::db
throw e;
}
}
}
void Session::migrateIfNeeded()
{
Migration::doDbMigration(*this);
// Indexes
{
LMS_SCOPED_TRACE_OVERVIEW("Database", "IndexCreation");
auto transaction{ createWriteTransaction() };
_session.execute("CREATE INDEX IF NOT EXISTS artist_id_idx ON artist(id)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_user_idx ON auth_token(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_expiry_idx ON auth_token(expiry)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_value_idx ON auth_token(value)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_user_backend_idx ON listen(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_track_user_backend_idx ON listen(track_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_user_track_backend_date_time_idx ON listen(user_id,track_id,backend,date_time)");
_session.execute("CREATE INDEX IF NOT EXISTS release_id_idx ON release(id)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS release_type_name_idx ON release_type(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_id_idx ON track(id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_absolute_path_idx ON track(absolute_file_path)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_recording_mbid_idx ON track(recording_mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_file_last_write_idx ON track(file_last_write)");
_session.execute("CREATE INDEX IF NOT EXISTS track_date_idx ON track(date)");
_session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)");
_session.execute("CREATE INDEX IF NOT EXISTS track_original_date_idx ON track(original_date)");
_session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)");
_session.execute("CREATE INDEX IF NOT EXISTS track_media_library_idx ON track(media_library_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_media_library_release_idx ON track(media_library_id, release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_idx ON track_artist_link(artist_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_type_idx ON track_artist_link(artist_id,type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_idx ON track_artist_link(track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_type_idx ON track_artist_link(track_id,type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_type_idx ON track_artist_link(type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_idx ON track_bookmark(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_track_idx ON track_bookmark(user_id,track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_artist_user_backend_idx ON starred_artist(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_artist_artist_user_backend_idx ON starred_artist(artist_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_release_user_backend_idx ON starred_release(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_release_release_user_backend_idx ON starred_release(release_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_track_user_backend_idx ON starred_track(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_track_track_user_backend_idx ON starred_track(track_id,user_id,backend)");
}
// Singletons
// TODO: move this elsewhere
{
auto uniqueTransaction{ createWriteTransaction() };
ScanSettings::init(*this);
}
}
void Session::createIndexesIfNeeded()
{
LMS_SCOPED_TRACE_OVERVIEW("Database", "IndexCreation");
auto transaction{ createWriteTransaction() };
_session.execute("CREATE INDEX IF NOT EXISTS artist_id_idx ON artist(id)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_user_idx ON auth_token(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_expiry_idx ON auth_token(expiry)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_value_idx ON auth_token(value)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_user_backend_idx ON listen(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_track_user_backend_idx ON listen(track_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_user_track_backend_date_time_idx ON listen(user_id,track_id,backend,date_time)");
_session.execute("CREATE INDEX IF NOT EXISTS release_id_idx ON release(id)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS release_type_name_idx ON release_type(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_id_idx ON track(id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_absolute_path_idx ON track(absolute_file_path)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_recording_mbid_idx ON track(recording_mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_file_last_write_idx ON track(file_last_write)");
_session.execute("CREATE INDEX IF NOT EXISTS track_date_idx ON track(date)");
_session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)");
_session.execute("CREATE INDEX IF NOT EXISTS track_original_date_idx ON track(original_date)");
_session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)");
_session.execute("CREATE INDEX IF NOT EXISTS track_media_library_idx ON track(media_library_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_media_library_release_idx ON track(media_library_id, release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_idx ON track_artist_link(artist_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_type_idx ON track_artist_link(artist_id,type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_idx ON track_artist_link(track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_type_idx ON track_artist_link(track_id,type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_type_idx ON track_artist_link(type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_idx ON track_bookmark(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_track_idx ON track_bookmark(user_id,track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_artist_user_backend_idx ON starred_artist(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_artist_artist_user_backend_idx ON starred_artist(artist_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_release_user_backend_idx ON starred_release(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_release_release_user_backend_idx ON starred_release(release_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_track_user_backend_idx ON starred_track(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_track_track_user_backend_idx ON starred_track(track_id,user_id,backend)");
}
void Session::vacuumIfNeeded()
{
long pageCount{};
long freeListCount{};
{
auto transaction{ createReadTransaction() };
pageCount = utils::fetchQuerySingleResult(_session.query<long>("SELECT page_count FROM pragma_page_count"));
freeListCount = utils::fetchQuerySingleResult(_session.query<long>("SELECT freelist_count FROM pragma_freelist_count"));
}
LMS_LOG(DB, INFO, "page stats: page_count = " << pageCount << ", freelist_count = " << freeListCount);
if (freeListCount >= (pageCount / 10))
{
LMS_SCOPED_TRACE_OVERVIEW("Database", "Vacuum");
LMS_LOG(DB, INFO, "Performing vacuum... This may take a while...");
// We manually take a lock here since vacuum cannot be inside a transaction
{
std::unique_lock lock{ _db.getMutex() };
_db.executeSql("VACUUM");
}
LMS_LOG(DB, INFO, "Vacuum complete!");
}
}
void Session::refreshTracingLoggerStats()
{
auto* traceLogger{ core::Service<core::tracing::ITraceLogger>::get() };
if (!traceLogger)
return;
auto& dbSession{ _db.getTLSSession() };
auto transaction{ dbSession.createReadTransaction() };
auto transaction{ createReadTransaction() };
traceLogger->setMetadata("db_artist_count", std::to_string(db::Artist::getCount(dbSession)));
traceLogger->setMetadata("db_cluster_count", std::to_string(db::Cluster::getCount(dbSession)));
traceLogger->setMetadata("db_cluster_type_count", std::to_string(db::ClusterType::getCount(dbSession)));
traceLogger->setMetadata("db_starred_artist_count", std::to_string(db::StarredArtist::getCount(dbSession)));
traceLogger->setMetadata("db_starred_release_count", std::to_string(db::StarredRelease::getCount(dbSession)));
traceLogger->setMetadata("db_starred_track_count", std::to_string(db::StarredTrack::getCount(dbSession)));
traceLogger->setMetadata("db_track_bookmark_count", std::to_string(db::TrackBookmark::getCount(dbSession)));
traceLogger->setMetadata("db_listen_count", std::to_string(db::Listen::getCount(dbSession)));
traceLogger->setMetadata("db_release_count", std::to_string(db::Release::getCount(dbSession)));
traceLogger->setMetadata("db_track_count", std::to_string(db::Track::getCount(dbSession)));
traceLogger->setMetadata("db_artist_count", std::to_string(db::Artist::getCount(*this)));
traceLogger->setMetadata("db_cluster_count", std::to_string(db::Cluster::getCount(*this)));
traceLogger->setMetadata("db_cluster_type_count", std::to_string(db::ClusterType::getCount(*this)));
traceLogger->setMetadata("db_starred_artist_count", std::to_string(db::StarredArtist::getCount(*this)));
traceLogger->setMetadata("db_starred_release_count", std::to_string(db::StarredRelease::getCount(*this)));
traceLogger->setMetadata("db_starred_track_count", std::to_string(db::StarredTrack::getCount(*this)));
traceLogger->setMetadata("db_track_bookmark_count", std::to_string(db::TrackBookmark::getCount(*this)));
traceLogger->setMetadata("db_listen_count", std::to_string(db::Listen::getCount(*this)));
traceLogger->setMetadata("db_release_count", std::to_string(db::Release::getCount(*this)));
traceLogger->setMetadata("db_track_count", std::to_string(db::Track::getCount(*this)));
}
void Session::analyze()
void Session::fullAnalyze()
{
LMS_SCOPED_TRACE_DETAILED("Database", "Analyze");
LMS_LOG(DB, INFO, "Analyzing database...");
LMS_SCOPED_TRACE_OVERVIEW("Database", "Analyze");
LMS_LOG(DB, INFO, "Performing database analyze... This may take a while...");
// first select all the tables and indexes, and then analyze one by one in order to not have a big lock
std::vector<std::string> entries;
retrieveEntriesToAnalyze(entries);
for (const std::string& entry : entries)
analyzeEntry(entry);
LMS_LOG(DB, INFO, "Analyze complete!");
}
void Session::retrieveEntriesToAnalyze(std::vector<std::string>& entryList)
{
auto transaction{ createReadTransaction() };
entryList = utils::fetchQueryResults(_session.query<std::string>("SELECT name FROM sqlite_master WHERE type='table' OR type ='index'"));
}
void Session::analyzeEntry(const std::string& entry)
{
LMS_SCOPED_TRACE_DETAILED_WITH_ARG("Database", "AnalyzeEntry", "Entry", entry);
LMS_LOG(DB, DEBUG, "Analyzing " << entry);
{
auto transaction{ createWriteTransaction() };
_session.execute("ANALYZE");
_session.execute("ANALYZE " + entry);
}
LMS_LOG(DB, INFO, "Database Analyze complete");
LMS_LOG(DB, DEBUG, "Analyzing " << entry << ": done!");
}
void Session::optimize()
{
LMS_SCOPED_TRACE_DETAILED("Database", "Optimize");
LMS_LOG(DB, INFO, "Optimizing database...");
{
auto transaction{ createWriteTransaction() };
_session.execute("PRAGMA optimize");
}
LMS_LOG(DB, INFO, "Database optimizing complete");
}
} // namespace lms::db
src/libs/database/include/database/Session.hpp
+10 -3
View File
@@ -22,6 +22,8 @@
#include <Wt/Dbo/Dbo.h>
#include <Wt/Dbo/SqlConnectionPool.h>
#include <string>
#include <vector>
#include "core/ITraceLogger.hpp"
#include "core/RecursiveSharedMutex.hpp"
#include "database/Object.hpp"
@@ -84,10 +86,15 @@ namespace lms::db
#endif
}
void analyze();
void optimize();
// All these methods will acquire transactions
void fullAnalyze(); // helper for retrieveEntriesToAnalyze + analyzeEntry
void retrieveEntriesToAnalyze(std::vector<std::string>& entryList);
void analyzeEntry(const std::string& entry);
void prepareTables(); // need to run only once at startup
void prepareTablesIfNeeded(); // need to run only once at startup
void migrateIfNeeded();
void createIndexesIfNeeded();
void vacuumIfNeeded();
void refreshTracingLoggerStats();
// returning a ptr here to ease further wrapping using operator->
src/libs/database/test/Common.cpp
+2 -2
View File
@@ -60,8 +60,8 @@ namespace lms::db::tests
_tmpDb = std::make_unique<TmpDatabase>();
{
db::Session s{ _tmpDb->getDb() };
s.prepareTables();
s.analyze();
s.prepareTablesIfNeeded();
s.createIndexesIfNeeded();
}
}
src/libs/services/scanner/CMakeLists.txt
+1
View File
@@ -2,6 +2,7 @@
add_library(lmsscanner SHARED
impl/ScannerService.cpp
impl/ScannerStats.cpp
impl/ScanStepAnalyze.cpp
impl/ScanStepCheckDuplicatedDbFiles.cpp
impl/ScanStepComputeClusterStats.cpp
impl/ScanStepDiscoverFiles.cpp
src/libs/services/scanner/impl/IScanStep.hpp
+2 -2
View File
@@ -19,9 +19,9 @@
#pragma once
#include <string_view>
#include <vector>
#include "core/LiteralString.hpp"
#include "services/scanner/ScannerStats.hpp"
namespace lms::scanner
@@ -32,7 +32,7 @@ namespace lms::scanner
virtual ~IScanStep() = default;
virtual ScanStep getStep() const = 0;
virtual std::string_view getStepName() const = 0;
virtual core::LiteralString getStepName() const = 0;
struct ScanContext
{
src/libs/services/scanner/impl/ScanStepAnalyze.cpp
+53
View File
@@ -0,0 +1,53 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepAnalyze.hpp"
#include "core/ILogger.hpp"
#include "database/Db.hpp"
#include "database/Session.hpp"
namespace lms::scanner
{
void ScanStepAnalyze::process(ScanContext& context)
{
ScanStats& stats{ context.stats };
if (stats.nbChanges() > (stats.nbFiles() / 5))
{
LMS_LOG(DBUPDATER, INFO, "Database changed substantially: triggering full analyze");
auto& session{ _db.getTLSSession() };
std::vector<std::string> entries;
session.retrieveEntriesToAnalyze(entries);
context.currentStepStats.totalElems = entries.size();
for (const std::string& entry : entries)
{
if (_abortScan)
break;
_db.getTLSSession().analyzeEntry(entry);
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
}
}
}
src/libs/services/scanner/impl/ScanStepAnalyze.hpp
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ScanStepBase.hpp"
namespace lms::scanner
{
class ScanStepAnalyze : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::Analyze; }
core::LiteralString getStepName() const override { return "Analyze"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.hpp
+1 -1
View File
@@ -29,7 +29,7 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
std::string_view getStepName() const override { return "Checking for duplicated files"; }
core::LiteralString getStepName() const override { return "Checking for duplicated files"; }
ScanStep getStep() const override { return ScanStep::CheckingForDuplicateFiles; }
void process(ScanContext& context) override;
};
src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp
+1
View File
@@ -79,6 +79,7 @@ namespace lms::scanner
}
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
return true;
src/libs/services/scanner/impl/ScanStepComputeClusterStats.hpp
+1 -1
View File
@@ -30,7 +30,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::ComputeClusterStats; }
std::string_view getStepName() const override { return "Compute cluster stats"; }
core::LiteralString getStepName() const override { return "Compute cluster stats"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepDiscoverFiles.hpp
+1 -1
View File
@@ -30,7 +30,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::DiscoveringFiles; }
std::string_view getStepName() const override { return "DiscoveringFiles"; }
core::LiteralString getStepName() const override { return "Discovering files"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.hpp
+1 -1
View File
@@ -31,7 +31,7 @@ namespace lms::scanner
using ScanStepBase::ScanStepBase;
private:
std::string_view getStepName() const override { return "Checking orphaned entries"; }
core::LiteralString getStepName() const override { return "Checking orphaned entries"; }
ScanStep getStep() const override { return ScanStep::ChekingForMissingFiles; }
void process(ScanContext& context) override;
src/libs/services/scanner/impl/ScanStepScanFiles.cpp
-4
View File
@@ -518,10 +518,6 @@ namespace lms::scanner
context.stats.scans++;
processFileMetaData(context, scanResult.path, *scanResult.trackMetaData, libraryInfo);
// optimize the database during scan (if we import a very large database, it may be too late to do it once at end)
if ((context.stats.scans % 1'000) == 0)
_db.getTLSSession().optimize();
}
else
{
src/libs/services/scanner/impl/ScanStepScanFiles.hpp
+1 -1
View File
@@ -40,7 +40,7 @@ namespace lms::scanner
private:
ScanStep getStep() const override { return ScanStep::ScanningFiles; }
std::string_view getStepName() const override { return "Scanning files"; }
core::LiteralString getStepName() const override { return "Scanning files"; }
void process(ScanContext& context) override;
bool checkFileNeedScan(ScanContext& context, const std::filesystem::path& file, const ScannerSettings::MediaLibraryInfo& libraryInfo);
src/libs/services/scanner/impl/ScannerService.cpp
+9 -4
View File
@@ -25,15 +25,17 @@
#include "database/TrackFeatures.hpp"
#include "database/ScanSettings.hpp"
#include "core/Exception.hpp"
#include "core/Path.hpp"
#include "core/IConfig.hpp"
#include "core/ILogger.hpp"
#include "core/Path.hpp"
#include "core/ITraceLogger.hpp"
#include "ScanStepAnalyze.hpp"
#include "ScanStepCheckDuplicatedDbFiles.hpp"
#include "ScanStepComputeClusterStats.hpp"
#include "ScanStepDiscoverFiles.hpp"
#include "ScanStepRemoveOrphanDbFiles.hpp"
#include "ScanStepScanFiles.hpp"
#include "ScanStepComputeClusterStats.hpp"
namespace lms::scanner
{
@@ -259,6 +261,8 @@ namespace lms::scanner
void ScannerService::scan(bool forceScan)
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", "Scan");
_events.scanStarted.emit();
{
@@ -278,6 +282,8 @@ namespace lms::scanner
for (auto& scanStep : _scanSteps)
{
LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() };
@@ -289,8 +295,6 @@ namespace lms::scanner
LMS_LOG(DBUPDATER, INFO, "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size());
_db.getTLSSession().analyze();
if (!_abortScan)
{
stats.stopTime = Wt::WDateTime::currentDateTime();
@@ -348,6 +352,7 @@ namespace lms::scanner
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepAnalyze>(params));
}
ScannerSettings ScannerService::readSettings()
src/libs/services/scanner/include/services/scanner/ScannerStats.hpp
+2 -1
View File
@@ -66,8 +66,9 @@ namespace lms::scanner
FetchingTrackFeatures,
ReloadingSimilarityEngine,
ComputeClusterStats,
Analyze,
};
static inline constexpr unsigned ScanProgressStepCount{ 7 };
static inline constexpr unsigned ScanProgressStepCount{ 8 };
// reduced scan stats
struct ScanStepStats
src/lms/CMakeLists.txt
+1
View File
@@ -4,6 +4,7 @@ add_executable(lms
ui/Auth.cpp
ui/LmsApplication.cpp
ui/LmsApplicationManager.cpp
ui/LmsInitApplication.cpp
ui/LmsTheme.cpp
ui/MediaPlayer.cpp
ui/ModalManager.cpp
src/lms/main.cpp
+25 -6
View File
@@ -40,6 +40,7 @@
#include "subsonic/SubsonicResource.hpp"
#include "ui/LmsApplication.hpp"
#include "ui/LmsApplicationManager.hpp"
#include "ui/LmsInitApplication.hpp"
#include "core/IChildProcessManager.hpp"
#include "core/IConfig.hpp"
#include "core/IOContextRunner.hpp"
@@ -275,20 +276,34 @@ namespace lms
Wt::WServer server{ argv[0] };
server.setServerConfiguration(wtServerArgs.size(), const_cast<char**>(&wtArgv[0]));
// As initialization can take a while (db migration, analyze, etc.), we bind a temporary init entry point to warn the user
server.addEntryPoint(Wt::EntryPointType::Application,
[&](const Wt::WEnvironment& env)
{
return ui::LmsInitApplication::create(env);
});
LMS_LOG(MAIN, INFO, "Starting init web server...");
server.start();
core::IOContextRunner ioContextRunner{ ioContext, getThreadCount(), "Misc" };
// Connection pool size must be twice the number of threads: we have at least 2 io pools with getThreadCount() each and they all may access the database
db::Db database{ config->getPath("working-dir") / "lms.db", getThreadCount() * 2 };
{
db::Session session{ database };
session.prepareTables();
session.prepareTablesIfNeeded();
session.createIndexesIfNeeded();
// As this may be quite long, we only do it during startup
session.vacuumIfNeeded();
// force optimize in case scanner aborted during a large import:
// queries may be too slow to even be able to relaunch a scan sing the web interface
session.analyze();
// queries may be too slow to even be able to relaunch a scan using the web interface
session.fullAnalyze();
database.getTLSSession().refreshTracingLoggerStats();
}
ui::LmsApplicationManager appManager;
// Service initialization order is important (reverse-order for deinit)
@@ -327,8 +342,12 @@ namespace lms
core::Service<feedback::IFeedbackService> feedbackService{ feedback::createFeedbackService(ioContext, database) };
core::Service<scrobbling::IScrobblingService> scrobblingService{ scrobbling::createScrobblingService(ioContext, database) };
std::unique_ptr<Wt::WResource> subsonicResource;
LMS_LOG(MAIN, INFO, "Stopping init web server...");
server.stop();
server.removeEntryPoint("");
std::unique_ptr<Wt::WResource> subsonicResource;
// bind API resources
if (config->getBool("api-subsonic", true))
{
@@ -345,7 +364,7 @@ namespace lms
proxyScannerEventsToApplication(*scannerService, server);
LMS_LOG(MAIN, INFO, "Starting server...");
LMS_LOG(MAIN, INFO, "Starting init web server...");
server.start();
LMS_LOG(MAIN, INFO, "Now running...");
src/lms/ui/LmsInitApplication.cpp
+38
View File
@@ -0,0 +1,38 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include <Wt/WContainerWidget.h>
#include <Wt/WText.h>
#include "LmsInitApplication.hpp"
namespace lms::ui
{
LmsInitApplication::LmsInitApplication(const Wt::WEnvironment& env)
: Wt::WApplication{ env }
{
enableUpdates(true);
root()->addNew<Wt::WText>("LMS is initializing. This may take a while, please wait...");
}
std::unique_ptr<Wt::WApplication> LmsInitApplication::create(const Wt::WEnvironment& env)
{
return std::make_unique<LmsInitApplication>(env);
}
} // namespace lms::ui
src/lms/ui/LmsInitApplication.hpp
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2024 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <Wt/WApplication.h>
namespace lms::ui
{
class LmsInitApplication : public Wt::WApplication
{
public:
LmsInitApplication(const Wt::WEnvironment& env);
static std::unique_ptr<Wt::WApplication> create(const Wt::WEnvironment& env);
};
} // namespace lms::ui
src/lms/ui/admin/ScannerController.cpp
+7
View File
@@ -274,6 +274,13 @@ namespace lms::ui
case scanner::ScanStep::ComputeClusterStats:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-compute-cluster-stats")
.arg(status.currentScanStepStats->progress()));
case scanner::ScanStep:: Analyze:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-analyze")
.arg(status.currentScanStepStats->processedElems)
.arg(status.currentScanStepStats->totalElems)
.arg(status.currentScanStepStats->progress()));
break;
}
break;
}