owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changed db optimization strategy: now perform a full analyze (only after scan if enough changes has been made, and during application startup). Also perform a vacuum if needed during startup. Added an init screen for the web interface since it can take a while to complete

Browse Source
This commit is contained in:
emeric
2024-04-18 18:46:35 +02:00
parent c63f9fa51c
commit 1862cd23c2
27 changed files with 382 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/database/impl/Db.cpp
+6 -6
View File
@@ -25,8 +25,9 @@
#include "database/Session.hpp"
#include "database/User.hpp"
#include "core/IConfig.hpp"
#include "core/Service.hpp"
#include "core/ILogger.hpp"
#include "core/ITraceLogger.hpp"
#include "core/Service.hpp"
namespace lms::db
{
@@ -49,7 +50,7 @@ namespace lms::db
prepare();
}
~Connection()
~Connection() override
{
// make use of per-connection usage stats to optimize
optimize();
@@ -66,16 +67,15 @@ namespace lms::db
void prepare()
{
LMS_LOG(DB, DEBUG, "Setting per-connection settings...");
executeSql("pragma journal_mode=WAL");
executeSql("pragma synchronous=normal");
executeSql("pragma analysis_limit=2000"); // to help make analyze command faster, 1000 does not seem to be enough to speed up all queries
executeSql("PRAGMA journal_mode=WAL");
executeSql("PRAGMA synchronous=normal");
LMS_LOG(DB, DEBUG, "Setting per-connection settings done!");
}
void optimize()
{
LMS_LOG(DB, DEBUG, "connection close: Running pragma optimize...");
executeSql("pragma optimize");
executeSql("PRAGMA optimize");
LMS_LOG(DB, DEBUG, "connection close: pragma optimize complete");
}
src/libs/database/impl/Session.cpp
+140 -101
View File
@@ -46,6 +46,7 @@
#include "EnumSetTraits.hpp"
#include "PathTraits.hpp"
#include "Migration.hpp"
#include "Utils.hpp"
namespace lms::db
{
@@ -120,7 +121,7 @@ namespace lms::db
return ReadTransaction{ _session };
}
void Session::prepareTables()
void Session::prepareTablesIfNeeded()
{
LMS_LOG(DB, INFO, "Preparing tables...");
@@ -140,127 +141,165 @@ namespace lms::db
throw e;
}
}
}
void Session::migrateIfNeeded()
{
Migration::doDbMigration(*this);
// Indexes
{
LMS_SCOPED_TRACE_OVERVIEW("Database", "IndexCreation");
auto transaction{ createWriteTransaction() };
_session.execute("CREATE INDEX IF NOT EXISTS artist_id_idx ON artist(id)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_user_idx ON auth_token(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_expiry_idx ON auth_token(expiry)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_value_idx ON auth_token(value)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_user_backend_idx ON listen(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_track_user_backend_idx ON listen(track_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_user_track_backend_date_time_idx ON listen(user_id,track_id,backend,date_time)");
_session.execute("CREATE INDEX IF NOT EXISTS release_id_idx ON release(id)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS release_type_name_idx ON release_type(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_id_idx ON track(id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_absolute_path_idx ON track(absolute_file_path)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_recording_mbid_idx ON track(recording_mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_file_last_write_idx ON track(file_last_write)");
_session.execute("CREATE INDEX IF NOT EXISTS track_date_idx ON track(date)");
_session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)");
_session.execute("CREATE INDEX IF NOT EXISTS track_original_date_idx ON track(original_date)");
_session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)");
_session.execute("CREATE INDEX IF NOT EXISTS track_media_library_idx ON track(media_library_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_media_library_release_idx ON track(media_library_id, release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_idx ON track_artist_link(artist_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_type_idx ON track_artist_link(artist_id,type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_idx ON track_artist_link(track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_type_idx ON track_artist_link(track_id,type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_type_idx ON track_artist_link(type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_idx ON track_bookmark(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_track_idx ON track_bookmark(user_id,track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_artist_user_backend_idx ON starred_artist(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_artist_artist_user_backend_idx ON starred_artist(artist_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_release_user_backend_idx ON starred_release(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_release_release_user_backend_idx ON starred_release(release_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_track_user_backend_idx ON starred_track(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_track_track_user_backend_idx ON starred_track(track_id,user_id,backend)");
}
// Singletons
// TODO: move this elsewhere
{
auto uniqueTransaction{ createWriteTransaction() };
ScanSettings::init(*this);
}
}
void Session::createIndexesIfNeeded()
{
LMS_SCOPED_TRACE_OVERVIEW("Database", "IndexCreation");
auto transaction{ createWriteTransaction() };
_session.execute("CREATE INDEX IF NOT EXISTS artist_id_idx ON artist(id)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_sort_name_nocase_idx ON artist(sort_name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS artist_mbid_idx ON artist(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_user_idx ON auth_token(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_expiry_idx ON auth_token(expiry)");
_session.execute("CREATE INDEX IF NOT EXISTS auth_token_value_idx ON auth_token(value)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_cluster_type_idx ON cluster(cluster_type_id)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_backend_idx ON listen(backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_id_idx ON listen(id)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_user_backend_idx ON listen(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_track_user_backend_idx ON listen(track_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS listen_user_track_backend_date_time_idx ON listen(user_id,track_id,backend,date_time)");
_session.execute("CREATE INDEX IF NOT EXISTS release_id_idx ON release(id)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_nocase_idx ON release(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS release_mbid_idx ON release(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS release_type_name_idx ON release_type(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_id_idx ON track(id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_absolute_path_idx ON track(absolute_file_path)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_idx ON track(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_name_nocase_idx ON track(name COLLATE NOCASE)");
_session.execute("CREATE INDEX IF NOT EXISTS track_mbid_idx ON track(mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_recording_mbid_idx ON track(recording_mbid)");
_session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_file_last_write_idx ON track(file_last_write)");
_session.execute("CREATE INDEX IF NOT EXISTS track_date_idx ON track(date)");
_session.execute("CREATE INDEX IF NOT EXISTS track_year_idx ON track(year)");
_session.execute("CREATE INDEX IF NOT EXISTS track_original_date_idx ON track(original_date)");
_session.execute("CREATE INDEX IF NOT EXISTS track_original_year_idx ON track(original_year)");
_session.execute("CREATE INDEX IF NOT EXISTS track_media_library_idx ON track(media_library_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_media_library_release_idx ON track(media_library_id, release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS tracklist_user_idx ON tracklist(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_idx ON track_artist_link(artist_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_artist_type_idx ON track_artist_link(artist_id,type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_idx ON track_artist_link(track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_track_type_idx ON track_artist_link(track_id,type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_link_type_idx ON track_artist_link(type)");
_session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_idx ON track_bookmark(user_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_bookmark_user_track_idx ON track_bookmark(user_id,track_id)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_artist_user_backend_idx ON starred_artist(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_artist_artist_user_backend_idx ON starred_artist(artist_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_release_user_backend_idx ON starred_release(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_release_release_user_backend_idx ON starred_release(release_id,user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_track_user_backend_idx ON starred_track(user_id,backend)");
_session.execute("CREATE INDEX IF NOT EXISTS starred_track_track_user_backend_idx ON starred_track(track_id,user_id,backend)");
}
void Session::vacuumIfNeeded()
{
long pageCount{};
long freeListCount{};
{
auto transaction{ createReadTransaction() };
pageCount = utils::fetchQuerySingleResult(_session.query<long>("SELECT page_count FROM pragma_page_count"));
freeListCount = utils::fetchQuerySingleResult(_session.query<long>("SELECT freelist_count FROM pragma_freelist_count"));
}
LMS_LOG(DB, INFO, "page stats: page_count = " << pageCount << ", freelist_count = " << freeListCount);
if (freeListCount >= (pageCount / 10))
{
LMS_SCOPED_TRACE_OVERVIEW("Database", "Vacuum");
LMS_LOG(DB, INFO, "Performing vacuum... This may take a while...");
// We manually take a lock here since vacuum cannot be inside a transaction
{
std::unique_lock lock{ _db.getMutex() };
_db.executeSql("VACUUM");
}
LMS_LOG(DB, INFO, "Vacuum complete!");
}
}
void Session::refreshTracingLoggerStats()
{
auto* traceLogger{ core::Service<core::tracing::ITraceLogger>::get() };
if (!traceLogger)
return;
auto& dbSession{ _db.getTLSSession() };
auto transaction{ dbSession.createReadTransaction() };
auto transaction{ createReadTransaction() };
traceLogger->setMetadata("db_artist_count", std::to_string(db::Artist::getCount(dbSession)));
traceLogger->setMetadata("db_cluster_count", std::to_string(db::Cluster::getCount(dbSession)));
traceLogger->setMetadata("db_cluster_type_count", std::to_string(db::ClusterType::getCount(dbSession)));
traceLogger->setMetadata("db_starred_artist_count", std::to_string(db::StarredArtist::getCount(dbSession)));
traceLogger->setMetadata("db_starred_release_count", std::to_string(db::StarredRelease::getCount(dbSession)));
traceLogger->setMetadata("db_starred_track_count", std::to_string(db::StarredTrack::getCount(dbSession)));
traceLogger->setMetadata("db_track_bookmark_count", std::to_string(db::TrackBookmark::getCount(dbSession)));
traceLogger->setMetadata("db_listen_count", std::to_string(db::Listen::getCount(dbSession)));
traceLogger->setMetadata("db_release_count", std::to_string(db::Release::getCount(dbSession)));
traceLogger->setMetadata("db_track_count", std::to_string(db::Track::getCount(dbSession)));
traceLogger->setMetadata("db_artist_count", std::to_string(db::Artist::getCount(*this)));
traceLogger->setMetadata("db_cluster_count", std::to_string(db::Cluster::getCount(*this)));
traceLogger->setMetadata("db_cluster_type_count", std::to_string(db::ClusterType::getCount(*this)));
traceLogger->setMetadata("db_starred_artist_count", std::to_string(db::StarredArtist::getCount(*this)));
traceLogger->setMetadata("db_starred_release_count", std::to_string(db::StarredRelease::getCount(*this)));
traceLogger->setMetadata("db_starred_track_count", std::to_string(db::StarredTrack::getCount(*this)));
traceLogger->setMetadata("db_track_bookmark_count", std::to_string(db::TrackBookmark::getCount(*this)));
traceLogger->setMetadata("db_listen_count", std::to_string(db::Listen::getCount(*this)));
traceLogger->setMetadata("db_release_count", std::to_string(db::Release::getCount(*this)));
traceLogger->setMetadata("db_track_count", std::to_string(db::Track::getCount(*this)));
}
void Session::analyze()
void Session::fullAnalyze()
{
LMS_SCOPED_TRACE_DETAILED("Database", "Analyze");
LMS_LOG(DB, INFO, "Analyzing database...");
LMS_SCOPED_TRACE_OVERVIEW("Database", "Analyze");
LMS_LOG(DB, INFO, "Performing database analyze... This may take a while...");
// first select all the tables and indexes, and then analyze one by one in order to not have a big lock
std::vector<std::string> entries;
retrieveEntriesToAnalyze(entries);
for (const std::string& entry : entries)
analyzeEntry(entry);
LMS_LOG(DB, INFO, "Analyze complete!");
}
void Session::retrieveEntriesToAnalyze(std::vector<std::string>& entryList)
{
auto transaction{ createReadTransaction() };
entryList = utils::fetchQueryResults(_session.query<std::string>("SELECT name FROM sqlite_master WHERE type='table' OR type ='index'"));
}
void Session::analyzeEntry(const std::string& entry)
{
LMS_SCOPED_TRACE_DETAILED_WITH_ARG("Database", "AnalyzeEntry", "Entry", entry);
LMS_LOG(DB, DEBUG, "Analyzing " << entry);
{
auto transaction{ createWriteTransaction() };
_session.execute("ANALYZE");
_session.execute("ANALYZE " + entry);
}
LMS_LOG(DB, INFO, "Database Analyze complete");
LMS_LOG(DB, DEBUG, "Analyzing " << entry << ": done!");
}
void Session::optimize()
{
LMS_SCOPED_TRACE_DETAILED("Database", "Optimize");
LMS_LOG(DB, INFO, "Optimizing database...");
{
auto transaction{ createWriteTransaction() };
_session.execute("PRAGMA optimize");
}
LMS_LOG(DB, INFO, "Database optimizing complete");
}
} // namespace lms::db
src/libs/database/include/database/Session.hpp
+10 -3
View File
@@ -22,6 +22,8 @@
#include <Wt/Dbo/Dbo.h>
#include <Wt/Dbo/SqlConnectionPool.h>
#include <string>
#include <vector>
#include "core/ITraceLogger.hpp"
#include "core/RecursiveSharedMutex.hpp"
#include "database/Object.hpp"
@@ -84,10 +86,15 @@ namespace lms::db
#endif
}
void analyze();
void optimize();
// All these methods will acquire transactions
void fullAnalyze(); // helper for retrieveEntriesToAnalyze + analyzeEntry
void retrieveEntriesToAnalyze(std::vector<std::string>& entryList);
void analyzeEntry(const std::string& entry);
void prepareTables(); // need to run only once at startup
void prepareTablesIfNeeded(); // need to run only once at startup
void migrateIfNeeded();
void createIndexesIfNeeded();
void vacuumIfNeeded();
void refreshTracingLoggerStats();
// returning a ptr here to ease further wrapping using operator->
src/libs/database/test/Common.cpp
+2 -2
View File
@@ -60,8 +60,8 @@ namespace lms::db::tests
_tmpDb = std::make_unique<TmpDatabase>();
{
db::Session s{ _tmpDb->getDb() };
s.prepareTables();
s.analyze();
s.prepareTablesIfNeeded();
s.createIndexesIfNeeded();
}
}