From 6704762cc31b402ce024b9161cd1373bf034fffa Mon Sep 17 00:00:00 2001 From: emeric Date: Sun, 13 Jul 2025 00:06:10 +0200 Subject: [PATCH] Tentative fix for bad query perf with recent sqlite versions: limit 1 seems to be culprit --- src/libs/database/impl/Db.cpp | 88 ++++++++++++++++++- src/libs/database/impl/Db.hpp | 3 + src/libs/database/impl/Utils.hpp | 3 +- .../impl/objects/TrackEmbeddedImage.cpp | 2 +- .../steps/ScanStepAssociateReleaseImages.cpp | 14 +-- .../steps/ScanStepAssociateTrackImages.cpp | 21 +++-- 6 files changed, 116 insertions(+), 15 deletions(-) diff --git a/src/libs/database/impl/Db.cpp b/src/libs/database/impl/Db.cpp index daa9fccd..057974ab 100644 --- a/src/libs/database/impl/Db.cpp +++ b/src/libs/database/impl/Db.cpp @@ -114,7 +114,7 @@ namespace lms::db std::string table; std::string foreignTable; - // see https://www.sqlite.org/pragma.html#pragma_foreign_key_check for exepcted result + // see https://www.sqlite.org/pragma.html#pragma_foreign_key_check for expected result while (statement->nextRow()) { foreignKeyConstraintsPassed = false; @@ -132,6 +132,57 @@ namespace lms::db return foreignKeyConstraintsPassed; } + + std::optional getPageSize(Wt::Dbo::SqlConnection& connection) + { + auto statement = connection.prepareStatement("PRAGMA page_size"); + statement->execute(); + + std::optional res; + while (statement->nextRow()) + { + assert(!res); + int value{}; + if (statement->getResult(0, &value)) + res = value; + break; + } + + return res; + } + + std::optional getCacheSize(Wt::Dbo::SqlConnection& connection) + { + auto statement = connection.prepareStatement("PRAGMA cache_size"); + statement->execute(); + + std::optional res; + while (statement->nextRow()) + { + assert(!res); + int value{}; + if (statement->getResult(0, &value)) + res = value; + break; + } + + return res; + } + + void getCompileOptions(Wt::Dbo::SqlConnection& connection, std::function callback) + { + auto statement = connection.prepareStatement("PRAGMA compile_options"); + statement->execute(); + + std::string res; + while (statement->nextRow()) + { + res.clear(); + + if (statement->getResult(0, &res, static_cast(res.capacity()))) + callback(res); + } + } } // namespace std::unique_ptr createDb(const std::filesystem::path& dbPath, std::size_t connectionCount) @@ -157,6 +208,13 @@ namespace lms::db _connectionPool = std::move(connectionPool); + executeSql("PRAGMA temp_store=MEMORY"); + executeSql("PRAGMA cache_size=-8000"); + executeSql("PRAGMA automatic_index=0"); + + logPageSize(); + logCacheSize(); + logCompileOptions(); if (checkType == "quick") { performQuickCheck(); @@ -199,6 +257,34 @@ namespace lms::db return *tlsSession; } + void Db::logPageSize() + { + ScopedConnection connection{ *_connectionPool }; + const std::optional pageSize{ getPageSize(*connection) }; + + if (pageSize) + LMS_LOG(DB, INFO, "Page size set to " << *pageSize); + } + + void Db::logCacheSize() + { + ScopedConnection connection{ *_connectionPool }; + const std::optional cacheSize{ getCacheSize(*connection) }; + + if (cacheSize) + LMS_LOG(DB, INFO, "Cache size set to " << *cacheSize); + } + + void Db::logCompileOptions() + { + ScopedConnection connection{ *_connectionPool }; + + LMS_LOG(DB, INFO, "Sqlite3 compile options:"); + getCompileOptions(*connection, [](std::string_view compileOption) { + LMS_LOG(DB, INFO, compileOption); + }); + } + void Db::performQuickCheck() { ScopedConnection connection{ *_connectionPool }; diff --git a/src/libs/database/impl/Db.hpp b/src/libs/database/impl/Db.hpp index 4d39b4b9..63890021 100644 --- a/src/libs/database/impl/Db.hpp +++ b/src/libs/database/impl/Db.hpp @@ -49,6 +49,9 @@ namespace lms::db core::RecursiveSharedMutex& getMutex() { return _sharedMutex; } Wt::Dbo::SqlConnectionPool& getConnectionPool() { return *_connectionPool; } + void logPageSize(); + void logCacheSize(); + void logCompileOptions(); void performQuickCheck(); void performIntegrityCheck(); void performForeignKeyConstraintsCheck(); diff --git a/src/libs/database/impl/Utils.hpp b/src/libs/database/impl/Utils.hpp index 7cbd1f03..bb377756 100644 --- a/src/libs/database/impl/Utils.hpp +++ b/src/libs/database/impl/Utils.hpp @@ -58,7 +58,8 @@ namespace lms::db::utils if (range) { query.limit(static_cast(range->size)); - query.offset(static_cast(range->offset)); + if (range->offset != 0) + query.offset(static_cast(range->offset)); } } diff --git a/src/libs/database/impl/objects/TrackEmbeddedImage.cpp b/src/libs/database/impl/objects/TrackEmbeddedImage.cpp index c3d2bb05..aebb8580 100644 --- a/src/libs/database/impl/objects/TrackEmbeddedImage.cpp +++ b/src/libs/database/impl/objects/TrackEmbeddedImage.cpp @@ -91,7 +91,7 @@ namespace lms::db } if (params.imageType.has_value()) - query.where("+t_e_i_l.type = ?").bind(params.imageType.value()); // hack: type is a bad way to reduce the result space + query.where("t_e_i_l.type = ?").bind(params.imageType.value()); } switch (params.sortMethod) diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp index 17572122..f5a589a8 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateReleaseImages.cpp @@ -142,9 +142,10 @@ namespace lms::scanner params.setRelease(release->getId()); params.setImageType(db::ImageType::FrontCover); params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc); - params.setRange(db::Range{ .offset = 0, .size = 1 }); - - db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); }); + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { + if (!artwork) + artwork = db::Artwork::find(session, image->getId()); + }); } if (artwork) @@ -156,9 +157,10 @@ namespace lms::scanner params.setRelease(release->getId()); params.setImageType(db::ImageType::Media); params.setSortMethod(db::TrackEmbeddedImageSortMethod::DiscNumberThenTrackNumberThenSizeDesc); - params.setRange(db::Range{ .offset = 0, .size = 1 }); - - db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { artwork = db::Artwork::find(session, image->getId()); }); + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { + if (!artwork) + artwork = db::Artwork::find(session, image->getId()); + }); } return artwork; diff --git a/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp b/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp index 6d4175f0..d7d7d456 100644 --- a/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp +++ b/src/libs/services/scanner/impl/steps/ScanStepAssociateTrackImages.cpp @@ -65,8 +65,11 @@ namespace lms::scanner params.setTrack(track->getId()); params.setImageType(db::ImageType::FrontCover); params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc); - params.setRange(db::Range{ .offset = 0, .size = 1 }); - db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); }); + + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { + if (!res) + res = db::Artwork::find(session, image->getId()); + }); } if (res) @@ -92,8 +95,11 @@ namespace lms::scanner params.setTrack(track->getId()); params.setImageType(db::ImageType::Media); params.setSortMethod(db::TrackEmbeddedImageSortMethod::SizeDesc); - params.setRange(db::Range{ .offset = 0, .size = 1 }); - db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); }); + + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { + if (!res) + res = db::Artwork::find(session, image->getId()); + }); } if (res) @@ -111,8 +117,11 @@ namespace lms::scanner params.setDiscNumber(discNumber); params.setImageType(db::ImageType::Media); params.setSortMethod(db::TrackEmbeddedImageSortMethod::TrackNumberThenSizeDesc); - params.setRange(db::Range{ .offset = 0, .size = 1 }); - db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { res = db::Artwork::find(session, image->getId()); }); + + db::TrackEmbeddedImage::find(session, params, [&](const db::TrackEmbeddedImage::pointer& image) { + if (!res) + res = db::Artwork::find(session, image->getId()); + }); } return res;