owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimized library assignation to directories

Browse Source
This commit is contained in:
emeric
2024-07-27 14:27:37 +02:00
parent cbc387b8b7
commit a2a2041f49
17 changed files with 215 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/database/impl/Directory.cpp
+30
View File
@@ -83,6 +83,25 @@ namespace lms::db
return query;
}
std::filesystem::path getPathWithTrailingSeparator(const std::filesystem::path& path)
{
if (path.empty())
return path;
// Convert the path to string
std::string pathStr{ path.string() };
// Check if the last character is a directory separator
if (pathStr.back() != std::filesystem::path::preferred_separator)
{
// If not, add the preferred separator
pathStr += std::filesystem::path::preferred_separator;
}
// Return the new path
return std::filesystem::path{ pathStr };
}
} // namespace
Directory::Directory(const std::filesystem::path& p)
@@ -157,6 +176,17 @@ namespace lms::db
return utils::execRangeQuery<DirectoryId>(query, range);
}
RangeResults<DirectoryId> Directory::findMismatchedLibrary(Session& session, std::optional<Range> range, const std::filesystem::path& rootPath, MediaLibraryId expectedLibraryId)
{
session.checkReadTransaction();
auto query{ session.getDboSession()->query<DirectoryId>("SELECT d.id FROM directory d") };
query.where("d.absolute_path = ? OR d.absolute_path LIKE ?").bind(rootPath).bind(getPathWithTrailingSeparator(rootPath).string() + "%");
query.where("d.media_library_id <> ? OR d.media_library_id IS NULL").bind(expectedLibraryId);
return utils::execRangeQuery<DirectoryId>(query, range);
}
RangeResults<Directory::pointer> Directory::findRootDirectories(Session& session, std::optional<Range> range)
{
auto query{ session.getDboSession()->query<Wt::Dbo::ptr<Directory>>("SELECT d from directory d").where("d.parent_directory_id IS NULL") };
src/libs/database/include/database/Directory.hpp
+1
View File
@@ -102,6 +102,7 @@ namespace lms::db
static RangeResults<Directory::pointer> find(Session& session, const FindParameters& params);
static void find(Session& session, const FindParameters& parameters, const std::function<void(const Directory::pointer&)>& func);
static RangeResults<DirectoryId> findOrphanIds(Session& session, std::optional<Range> range = std::nullopt);
static RangeResults<DirectoryId> findMismatchedLibrary(Session& session, std::optional<Range> range, const std::filesystem::path& rootPath, MediaLibraryId expectedLibraryId);
static RangeResults<pointer> findRootDirectories(Session& session, std::optional<Range> range = std::nullopt);
// getters
src/libs/database/test/Directory.cpp
+33
View File
@@ -252,4 +252,37 @@ namespace lms::db::tests
EXPECT_EQ(res[1]->getId(), child2.getId());
}
}
TEST_F(DatabaseFixture, Directory_findMismatchedLibrary)
{
ScopedDirectory parent1{ session, "/root" };
ScopedDirectory child1{ session, "/root/foo" };
ScopedDirectory parent2{ session, "/root_1" };
ScopedDirectory child2{ session, "/root_1/foo" };
ScopedMediaLibrary library{ session, "/root" };
{
auto transaction{ session.createReadTransaction() };
const auto res{ Directory::findMismatchedLibrary(session, std::nullopt, library->getPath(), library->getId()).results };
ASSERT_EQ(res.size(), 2);
EXPECT_EQ(res[0], parent1.getId());
EXPECT_EQ(res[1], child1.getId());
}
{
auto transaction{ session.createWriteTransaction() };
parent1.get().modify()->setMediaLibrary(library.get());
child1.get().modify()->setMediaLibrary(library.get());
}
{
auto transaction{ session.createReadTransaction() };
const auto res{ Directory::findMismatchedLibrary(session, std::nullopt, library->getPath(), library->getId()).results };
EXPECT_EQ(res.size(), 0);
}
}
} // namespace lms::db::tests
src/libs/services/scanner/CMakeLists.txt
+1
View File
@@ -12,6 +12,7 @@ add_library(lmsscanner SHARED
impl/ScanStepOptimize.cpp
impl/ScanStepRemoveOrphanedDbEntries.cpp
impl/ScanStepScanFiles.cpp
impl/ScanStepUpdateLibraryFields.cpp
)
target_include_directories(lmsscanner INTERFACE
src/libs/services/scanner/impl/ScanStepScanFiles.cpp
-41
View File
@@ -383,17 +383,6 @@ namespace lms::scanner
if (fileMatched)
{
// Not very efficient way to update media_library for directories
if (path.has_parent_path())
{
const std::filesystem::path directory{ path.parent_path() };
if (directory != currentDirectory)
{
updateDirectoryIfNeeded(currentDirectory, mediaLibrary);
currentDirectory = directory;
}
}
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
@@ -792,34 +781,4 @@ namespace lms::scanner
stats.updates++;
}
}
void ScanStepScanFiles::updateDirectoryIfNeeded(const std::filesystem::path& dirPath, const ScannerSettings::MediaLibraryInfo& libraryInfo)
{
db::Session& dbSession{ _db.getTLSSession() };
const bool needUpdateLibrary{ [&] {
auto transaction{ dbSession.createReadTransaction() };
Directory::pointer directory{ Directory::find(dbSession, dirPath) };
if (!directory)
return false; // we create directories only of we find images or tracks inside
MediaLibrary::pointer currentLibrary{ directory->getMediaLibrary() };
return !currentLibrary || currentLibrary->getId() != libraryInfo.id;
}() };
if (needUpdateLibrary)
{
auto transaction{ dbSession.createWriteTransaction() };
Directory::pointer directory{ Directory::find(dbSession, dirPath) };
assert(directory);
directory.modify()->setMediaLibrary(MediaLibrary::find(dbSession, libraryInfo.id)); // may be null if settings are updated in // => next scan will correct this
}
if (dirPath != libraryInfo.rootDirectory && dirPath.has_parent_path())
{
updateDirectoryIfNeeded(dirPath.parent_path(), libraryInfo);
}
}
} // namespace lms::scanner
src/libs/services/scanner/impl/ScanStepScanFiles.hpp
-2
View File
@@ -49,8 +49,6 @@ namespace lms::scanner
void processAudioFileScanData(ScanContext& context, const std::filesystem::path& path, const metadata::Track* trackMetadata, const ScannerSettings::MediaLibraryInfo& libraryInfo);
void processImageFileScanData(ScanContext& context, const std::filesystem::path& path, const ImageInfo* imageInfo, const ScannerSettings::MediaLibraryInfo& libraryInfo);
void updateDirectoryIfNeeded(const std::filesystem::path& directory, const ScannerSettings::MediaLibraryInfo& libraryInfo);
std::unique_ptr<metadata::IParser> _metadataParser;
const std::vector<std::string> _extraTagsToParse;
src/libs/services/scanner/impl/ScanStepUpdateLibraryFields.cpp
+87
View File
@@ -0,0 +1,87 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepUpdateLibraryFields.hpp"
#include "core/ILogger.hpp"
#include "core/Path.hpp"
#include "database/Db.hpp"
#include "database/Directory.hpp"
#include "database/MediaLibrary.hpp"
#include "database/Session.hpp"
namespace lms::scanner
{
void ScanStepUpdateLibraryFields::process(ScanContext& context)
{
processDirectories(context);
}
void ScanStepUpdateLibraryFields::processDirectories(ScanContext& context)
{
for (const ScannerSettings::MediaLibraryInfo& mediaLibrary : _settings.mediaLibraries)
{
if (_abortScan)
break;
processDirectory(context, mediaLibrary);
}
}
void ScanStepUpdateLibraryFields::processDirectory(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary)
{
db::Session& session{ _db.getTLSSession() };
constexpr std::size_t batchSize = 100;
db::RangeResults<db::DirectoryId> entries;
while (!_abortScan)
{
{
auto transaction{ session.createReadTransaction() };
entries = db::Directory::findMismatchedLibrary(session, db::Range{ 0, batchSize }, mediaLibrary.rootDirectory, mediaLibrary.id);
};
if (entries.results.empty())
break;
{
auto transaction{ session.createWriteTransaction() };
db::MediaLibrary::pointer library{ db::MediaLibrary::find(session, mediaLibrary.id) };
if (!library) // may be legit
break;
for (const db::DirectoryId directoryId : entries.results)
{
if (_abortScan)
break;
db::Directory::pointer directory{ db::Directory::find(session, directoryId) };
directory.modify()->setMediaLibrary(library);
}
}
context.currentStepStats.processedElems += entries.results.size();
_progressCallback(context.currentStepStats);
}
}
} // namespace lms::scanner
src/libs/services/scanner/impl/ScanStepUpdateLibraryFields.hpp
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "database/DirectoryId.hpp"
#include "ScanStepBase.hpp"
namespace lms::scanner
{
class ScanStepUpdateLibraryFields : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
core::LiteralString getStepName() const override { return "Update Library fields"; }
ScanStep getStep() const override { return ScanStep::UpdateLibraryFields; }
void process(ScanContext& context) override;
void processDirectories(ScanContext& context);
void processDirectory(ScanContext& context, const ScannerSettings::MediaLibraryInfo& mediaLibrary);
};
} // namespace lms::scanner
src/libs/services/scanner/impl/ScannerService.cpp
+4 -2
View File
@@ -40,6 +40,7 @@
#include "ScanStepOptimize.hpp"
#include "ScanStepRemoveOrphanedDbEntries.hpp"
#include "ScanStepScanFiles.hpp"
#include "ScanStepUpdateLibraryFields.hpp"
namespace lms::scanner
{
@@ -279,7 +280,7 @@ namespace lms::scanner
LMS_SCOPED_TRACE_OVERVIEW("Scanner", scanStep->getStepName());
LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
scanContext.currentStepStats = ScanStepStats{ .startTime = Wt::WDateTime::currentDateTime(), .stepIndex = stepIndex++, .currentStep = scanStep->getStep() };
scanContext.currentStepStats = ScanStepStats{ .startTime = Wt::WDateTime::currentDateTime(), .stepCount = _scanSteps.size(), .stepIndex = stepIndex++, .currentStep = scanStep->getStep() };
notifyInProgress(scanContext.currentStepStats);
scanStep->process(scanContext);
@@ -339,11 +340,12 @@ namespace lms::scanner
_db
};
// Order is important
// Order is important, steps are sequential
_scanSteps.clear();
_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepCheckForRemovedFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepUpdateLibraryFields>(params));
_scanSteps.push_back(std::make_unique<ScanStepAssociateArtistImages>(params));
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanedDbEntries>(params));
_scanSteps.push_back(std::make_unique<ScanStepCompact>(params));
src/libs/services/scanner/include/services/scanner/ScannerStats.hpp
+2 -1
View File
@@ -72,14 +72,15 @@ namespace lms::scanner
ReloadSimilarityEngine,
RemoveOrphanedDbEntries,
ScanFiles,
UpdateLibraryFields,
};
static inline constexpr unsigned ScanProgressStepCount{ 11 };
// reduced scan stats
struct ScanStepStats
{
Wt::WDateTime startTime;
std::size_t stepCount{};
std::size_t stepIndex{};
ScanStep currentStep;
src/lms/ui/admin/ScannerController.cpp
+6 -1
View File
@@ -256,7 +256,7 @@ namespace lms::ui
_status->setText(Wt::WString::tr("Lms.Admin.ScannerController.status-in-progress")
.arg(status.currentScanStepStats->stepIndex + 1)
.arg(scanner::ScanProgressStepCount));
.arg(status.currentScanStepStats->stepCount));
refreshCurrentStep(*status.currentScanStepStats);
break;
@@ -326,6 +326,11 @@ namespace lms::ui
.arg(stepStats.totalElems)
.arg(stepStats.progress()));
break;
case ScanStep::UpdateLibraryFields:
_stepStatus->setText(Wt::WString::tr("Lms.Admin.ScannerController.step-updating-library-fields")
.arg(stepStats.processedElems));
break;
}
}
} // namespace lms::ui