owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Subsonic API: cached cluster stats for track and releases, in order to make the getGenre entrypoint efficient on large databases

Browse Source
This commit is contained in:
emeric
2023-11-08 20:26:24 +01:00
parent c2e1c5d9df
commit 121d27b5cd
20 changed files with 301 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/scanner/CMakeLists.txt
+1
View File
@@ -3,6 +3,7 @@ add_library(lmsscanner SHARED
impl/ScannerService.cpp
impl/ScannerStats.cpp
impl/ScanStepCheckDuplicatedDbFiles.cpp
impl/ScanStepComputeClusterStats.cpp
impl/ScanStepDiscoverFiles.cpp
impl/ScanStepRemoveOrphanDbFiles.cpp
impl/ScanStepScanFiles.cpp
src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp
+86
View File
@@ -0,0 +1,86 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepComputeClusterStats.hpp"
#include "services/database/Db.hpp"
#include "services/database/Cluster.hpp"
#include "services/database/Session.hpp"
#include "utils/Logger.hpp"
#include "utils/Path.hpp"
namespace Scanner
{
void ScanStepComputeClusterStats::process(ScanContext& context)
{
using namespace Database;
if (context.stats.nbChanges() == 0)
return;
Session& dbSession{ _db.getTLSSession() };
const std::size_t clusterCount{ [&] {
auto transaction{ dbSession.createSharedTransaction() };
return Cluster::getCount(dbSession);
}() };
context.currentStepStats.totalElems = clusterCount;
foreachSubRange(Range{ 0, clusterCount }, 100, [&](Range range)
{
const std::vector<ClusterId> clusterIds{ [&]
{
Cluster::FindParameters params;
params.setRange(range);
{
auto transaction{ dbSession.createSharedTransaction() };
return std::move(Cluster::findIds(dbSession, params).results);
}
}() };
for (const ClusterId clusterId : clusterIds)
{
std::size_t trackCount;
std::size_t releaseCount;
{
auto transaction{ dbSession.createSharedTransaction() };
trackCount = Cluster::computeTrackCount(dbSession, clusterId);
releaseCount = Cluster::computeReleaseCount(dbSession, clusterId);
}
{
auto transaction{ dbSession.createUniqueTransaction() };
auto cluster{ Cluster::find(dbSession, clusterId) };
cluster.modify()->setTrackCount(trackCount);
cluster.modify()->setReleaseCount(releaseCount);
}
context.currentStepStats.processedElems++;
}
return true;
});
LMS_LOG(DBUPDATER, DEBUG) << "Recomputed stats for " << clusterCount << " clusters!";
}
}
src/libs/services/scanner/impl/ScanStepComputeClusterStats.hpp
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright (C) 2023 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "ScanStepBase.hpp"
namespace Scanner
{
class ScanStepComputeClusterStats : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::ComputeClusterStats; }
std::string_view getStepName() const override { return "Compute cluster stats"; }
void process(ScanContext& context) override;
};
}
src/libs/services/scanner/impl/ScannerService.cpp
+2
View File
@@ -36,6 +36,7 @@
#include "ScanStepDiscoverFiles.hpp"
#include "ScanStepRemoveOrphanDbFiles.hpp"
#include "ScanStepScanFiles.hpp"
#include "ScanStepComputeClusterStats.hpp"
using namespace Database;
@@ -357,6 +358,7 @@ ScannerService::refreshScanSettings()
_scanSteps.push_back(std::make_unique<ScanStepDiscoverFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepScanFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepRemoveOrphanDbFiles>(params));
_scanSteps.push_back(std::make_unique<ScanStepComputeClusterStats>(params));
_scanSteps.push_back(std::make_unique<ScanStepCheckDuplicatedDbFiles>(params));
}
src/libs/services/scanner/include/services/scanner/ScannerStats.hpp
+60 -59
View File
@@ -28,80 +28,81 @@
namespace Scanner
{
enum class ScanErrorType
{
CannotReadFile, // cannot read file
CannotParseFile, // cannot parse file
NoAudioTrack, // no audio track found
BadDuration, // bad duration
};
enum class ScanErrorType
{
CannotReadFile, // cannot read file
CannotParseFile, // cannot parse file
NoAudioTrack, // no audio track found
BadDuration, // bad duration
};
enum class DuplicateReason
{
SameHash,
SameTrackMBID,
};
enum class DuplicateReason
{
SameHash,
SameTrackMBID,
};
struct ScanError
{
std::filesystem::path file;
ScanErrorType error;
std::string systemError;
struct ScanError
{
std::filesystem::path file;
ScanErrorType error;
std::string systemError;
ScanError(const std::filesystem::path& file, ScanErrorType error, const std::string& systemError = "");
};
ScanError(const std::filesystem::path& file, ScanErrorType error, const std::string& systemError = "");
};
struct ScanDuplicate
{
Database::TrackId trackId;
DuplicateReason reason;
};
struct ScanDuplicate
{
Database::TrackId trackId;
DuplicateReason reason;
};
enum class ScanStep
{
DiscoveringFiles,
ScanningFiles,
ChekingForMissingFiles,
CheckingForDuplicateFiles,
FetchingTrackFeatures,
ReloadingSimilarityEngine,
};
static inline constexpr unsigned ScanProgressStepCount {5};
enum class ScanStep
{
DiscoveringFiles,
ScanningFiles,
ChekingForMissingFiles,
CheckingForDuplicateFiles,
FetchingTrackFeatures,
ReloadingSimilarityEngine,
ComputeClusterStats,
};
static inline constexpr unsigned ScanProgressStepCount{ 6 };
// reduced scan stats
struct ScanStepStats
{
Wt::WDateTime startTime;
// reduced scan stats
struct ScanStepStats
{
Wt::WDateTime startTime;
ScanStep currentStep;
ScanStep currentStep;
std::size_t totalElems {};
std::size_t processedElems {};
std::size_t totalElems{};
std::size_t processedElems{};
unsigned progress() const;
};
unsigned progress() const;
};
struct ScanStats
{
Wt::WDateTime startTime;
Wt::WDateTime stopTime;
struct ScanStats
{
Wt::WDateTime startTime;
Wt::WDateTime stopTime;
std::size_t filesScanned {}; // Total number of files scanned (estimated)
std::size_t filesScanned{}; // Total number of files scanned (estimated)
std::size_t skips {}; // no change since last scan
std::size_t scans {}; // actually scanned filed
std::size_t skips{}; // no change since last scan
std::size_t scans{}; // actually scanned filed
std::size_t additions {}; // added in DB
std::size_t deletions {}; // removed from DB
std::size_t updates {}; // updated file in DB
std::size_t additions{}; // added in DB
std::size_t deletions{}; // removed from DB
std::size_t updates{}; // updated file in DB
std::size_t featuresFetched {}; // features fetched in DB
std::size_t featuresFetched{}; // features fetched in DB
std::vector<ScanError> errors;
std::vector<ScanDuplicate> duplicates;
std::vector<ScanError> errors;
std::vector<ScanDuplicate> duplicates;
std::size_t nbFiles() const;
std::size_t nbChanges() const;
};
std::size_t nbFiles() const;
std::size_t nbChanges() const;
};
} // namespace Scanner