owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extracted genre, mood, language and grouping from generic clusters

Browse Source
This commit is contained in:
emeric
2026-06-17 21:34:29 +02:00
parent 77f9b033ed
commit dbd6e57160
82 changed files with 3648 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/services/scanner/impl/ScannerService.cpp
+2 -2
View File
@@ -49,7 +49,7 @@
#include "steps/ScanStepCheckForDuplicatedFiles.hpp"
#include "steps/ScanStepCheckForRemovedFiles.hpp"
#include "steps/ScanStepCompact.hpp"
#include "steps/ScanStepComputeClusterStats.hpp"
#include "steps/ScanStepComputeGenreStats.hpp"
#include "steps/ScanStepExtractMusicNNEmbeddings.hpp"
#include "steps/ScanStepOptimize.hpp"
#include "steps/ScanStepRemoveOrphanedDbEntries.hpp"
@@ -528,7 +528,7 @@ namespace lms::scanner
_scanSteps.emplace_back(std::make_unique<ScanStepRemoveOrphanedDbEntries>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepCompact>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepOptimize>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepComputeClusterStats>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepComputeGenreStats>(params));
_scanSteps.emplace_back(std::make_unique<ScanStepCheckForDuplicatedFiles>(params));
// Audio extraction scan step must be last as it is the most long running
src/libs/services/scanner/impl/scanners/audiofile/AudioFileScanOperation.cpp
+64 -6
View File
@@ -35,8 +35,12 @@
#include "database/objects/Artwork.hpp"
#include "database/objects/Cluster.hpp"
#include "database/objects/Directory.hpp"
#include "database/objects/Genre.hpp"
#include "database/objects/Grouping.hpp"
#include "database/objects/Language.hpp"
#include "database/objects/MediaLibrary.hpp"
#include "database/objects/Medium.hpp"
#include "database/objects/Mood.hpp"
#include "database/objects/Release.hpp"
#include "database/objects/ReleaseArtistLink.hpp"
#include "database/objects/Track.hpp"
@@ -308,6 +312,62 @@ namespace lms::scanner
return dbMedium;
}
std::vector<db::Genre::pointer> getOrCreateGenres(db::Session& session, std::span<const std::string> names)
{
std::vector<db::Genre::pointer> genres;
genres.reserve(names.size());
for (const std::string& name : names)
{
db::Genre::pointer genre{ db::Genre::find(session, name) };
if (!genre)
genre = session.create<db::Genre>(name);
genres.push_back(genre);
}
return genres;
}
std::vector<db::Grouping::pointer> getOrCreateGroupings(db::Session& session, std::span<const std::string> names)
{
std::vector<db::Grouping::pointer> groupings;
groupings.reserve(names.size());
for (const std::string& name : names)
{
db::Grouping::pointer grouping{ db::Grouping::find(session, name) };
if (!grouping)
grouping = session.create<db::Grouping>(name);
groupings.push_back(grouping);
}
return groupings;
}
std::vector<db::Language::pointer> getOrCreateLanguages(db::Session& session, std::span<const std::string> names)
{
std::vector<db::Language::pointer> languages;
languages.reserve(names.size());
for (const std::string& name : names)
{
db::Language::pointer language{ db::Language::find(session, name) };
if (!language)
language = session.create<db::Language>(name);
languages.push_back(language);
}
return languages;
}
std::vector<db::Mood::pointer> getOrCreateMoods(db::Session& session, std::span<const std::string> names)
{
std::vector<db::Mood::pointer> moods;
moods.reserve(names.size());
for (const std::string& name : names)
{
db::Mood::pointer mood{ db::Mood::find(session, name) };
if (!mood)
mood = session.create<db::Mood>(name);
moods.push_back(mood);
}
return moods;
}
std::vector<db::Cluster::pointer> getOrCreateClusters(db::Session& session, const Track& track)
{
std::vector<db::Cluster::pointer> clusters;
@@ -327,12 +387,6 @@ namespace lms::scanner
}
} };
// TODO: migrate these fields in dedicated tables in DB
getOrCreateClusters("GENRE", track.genres);
getOrCreateClusters("MOOD", track.moods);
getOrCreateClusters("LANGUAGE", track.languages);
getOrCreateClusters("GROUPING", track.groupings);
for (const auto& [tag, values] : track.userExtraTags)
getOrCreateClusters(tag, values);
@@ -778,6 +832,10 @@ namespace lms::scanner
createTrackArtistLinks(dbSession, track, db::TrackArtistLinkType::Performer, role, performers, allowFallback);
track.modify()->setClusters(getOrCreateClusters(dbSession, _file->track));
track.modify()->setGenres(getOrCreateGenres(dbSession, _file->track.genres));
track.modify()->setGroupings(getOrCreateGroupings(dbSession, _file->track.groupings));
track.modify()->setLanguages(getOrCreateLanguages(dbSession, _file->track.languages));
track.modify()->setMoods(getOrCreateMoods(dbSession, _file->track.moods));
track.modify()->setName(title);
track.modify()->setTrackNumber(_file->track.position);
track.modify()->setDate(_file->track.date);
src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.cpp → src/libs/services/scanner/impl/steps/ScanStepComputeGenreStats.cpp
+18 -29
View File
@@ -17,68 +17,59 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "ScanStepComputeClusterStats.hpp"
#include "core/ILogger.hpp"
#include "ScanStepComputeGenreStats.hpp"
#include "database/IDb.hpp"
#include "database/Session.hpp"
#include "database/objects/Cluster.hpp"
#include "database/objects/Genre.hpp"
#include "ScanContext.hpp"
namespace lms::scanner
{
bool ScanStepComputeClusterStats::needProcess(const ScanContext& context) const
bool ScanStepComputeGenreStats::needProcess(const ScanContext& context) const
{
return context.stats.getChangesCount() > 0;
}
void ScanStepComputeClusterStats::process(ScanContext& context)
void ScanStepComputeGenreStats::process(ScanContext& context)
{
using namespace db;
Session& dbSession{ _db.getTLSSession() };
const std::size_t clusterCount{ [&] {
const std::size_t genreCount{ [&] {
auto transaction{ dbSession.createReadTransaction() };
return Cluster::getCount(dbSession);
return Genre::getCount(dbSession);
}() };
context.currentStepStats.totalElems = clusterCount;
context.currentStepStats.totalElems = genreCount;
foreachSubRange(Range{ 0, clusterCount }, 100, [&](Range range) {
const std::vector<ClusterId> clusterIds{ [&] {
Cluster::FindParameters params;
foreachSubRange(Range{ 0, genreCount }, 100, [&](Range range) {
const std::vector<GenreId> genreIds{ [&] {
Genre::FindParameters params;
params.setRange(range);
{
auto transaction{ dbSession.createReadTransaction() };
return std::move(Cluster::findIds(dbSession, params).results);
}
auto transaction{ dbSession.createReadTransaction() };
return std::move(Genre::findIds(dbSession, params).results);
}() };
for (const ClusterId clusterId : clusterIds)
for (const GenreId genreId : genreIds)
{
if (_abortScan)
break;
std::size_t trackCount;
std::size_t releaseCount;
{
auto transaction{ dbSession.createReadTransaction() };
trackCount = Cluster::computeTrackCount(dbSession, clusterId);
releaseCount = Cluster::computeReleaseCount(dbSession, clusterId);
trackCount = Genre::computeTrackCount(dbSession, genreId);
releaseCount = Genre::computeReleaseCount(dbSession, genreId);
}
{
auto transaction{ dbSession.createWriteTransaction() };
auto cluster{ Cluster::find(dbSession, clusterId) };
cluster.modify()->setTrackCount(trackCount);
cluster.modify()->setReleaseCount(releaseCount);
auto genre{ Genre::find(dbSession, genreId) };
genre.modify()->setTrackCount(trackCount);
genre.modify()->setReleaseCount(releaseCount);
}
context.currentStepStats.processedElems++;
@@ -87,7 +78,5 @@ namespace lms::scanner
return true;
});
LMS_LOG(DBUPDATER, DEBUG, "Recomputed stats for " << context.currentStepStats.processedElems << " clusters!");
}
} // namespace lms::scanner
src/libs/services/scanner/impl/steps/ScanStepComputeClusterStats.hpp → src/libs/services/scanner/impl/steps/ScanStepComputeGenreStats.hpp
+3 -3
View File
@@ -23,14 +23,14 @@
namespace lms::scanner
{
class ScanStepComputeClusterStats : public ScanStepBase
class ScanStepComputeGenreStats : public ScanStepBase
{
public:
using ScanStepBase::ScanStepBase;
private:
ScanStep getStep() const override { return ScanStep::ComputeClusterStats; }
core::LiteralString getStepName() const override { return "Compute cluster stats"; }
ScanStep getStep() const override { return ScanStep::ComputeGenreStats; }
core::LiteralString getStepName() const override { return "Compute genre stats"; }
bool needProcess(const ScanContext& context) const override;
void process(ScanContext& context) override;
};
src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.cpp
+32
View File
@@ -25,7 +25,11 @@
#include "database/objects/Artist.hpp"
#include "database/objects/Cluster.hpp"
#include "database/objects/Directory.hpp"
#include "database/objects/Genre.hpp"
#include "database/objects/Grouping.hpp"
#include "database/objects/Language.hpp"
#include "database/objects/Medium.hpp"
#include "database/objects/Mood.hpp"
#include "database/objects/Release.hpp"
#include "database/objects/Track.hpp"
#include "database/objects/TrackEmbeddedImage.hpp"
@@ -44,6 +48,10 @@ namespace lms::scanner
{
removeOrphanedClusters(context);
removeOrphanedClusterTypes(context);
removeOrphanedGenres(context);
removeOrphanedGroupings(context);
removeOrphanedLanguages(context);
removeOrphanedMoods(context);
removeOrphanedArtists(context);
removeOrphanedReleases(context);
removeOrphanedMediums(context); // after release so that most entries are removed using the medium foreign key
@@ -66,6 +74,30 @@ namespace lms::scanner
removeOrphanedEntries<db::ClusterType>(context);
}
void ScanStepRemoveOrphanedDbEntries::removeOrphanedGenres(ScanContext& context)
{
LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned genres...");
removeOrphanedEntries<db::Genre>(context);
}
void ScanStepRemoveOrphanedDbEntries::removeOrphanedGroupings(ScanContext& context)
{
LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned groupings...");
removeOrphanedEntries<db::Grouping>(context);
}
void ScanStepRemoveOrphanedDbEntries::removeOrphanedLanguages(ScanContext& context)
{
LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned languages...");
removeOrphanedEntries<db::Language>(context);
}
void ScanStepRemoveOrphanedDbEntries::removeOrphanedMoods(ScanContext& context)
{
LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned moods...");
removeOrphanedEntries<db::Mood>(context);
}
void ScanStepRemoveOrphanedDbEntries::removeOrphanedArtists(ScanContext& context)
{
LMS_LOG(DBUPDATER, DEBUG, "Checking orphaned artists...");
src/libs/services/scanner/impl/steps/ScanStepRemoveOrphanedDbEntries.hpp
+4
View File
@@ -36,6 +36,10 @@ namespace lms::scanner
void removeOrphanedClusters(ScanContext& context);
void removeOrphanedClusterTypes(ScanContext& context);
void removeOrphanedGenres(ScanContext& context);
void removeOrphanedGroupings(ScanContext& context);
void removeOrphanedLanguages(ScanContext& context);
void removeOrphanedMoods(ScanContext& context);
void removeOrphanedArtists(ScanContext& context);
void removeOrphanedMediums(ScanContext& context);
void removeOrphanedReleases(ScanContext& context);