owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Started to restore clusters based recommendation engine

Browse Source
This commit is contained in:
emeric
2020-02-15 15:52:39 +01:00
parent 7f05a1d4e6
commit d1bdaf6203
26 changed files with 326 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -17,7 +17,7 @@ A [demo](http://lms.demo.poupon.io) instance is available, with the following li
* Audio transcode for maximum interoperability and low bandwith requirements
* Persistent play queue across sessions
* Subsonic API
* Album artist
* Compilation support
* Multi-value tags: artists, genres, ...
* Custom tags (ex: _mood_, _genre_, _albummood_, _albumgrouping_, ...)
* MusicBrainzID support to handle duplicated artist and release names
approot/messages.xml
+3 -3
View File
@@ -39,9 +39,9 @@
<message id="Lms.Admin.Database.monthly">Monthly</message>
<message id="Lms.Admin.Database.never">Never</message>
<message id="Lms.Admin.Database.path">Media root directory</message>
<message id="Lms.Admin.Database.similarity-engine-type">Recommendation engine</message>
<message id="Lms.Admin.Database.similarity-engine-type.clusters">Tags based</message>
<message id="Lms.Admin.Database.similarity-engine-type.features">Audio analysis based</message>
<message id="Lms.Admin.Database.recommendation-engine-type">Recommendation engine</message>
<message id="Lms.Admin.Database.recommendation-engine-type.clusters">Tags based</message>
<message id="Lms.Admin.Database.recommendation-engine-type.features">Audio analysis based</message>
<message id="Lms.Admin.Database.scan-complete">Scan complete: {1} total files, {2} additions, {3} updates, {4} deletions, {5} duplicates, {6} errors</message>
<message id="Lms.Admin.Database.scan-launched">Scan launched!</message>
<message id="Lms.Admin.Database.scan-options">Scan options</message>
approot/messages_fr.xml
+3 -3
View File
@@ -39,9 +39,9 @@
<message id="Lms.Admin.Database.monthly">Tous les mois</message>
<message id="Lms.Admin.Database.never">Jamais</message>
<message id="Lms.Admin.Database.path">Dossier racine des fichiers de musique</message>
<message id="Lms.Admin.Database.similarity-engine-type">Moteur de recommandation</message>
<message id="Lms.Admin.Database.similarity-engine-type.clusters">Basé sur les tags</message>
<message id="Lms.Admin.Database.similarity-engine-type.features">Basé sur l'analyse audio</message>
<message id="Lms.Admin.Database.recommendation-engine-type">Moteur de recommandation</message>
<message id="Lms.Admin.Database.recommendation-engine-type.clusters">Basé sur les tags</message>
<message id="Lms.Admin.Database.recommendation-engine-type.features">Basé sur l'analyse audio</message>
<message id="Lms.Admin.Database.scan-complete">Scan terminé : {1} fichiers, {2} ajouts, {3} mises à jour, {4} suppressions, {5} duplicatas, {6} erreurs</message>
<message id="Lms.Admin.Database.scan-launched">Scan lancé !</message>
<message id="Lms.Admin.Database.scan-options">Options </message>
src/libs/database/impl/Artist.cpp
+15
View File
@@ -104,6 +104,21 @@ Artist::getAllOrphans(Session& session)
return std::vector<pointer>(res.begin(), res.end());
}
std::vector<IdType>
Artist::getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit)
{
session.checkSharedLocked();
Wt::Dbo::collection<IdType> res = session.getDboSession().query<IdType>
("SELECT DISTINCT a.id FROM artist a"
" INNER JOIN track t ON t.id = t_a_l.track_id INNER JOIN track_artist_link t_a_l ON t_a_l.artist_id = a.id"
" INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.limit(limit ? static_cast<int>(*limit) : -1);
return std::vector<IdType>(res.begin(), res.end());
}
static
Wt::Dbo::Query<Artist::pointer>
getQuery(Session& session,
src/libs/database/impl/Release.cpp
+15
View File
@@ -252,6 +252,21 @@ Release::getByFilter(Session& session,
return res;
}
std::vector<IdType>
Release::getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit)
{
session.checkSharedLocked();
Wt::Dbo::collection<IdType> res = session.getDboSession().query<IdType>
("SELECT DISTINCT r.id FROM release r"
" INNER JOIN track t ON t.release_id = r.id"
" INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.limit(limit ? static_cast<int>(*limit) : -1);
return std::vector<IdType>(res.begin(), res.end());
}
std::optional<std::size_t>
Release::getTotalTrackNumber(void) const
{
src/libs/database/impl/Session.cpp
+1 -1
View File
@@ -118,7 +118,7 @@ Session::doDatabaseMigrationIfNeeded()
{
_session.execute("DROP TABLE similarity_settings");
_session.execute("DROP TABLE similarity_settings_feature");
_session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast<int>(ScanSettings::SimilarityEngineType::Clusters)) + ")");
_session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast<int>(ScanSettings::RecommendationEngineType::Clusters)) + ")");
}
else if (version == 8)
{
src/libs/database/impl/Track.cpp
+14 -1
View File
@@ -163,6 +163,19 @@ Track::getAllIdsWithFeatures(Session& session, std::optional<std::size_t> limit)
return std::vector<IdType>(res.begin(), res.end());
}
std::vector<IdType>
Track::getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit)
{
session.checkSharedLocked();
Wt::Dbo::collection<IdType> res = session.getDboSession().query<IdType>
("SELECT DISTINCT t.id FROM track t"
" INNER JOIN track_cluster t_c ON t_c.track_id = t.id")
.limit(limit ? static_cast<int>(*limit) : -1);
return std::vector<IdType>(res.begin(), res.end());
}
std::vector<Cluster::pointer>
Track::getClusters(void) const
{
@@ -263,7 +276,7 @@ Track::getByFilter(Session& session,
std::vector<Track::pointer>
Track::getSimilarTracks(Session& session,
const std::set<IdType>& tracks,
const std::unordered_set<IdType>& tracks,
std::optional<std::size_t> offset,
std::optional<std::size_t> size)
{
src/libs/database/include/database/Artist.hpp
+1
View File
@@ -68,6 +68,7 @@ class Artist : public Wt::Dbo::Dbo<Artist>
static std::vector<IdType> getAllIds(Session& session);
static std::vector<pointer> getAllOrphans(Session& session); // No track related
static std::vector<pointer> getLastAdded(Session& session, Wt::WDateTime after, std::optional<std::size_t> size = {});
static std::vector<IdType> getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit = {});
// Accessors
const std::string& getName(void) const { return _name; }
src/libs/database/include/database/Release.hpp
+1
View File
@@ -66,6 +66,7 @@ class Release : public Wt::Dbo::Dbo<Release>
std::optional<std::size_t> offset,
std::optional<std::size_t> size,
bool& moreExpected);
static std::vector<IdType> getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit = {});
std::vector<Wt::Dbo::ptr<Track>> getTracks(const std::set<IdType>& clusters = std::set<IdType>()) const;
std::size_t getTracksCount() const;
src/libs/database/include/database/ScanSettings.hpp
+5 -5
View File
@@ -43,7 +43,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
};
// Do not modify values (just add)
enum class SimilarityEngineType
enum class RecommendationEngineType
{
Clusters = 0,
Features,
@@ -60,7 +60,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
UpdatePeriod getUpdatePeriod() const { return _updatePeriod; }
std::vector<Wt::Dbo::ptr<ClusterType>> getClusterTypes() const;
std::set<std::filesystem::path> getAudioFileExtensions() const;
SimilarityEngineType getSimilarityEngineType() const { return _similarityEngineType; }
RecommendationEngineType getRecommendationEngineType() const { return _recommendationEngineType; }
// Setters
void addAudioFileExtension(const std::filesystem::path& ext);
@@ -68,7 +68,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
void setUpdateStartTime(Wt::WTime t) { _startTime = t; }
void setUpdatePeriod(UpdatePeriod p) { _updatePeriod = p; }
void setClusterTypes(Session& session, const std::set<std::string>& clusterTypeNames);
void setSimilarityEngineType(SimilarityEngineType type) { _similarityEngineType = type; }
void setRecommendationEngineType(RecommendationEngineType type) { _recommendationEngineType = type; }
void incScanVersion();
template<class Action>
@@ -79,7 +79,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
Wt::Dbo::field(a, _startTime, "start_time");
Wt::Dbo::field(a, _updatePeriod, "update_period");
Wt::Dbo::field(a, _audioFileExtensions, "audio_file_extensions");
Wt::Dbo::field(a, _similarityEngineType,"similarity_engine_type");
Wt::Dbo::field(a, _recommendationEngineType,"similarity_engine_type");
Wt::Dbo::hasMany(a, _clusterTypes, Wt::Dbo::ManyToOne, "scan_settings");
}
@@ -89,7 +89,7 @@ class ScanSettings : public Wt::Dbo::Dbo<ScanSettings>
std::string _mediaDirectory;
Wt::WTime _startTime = Wt::WTime {0,0,0};
UpdatePeriod _updatePeriod {UpdatePeriod::Never};
SimilarityEngineType _similarityEngineType {SimilarityEngineType::Clusters};
RecommendationEngineType _recommendationEngineType {RecommendationEngineType::Clusters};
std::string _audioFileExtensions {".alac .mp3 .ogg .oga .aac .m4a .m4b .flac .wav .wma .aif .aiff .ape .mpc .shn .opus"};
Wt::Dbo::collection<Wt::Dbo::ptr<ClusterType>> _clusterTypes;
};
src/libs/database/include/database/Track.hpp
+4 -2
View File
@@ -22,8 +22,9 @@
#include <chrono>
#include <filesystem>
#include <optional>
#include <vector>
#include <string>
#include <unordered_set>
#include <vector>
#include <Wt/Dbo/Dbo.h>
#include <Wt/WDateTime.h>
@@ -58,7 +59,7 @@ class Track : public Wt::Dbo::Dbo<Track>
static pointer getById(Session& session, IdType id);
static pointer getByMBID(Session& session, const UUID& MBID);
static std::vector<pointer> getSimilarTracks(Session& session,
const std::set<IdType>& trackIds,
const std::unordered_set<IdType>& trackIds,
std::optional<std::size_t> offset = {},
std::optional<std::size_t> size = {});
static std::vector<pointer> getByClusters(Session& session,
@@ -78,6 +79,7 @@ class Track : public Wt::Dbo::Dbo<Track>
static std::vector<pointer> getLastAdded(Session& session, const Wt::WDateTime& after, std::optional<std::size_t> size = 1);
static std::vector<pointer> getAllWithMBIDAndMissingFeatures(Session& session);
static std::vector<IdType> getAllIdsWithFeatures(Session& session, std::optional<std::size_t> limit = {});
static std::vector<IdType> getAllIdsWithClusters(Session& session, std::optional<std::size_t> limit = {});
// Create utility
static pointer create(Session& session, const std::filesystem::path& p);
src/libs/recommendation/CMakeLists.txt
+1
View File
@@ -1,5 +1,6 @@
add_library(lmsrecommendation SHARED
impl/clusters/ClustersClassifier.cpp
impl/Engine.cpp
impl/ClassifierCreator.cpp
)
src/libs/recommendation/impl/ClassifierCreator.cpp
+2 -7
View File
@@ -17,22 +17,17 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "recommendation/ClustersClassifierCreator.hpp"
#include "recommendation/FeaturesClassifierCreator.hpp"
#include "recommendation/Classifier.hpp"
#include "recommendation/IClassifier.hpp"
namespace Recommendation
{
std::unique_ptr<Classifier> createClustersClassifier()
std::unique_ptr<IClassifier> createFeaturesClassifier()
{
return {};
}
std::unique_ptr<Classifier> createFeaturesClassifier()
{
return {};
}
}
src/libs/recommendation/impl/Engine.cpp
+58 -47
View File
@@ -19,30 +19,49 @@
#include "Engine.hpp"
//#include "features/SimilarityFeaturesScannerAddon.hpp"
//#include "cluster/SimilarityClusterSearcher.hpp"
#include "recommendation/ClustersClassifierCreator.hpp"
#include "recommendation/FeaturesClassifierCreator.hpp"
#include "database/ScanSettings.hpp"
#include "database/Session.hpp"
#include "database/TrackList.hpp"
namespace Recommendation {
std::unique_ptr<IEngine>
createEngine()
createEngine(Database::Session& session)
{
return std::make_unique<Engine>();
return std::make_unique<Engine>(session);
}
Engine::Engine(Database::Session& session)
{
reloadSettings(session);
}
void
Engine::clearClassifiers()
Engine::reloadSettings(Database::Session& session)
{
_classifiers.clear();
}
using namespace Database;
void
Engine::addClassifier(std::unique_ptr<Classifier> classifier, unsigned priority)
{
_classifiers.emplace(priority, std::move(classifier));
const ScanSettings::RecommendationEngineType engineType {[&]()
{
auto transaction {session.createSharedTransaction()};
return ScanSettings::get(session)->getRecommendationEngineType();
}()};
clearClassifiers();
switch (engineType)
{
case ScanSettings::RecommendationEngineType::Features:
// _classifiers.emplace(0, createFeaturesClassifier()); // higher priority
// [[fallthrough]];
case ScanSettings::RecommendationEngineType::Clusters:
_classifiers.emplace(1, createClustersClassifier(session)); // lower priority
break;
}
}
std::vector<Database::IdType>
@@ -79,60 +98,52 @@ Engine::getSimilarTracksFromTrackList(Database::Session& /*session*/, Database::
}
std::vector<Database::IdType>
Engine::getSimilarTracks(Database::Session& /*dbSession*/, const std::unordered_set<Database::IdType>& /*trackIds*/, std::size_t /*maxCount*/)
Engine::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount)
{
#if 0
auto engineType {getEngineType(dbSession)};
auto somSearcher {_somAddon.getSearcher()};
if (engineType == Database::ScanSettings::SimilarityEngineType::Features
&& somSearcher
&& std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return somSearcher->isTrackClassified(trackId); } ))
for (const auto& [priority, classifier] : _classifiers)
{
return somSearcher->getSimilarTracks(trackIds, maxCount);
if (std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return classifier->isTrackClassified(trackId); } ))
return classifier->getSimilarTracks(dbSession, trackIds, maxCount);
}
else
return ClusterEngine::getSimilarTracks(dbSession, trackIds, maxCount);
#endif
return {};
}
std::vector<Database::IdType>
Engine::getSimilarReleases(Database::Session& /*dbSession*/, Database::IdType /*releaseId*/, std::size_t /*maxCount*/)
Engine::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount)
{
#if 0
auto engineType {getEngineType(dbSession)};
auto somSearcher {_somAddon.getSearcher()};
if (engineType == Database::ScanSettings::SimilarityEngineType::Features
&& somSearcher
&& somSearcher->isReleaseClassified(releaseId))
for (const auto& [priority, classifier] : _classifiers)
{
return somSearcher->getSimilarReleases(releaseId, maxCount);
if (classifier->isReleaseClassified(releaseId))
return classifier->getSimilarReleases(dbSession, releaseId, maxCount);
}
else
return ClusterEngine::getSimilarReleases(dbSession, releaseId, maxCount);
#endif
return {};
}
std::vector<Database::IdType>
Engine::getSimilarArtists(Database::Session& /*dbSession*/, Database::IdType /*artistId*/, std::size_t /*maxCount*/)
Engine::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount)
{
#if 0
auto engineType {getEngineType(dbSession)};
auto somSearcher {_somAddon.getSearcher()};
if (engineType == Database::ScanSettings::SimilarityEngineType::Features
&& somSearcher
&& somSearcher->isArtistClassified(artistId))
for (const auto& [priority, classifier] : _classifiers)
{
return somSearcher->getSimilarArtists(artistId, maxCount);
if (classifier->isArtistClassified(artistId))
return classifier->getSimilarArtists(dbSession, artistId, maxCount);
}
else
return ClusterEngine::getSimilarArtists(dbSession, artistId, maxCount);
#endif
return {};
}
void
Engine::clearClassifiers()
{
_classifiers.clear();
}
void
Engine::addClassifier(std::unique_ptr<IClassifier> classifier, unsigned priority)
{
_classifiers.emplace(priority, std::move(classifier));
}
} // ns Similarity
src/libs/recommendation/impl/Engine.hpp
+9 -4
View File
@@ -22,7 +22,7 @@
#include <map>
#include "recommendation/IEngine.hpp"
#include "recommendation/Classifier.hpp"
#include "recommendation/IClassifier.hpp"
namespace Database
{
@@ -34,17 +34,22 @@ namespace Recommendation
class Engine : public IEngine
{
public:
Engine(Database::Session& session);
private:
void reloadSettings(Database::Session& session) override;
// Closest results first
std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) override;
std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) override;
std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) override;
std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) override;
private:
void clearClassifiers();
void addClassifier(std::unique_ptr<Classifier> classifier, unsigned priority);
void addClassifier(std::unique_ptr<IClassifier> classifier, unsigned priority);
std::map<unsigned, std::unique_ptr<Classifier>> _classifiers;
std::map<unsigned, std::unique_ptr<IClassifier>> _classifiers;
};
} // ns Recommendation
src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.hpp
-40
View File
@@ -1,40 +0,0 @@
/*
* Copyright (C) 2018 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <set>
#include "database/Types.hpp"
namespace Database {
class Session;
}
namespace Similarity {
namespace ClusterSearcher
{
std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount);
std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::set<Database::IdType>& tracksId, std::size_t maxCount);
std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount);
std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount);
};
} // namespace Similarity
src/libs/recommendation/impl/cluster/SimilarityClusterSearcher.cpp → src/libs/recommendation/impl/clusters/ClustersClassifier.cpp
+57 -9
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "SimilarityClusterSearcher.hpp"
#include "ClustersClassifier.hpp"
#include "database/Artist.hpp"
#include "database/Cluster.hpp"
@@ -26,11 +26,60 @@
#include "database/Track.hpp"
#include "database/TrackList.hpp"
namespace Similarity {
namespace ClusterSearcher {
namespace Recommendation {
std::unique_ptr<IClassifier> createClustersClassifier(Database::Session& session)
{
return std::make_unique<ClusterClassifier>(session);
}
ClusterClassifier::ClusterClassifier(Database::Session& session)
{
classify(session);
}
void
ClusterClassifier::classify(Database::Session& session)
{
auto transaction {session.createSharedTransaction()};
{
std::vector<Database::IdType> trackIds {Database::Track::getAllIdsWithClusters(session)};
_classifiedTracks = std::unordered_set<Database::IdType>(std::cbegin(trackIds), std::cend(trackIds));
}
{
std::vector<Database::IdType> releaseIds {Database::Release::getAllIdsWithClusters(session)};
_classifiedReleases = std::unordered_set<Database::IdType>(std::cbegin(releaseIds), std::cend(releaseIds));
}
{
std::vector<Database::IdType> artistIds {Database::Artist::getAllIdsWithClusters(session)};
_classifiedArtists = std::unordered_set<Database::IdType>(std::cbegin(artistIds), std::cend(artistIds));
}
}
bool
ClusterClassifier::isTrackClassified(Database::IdType trackId) const
{
return _classifiedTracks.find(trackId) != std::cend(_classifiedTracks);
}
bool
ClusterClassifier::isReleaseClassified(Database::IdType releaseId) const
{
return _classifiedReleases.find(releaseId) != std::cend(_classifiedReleases);
}
bool
ClusterClassifier::isArtistClassified(Database::IdType artistId) const
{
return _classifiedArtists.find(artistId) != std::cend(_classifiedArtists);
}
std::vector<Database::IdType>
getSimilarTracks(Database::Session& dbSession, const std::set<Database::IdType>& trackIds, std::size_t maxCount)
ClusterClassifier::getSimilarTracks(Database::Session& dbSession, const std::unordered_set<Database::IdType>& trackIds, std::size_t maxCount) const
{
auto transaction {dbSession.createSharedTransaction()};
@@ -43,7 +92,7 @@ getSimilarTracks(Database::Session& dbSession, const std::set<Database::IdType>&
}
std::vector<Database::IdType>
getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount)
ClusterClassifier::getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const
{
std::vector<Database::IdType> res;
@@ -62,7 +111,7 @@ getSimilarTracksFromTrackList(Database::Session& session, Database::IdType track
}
std::vector<Database::IdType>
getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount)
ClusterClassifier::getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std::size_t maxCount) const
{
std::vector<Database::IdType> res;
@@ -80,7 +129,7 @@ getSimilarReleases(Database::Session& dbSession, Database::IdType releaseId, std
}
std::vector<Database::IdType>
getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount)
ClusterClassifier::getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::size_t maxCount) const
{
std::vector<Database::IdType> res;
@@ -97,5 +146,4 @@ getSimilarArtists(Database::Session& dbSession, Database::IdType artistId, std::
return res;
}
} // namespace ClusterSearcher
} // namespace Similarity
} // namespace Recommendation
src/libs/recommendation/impl/clusters/ClustersClassifier.hpp
+56
View File
@@ -0,0 +1,56 @@
/*
* Copyright (C) 2018 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "recommendation/IClassifier.hpp"
namespace Recommendation
{
class ClusterClassifier : public IClassifier
{
public:
ClusterClassifier(Database::Session& session);
ClusterClassifier(const ClusterClassifier&) = delete;
ClusterClassifier(ClusterClassifier&&) = delete;
ClusterClassifier& operator=(const ClusterClassifier&) = delete;
ClusterClassifier& operator=(ClusterClassifier&&) = delete;
private:
bool isTrackClassified(Database::IdType trackId) const override;
bool isReleaseClassified(Database::IdType releaseId) const override;
bool isArtistClassified(Database::IdType artistId) const override;
std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const override;
std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const override;
std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const override;
std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const override;
void classify(Database::Session& session);
std::unordered_set<Database::IdType> _classifiedArtists;
std::unordered_set<Database::IdType> _classifiedReleases;
std::unordered_set<Database::IdType> _classifiedTracks;
};
} // namespace Recommendation
src/libs/recommendation/include/recommendation/ClustersClassifierCreator.hpp
+2 -7
View File
@@ -21,15 +21,10 @@
#include <memory>
namespace Database
{
class Session;
}
namespace Recommendation
{
class Classifier;
class IClassifier;
std::unique_ptr<Classifier> createClustersClassifier();
std::unique_ptr<IClassifier> createClustersClassifier(Database::Session& session);
}
src/libs/recommendation/include/recommendation/FeaturesClassifierCreator.hpp
+2 -7
View File
@@ -21,15 +21,10 @@
#include <memory>
namespace Database
{
class Session;
}
namespace Recommendation
{
class Classifier;
class IClassifier;
std::unique_ptr<Classifier> createFeaturesClassifier();
std::unique_ptr<IClassifier> createFeaturesClassifier();
}
src/libs/recommendation/include/recommendation/Classifier.hpp → src/libs/recommendation/include/recommendation/IClassifier.hpp
+5 -7
View File
@@ -32,21 +32,19 @@ namespace Database
namespace Recommendation
{
class Classifier
class IClassifier
{
public:
virtual ~Classifier() = default;
virtual void classify() = 0;
virtual ~IClassifier() = default;
virtual bool isTrackClassified(Database::IdType trackId) const = 0;
virtual bool isReleaseClassified(Database::IdType releaseId) const = 0;
virtual bool isArtistClassified(Database::IdType artistId) const = 0;
virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0;
virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) const = 0;
virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) const = 0;
virtual std::vector<Database::IdType> getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const = 0;
virtual std::vector<Database::IdType> getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const = 0;
virtual std::vector<Database::IdType> getSimilarReleases(Database::Session& session, Database::IdType releaseId, std::size_t maxCount) const = 0;
virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) const = 0;
};
} // ns Recommendation
src/libs/recommendation/include/recommendation/IEngine.hpp
+3 -3
View File
@@ -23,7 +23,6 @@
#include <unordered_set>
#include "database/Types.hpp"
#include "Classifier.hpp"
namespace Database
{
@@ -32,12 +31,13 @@ namespace Database
namespace Recommendation
{
class Classifier;
class IEngine
{
public:
virtual ~IEngine() = default;
virtual void reloadSettings(Database::Session& session) = 0;
// Closest results first
virtual std::vector<Database::IdType> getSimilarTracksFromTrackList(Database::Session& session, Database::IdType tracklistId, std::size_t maxCount) = 0;
virtual std::vector<Database::IdType> getSimilarTracks(Database::Session& session, const std::unordered_set<Database::IdType>& tracksId, std::size_t maxCount) = 0;
@@ -45,7 +45,7 @@ namespace Recommendation
virtual std::vector<Database::IdType> getSimilarArtists(Database::Session& session, Database::IdType artistId, std::size_t maxCount) = 0;
};
std::unique_ptr<IEngine> createEngine();
std::unique_ptr<IEngine> createEngine(Database::Session& session);
} // ns Recommendation
src/lms/main.cpp
+4 -1
View File
@@ -143,7 +143,10 @@ int main(int argc, char* argv[])
ServiceProvider<Auth::IPasswordService>::assign(Auth::createPasswordService(ServiceProvider<IConfig>::get()->getULong("login-throttler-max-entriees", 10000)));
Scanner::IMediaScanner& mediaScanner {ServiceProvider<Scanner::IMediaScanner>::assign(Scanner::createMediaScanner(database))};
ServiceProvider<Recommendation::IEngine>::assign(Recommendation::createEngine());
{
Database::Session session {database};
ServiceProvider<Recommendation::IEngine>::assign(Recommendation::createEngine(session));
}
CoverArt::IGrabber& coverArtGrabber {ServiceProvider<CoverArt::IGrabber>::assign(CoverArt::createGrabber(argv[0]))};
coverArtGrabber.setDefaultCover(server.appRoot() + "/images/unknown-cover.jpg");
src/lms/ui/admin/DatabaseSettingsView.cpp
+23 -23
View File
@@ -49,7 +49,7 @@ class DatabaseSettingsModel : public Wt::WFormModel
static const Field MediaDirectoryField;
static const Field UpdatePeriodField;
static const Field UpdateStartTimeField;
static const Field SimilarityEngineTypeField;
static const Field RecommendationEngineTypeField;
static const Field TagsField;
DatabaseSettingsModel()
@@ -60,7 +60,7 @@ class DatabaseSettingsModel : public Wt::WFormModel
addField(MediaDirectoryField);
addField(UpdatePeriodField);
addField(UpdateStartTimeField);
addField(SimilarityEngineTypeField);
addField(RecommendationEngineTypeField);
addField(TagsField);
auto dirValidator {std::make_shared<DirectoryValidator>()};
@@ -69,7 +69,7 @@ class DatabaseSettingsModel : public Wt::WFormModel
setValidator(UpdatePeriodField, createMandatoryValidator());
setValidator(UpdateStartTimeField, createMandatoryValidator());
setValidator(SimilarityEngineTypeField, createMandatoryValidator());
setValidator(RecommendationEngineTypeField, createMandatoryValidator());
setValidator(TagsField, createTagsValidator());
// populate the model with initial data
@@ -78,7 +78,7 @@ class DatabaseSettingsModel : public Wt::WFormModel
std::shared_ptr<Wt::WAbstractItemModel> updatePeriodModel() { return _updatePeriodModel; }
std::shared_ptr<Wt::WAbstractItemModel> updateStartTimeModel() { return _updateStartTimeModel; }
std::shared_ptr<Wt::WAbstractItemModel> similarityEngineTypeModel() { return _similarityEngineTypeModel; }
std::shared_ptr<Wt::WAbstractItemModel> recommendationEngineTypeModel() { return _recommendationEngineTypeModel; }
void loadData()
{
@@ -96,9 +96,9 @@ class DatabaseSettingsModel : public Wt::WFormModel
if (startTimeRow)
setValue(UpdateStartTimeField, _updateStartTimeModel->getString(*startTimeRow));
auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromValue(scanSettings->getSimilarityEngineType())};
if (similarityEngineTypeRow)
setValue(SimilarityEngineTypeField, _similarityEngineTypeModel->getString(*similarityEngineTypeRow));
auto recommendationEngineTypeRow {_recommendationEngineTypeModel->getRowFromValue(scanSettings->getRecommendationEngineType())};
if (recommendationEngineTypeRow)
setValue(RecommendationEngineTypeField, _recommendationEngineTypeModel->getString(*recommendationEngineTypeRow));
auto clusterTypes {scanSettings->getClusterTypes()};
if (!clusterTypes.empty())
@@ -125,9 +125,9 @@ class DatabaseSettingsModel : public Wt::WFormModel
if (startTimeRow)
scanSettings.modify()->setUpdateStartTime(_updateStartTimeModel->getValue(*startTimeRow));
auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromString(valueText(SimilarityEngineTypeField))};
if (similarityEngineTypeRow)
scanSettings.modify()->setSimilarityEngineType(_similarityEngineTypeModel->getValue(*similarityEngineTypeRow));
auto recommendationEngineTypeRow {_recommendationEngineTypeModel->getRowFromString(valueText(RecommendationEngineTypeField))};
if (recommendationEngineTypeRow)
scanSettings.modify()->setRecommendationEngineType(_recommendationEngineTypeModel->getValue(*recommendationEngineTypeRow));
auto clusterTypes {StringUtils::splitString(valueText(TagsField).toUTF8(), " ")};
scanSettings.modify()->setClusterTypes(LmsApp->getDbSession(), std::set<std::string>(clusterTypes.begin(), clusterTypes.end()));
@@ -156,22 +156,22 @@ class DatabaseSettingsModel : public Wt::WFormModel
_updateStartTimeModel->add(time.toString(), time);
}
_similarityEngineTypeModel = std::make_shared<ValueStringModel<ScanSettings::SimilarityEngineType>>();
_similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.clusters"), ScanSettings::SimilarityEngineType::Clusters);
_similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.features"), ScanSettings::SimilarityEngineType::Features);
_recommendationEngineTypeModel = std::make_shared<ValueStringModel<ScanSettings::RecommendationEngineType>>();
_recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.clusters"), ScanSettings::RecommendationEngineType::Clusters);
_recommendationEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.recommendation-engine-type.features"), ScanSettings::RecommendationEngineType::Features);
}
std::shared_ptr<ValueStringModel<ScanSettings::UpdatePeriod>> _updatePeriodModel;
std::shared_ptr<ValueStringModel<Wt::WTime>> _updateStartTimeModel;
std::shared_ptr<ValueStringModel<ScanSettings::SimilarityEngineType>> _similarityEngineTypeModel;
std::shared_ptr<ValueStringModel<ScanSettings::RecommendationEngineType>> _recommendationEngineTypeModel;
};
const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField = "media-directory";
const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField = "update-period";
const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField = "update-start-time";
const Wt::WFormModel::Field DatabaseSettingsModel::SimilarityEngineTypeField = "similarity-engine-type";
const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags";
const Wt::WFormModel::Field DatabaseSettingsModel::MediaDirectoryField = "media-directory";
const Wt::WFormModel::Field DatabaseSettingsModel::UpdatePeriodField = "update-period";
const Wt::WFormModel::Field DatabaseSettingsModel::UpdateStartTimeField = "update-start-time";
const Wt::WFormModel::Field DatabaseSettingsModel::RecommendationEngineTypeField = "recommendation-engine-type";
const Wt::WFormModel::Field DatabaseSettingsModel::TagsField = "tags";
DatabaseSettingsView::DatabaseSettingsView()
{
@@ -207,10 +207,10 @@ DatabaseSettingsView::refreshView()
updateStartTime->setModel(model->updateStartTimeModel());
t->setFormWidget(DatabaseSettingsModel::UpdateStartTimeField, std::move(updateStartTime));
// Similarity engine type
auto similarityEngineType {std::make_unique<Wt::WComboBox>()};
similarityEngineType->setModel(model->similarityEngineTypeModel());
t->setFormWidget(DatabaseSettingsModel::SimilarityEngineTypeField, std::move(similarityEngineType));
// recommendation engine type
auto recommendationEngineType {std::make_unique<Wt::WComboBox>()};
recommendationEngineType->setModel(model->recommendationEngineTypeModel());
t->setFormWidget(DatabaseSettingsModel::RecommendationEngineTypeField, std::move(recommendationEngineType));
// Tags
t->setFormWidget(DatabaseSettingsModel::TagsField, std::make_unique<Wt::WLineEdit>());
src/test/database/DatabaseTest.cpp
+36
View File
@@ -453,12 +453,24 @@ testSingleTrackSingleCluster(Session& session)
CHECK(track->getClusterIds().empty());
}
{
auto transaction {session.createSharedTransaction()};
CHECK(Track::getAllIdsWithClusters(session).empty());
}
{
auto transaction {session.createUniqueTransaction()};
cluster1.get().modify()->addTrack(track.get());
}
{
auto transaction {session.createSharedTransaction()};
auto tracks {Track::getAllIdsWithClusters(session)};
CHECK(tracks.size() == 1);
CHECK(tracks.front() == track.getId());
}
{
auto transaction {session.createSharedTransaction()};
auto clusters {Cluster::getAllOrphans(session)};
@@ -611,6 +623,11 @@ testSingleTrackSingleReleaseSingleCluster(Session& session)
ScopedClusterType clusterType {session, "MyClusterType"};
ScopedCluster cluster {session, clusterType .lockAndGet(), "MyCluster"};
{
auto transaction {session.createSharedTransaction()};
CHECK(Release::getAllIdsWithClusters(session).empty());
}
{
auto transaction {session.createUniqueTransaction()};
@@ -618,6 +635,13 @@ testSingleTrackSingleReleaseSingleCluster(Session& session)
cluster.get().modify()->addTrack(track.get());
}
{
auto transaction {session.createSharedTransaction()};
auto releases {Release::getAllIdsWithClusters(session)};
CHECK(releases.size() == 1);
CHECK(releases.front() == release.getId());
}
{
auto transaction {session.createSharedTransaction()};
@@ -858,6 +882,11 @@ testSingleTrackSingleReleaseSingleArtistSingleCluster(Session& session)
ScopedClusterType clusterType {session, "MyType"};
ScopedCluster cluster {session, clusterType.lockAndGet(), "MyCluster"};
{
auto transaction {session.createSharedTransaction()};
CHECK(Artist::getAllIdsWithClusters(session).empty());
}
{
auto transaction {session.createUniqueTransaction()};
@@ -875,6 +904,13 @@ testSingleTrackSingleReleaseSingleArtistSingleCluster(Session& session)
CHECK(Release::getAllOrphans(session).empty());
}
{
auto transaction {session.createSharedTransaction()};
auto artists {Artist::getAllIdsWithClusters(session)};
CHECK(artists.size() == 1);
CHECK(artists.front() == artist.getId());
}
{
auto transaction {session.createSharedTransaction()};
src/tools/recommendation-features/LmsRecommendationFeatures.cpp
+5 -6
View File
@@ -32,6 +32,7 @@
#include "utils/Service.hpp"
#include "utils/StreamLogger.hpp"
#include "recommendation/IEngine.hpp"
#include "recommendation/IClassifier.hpp"
#include "recommendation/FeaturesClassifierCreator.hpp"
int main(int argc, char *argv[])
@@ -50,11 +51,9 @@ int main(int argc, char *argv[])
Database::Db db {ServiceProvider<IConfig>::get()->getPath("working-dir") / "lms.db"};
Database::Session session {db};
auto classifier {Recommendation::createFeaturesClassifier()};
std::cout << "Classifying tracks..." << std::endl;
// may be long...
classifier->classify();
std::cout << "Classifying tracks..." << std::endl;
auto classifier {Recommendation::createFeaturesClassifier()};
std::cout << "Classifying tracks DONE" << std::endl;
const std::vector<Database::IdType> trackIds = std::invoke([&]()
@@ -106,7 +105,7 @@ int main(int argc, char *argv[])
};
std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl;
for (Database::IdType similarReleaseId : classifier->getSimilarReleases({releaseId}, 3))
for (Database::IdType similarReleaseId : classifier->getSimilarReleases(session, {releaseId}, 3))
std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl;
}
@@ -128,7 +127,7 @@ int main(int argc, char *argv[])
};
std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl;
for (Database::IdType similarArtistId : classifier->getSimilarArtists({artistId}, 3))
for (Database::IdType similarArtistId : classifier->getSimilarArtists(session, {artistId}, 3))
std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl;
}