diff --git a/configure.ac b/configure.ac index 9184ff02..53980dc6 100644 --- a/configure.ac +++ b/configure.ac @@ -15,7 +15,7 @@ fi AC_SUBST(MAGICKXX_CFLAGS) AC_SUBST(MAGICKXX_LIBS) -AC_CHECK_HEADERS([Wt/WApplication.h pstreams/pstream.h], +AC_CHECK_HEADERS([Wt/WApplication.h pstreams/pstream.h boost/asio.hpp], [], [AC_MSG_ERROR([Header not found or unusable !])]) @@ -80,6 +80,7 @@ AC_CONFIG_FILES([Makefile test/Makefile tools/Makefile tools/similarity/Makefile + tools/similarity-parameters/Makefile tools/metadata/Makefile]) AC_ARG_ENABLE([tools], diff --git a/src/Makefile.am b/src/Makefile.am index 8c459272..d4b3cfbd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -40,8 +40,8 @@ lms_SOURCES = \ $(srcdir)/database/ScanSettings.hpp \ $(srcdir)/database/Session.cpp \ $(srcdir)/database/Session.hpp \ - $(srcdir)/database/SimilaritySettings.cpp \ - $(srcdir)/database/SimilaritySettings.hpp \ + $(srcdir)/database/SessionPool.cpp \ + $(srcdir)/database/SessionPool.hpp \ $(srcdir)/database/SqlQuery.cpp \ $(srcdir)/database/SqlQuery.hpp \ $(srcdir)/database/Track.cpp \ @@ -69,6 +69,8 @@ lms_SOURCES = \ $(srcdir)/similarity/features/AcousticBrainzUtils.hpp \ $(srcdir)/similarity/features/SimilarityFeaturesCache.cpp \ $(srcdir)/similarity/features/SimilarityFeaturesCache.hpp \ + $(srcdir)/similarity/features/SimilarityFeaturesDefs.cpp \ + $(srcdir)/similarity/features/SimilarityFeaturesDefs.hpp \ $(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.cpp \ $(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.hpp \ $(srcdir)/similarity/features/SimilarityFeaturesSearcher.cpp \ @@ -151,8 +153,12 @@ lms_SOURCES = \ $(srcdir)/utils/Path.cpp \ $(srcdir)/utils/Path.hpp \ $(srcdir)/utils/Service.hpp \ + $(srcdir)/utils/StreamLogger.cpp \ + $(srcdir)/utils/StreamLogger.hpp \ $(srcdir)/utils/Utils.cpp \ - $(srcdir)/utils/Utils.hpp + $(srcdir)/utils/Utils.hpp \ + $(srcdir)/utils/WtLogger.cpp \ + $(srcdir)/utils/WtLogger.hpp lms_CXXFLAGS=-std=c++17 -I$(srcdir)/ui $(MAGICKXX_CFLAGS) -D_REENTRANT lms_LDADD=$(MAGICKXX_LIBS) diff --git a/src/api/subsonic/SubsonicResource.cpp b/src/api/subsonic/SubsonicResource.cpp index 52b2dbc2..30f1270c 100644 --- a/src/api/subsonic/SubsonicResource.cpp +++ b/src/api/subsonic/SubsonicResource.cpp @@ -33,6 +33,7 @@ #include "database/Cluster.hpp" #include "database/Db.hpp" #include "database/Release.hpp" +#include "database/Session.hpp" #include "database/Track.hpp" #include "database/TrackList.hpp" #include "database/User.hpp" @@ -128,42 +129,6 @@ struct RequestContext std::string userName; }; -using SessionMap = std::map>; -static std::map dbSessions; - -static -Session& -getOrCreateDbSession(Db& db) -{ - static std::mutex mutex; - - SessionMap* sessionMap {}; - - { - std::unique_lock lock {mutex}; - sessionMap = &dbSessions[std::this_thread::get_id()]; - } - - auto it {sessionMap->find(&db)}; - if (it != std::end(*sessionMap)) - return *it->second; - - auto res { sessionMap->try_emplace(&db, db.createSession())}; - assert(res.second); - - LMS_LOG(API_SUBSONIC, DEBUG) << "Created db session"; - - return *res.first->second; -} - -static -void -clearDbSessions() -{ - dbSessions.clear(); -} - - static std::string makeNameFilesystemCompatible(const std::string& name) @@ -274,16 +239,10 @@ struct MediaRetrievalResult }; SubsonicResource::SubsonicResource(Db& db) -: _db {db} +: _sessionPool {db} { } -SubsonicResource::~SubsonicResource() -{ - LMS_LOG(API_SUBSONIC, DEBUG) << "Cleaning db sessions..."; - clearDbSessions(); -} - static std::string parameterMapToDebugString(const Wt::Http::ParameterMap& parameterMap) { @@ -595,10 +554,10 @@ handleChangePassword(RequestContext& context) std::string username {getMandatoryParameterAs(context.parameters, "username")}; std::string password {decodePasswordIfNeeded(getMandatoryParameterAs(context.parameters, "password"))}; - if (!getService()->evaluatePasswordStrength(username, password)) + if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) throw PasswordTooWeakGenericError {}; - const User::PasswordHash hash {getService()->hashPassword(password)}; + const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; auto transaction {context.dbSession.createUniqueTransaction()}; @@ -677,10 +636,10 @@ handleCreateUserRequest(RequestContext& context) std::string password {decodePasswordIfNeeded(getMandatoryParameterAs(context.parameters, "password"))}; // Just ignore all the other fields as we don't handle them - if (!getService()->evaluatePasswordStrength(username, password)) + if (!ServiceProvider::get()->evaluatePasswordStrength(username, password)) throw PasswordTooWeakGenericError {}; - const User::PasswordHash hash {getService()->hashPassword(password)}; + const User::PasswordHash hash {ServiceProvider::get()->hashPassword(password)}; auto transaction {context.dbSession.createUniqueTransaction()}; @@ -960,7 +919,7 @@ handleGetArtistInfoRequestCommon(RequestContext& context, bool id3) artistInfoNode.createChild("musicBrainzId").setValue(artist->getMBID()); } - auto similarArtistsId {getService()->getSimilarArtists(context.dbSession, id.value, count)}; + auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, count)}; { auto transaction {context.dbSession.createSharedTransaction()}; @@ -1156,7 +1115,7 @@ handleGetSimilarSongsRequestCommon(RequestContext& context, bool id3) // Optional params std::size_t count {getParameterAs(context.parameters, "count").value_or(50)}; - auto similarArtistsId {getService()->getSimilarArtists(context.dbSession, id.value, 5)}; + auto similarArtistsId {ServiceProvider::get()->getSimilarArtists(context.dbSession, id.value, 5)}; auto transaction {context.dbSession.createSharedTransaction()}; @@ -1604,10 +1563,10 @@ handleUpdateUserRequest(RequestContext& context) if (password) { *password = decodePasswordIfNeeded(*password); - if (!getService()->evaluatePasswordStrength(username, *password)) + if (!ServiceProvider::get()->evaluatePasswordStrength(username, *password)) throw PasswordTooWeakGenericError {}; - hash = getService()->hashPassword(*password); + hash = ServiceProvider::get()->hashPassword(*password); } auto transaction {context.dbSession.createUniqueTransaction()}; @@ -1816,10 +1775,10 @@ handleGetCoverArt(RequestContext& context, Wt::Http::ResponseContinuation*) switch (id.type) { case Id::Type::Track: - res.data = getService()->getFromTrack(context.dbSession, id.value, Image::Format::JPEG, size); + res.data = ServiceProvider::get()->getFromTrack(context.dbSession, id.value, Image::Format::JPEG, size); break; case Id::Type::Release: - res.data = getService()->getFromRelease(context.dbSession, id.value, Image::Format::JPEG, size); + res.data = ServiceProvider::get()->getFromRelease(context.dbSession, id.value, Image::Format::JPEG, size); break; default: throw BadParameterGenericError {"id"}; @@ -1906,9 +1865,9 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp // Mandatory parameters const ClientInfo clientInfo {getClientInfo(parameters)}; - Session& dbSession {getOrCreateDbSession(_db)}; + SessionPool::ScopedSession dbSession {_sessionPool}; - switch (getService()->checkUserPassword(dbSession, + switch (ServiceProvider::get()->checkUserPassword(dbSession.get(), boost::asio::ip::address::from_string(request.clientAddress()), clientInfo.user, clientInfo.password)) { @@ -1920,16 +1879,16 @@ SubsonicResource::handleRequest(const Wt::Http::Request &request, Wt::Http::Resp throw LoginThrottledGenericError {}; } - RequestContext requestContext {.parameters = parameters, .dbSession = dbSession, .userName = clientInfo.user}; + RequestContext requestContext {.parameters = parameters, .dbSession = dbSession.get(), .userName = clientInfo.user}; auto itEntryPoint {requestEntryPoints.find(requestPath)}; if (itEntryPoint != requestEntryPoints.end()) { if (itEntryPoint->second.mustBeAdmin) { - auto transaction {dbSession.createSharedTransaction()}; + auto transaction {dbSession.get().createSharedTransaction()}; - User::pointer user {User::getByLoginName(dbSession, clientInfo.user)}; + User::pointer user {User::getByLoginName(dbSession.get(), clientInfo.user)}; if (!user || !user->isAdmin()) throw UserNotAuthorizedError {}; } diff --git a/src/api/subsonic/SubsonicResource.hpp b/src/api/subsonic/SubsonicResource.hpp index f96a92be..af9bc53c 100644 --- a/src/api/subsonic/SubsonicResource.hpp +++ b/src/api/subsonic/SubsonicResource.hpp @@ -21,6 +21,8 @@ #include #include +#include "database/SessionPool.hpp" + namespace Database { class Db; @@ -33,14 +35,13 @@ class SubsonicResource final : public Wt::WResource { public: SubsonicResource(Database::Db& db); - ~SubsonicResource(); static std::string getPath() { return "/rest/"; } private: void handleRequest(const Wt::Http::Request &request, Wt::Http::Response &response) override; - Database::Db& _db; + Database::SessionPool _sessionPool; }; } // namespace diff --git a/src/av/AvTranscoder.cpp b/src/av/AvTranscoder.cpp index fd13e053..1dac0797 100644 --- a/src/av/AvTranscoder.cpp +++ b/src/av/AvTranscoder.cpp @@ -38,7 +38,7 @@ static std::filesystem::path ffmpegPath; void Transcoder::init() { - ffmpegPath = getService()->getPath("ffmpeg-file", "/usr/bin/ffmpeg"); + ffmpegPath = ServiceProvider::get()->getPath("ffmpeg-file", "/usr/bin/ffmpeg"); if (!std::filesystem::exists(ffmpegPath)) throw LmsException {"File '" + ffmpegPath.string() + "' does not exist!"}; } diff --git a/src/database/Artist.cpp b/src/database/Artist.cpp index 1951b69a..0b457f1f 100644 --- a/src/database/Artist.cpp +++ b/src/database/Artist.cpp @@ -86,6 +86,15 @@ Artist::getAll(Session& session, std::optional offset, std::optiona return std::vector(res.begin(), res.end()); } +std::vector +Artist::getAllIds(Session& session) +{ + session.checkSharedLocked(); + + Wt::Dbo::collection res = session.getDboSession().query("SELECT id FROM artist"); + return std::vector(res.begin(), res.end()); +} + std::vector Artist::getAllOrphans(Session& session) { diff --git a/src/database/Artist.hpp b/src/database/Artist.hpp index b46cafb1..fb03bf8b 100644 --- a/src/database/Artist.hpp +++ b/src/database/Artist.hpp @@ -62,6 +62,7 @@ class Artist : public Wt::Dbo::Dbo bool& moreExpected); static std::vector getAll(Session& session, std::optional offset = {}, std::optional size = {}); + static std::vector getAllIds(Session& session); static std::vector getAllOrphans(Session& session); // No track related static std::vector getLastAdded(Session& session, Wt::WDateTime after, std::optional size = {}); diff --git a/src/database/Db.cpp b/src/database/Db.cpp index c6323e09..23edca35 100644 --- a/src/database/Db.cpp +++ b/src/database/Db.cpp @@ -40,18 +40,6 @@ Db::Db(const std::filesystem::path& dbPath) connectionPool->setTimeout(std::chrono::seconds(10)); _connectionPool = std::move(connectionPool); - - { - auto session {createSession()}; - session->prepareTables(); - } - -} - -std::unique_ptr -Db::createSession() -{ - return std::unique_ptr(new Session {_sharedMutex, *_connectionPool.get()}); } } // namespace Database diff --git a/src/database/Db.hpp b/src/database/Db.hpp index f279fe3e..fa07af26 100644 --- a/src/database/Db.hpp +++ b/src/database/Db.hpp @@ -24,8 +24,6 @@ #include -#include "Session.hpp" - namespace Database { // Session living class handling the database and the login @@ -35,9 +33,12 @@ class Db Db(const std::filesystem::path& dbPath); - std::unique_ptr createSession(); - private: + friend class Session; + + std::shared_mutex& getMutex() { return _sharedMutex; } + Wt::Dbo::SqlConnectionPool& getConnectionPool() { return *_connectionPool; } + std::shared_mutex _sharedMutex; std::unique_ptr _connectionPool; }; diff --git a/src/database/Release.cpp b/src/database/Release.cpp index 30587939..bbf26aaa 100644 --- a/src/database/Release.cpp +++ b/src/database/Release.cpp @@ -96,6 +96,15 @@ Release::getAll(Session& session, std::optional offset, std::option return std::vector(res.begin(), res.end()); } +std::vector +Release::getAllIds(Session& session) +{ + session.checkSharedLocked(); + + Wt::Dbo::collection res = session.getDboSession().query("SELECT id FROM release"); + return std::vector(res.begin(), res.end()); +} + std::vector Release::getAllOrderedByArtist(Session& session, std::optional offset, std::optional size) { diff --git a/src/database/Release.hpp b/src/database/Release.hpp index 494bee49..c2c6bbf1 100644 --- a/src/database/Release.hpp +++ b/src/database/Release.hpp @@ -52,6 +52,7 @@ class Release : public Wt::Dbo::Dbo static pointer getById(Session& session, IdType id); static std::vector getAllOrphans(Session& session); // no track related static std::vector getAll(Session& session, std::optional offset = {}, std::optional size = {}); + static std::vector getAllIds(Session& session); static std::vector getAllOrderedByArtist(Session& session, std::optional offset = {}, std::optional size = {}); static std::vector getAllRandom(Session& session, std::optional size = {}); static std::vector getLastAdded(Session& session, const Wt::WDateTime& after, std::optional offset = {}, std::optional size = {}); diff --git a/src/database/ScanSettings.hpp b/src/database/ScanSettings.hpp index bd908654..209e5829 100644 --- a/src/database/ScanSettings.hpp +++ b/src/database/ScanSettings.hpp @@ -34,6 +34,7 @@ class ScanSettings : public Wt::Dbo::Dbo public: using pointer = Wt::Dbo::ptr; + // Do not modify values (just add) enum class UpdatePeriod { Never = 0, Daily, @@ -41,6 +42,13 @@ class ScanSettings : public Wt::Dbo::Dbo Monthly }; + // Do not modify values (just add) + enum class SimilarityEngineType + { + Clusters = 0, + Features, + }; + static void init(Session& session); static pointer get(Session& session); @@ -52,12 +60,14 @@ class ScanSettings : public Wt::Dbo::Dbo UpdatePeriod getUpdatePeriod() const { return _updatePeriod; } std::vector> getClusterTypes() const; std::set getAudioFileExtensions() const; + SimilarityEngineType getSimilarityEngineType() const { return _similarityEngineType; } // Setters void setMediaDirectory(std::filesystem::path p); void setUpdateStartTime(Wt::WTime t) { _startTime = t; } void setUpdatePeriod(UpdatePeriod p) { _updatePeriod = p; } void setClusterTypes(Session& session, const std::set& clusterTypeNames); + void setSimilarityEngineType(SimilarityEngineType type) { _similarityEngineType = type; } void incScanVersion(); template @@ -68,6 +78,7 @@ class ScanSettings : public Wt::Dbo::Dbo Wt::Dbo::field(a, _startTime, "start_time"); Wt::Dbo::field(a, _updatePeriod, "update_period"); Wt::Dbo::field(a, _audioFileExtensions, "audio_file_extensions"); + Wt::Dbo::field(a, _similarityEngineType,"similarity_engine_type"); Wt::Dbo::hasMany(a, _clusterTypes, Wt::Dbo::ManyToOne, "scan_settings"); } @@ -77,6 +88,7 @@ class ScanSettings : public Wt::Dbo::Dbo std::string _mediaDirectory; Wt::WTime _startTime = Wt::WTime {0,0,0}; UpdatePeriod _updatePeriod {UpdatePeriod::Never}; + SimilarityEngineType _similarityEngineType {SimilarityEngineType::Clusters}; std::string _audioFileExtensions {".mp3 .ogg .oga .aac .m4a .flac .wav .wma .aif .aiff .ape .mpc .shn .opus"}; Wt::Dbo::collection> _clusterTypes; }; diff --git a/src/database/Session.cpp b/src/database/Session.cpp index 2d815fdb..71f8dbe6 100644 --- a/src/database/Session.cpp +++ b/src/database/Session.cpp @@ -19,14 +19,18 @@ #include "Session.hpp" +#include +#include +#include + #include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "Artist.hpp" #include "Cluster.hpp" +#include "Db.hpp" #include "Release.hpp" #include "ScanSettings.hpp" -#include "SimilaritySettings.hpp" #include "Track.hpp" #include "TrackArtistLink.hpp" #include "TrackList.hpp" @@ -35,7 +39,7 @@ namespace Database { -#define LMS_DATABASE_VERSION 7 +#define LMS_DATABASE_VERSION 8 using Version = std::size_t; @@ -96,30 +100,41 @@ Session::doDatabaseMigrationIfNeeded() throw LmsException {outdatedMsg}; } - switch (version) + while (version < LMS_DATABASE_VERSION) { - case 5: - LMS_LOG(DB, INFO) << "Migrating database from version 5..."; + LMS_LOG(DB, INFO) << "Migrating database from version " << version << "..."; + + if (version == 5) + { _session.execute("DELETE FROM auth_token"); // format has changed - break; - case 6: - LMS_LOG(DB, INFO) << "Migrating database from version 6..."; + } + else if (version == 6) + { // Just increment the scan version of the settings to make the next scheduled scan rescan everything ScanSettings::get(*this).modify()->incScanVersion(); - break; - - default: + } + else if (version == 7) + { + _session.execute("DROP TABLE similarity_settings"); + _session.execute("DROP TABLE similarity_settings_feature"); + _session.execute("ALTER TABLE scan_settings ADD similarity_engine_type INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(ScanSettings::SimilarityEngineType::Clusters)) + ")"); + } + else + { LMS_LOG(DB, ERROR) << "Database version " << version << " cannot be handled using migration"; throw LmsException { LMS_DATABASE_VERSION > version ? outdatedMsg : "Server binary outdated, please upgrade it to handle this database"}; + } + + ++version; } VersionInfo::get(*this).modify()->setVersion(LMS_DATABASE_VERSION); } -Session::Session(std::shared_mutex& mutex, Wt::Dbo::SqlConnectionPool& connectionPool) -: _mutex {mutex} +Session::Session(Db& db) +: _db {db} { - _session.setConnectionPool(connectionPool); + _session.setConnectionPool(_db.getConnectionPool()); _session.mapClass("version_info"); _session.mapClass("artist"); @@ -128,8 +143,6 @@ Session::Session(std::shared_mutex& mutex, Wt::Dbo::SqlConnectionPool& connectio _session.mapClass("cluster_type"); _session.mapClass("release"); _session.mapClass("scan_settings"); - _session.mapClass("similarity_settings"); - _session.mapClass("similarity_settings_feature"); _session.mapClass("track"); _session.mapClass("track_artist_link"); _session.mapClass("track_features"); @@ -179,25 +192,25 @@ SharedTransaction::~SharedTransaction() void Session::checkUniqueLocked() { - assert(lockDebug[&_mutex] == OwnedLock::Unique); + assert(lockDebug[&_db.getMutex()] == OwnedLock::Unique); } void Session::checkSharedLocked() { - assert(lockDebug[&_mutex] != OwnedLock::None); + assert(lockDebug[&_db.getMutex()] != OwnedLock::None); } UniqueTransaction Session::createUniqueTransaction() { - return UniqueTransaction{_mutex, _session}; + return UniqueTransaction{_db.getMutex(), _session}; } SharedTransaction Session::createSharedTransaction() { - return SharedTransaction{_mutex, _session}; + return SharedTransaction{_db.getMutex(), _session}; } void @@ -252,7 +265,6 @@ Session::prepareTables() auto uniqueTransaction {createUniqueTransaction()}; ScanSettings::init(*this); - SimilaritySettings::init(*this); } } diff --git a/src/database/Session.hpp b/src/database/Session.hpp index c102e256..427fc6fd 100644 --- a/src/database/Session.hpp +++ b/src/database/Session.hpp @@ -19,9 +19,11 @@ #pragma once -#include #include +#include #include +#include +#include #include #include @@ -54,9 +56,12 @@ class SharedTransaction Wt::Dbo::Transaction _transaction; }; +class Db; class Session { public: + Session (Db& database); + Session(const Session&) = delete; Session(Session&&) = delete; Session& operator=(const Session&) = delete; @@ -70,17 +75,16 @@ class Session void optimize(); + void prepareTables(); // need to run only once at startup + Wt::Dbo::Session& getDboSession() { return _session; } private: - friend class Db; - Session(std::shared_mutex& mutex, Wt::Dbo::SqlConnectionPool& connectionPool); void doDatabaseMigrationIfNeeded(); - void prepareTables(); // need to run only once at startup - std::shared_mutex& _mutex; + Db& _db; Wt::Dbo::Session _session; }; diff --git a/src/database/SessionPool.cpp b/src/database/SessionPool.cpp new file mode 100644 index 00000000..f6526aca --- /dev/null +++ b/src/database/SessionPool.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "SessionPool.hpp" + +#include "utils/Exception.hpp" +#include "utils/Logger.hpp" + +#include "Session.hpp" + +namespace Database { + +SessionPool::SessionPool(Db& database, std::size_t maxSessionCount) +: _db {database}, +_maxSessionCount {maxSessionCount} +{ +} + +Session& +SessionPool::acquireSession() +{ + std::scoped_lock lock {_mutex}; + + if (_freeSessions.empty()) + { + if (_acquiredSessions.size() == _maxSessionCount) + throw LmsException {"Too many database sessions!"}; + + _freeSessions.emplace_back(std::make_unique(_db)); + } + + std::unique_ptr session {std::move(_freeSessions.back())}; + _freeSessions.pop_back(); + _acquiredSessions.push_back(std::move(session)); + + return *_acquiredSessions.back().get(); +} + +void +SessionPool::releaseSession(Session& sessionToRelease) +{ + std::scoped_lock lock {_mutex}; + + auto it {std::find_if(std::begin(_acquiredSessions), std::end(_acquiredSessions), [&](const std::unique_ptr& session) { return session.get() == &sessionToRelease; })}; + if (it == std::end(_acquiredSessions)) + throw LmsException {"Unknown released Session!"}; + + std::unique_ptr session {std::move(*it)}; + _acquiredSessions.erase(it); + _freeSessions.push_back(std::move(session)); +} + +} // namespace Database diff --git a/src/database/SessionPool.hpp b/src/database/SessionPool.hpp new file mode 100644 index 00000000..f9019c36 --- /dev/null +++ b/src/database/SessionPool.hpp @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2013 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include "Session.hpp" + +namespace Database { + +class SessionPool +{ + public: + class ScopedSession + { + public: + ScopedSession(SessionPool& pool) : _pool {pool}, _session {_pool.acquireSession()} {} + ~ScopedSession() { _pool.releaseSession(_session); } + + ScopedSession(const ScopedSession&) = delete; + ScopedSession(ScopedSession&&) = delete; + ScopedSession& operator=(const ScopedSession&) = delete; + ScopedSession& operator=(ScopedSession&&) = delete; + + Session& get() { return _session; } + + private: + SessionPool& _pool; + Session& _session; + }; + + SessionPool(Db& database, std::size_t maxSessionCount = 30); + + SessionPool(const SessionPool&) = delete; + SessionPool(SessionPool&&) = delete; + SessionPool& operator=(const SessionPool&) = delete; + SessionPool& operator=(SessionPool&&) = delete; + + private: + friend class ScopedSession; + Session& acquireSession(); + void releaseSession(Session& session); + + std::mutex _mutex; + Db& _db; + std::size_t _maxSessionCount; + std::vector> _freeSessions; + std::vector> _acquiredSessions; +}; + +} // namespace Database + + diff --git a/src/database/SimilaritySettings.cpp b/src/database/SimilaritySettings.cpp deleted file mode 100644 index 27418e76..00000000 --- a/src/database/SimilaritySettings.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "SimilaritySettings.hpp" - -#include "utils/Logger.hpp" -#include "utils/Utils.hpp" - -#include "Session.hpp" -#include "TrackFeatures.hpp" - -namespace Database { - -struct TrackFeatureInfo -{ - std::string name; - std::size_t nbDimensions; - double weight; -}; - -static const std::vector defaultFeatures = -{ - { "lowlevel.spectral_contrast_coeffs.median", 6, 1. }, - { "lowlevel.erbbands.median", 40, 1. }, - { "tonal.hpcp.median", 36, 1. }, - { "lowlevel.melbands.median", 40, 1. }, - { "lowlevel.barkbands.median", 27, 1. }, - { "lowlevel.mfcc.mean", 13, 1. }, - { "lowlevel.gfcc.mean", 13, 1. }, -}; - -SimilaritySettingsFeature::SimilaritySettingsFeature(Wt::Dbo::ptr settings, const std::string& name, std::size_t nbDimensions, double weight) -: _name(name), -_nbDimensions(nbDimensions), -_weight(weight), -_settings(settings) -{ -} - -SimilaritySettingsFeature::pointer -SimilaritySettingsFeature::create(Session& session, Wt::Dbo::ptr settings, const std::string& name, std::size_t nbDimensions, double weight) -{ - session.checkUniqueLocked(); - - SimilaritySettingsFeature::pointer res {session.getDboSession().add(std::make_unique(settings, name, nbDimensions, weight))}; - session.getDboSession().flush(); - - return res; -} - -void -SimilaritySettings::init(Session& session) -{ - session.checkUniqueLocked(); - - pointer settings {session.getDboSession().find()}; - if (settings) - return; - - settings = session.getDboSession().add(std::make_unique()); - for (const auto& feature : defaultFeatures) - SimilaritySettingsFeature::create(session, settings, feature.name, feature.nbDimensions, feature.weight); -} - - -SimilaritySettings::pointer -SimilaritySettings::get(Session& session) -{ - session.checkSharedLocked(); - - return session.getDboSession().find(); -} - -std::vector> -SimilaritySettings::getFeatures() const -{ - return std::vector>(_features.begin(), _features.end()); -} - -} // namespace Database - diff --git a/src/database/SimilaritySettings.hpp b/src/database/SimilaritySettings.hpp deleted file mode 100644 index 9e22f36d..00000000 --- a/src/database/SimilaritySettings.hpp +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include - -namespace Database { - -class Session; -class SimilaritySettings; - -class SimilaritySettingsFeature : public Wt::Dbo::Dbo -{ - public: - using pointer = Wt::Dbo::ptr; - - SimilaritySettingsFeature() = default; - SimilaritySettingsFeature(Wt::Dbo::ptr settings, const std::string& name, std::size_t nbDimensions, double weight); - - static pointer create(Session& session, Wt::Dbo::ptr settings, const std::string& name, std::size_t nbDimensions, double weight = 1); - - const std::string& getName() const { return _name; } ; - std::size_t getNbDimensions() const { return static_cast(_nbDimensions); } - double getWeight() const { return _weight; } - - template - void persist(Action& a) - { - Wt::Dbo::field(a, _name, "name"); - Wt::Dbo::field(a, _nbDimensions, "dimension_count"); - Wt::Dbo::field(a, _weight, "weight"); - - Wt::Dbo::belongsTo(a, _settings, "similarity_settings", Wt::Dbo::OnDeleteCascade); - } - - private: - std::string _name; - int _nbDimensions; - double _weight; - - Wt::Dbo::ptr _settings; -}; - -class SimilaritySettings : public Wt::Dbo::Dbo -{ - public: - - enum class EngineType - { - Features = 0, - Clusters = 1, - }; - - using pointer = Wt::Dbo::ptr; - - // Utils - static void init(Session& session); - static pointer get(Session& session); - - // Accessors Read - std::size_t getVersion() const { return _settingsVersion; } - EngineType getEngineType() const { return _engineType; } - std::vector> getFeatures() const; - - // Setters - void setEngineType(EngineType type) { _engineType = type; } - - template - void persist(Action& a) - { - Wt::Dbo::field(a, _settingsVersion, "settings_version"); - Wt::Dbo::field(a, _engineType, "engine_type"); - - Wt::Dbo::hasMany(a, _features, Wt::Dbo::ManyToOne, "similarity_settings"); - } - - private: - - int _settingsVersion {}; - EngineType _engineType {EngineType::Clusters}; - - Wt::Dbo::collection> _features; -}; - - -} // namespace Database - diff --git a/src/database/Track.cpp b/src/database/Track.cpp index f188c7ed..f28d9d9c 100644 --- a/src/database/Track.cpp +++ b/src/database/Track.cpp @@ -171,6 +171,20 @@ Track::getClusters(void) const return clusters; } +std::vector +Track::getClusterIds(void) const +{ + assert(self()); + assert(IdIsValid(self()->id())); + assert(session()); + + Wt::Dbo::collection res = session()->query + ("SELECT DISTINCT c.id FROM cluster c INNER JOIN track_cluster t_c ON t_c.cluster_id = c.id INNER JOIN track t ON t.id = t_c.track_id") + .where("t.id = ?").bind(self()->id()); + + return std::vector(res.begin(), res.end()); +} + bool Track::hasTrackFeatures() const { @@ -377,6 +391,20 @@ Track::getArtists(TrackArtistLink::Type type) const return std::vector>(artists.begin(), artists.end()); } +std::vector +Track::getArtistIds(TrackArtistLink::Type type) const +{ + assert(self()); + assert(IdIsValid(self()->id())); + assert(session()); + + Wt::Dbo::collection artists {session()->query("SELECT a.id from artist a INNER JOIN track_artist_link t_a_l ON a.id = t_a_l.artist_id INNER JOIN track t ON t.id = t_a_l.track_id") + .where("t.id = ?").bind(self()->id()) + .where("t_a_l.type = ?").bind(type)}; + + return std::vector(artists.begin(), artists.end()); +} + std::vector> Track::getArtistLinks() const { diff --git a/src/database/Track.hpp b/src/database/Track.hpp index 2ed74a11..898eacc3 100644 --- a/src/database/Track.hpp +++ b/src/database/Track.hpp @@ -70,8 +70,8 @@ class Track : public Wt::Dbo::Dbo static std::vector getAll(Session& session, std::optional limit = {}); static std::vector getAllRandom(Session& session, std::optional limit = {}); - static std::vector getAllIds(Session& session); // nested transaction - static std::vector getAllPaths(Session& session); // nested transaction + static std::vector getAllIds(Session& session); + static std::vector getAllPaths(Session& session); static std::vector getMBIDDuplicates(Session& session); static std::vector getLastAdded(Session& session, const Wt::WDateTime& after, std::optional size = 1); static std::vector getAllWithMBIDAndMissingFeatures(Session& session); @@ -115,9 +115,11 @@ class Track : public Wt::Dbo::Dbo std::optional getCopyright() const; std::optional getCopyrightURL() const; std::vector> getArtists(TrackArtistLink::Type type = TrackArtistLink::Type::Artist) const; + std::vector getArtistIds(TrackArtistLink::Type type = TrackArtistLink::Type::Artist) const; std::vector> getArtistLinks() const; Wt::Dbo::ptr getRelease() const { return _release; } std::vector> getClusters() const; + std::vector getClusterIds() const; bool hasTrackFeatures() const; Wt::Dbo::ptr getTrackFeatures() const; diff --git a/src/database/TrackFeatures.cpp b/src/database/TrackFeatures.cpp index 7cdc6be9..6a812bc9 100644 --- a/src/database/TrackFeatures.cpp +++ b/src/database/TrackFeatures.cpp @@ -41,53 +41,47 @@ TrackFeatures::create(Session& session, Wt::Dbo::ptr track, const std::st return session.getDboSession().add(std::make_unique(track, jsonEncodedFeatures)); } -std::vector -TrackFeatures::getFeatures(const std::string& featureNode) const +FeatureValues +TrackFeatures::getFeatureValues(const FeatureName& featureNode) const { - std::vector res; - - std::map> features = { {featureNode, {}} }; - if (!getFeatures( features )) - return res; - - res = std::move(features[featureNode]); - - return res; + FeatureValuesMap featuresValuesMap {getFeatureValuesMap({featureNode})}; + return std::move(featuresValuesMap[featureNode]); } -bool -TrackFeatures::getFeatures(std::map /*values*/>& features) const +FeatureValuesMap +TrackFeatures::getFeatureValuesMap(const std::unordered_set& featureNames) const { try { + std::istringstream iss {_data}; boost::property_tree::ptree root; - std::istringstream iss(_data); boost::property_tree::read_json(iss, root); - for (auto& featureNode : features) + FeatureValuesMap res; + for (const FeatureName& featureName : featureNames) { - auto node = root.get_child(featureNode.first); + FeatureValues& featureValues {res[featureName]}; + + auto node {root.get_child(featureName)}; bool hasChildren = false; for (const auto& child : node.get_child("")) { hasChildren = true; - featureNode.second.push_back(child.second.get_value()); + featureValues.push_back(child.second.get_value()); } if (!hasChildren) - { - featureNode.second.push_back(node.get_value()); - } + featureValues.push_back(node.get_value()); } - return true; + return res; } catch (boost::property_tree::ptree_error& error) { - LMS_LOG(SIMILARITY, ERROR) << "Track " << _track.id() << ": ptree exception: " << error.what(); - return false; + LMS_LOG(DB, ERROR) << "Track " << _track.id() << ": ptree exception: " << error.what(); + return {}; } } diff --git a/src/database/TrackFeatures.hpp b/src/database/TrackFeatures.hpp index 69e5f185..86b58679 100644 --- a/src/database/TrackFeatures.hpp +++ b/src/database/TrackFeatures.hpp @@ -20,6 +20,9 @@ #pragma once #include +#include +#include +#include #include @@ -30,6 +33,10 @@ namespace Database { class Session; class Track; +using FeatureName = std::string; +using FeatureValues = std::vector; +using FeatureValuesMap = std::unordered_map; + class TrackFeatures : public Wt::Dbo::Dbo { public: @@ -42,8 +49,8 @@ class TrackFeatures : public Wt::Dbo::Dbo // Create utility static pointer create(Session& session, Wt::Dbo::ptr track, const std::string& jsonEncodedFeatures); - std::vector getFeatures(const std::string& featureNode) const; - bool getFeatures(std::map /*values*/>& featureNodes) const; + FeatureValues getFeatureValues(const FeatureName& feature) const; + FeatureValuesMap getFeatureValuesMap(const std::unordered_set& featureNames) const; template void persist(Action& a) diff --git a/src/main/main.cpp b/src/main/main.cpp index b09f044d..805de6c0 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -35,35 +35,35 @@ #include "similarity/SimilaritySearcher.hpp" #include "ui/LmsApplication.hpp" #include "utils/Config.hpp" -#include "utils/Logger.hpp" #include "utils/Service.hpp" +#include "utils/WtLogger.hpp" std::vector generateWtConfig(std::string execPath) { std::vector args; - const std::filesystem::path wtConfigPath {getService()->getPath("working-dir") / "wt_config.xml"}; - const std::filesystem::path wtLogFilePath {getService()->getPath("log-file", "/var/log/lms.log")}; - const std::filesystem::path wtAccessLogFilePath {getService()->getPath("access-log-file", "/var/log/lms.access.log")}; + const std::filesystem::path wtConfigPath {ServiceProvider::get()->getPath("working-dir") / "wt_config.xml"}; + const std::filesystem::path wtLogFilePath {ServiceProvider::get()->getPath("log-file", "/var/log/lms.log")}; + const std::filesystem::path wtAccessLogFilePath {ServiceProvider::get()->getPath("access-log-file", "/var/log/lms.access.log")}; args.push_back(execPath); args.push_back("--config=" + wtConfigPath.string()); - args.push_back("--docroot=" + getService()->getString("docroot")); - args.push_back("--approot=" + getService()->getString("approot")); - args.push_back("--resources-dir=" + getService()->getString("wt-resources")); + args.push_back("--docroot=" + ServiceProvider::get()->getString("docroot")); + args.push_back("--approot=" + ServiceProvider::get()->getString("approot")); + args.push_back("--resources-dir=" + ServiceProvider::get()->getString("wt-resources")); - if (getService()->getBool("tls-enable", false)) + if (ServiceProvider::get()->getBool("tls-enable", false)) { - args.push_back("--https-port=" + std::to_string( getService()->getULong("listen-port", 5082))); - args.push_back("--https-address=" + getService()->getString("listen-addr", "0.0.0.0")); - args.push_back("--ssl-certificate=" + getService()->getString("tls-cert")); - args.push_back("--ssl-private-key=" + getService()->getString("tls-key")); - args.push_back("--ssl-tmp-dh=" + getService()->getString("tls-dh")); + args.push_back("--https-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); + args.push_back("--https-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); + args.push_back("--ssl-certificate=" + ServiceProvider::get()->getString("tls-cert")); + args.push_back("--ssl-private-key=" + ServiceProvider::get()->getString("tls-key")); + args.push_back("--ssl-tmp-dh=" + ServiceProvider::get()->getString("tls-dh")); } else { - args.push_back("--http-port=" + std::to_string( getService()->getULong("listen-port", 5082))); - args.push_back("--http-address=" + getService()->getString("listen-addr", "0.0.0.0")); + args.push_back("--http-port=" + std::to_string( ServiceProvider::get()->getULong("listen-port", 5082))); + args.push_back("--http-address=" + ServiceProvider::get()->getString("listen-addr", "0.0.0.0")); } if (!wtAccessLogFilePath.empty()) @@ -74,8 +74,8 @@ std::vector generateWtConfig(std::string execPath) pt.put("server.application-settings..location", "*"); pt.put("server.application-settings.log-file", wtLogFilePath.string()); - pt.put("server.application-settings.log-config", getService()->getString("log-config", "* -debug -info:WebRequest")); - pt.put("server.application-settings.behind-reverse-proxy", getService()->getBool("behind-reverse-proxy", false)); + pt.put("server.application-settings.log-config", ServiceProvider::get()->getString("log-config", "* -debug -info:WebRequest")); + pt.put("server.application-settings.behind-reverse-proxy", ServiceProvider::get()->getBool("behind-reverse-proxy", false)); pt.put("server.application-settings.progressive-bootstrap", true); std::ofstream oss(wtConfigPath.string().c_str(), std::ios::out); @@ -109,10 +109,11 @@ int main(int argc, char* argv[]) close(STDIN_FILENO); ServiceProvider::create(configFilePath); + ServiceProvider::create(); // Make sure the working directory exists - std::filesystem::create_directories(getService()->getPath("working-dir")); - std::filesystem::create_directories(getService()->getPath("working-dir") / "cache"); + std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir")); + std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache"); // Construct WT configuration and get the argc/argv back std::vector wtServerArgs = generateWtConfig(argv[0]); @@ -132,16 +133,20 @@ int main(int argc, char* argv[]) Av::Transcoder::init(); // Initializing a connection pool to the database that will be shared along services - Database::Db database {getService()->getPath("working-dir") / "lms.db"}; + Database::Db database {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; + { + Database::Session session {database}; + session.prepareTables(); + } UserInterface::LmsApplicationGroupContainer appGroups; // Service initialization order is important - ServiceProvider::create(getService()->getULong("login-throttler-max-entriees", 10000)); - ServiceProvider::create(getService()->getULong("login-throttler-max-entriees", 10000)); - Scanner::MediaScanner& mediaScanner {ServiceProvider::create(database.createSession())}; + ServiceProvider::create(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000)); + ServiceProvider::create(ServiceProvider::get()->getULong("login-throttler-max-entriees", 10000)); + Scanner::MediaScanner& mediaScanner {ServiceProvider::create(database)}; - Similarity::FeaturesScannerAddon similarityFeaturesScannerAddon {database.createSession()}; + Similarity::FeaturesScannerAddon similarityFeaturesScannerAddon {database}; mediaScanner.setAddon(similarityFeaturesScannerAddon); @@ -153,7 +158,7 @@ int main(int argc, char* argv[]) API::Subsonic::SubsonicResource subsonicResource {database}; // bind API resources - if (getService()->getBool("api-subsonic", true)) + if (ServiceProvider::get()->getBool("api-subsonic", true)) server.addResource(&subsonicResource, subsonicResource.getPath()); // bind UI entry point diff --git a/src/scanner/MediaScanner.cpp b/src/scanner/MediaScanner.cpp index 3f18075f..76bfa9be 100644 --- a/src/scanner/MediaScanner.cpp +++ b/src/scanner/MediaScanner.cpp @@ -191,8 +191,8 @@ getOrCreateClusters(Session& session, const MetaData::Clusters& clustersNames) namespace Scanner { -MediaScanner::MediaScanner(std::unique_ptr dbSession) -: _dbSession {std::move(dbSession)} +MediaScanner::MediaScanner(Database::Db& db) +: _dbSession {db} { _ioService.setThreadCount(1); @@ -444,7 +444,7 @@ MediaScanner::scan(boost::system::error_code err) } LMS_LOG(DBUPDATER, INFO) << "Optimizing db..."; - _dbSession->optimize(); + _dbSession.optimize(); LMS_LOG(DBUPDATER, INFO) << "Optimize db done!"; } @@ -452,9 +452,9 @@ void MediaScanner::refreshScanSettings() { { - auto transaction {_dbSession->createSharedTransaction()}; + auto transaction {_dbSession.createSharedTransaction()}; - ScanSettings::pointer scanSettings {ScanSettings::get(*_dbSession)}; + ScanSettings::pointer scanSettings {ScanSettings::get(_dbSession)}; LMS_LOG(DBUPDATER, INFO) << "Using scan settings version " << scanSettings->getScanVersion(); @@ -521,9 +521,9 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S if (!forceScan) { // Skip file if last write is the same - auto transaction {_dbSession->createSharedTransaction()}; + auto transaction {_dbSession.createSharedTransaction()}; - const Track::pointer track {Track::getByPath(*_dbSession, file)}; + const Track::pointer track {Track::getByPath(_dbSession, file)}; if (track && track->getLastWriteTime().toTime_t() == lastWriteTime.toTime_t() && track->getScanVersion() == _scanVersion) @@ -542,9 +542,9 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S stats.scans++; - auto uniqueTransaction {_dbSession->createUniqueTransaction()}; + auto uniqueTransaction {_dbSession.createUniqueTransaction()}; - Track::pointer track {Track::getByPath(*_dbSession, file) }; + Track::pointer track {Track::getByPath(_dbSession, file) }; // We estimate this is an audio file if: // - we found a least one audio stream @@ -588,25 +588,25 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S } // ***** Clusters - std::vector clusters {getOrCreateClusters(*_dbSession, trackInfo->clusters)}; + std::vector clusters {getOrCreateClusters(_dbSession, trackInfo->clusters)}; // ***** Artists - std::vector artists {getOrCreateArtists(*_dbSession, trackInfo->artists)}; + std::vector artists {getOrCreateArtists(_dbSession, trackInfo->artists)}; // ***** Release artists - std::vector releaseArtists {getOrCreateArtists(*_dbSession, trackInfo->albumArtists)}; + std::vector releaseArtists {getOrCreateArtists(_dbSession, trackInfo->albumArtists)}; // ***** Release Release::pointer release; if (trackInfo->album) - release = getOrCreateRelease(*_dbSession, *trackInfo->album); + release = getOrCreateRelease(_dbSession, *trackInfo->album); // If file already exist, update data // Otherwise, create it if (!track) { // Create a new song - track = Track::create(*_dbSession, file); + track = Track::create(_dbSession, file); LMS_LOG(DBUPDATER, INFO) << "Adding '" << file.string() << "'"; stats.additions++; } @@ -629,10 +629,10 @@ MediaScanner::scanAudioFile(const std::filesystem::path& file, bool forceScan, S track.modify()->clearArtistLinks(); for (const auto& artist : artists) - track.modify()->addArtistLink(Database::TrackArtistLink::create(*_dbSession, track, artist, Database::TrackArtistLink::Type::Artist)); + track.modify()->addArtistLink(Database::TrackArtistLink::create(_dbSession, track, artist, Database::TrackArtistLink::Type::Artist)); for (const auto& releaseArtist : releaseArtists) - track.modify()->addArtistLink(Database::TrackArtistLink::create(*_dbSession, track, releaseArtist, Database::TrackArtistLink::Type::ReleaseArtist)); + track.modify()->addArtistLink(Database::TrackArtistLink::create(_dbSession, track, releaseArtist, Database::TrackArtistLink::Type::ReleaseArtist)); track.modify()->setScanVersion(_scanVersion); track.modify()->setRelease(release); @@ -733,8 +733,8 @@ MediaScanner::removeMissingTracks(ScanStats& stats) { std::vector trackPaths; { - auto transaction {_dbSession->createSharedTransaction()}; - trackPaths = Track::getAllPaths(*_dbSession);; + auto transaction {_dbSession.createSharedTransaction()}; + trackPaths = Track::getAllPaths(_dbSession);; } LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks..."; @@ -745,9 +745,9 @@ MediaScanner::removeMissingTracks(ScanStats& stats) if (!checkFile(trackPath, _mediaDirectory, _fileExtensions)) { - auto transaction {_dbSession->createUniqueTransaction()}; + auto transaction {_dbSession.createUniqueTransaction()}; - Track::pointer track {Track::getByPath(*_dbSession, trackPath)}; + Track::pointer track {Track::getByPath(_dbSession, trackPath)}; if (track) { track.remove(); @@ -762,10 +762,10 @@ MediaScanner::removeOrphanEntries() { LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters..."; { - auto transaction {_dbSession->createUniqueTransaction()}; + auto transaction {_dbSession.createUniqueTransaction()}; // Now process orphan Cluster (no track) - auto clusters {Cluster::getAllOrphans(*_dbSession)}; + auto clusters {Cluster::getAllOrphans(_dbSession)}; for (auto& cluster : clusters) { LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'"; @@ -775,9 +775,9 @@ MediaScanner::removeOrphanEntries() LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists..."; { - auto transaction {_dbSession->createUniqueTransaction()}; + auto transaction {_dbSession.createUniqueTransaction()}; - auto artists {Artist::getAllOrphans(*_dbSession)}; + auto artists {Artist::getAllOrphans(_dbSession)}; for (auto& artist : artists) { LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'"; @@ -787,9 +787,9 @@ MediaScanner::removeOrphanEntries() LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases..."; { - auto transaction {_dbSession->createUniqueTransaction()}; + auto transaction {_dbSession.createUniqueTransaction()}; - auto releases {Release::getAllOrphans(*_dbSession)}; + auto releases {Release::getAllOrphans(_dbSession)}; for (auto& release : releases) { LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'"; @@ -805,9 +805,9 @@ MediaScanner::checkDuplicatedAudioFiles(ScanStats& stats) { LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files"; - auto transaction {_dbSession->createSharedTransaction()}; + auto transaction {_dbSession.createSharedTransaction()}; - const std::vector tracks = Database::Track::getMBIDDuplicates(*_dbSession); + const std::vector tracks = Database::Track::getMBIDDuplicates(_dbSession); for (const Track::pointer& track : tracks) { LMS_LOG(DBUPDATER, INFO) << "Found duplicated MBID [" << track->getMBID() << "], file: " << track->getPath().string() << " - " << track->getName(); diff --git a/src/scanner/MediaScanner.hpp b/src/scanner/MediaScanner.hpp index 9d4c2f0c..0ad1586a 100644 --- a/src/scanner/MediaScanner.hpp +++ b/src/scanner/MediaScanner.hpp @@ -41,7 +41,7 @@ namespace Scanner { class MediaScanner { public: - MediaScanner(std::unique_ptr dbSession); + MediaScanner(Database::Db& db); void setAddon(MediaScannerAddon& addon); @@ -110,7 +110,7 @@ class MediaScanner Wt::Signal _sigScanInProgress; std::chrono::system_clock::time_point _lastScanInProgressEmit {}; Wt::Signal _sigScheduled; - std::unique_ptr _dbSession; + Database::Session _dbSession; MetaData::TagLibParser _metadataParser; std::vector _addons; diff --git a/src/similarity/SimilaritySearcher.cpp b/src/similarity/SimilaritySearcher.cpp index 34e43835..cf2513d0 100644 --- a/src/similarity/SimilaritySearcher.cpp +++ b/src/similarity/SimilaritySearcher.cpp @@ -22,7 +22,7 @@ #include "features/SimilarityFeaturesScannerAddon.hpp" #include "cluster/SimilarityClusterSearcher.hpp" -#include "database/SimilaritySettings.hpp" +#include "database/ScanSettings.hpp" #include "database/TrackList.hpp" namespace Similarity { @@ -32,10 +32,11 @@ Searcher::Searcher(FeaturesScannerAddon& somAddon) {} static -Database::SimilaritySettings::EngineType getEngineType(Database::Session& dbSession) +Database::ScanSettings::SimilarityEngineType +getEngineType(Database::Session& dbSession) { auto transaction {dbSession.createSharedTransaction()}; - return Database::SimilaritySettings::get(dbSession)->getEngineType(); + return Database::ScanSettings::get(dbSession)->getSimilarityEngineType(); } std::vector @@ -58,7 +59,7 @@ Searcher::getSimilarTracksFromTrackList(Database::Session& session, Database::Id if (trackIds.empty()) return {}; - if (engineType == Database::SimilaritySettings::EngineType::Features + if (engineType == Database::ScanSettings::SimilarityEngineType::Features && somSearcher && std::any_of(std::cbegin(trackIds), std::cend(trackIds), [&](Database::IdType trackId) { return somSearcher->isTrackClassified(trackId); } )) { @@ -74,7 +75,7 @@ Searcher::getSimilarTracks(Database::Session& dbSession, const std::setisTrackClassified(trackId); } )) { @@ -90,7 +91,7 @@ Searcher::getSimilarReleases(Database::Session& dbSession, Database::IdType rele auto engineType {getEngineType(dbSession)}; auto somSearcher {_somAddon.getSearcher()}; - if (engineType == Database::SimilaritySettings::EngineType::Features + if (engineType == Database::ScanSettings::SimilarityEngineType::Features && somSearcher && somSearcher->isReleaseClassified(releaseId)) { @@ -106,7 +107,7 @@ Searcher::getSimilarArtists(Database::Session& dbSession, Database::IdType artis auto engineType {getEngineType(dbSession)}; auto somSearcher {_somAddon.getSearcher()}; - if (engineType == Database::SimilaritySettings::EngineType::Features + if (engineType == Database::ScanSettings::SimilarityEngineType::Features && somSearcher && somSearcher->isArtistClassified(artistId)) { diff --git a/src/similarity/features/AcousticBrainzUtils.cpp b/src/similarity/features/AcousticBrainzUtils.cpp index e58121c6..26f74f3b 100644 --- a/src/similarity/features/AcousticBrainzUtils.cpp +++ b/src/similarity/features/AcousticBrainzUtils.cpp @@ -38,7 +38,7 @@ getJsonData(const std::string& mbid) { static const std::string defaultAPIURL = "https://acousticbrainz.org/api/v1/"; - const std::string url {getService()->getString("acousticbrainz-api-url", defaultAPIURL) + mbid + "/low-level"}; + const std::string url {ServiceProvider::get()->getString("acousticbrainz-api-url", defaultAPIURL) + mbid + "/low-level"}; boost::asio::io_service ioService; diff --git a/src/similarity/features/SimilarityFeaturesCache.cpp b/src/similarity/features/SimilarityFeaturesCache.cpp index 1c9ed6b6..8c5589ab 100644 --- a/src/similarity/features/SimilarityFeaturesCache.cpp +++ b/src/similarity/features/SimilarityFeaturesCache.cpp @@ -33,7 +33,7 @@ namespace Similarity { static std::filesystem::path getCacheDirectory() { - return getService()->getPath("working-dir") / "cache" / "features"; + return ServiceProvider::get()->getPath("working-dir") / "cache" / "features"; } static std::filesystem::path getCacheNetworkFilePath() @@ -243,7 +243,7 @@ FeaturesCache::read() void FeaturesCache::write() { - std::filesystem::create_directories(getService()->getPath("working-dir") / "cache" / "features"); + std::filesystem::create_directories(ServiceProvider::get()->getPath("working-dir") / "cache" / "features"); if (!networkToCacheFile(_network, getCacheNetworkFilePath()) || !objectPositionToCacheFile(_trackPositions, getCacheTrackPositionsFilePath())) diff --git a/src/similarity/features/SimilarityFeaturesDefs.cpp b/src/similarity/features/SimilarityFeaturesDefs.cpp new file mode 100644 index 00000000..a0aecc4d --- /dev/null +++ b/src/similarity/features/SimilarityFeaturesDefs.cpp @@ -0,0 +1,402 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "SimilarityFeaturesDefs.hpp" + +#include +#include + +#include "utils/Exception.hpp" + +namespace Similarity { + +static const std::unordered_map featureDefinitions +{ + { "lowlevel.average_loudness", {1}}, + { "lowlevel.barkbands.dmean", {27}}, + { "lowlevel.barkbands.dmean2", {27}}, + { "lowlevel.barkbands.dvar", {27}}, + { "lowlevel.barkbands.dvar2", {27}}, + { "lowlevel.barkbands.max", {27}}, + { "lowlevel.barkbands.mean", {27}}, + { "lowlevel.barkbands.median", {27}}, + { "lowlevel.barkbands.min", {27}}, + { "lowlevel.barkbands.var", {27}}, + { "lowlevel.barkbands_crest.dmean", {1}}, + { "lowlevel.barkbands_crest.dmean2", {1}}, + { "lowlevel.barkbands_crest.dvar", {1}}, + { "lowlevel.barkbands_crest.dvar2", {1}}, + { "lowlevel.barkbands_crest.max", {1}}, + { "lowlevel.barkbands_crest.mean", {1}}, + { "lowlevel.barkbands_crest.median", {1}}, + { "lowlevel.barkbands_crest.min", {1}}, + { "lowlevel.barkbands_crest.var", {1}}, + { "lowlevel.barkbands_flatness_db.dmean", {1}}, + { "lowlevel.barkbands_flatness_db.dmean2", {1}}, + { "lowlevel.barkbands_flatness_db.dvar", {1}}, + { "lowlevel.barkbands_flatness_db.dvar2", {1}}, + { "lowlevel.barkbands_flatness_db.max", {1}}, + { "lowlevel.barkbands_flatness_db.mean", {1}}, + { "lowlevel.barkbands_flatness_db.median", {1}}, + { "lowlevel.barkbands_flatness_db.min", {1}}, + { "lowlevel.barkbands_flatness_db.var", {1}}, + { "lowlevel.barkbands_kurtosis.dmean", {1}}, + { "lowlevel.barkbands_kurtosis.dmean2", {1}}, + { "lowlevel.barkbands_kurtosis.dvar", {1}}, + { "lowlevel.barkbands_kurtosis.dvar2", {1}}, + { "lowlevel.barkbands_kurtosis.max", {1}}, + { "lowlevel.barkbands_kurtosis.mean", {1}}, + { "lowlevel.barkbands_kurtosis.median", {1}}, + { "lowlevel.barkbands_kurtosis.min", {1}}, + { "lowlevel.barkbands_kurtosis.var", {1}}, + { "lowlevel.barkbands_skewness.dmean", {1}}, + { "lowlevel.barkbands_skewness.dmean2", {1}}, + { "lowlevel.barkbands_skewness.dvar", {1}}, + { "lowlevel.barkbands_skewness.dvar2", {1}}, + { "lowlevel.barkbands_skewness.max", {1}}, + { "lowlevel.barkbands_skewness.mean", {1}}, + { "lowlevel.barkbands_skewness.median", {1}}, + { "lowlevel.barkbands_skewness.min", {1}}, + { "lowlevel.barkbands_skewness.var", {1}}, + { "lowlevel.barkbands_spread.dmean", {1}}, + { "lowlevel.barkbands_spread.dmean2", {1}}, + { "lowlevel.barkbands_spread.dvar", {1}}, + { "lowlevel.barkbands_spread.dvar2", {1}}, + { "lowlevel.barkbands_spread.max", {1}}, + { "lowlevel.barkbands_spread.mean", {1}}, + { "lowlevel.barkbands_spread.median", {1}}, + { "lowlevel.barkbands_spread.min", {1}}, + { "lowlevel.barkbands_spread.var", {1}}, + { "lowlevel.dissonance.dmean", {1}}, + { "lowlevel.dissonance.dmean2", {1}}, + { "lowlevel.dissonance.dvar", {1}}, + { "lowlevel.dissonance.dvar2", {1}}, + { "lowlevel.dissonance.max", {1}}, + { "lowlevel.dissonance.mean", {1}}, + { "lowlevel.dissonance.median", {1}}, + { "lowlevel.dissonance.min", {1}}, + { "lowlevel.dissonance.var", {1}}, + { "lowlevel.dynamic_complexity", {1}}, + { "lowlevel.spectral_contrast_coeffs.dmean", {6}}, + { "lowlevel.spectral_contrast_coeffs.dmean2", {6}}, + { "lowlevel.spectral_contrast_coeffs.dvar", {6}}, + { "lowlevel.spectral_contrast_coeffs.dvar2", {6}}, + { "lowlevel.spectral_contrast_coeffs.max", {6}}, + { "lowlevel.spectral_contrast_coeffs.mean", {6}}, + { "lowlevel.spectral_contrast_coeffs.median", {6}}, + { "lowlevel.spectral_contrast_coeffs.min", {6}}, + { "lowlevel.spectral_contrast_coeffs.var", {6}}, + { "lowlevel.erbbands.dmean", {40}}, + { "lowlevel.erbbands.dmean2", {40}}, + { "lowlevel.erbbands.dvar", {40}}, + { "lowlevel.erbbands.dvar2", {40}}, + { "lowlevel.erbbands.max", {40}}, + { "lowlevel.erbbands.mean", {40}}, + { "lowlevel.erbbands.median", {40}}, + { "lowlevel.erbbands.min", {40}}, + { "lowlevel.erbbands.var", {40}}, + { "lowlevel.gfcc.mean", {13}}, + { "lowlevel.hfc.dmean", {1}}, + { "lowlevel.hfc.dmean2", {1}}, + { "lowlevel.hfc.dvar", {1}}, + { "lowlevel.hfc.dvar2", {1}}, + { "lowlevel.hfc.max", {1}}, + { "lowlevel.hfc.mean", {1}}, + { "lowlevel.hfc.median", {1}}, + { "lowlevel.hfc.min", {1}}, + { "lowlevel.hfc.var", {1}}, + { "tonal.hpcp.median", {36}}, + { "lowlevel.melbands.dmean", {40}}, + { "lowlevel.melbands.dmean2", {40}}, + { "lowlevel.melbands.dvar", {40}}, + { "lowlevel.melbands.dvar2", {40}}, + { "lowlevel.melbands.max", {40}}, + { "lowlevel.melbands.mean", {40}}, + { "lowlevel.melbands.median", {40}}, + { "lowlevel.melbands.min", {40}}, + { "lowlevel.melbands.var", {40}}, + { "lowlevel.melbands_crest.dmean", {1}}, + { "lowlevel.melbands_crest.dmean2", {1}}, + { "lowlevel.melbands_crest.dvar", {1}}, + { "lowlevel.melbands_crest.dvar2", {1}}, + { "lowlevel.melbands_crest.max", {1}}, + { "lowlevel.melbands_crest.mean", {1}}, + { "lowlevel.melbands_crest.median", {1}}, + { "lowlevel.melbands_crest.min", {1}}, + { "lowlevel.melbands_crest.var", {1}}, + { "lowlevel.melbands_flatness_db.dmean", {1}}, + { "lowlevel.melbands_flatness_db.dmean2", {1}}, + { "lowlevel.melbands_flatness_db.dvar", {1}}, + { "lowlevel.melbands_flatness_db.dvar2", {1}}, + { "lowlevel.melbands_flatness_db.max", {1}}, + { "lowlevel.melbands_flatness_db.mean", {1}}, + { "lowlevel.melbands_flatness_db.median", {1}}, + { "lowlevel.melbands_flatness_db.min", {1}}, + { "lowlevel.melbands_flatness_db.var", {1}}, + { "lowlevel.melbands_kurtosis.dmean", {1}}, + { "lowlevel.melbands_kurtosis.dmean2", {1}}, + { "lowlevel.melbands_kurtosis.dvar", {1}}, + { "lowlevel.melbands_kurtosis.dvar2", {1}}, + { "lowlevel.melbands_kurtosis.max", {1}}, + { "lowlevel.melbands_kurtosis.mean", {1}}, + { "lowlevel.melbands_kurtosis.median", {1}}, + { "lowlevel.melbands_kurtosis.min", {1}}, + { "lowlevel.melbands_kurtosis.var", {1}}, + { "lowlevel.melbands_skewness.dmean", {1}}, + { "lowlevel.melbands_skewness.dmean2", {1}}, + { "lowlevel.melbands_skewness.dvar", {1}}, + { "lowlevel.melbands_skewness.dvar2", {1}}, + { "lowlevel.melbands_skewness.max", {1}}, + { "lowlevel.melbands_skewness.mean", {1}}, + { "lowlevel.melbands_skewness.median", {1}}, + { "lowlevel.melbands_skewness.min", {1}}, + { "lowlevel.melbands_skewness.var", {1}}, + { "lowlevel.melbands_spread.dmean", {1}}, + { "lowlevel.melbands_spread.dmean2", {1}}, + { "lowlevel.melbands_spread.dvar", {1}}, + { "lowlevel.melbands_spread.dvar2", {1}}, + { "lowlevel.melbands_spread.max", {1}}, + { "lowlevel.melbands_spread.mean", {1}}, + { "lowlevel.melbands_spread.median", {1}}, + { "lowlevel.melbands_spread.min", {1}}, + { "lowlevel.melbands_spread.var", {1}}, + { "lowlevel.mfcc.mean", {13}}, + { "lowlevel.pitch_salience.dmean", {1}}, + { "lowlevel.pitch_salience.dmean2", {1}}, + { "lowlevel.pitch_salience.dvar", {1}}, + { "lowlevel.pitch_salience.dvar2", {1}}, + { "lowlevel.pitch_salience.max", {1}}, + { "lowlevel.pitch_salience.mean", {1}}, + { "lowlevel.pitch_salience.median", {1}}, + { "lowlevel.pitch_salience.min", {1}}, + { "lowlevel.pitch_salience.var", {1}}, + { "lowlevel.silence_rate_30dB.dmean", {1}}, + { "lowlevel.silence_rate_30dB.dmean2", {1}}, + { "lowlevel.silence_rate_30dB.dvar", {1}}, + { "lowlevel.silence_rate_30dB.dvar2", {1}}, + { "lowlevel.silence_rate_30dB.max", {1}}, + { "lowlevel.silence_rate_30dB.mean", {1}}, + { "lowlevel.silence_rate_30dB.median", {1}}, + { "lowlevel.silence_rate_30dB.min", {1}}, + { "lowlevel.silence_rate_30dB.var", {1}}, + { "lowlevel.silence_rate_60dB.dmean", {1}}, + { "lowlevel.silence_rate_60dB.dmean2", {1}}, + { "lowlevel.silence_rate_60dB.dvar", {1}}, + { "lowlevel.silence_rate_60dB.dvar2", {1}}, + { "lowlevel.silence_rate_60dB.max", {1}}, + { "lowlevel.silence_rate_60dB.mean", {1}}, + { "lowlevel.silence_rate_60dB.median", {1}}, + { "lowlevel.silence_rate_60dB.min", {1}}, + { "lowlevel.silence_rate_60dB.var", {1}}, + { "lowlevel.spectral_centroid.dmean", {1}}, + { "lowlevel.spectral_centroid.dmean2", {1}}, + { "lowlevel.spectral_centroid.dvar", {1}}, + { "lowlevel.spectral_centroid.dvar2", {1}}, + { "lowlevel.spectral_centroid.max", {1}}, + { "lowlevel.spectral_centroid.mean", {1}}, + { "lowlevel.spectral_centroid.median", {1}}, + { "lowlevel.spectral_centroid.min", {1}}, + { "lowlevel.spectral_centroid.var", {1}}, + { "lowlevel.spectral_complexity.dmean", {1}}, + { "lowlevel.spectral_complexity.dmean2", {1}}, + { "lowlevel.spectral_complexity.dvar", {1}}, + { "lowlevel.spectral_complexity.dvar2", {1}}, + { "lowlevel.spectral_complexity.max", {1}}, + { "lowlevel.spectral_complexity.mean", {1}}, + { "lowlevel.spectral_complexity.median", {1}}, + { "lowlevel.spectral_complexity.min", {1}}, + { "lowlevel.spectral_complexity.var", {1}}, + { "lowlevel.spectral_contrast_coeffs.dmean", {6}}, + { "lowlevel.spectral_contrast_coeffs.dmean2", {6}}, + { "lowlevel.spectral_contrast_coeffs.dvar", {6}}, + { "lowlevel.spectral_contrast_coeffs.dvar2", {6}}, + { "lowlevel.spectral_contrast_coeffs.max", {6}}, + { "lowlevel.spectral_contrast_coeffs.mean", {6}}, + { "lowlevel.spectral_contrast_coeffs.median", {6}}, + { "lowlevel.spectral_contrast_coeffs.min", {6}}, + { "lowlevel.spectral_contrast_coeffs.var", {6}}, + { "lowlevel.spectral_contrast_valleys.dmean", {6}}, + { "lowlevel.spectral_contrast_valleys.dmean2", {6}}, + { "lowlevel.spectral_contrast_valleys.dvar", {6}}, + { "lowlevel.spectral_contrast_valleys.dvar2", {6}}, + { "lowlevel.spectral_contrast_valleys.max", {6}}, + { "lowlevel.spectral_contrast_valleys.mean", {6}}, + { "lowlevel.spectral_contrast_valleys.median", {6}}, + { "lowlevel.spectral_contrast_valleys.min", {6}}, + { "lowlevel.spectral_contrast_valleys.var", {6}}, + { "lowlevel.spectral_decrease.dmean", {1}}, + { "lowlevel.spectral_decrease.dmean2", {1}}, + { "lowlevel.spectral_decrease.dvar", {1}}, + { "lowlevel.spectral_decrease.dvar2", {1}}, + { "lowlevel.spectral_decrease.max", {1}}, + { "lowlevel.spectral_decrease.mean", {1}}, + { "lowlevel.spectral_decrease.median", {1}}, + { "lowlevel.spectral_decrease.min", {1}}, + { "lowlevel.spectral_decrease.var", {1}}, + { "lowlevel.spectral_energy.dmean", {1}}, + { "lowlevel.spectral_energy.dmean2", {1}}, + { "lowlevel.spectral_energy.dvar", {1}}, + { "lowlevel.spectral_energy.dvar2", {1}}, + { "lowlevel.spectral_energy.max", {1}}, + { "lowlevel.spectral_energy.mean", {1}}, + { "lowlevel.spectral_energy.median", {1}}, + { "lowlevel.spectral_energy.min", {1}}, + { "lowlevel.spectral_energy.var", {1}}, + { "lowlevel.spectral_energyband_high.dmean", {1}}, + { "lowlevel.spectral_energyband_high.dmean2", {1}}, + { "lowlevel.spectral_energyband_high.dvar", {1}}, + { "lowlevel.spectral_energyband_high.dvar2", {1}}, + { "lowlevel.spectral_energyband_high.max", {1}}, + { "lowlevel.spectral_energyband_high.mean", {1}}, + { "lowlevel.spectral_energyband_high.median", {1}}, + { "lowlevel.spectral_energyband_high.min", {1}}, + { "lowlevel.spectral_energyband_high.var", {1}}, + { "lowlevel.spectral_energyband_low.dmean", {1}}, + { "lowlevel.spectral_energyband_low.dmean2", {1}}, + { "lowlevel.spectral_energyband_low.dvar", {1}}, + { "lowlevel.spectral_energyband_low.dvar2", {1}}, + { "lowlevel.spectral_energyband_low.max", {1}}, + { "lowlevel.spectral_energyband_low.mean", {1}}, + { "lowlevel.spectral_energyband_low.median", {1}}, + { "lowlevel.spectral_energyband_low.min", {1}}, + { "lowlevel.spectral_energyband_low.var", {1}}, + { "lowlevel.spectral_energyband_middle_high.dmean", {1}}, + { "lowlevel.spectral_energyband_middle_high.dmean2", {1}}, + { "lowlevel.spectral_energyband_middle_high.dvar", {1}}, + { "lowlevel.spectral_energyband_middle_high.dvar2", {1}}, + { "lowlevel.spectral_energyband_middle_high.max", {1}}, + { "lowlevel.spectral_energyband_middle_high.mean", {1}}, + { "lowlevel.spectral_energyband_middle_high.median", {1}}, + { "lowlevel.spectral_energyband_middle_high.min", {1}}, + { "lowlevel.spectral_energyband_middle_high.var", {1}}, + { "lowlevel.spectral_energyband_middle_low.dmean", {1}}, + { "lowlevel.spectral_energyband_middle_low.dmean2", {1}}, + { "lowlevel.spectral_energyband_middle_low.dvar", {1}}, + { "lowlevel.spectral_energyband_middle_low.dvar2", {1}}, + { "lowlevel.spectral_energyband_middle_low.max", {1}}, + { "lowlevel.spectral_energyband_middle_low.mean", {1}}, + { "lowlevel.spectral_energyband_middle_low.median", {1}}, + { "lowlevel.spectral_energyband_middle_low.min", {1}}, + { "lowlevel.spectral_energyband_middle_low.var", {1}}, + { "lowlevel.spectral_entropy.dmean", {1}}, + { "lowlevel.spectral_entropy.dmean2", {1}}, + { "lowlevel.spectral_entropy.dvar", {1}}, + { "lowlevel.spectral_entropy.dvar2", {1}}, + { "lowlevel.spectral_entropy.max", {1}}, + { "lowlevel.spectral_entropy.mean", {1}}, + { "lowlevel.spectral_entropy.median", {1}}, + { "lowlevel.spectral_entropy.min", {1}}, + { "lowlevel.spectral_entropy.var", {1}}, + { "lowlevel.spectral_flux.dmean", {1}}, + { "lowlevel.spectral_flux.dmean2", {1}}, + { "lowlevel.spectral_flux.dvar", {1}}, + { "lowlevel.spectral_flux.dvar2", {1}}, + { "lowlevel.spectral_flux.max", {1}}, + { "lowlevel.spectral_flux.mean", {1}}, + { "lowlevel.spectral_flux.median", {1}}, + { "lowlevel.spectral_flux.min", {1}}, + { "lowlevel.spectral_flux.var", {1}}, + { "lowlevel.spectral_kurtosis.dmean", {1}}, + { "lowlevel.spectral_kurtosis.dmean2", {1}}, + { "lowlevel.spectral_kurtosis.dvar", {1}}, + { "lowlevel.spectral_kurtosis.dvar2", {1}}, + { "lowlevel.spectral_kurtosis.max", {1}}, + { "lowlevel.spectral_kurtosis.mean", {1}}, + { "lowlevel.spectral_kurtosis.median", {1}}, + { "lowlevel.spectral_kurtosis.min", {1}}, + { "lowlevel.spectral_kurtosis.var", {1}}, + { "lowlevel.spectral_rms.dmean", {1}}, + { "lowlevel.spectral_rms.dmean2", {1}}, + { "lowlevel.spectral_rms.dvar", {1}}, + { "lowlevel.spectral_rms.dvar2", {1}}, + { "lowlevel.spectral_rms.max", {1}}, + { "lowlevel.spectral_rms.mean", {1}}, + { "lowlevel.spectral_rms.median", {1}}, + { "lowlevel.spectral_rms.min", {1}}, + { "lowlevel.spectral_rms.var", {1}}, + { "lowlevel.spectral_rolloff.dmean", {1}}, + { "lowlevel.spectral_rolloff.dmean2", {1}}, + { "lowlevel.spectral_rolloff.dvar", {1}}, + { "lowlevel.spectral_rolloff.dvar2", {1}}, + { "lowlevel.spectral_rolloff.max", {1}}, + { "lowlevel.spectral_rolloff.mean", {1}}, + { "lowlevel.spectral_rolloff.median", {1}}, + { "lowlevel.spectral_rolloff.min", {1}}, + { "lowlevel.spectral_rolloff.var", {1}}, + { "lowlevel.spectral_skewness.dmean", {1}}, + { "lowlevel.spectral_skewness.dmean2", {1}}, + { "lowlevel.spectral_skewness.dvar", {1}}, + { "lowlevel.spectral_skewness.dvar2", {1}}, + { "lowlevel.spectral_skewness.max", {1}}, + { "lowlevel.spectral_skewness.mean", {1}}, + { "lowlevel.spectral_skewness.median", {1}}, + { "lowlevel.spectral_skewness.min", {1}}, + { "lowlevel.spectral_skewness.var", {1}}, + { "lowlevel.spectral_spread.dmean", {1}}, + { "lowlevel.spectral_spread.dmean2", {1}}, + { "lowlevel.spectral_spread.dvar", {1}}, + { "lowlevel.spectral_spread.dvar2", {1}}, + { "lowlevel.spectral_spread.max", {1}}, + { "lowlevel.spectral_spread.mean", {1}}, + { "lowlevel.spectral_spread.median", {1}}, + { "lowlevel.spectral_spread.min", {1}}, + { "lowlevel.spectral_spread.var", {1}}, + { "lowlevel.spectral_strongpeak.dmean", {1}}, + { "lowlevel.spectral_strongpeak.dmean2", {1}}, + { "lowlevel.spectral_strongpeak.dvar", {1}}, + { "lowlevel.spectral_strongpeak.dvar2", {1}}, + { "lowlevel.spectral_strongpeak.max", {1}}, + { "lowlevel.spectral_strongpeak.mean", {1}}, + { "lowlevel.spectral_strongpeak.median", {1}}, + { "lowlevel.spectral_strongpeak.min", {1}}, + { "lowlevel.spectral_strongpeak.var", {1}}, + { "lowlevel.zerocrossingrate.dmean", {1}}, + { "lowlevel.zerocrossingrate.dmean2", {1}}, + { "lowlevel.zerocrossingrate.dvar", {1}}, + { "lowlevel.zerocrossingrate.dvar2", {1}}, + { "lowlevel.zerocrossingrate.max", {1}}, + { "lowlevel.zerocrossingrate.mean", {1}}, + { "lowlevel.zerocrossingrate.median", {1}}, + { "lowlevel.zerocrossingrate.min", {1}}, + { "lowlevel.zerocrossingrate.var", {1}}, +}; + +FeatureDef +getFeatureDef(const FeatureName& featureName) +{ + auto it {featureDefinitions.find(featureName)}; + if (it == std::cend(featureDefinitions)) + throw LmsException {"Unhandled requested feature '" + featureName + "'"}; + + return it->second; +} + +FeatureNames +getFeatureNames() +{ + FeatureNames res; + + std::transform(std::cbegin(featureDefinitions), std::cend(featureDefinitions), + std::inserter(res, std::begin(res)), [](auto itFeature) { return itFeature.first; }); + + return res; +} + +} // namespace Similarity + diff --git a/src/similarity/features/SimilarityFeaturesDefs.hpp b/src/similarity/features/SimilarityFeaturesDefs.hpp new file mode 100644 index 00000000..58eed273 --- /dev/null +++ b/src/similarity/features/SimilarityFeaturesDefs.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include +#include + +namespace Similarity { + +using FeatureName = std::string; +using FeatureNames = std::unordered_set; +using FeatureValue = double; +using FeatureValues = std::vector; +using FeatureValuesMap = std::unordered_map; + +struct FeatureDef +{ + std::size_t nbDimensions {}; +}; + +FeatureDef getFeatureDef(const FeatureName& featureName); +FeatureNames getFeatureNames(); + +struct FeatureSettings +{ + double weight {}; +}; +using FeatureSettingsMap = std::unordered_map; + +} // namespace Similarity diff --git a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp b/src/similarity/features/SimilarityFeaturesScannerAddon.cpp index f8821934..c0f3107e 100644 --- a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp +++ b/src/similarity/features/SimilarityFeaturesScannerAddon.cpp @@ -20,8 +20,8 @@ #include "SimilarityFeaturesScannerAddon.hpp" #include "AcousticBrainzUtils.hpp" +#include "database/ScanSettings.hpp" #include "database/Track.hpp" -#include "database/SimilaritySettings.hpp" #include "database/TrackFeatures.hpp" #include "similarity/features/SimilarityFeaturesCache.hpp" #include "utils/Config.hpp" @@ -29,7 +29,13 @@ namespace Similarity { -namespace { +static +bool +hasAtLeastOneTrackWithFeatures(Database::Session& session) +{ + auto transaction {session.createSharedTransaction()}; + return !Database::Track::getAllIdsWithFeatures(session, 1).empty(); +} struct TrackInfo { @@ -37,6 +43,7 @@ struct TrackInfo std::string mbid; }; +static std::vector getTracksWithMBIDAndMissingFeatures(Database::Session& dbSession) { @@ -51,15 +58,13 @@ getTracksWithMBIDAndMissingFeatures(Database::Session& dbSession) return res; } -} // namespace - -FeaturesScannerAddon::FeaturesScannerAddon(std::unique_ptr dbSession) -: _dbSession {std::move(dbSession)} +FeaturesScannerAddon::FeaturesScannerAddon(Database::Db& db) +: _dbSession {db} { std::optional cache {Similarity::FeaturesCache::read()}; if (cache) { - auto searcher {std::make_shared(*_dbSession.get(), *cache, [&]() { return _stopRequested; })}; + auto searcher {std::make_shared(_dbSession, *cache, [&]() { return _stopRequested; })}; if (searcher->isValid()) std::atomic_store(&_searcher, searcher); } @@ -80,9 +85,9 @@ FeaturesScannerAddon::requestStop() void FeaturesScannerAddon::trackUpdated(Database::IdType trackId) { - auto uniqueTransaction {_dbSession->createUniqueTransaction()}; + auto uniqueTransaction {_dbSession.createUniqueTransaction()}; - auto track {Database::Track::getById(*_dbSession, trackId)}; + auto track {Database::Track::getById(_dbSession, trackId)}; if (!track) return; @@ -93,9 +98,9 @@ void FeaturesScannerAddon::preScanComplete() { { - auto transaction {_dbSession->createSharedTransaction()}; + auto transaction {_dbSession.createSharedTransaction()}; - if (Database::SimilaritySettings::get(*_dbSession)->getEngineType() != Database::SimilaritySettings::EngineType::Features) + if (Database::ScanSettings::get(_dbSession)->getSimilarityEngineType() != Database::ScanSettings::SimilarityEngineType::Features) { LMS_LOG(DBUPDATER, INFO) << "Do not fetch features since the engine type does not make use of them"; return; @@ -103,7 +108,7 @@ FeaturesScannerAddon::preScanComplete() } LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features..."; - const std::vector tracksInfo {getTracksWithMBIDAndMissingFeatures(*_dbSession)}; + const std::vector tracksInfo {getTracksWithMBIDAndMissingFeatures(_dbSession)}; LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features DONE (found " << tracksInfo.size() << ")"; if (!tracksInfo.empty()) @@ -125,20 +130,17 @@ FeaturesScannerAddon::updateSearcher() { LMS_LOG(SIMILARITY, INFO) << "Updating searcher..."; - std::vector trackIds; + if (!hasAtLeastOneTrackWithFeatures(_dbSession)) { - auto transaction {_dbSession->createSharedTransaction()}; - trackIds = Database::Track::getAllIdsWithFeatures(*_dbSession); - } - - if (trackIds.empty()) - { - LMS_LOG(DBUPDATER, INFO) << "No track suitable for features similarity clustering"; + LMS_LOG(DBUPDATER, INFO) << "No track found with features!"; std::atomic_store(&_searcher, std::shared_ptr{}); return; } - auto searcher {std::make_shared(*_dbSession, [&]() { return _stopRequested; })}; + Similarity::FeaturesSearcher::TrainSettings trainSettings; + trainSettings.featureSettingsMap = FeaturesSearcher::getDefaultTrainFeatureSettings(); + + auto searcher {std::make_shared(_dbSession, trainSettings, [&]() { return _stopRequested; })}; if (searcher->isValid()) { std::atomic_store(&_searcher, searcher); @@ -168,13 +170,13 @@ FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const std::string& } { - auto uniqueTransaction {_dbSession->createUniqueTransaction()}; + auto uniqueTransaction {_dbSession.createUniqueTransaction()}; - Wt::Dbo::ptr track {Database::Track::getById(*_dbSession, trackId)}; + Wt::Dbo::ptr track {Database::Track::getById(_dbSession, trackId)}; if (!track) return false; - Database::TrackFeatures::create(*_dbSession, track, data); + Database::TrackFeatures::create(_dbSession, track, data); } return true; diff --git a/src/similarity/features/SimilarityFeaturesScannerAddon.hpp b/src/similarity/features/SimilarityFeaturesScannerAddon.hpp index 34945db7..7bbc8900 100644 --- a/src/similarity/features/SimilarityFeaturesScannerAddon.hpp +++ b/src/similarity/features/SimilarityFeaturesScannerAddon.hpp @@ -24,13 +24,17 @@ #include "SimilarityFeaturesSearcher.hpp" +namespace Database { + class Db; +} + namespace Similarity { class FeaturesScannerAddon final : public Scanner::MediaScannerAddon { public: - FeaturesScannerAddon(std::unique_ptr dbSession); + FeaturesScannerAddon(Database::Db& db); std::shared_ptr getSearcher(); @@ -48,7 +52,7 @@ class FeaturesScannerAddon final : public Scanner::MediaScannerAddon void updateSearcher(); - std::unique_ptr _dbSession; + Database::Session _dbSession; std::shared_ptr _searcher; bool _stopRequested {}; }; diff --git a/src/similarity/features/SimilarityFeaturesSearcher.cpp b/src/similarity/features/SimilarityFeaturesSearcher.cpp index fb72fbce..60414b31 100644 --- a/src/similarity/features/SimilarityFeaturesSearcher.cpp +++ b/src/similarity/features/SimilarityFeaturesSearcher.cpp @@ -20,9 +20,9 @@ #include "SimilarityFeaturesSearcher.hpp" #include +#include #include "database/Artist.hpp" -#include "database/SimilaritySettings.hpp" #include "database/Release.hpp" #include "database/Session.hpp" #include "database/Track.hpp" @@ -34,71 +34,68 @@ namespace Similarity { -struct FeatureInfo +const FeatureSettingsMap& +FeaturesSearcher::getDefaultTrainFeatureSettings() { - std::size_t nbDimensions; - double weight; -}; - -using FeatureInfoMap = std::map; - -static -FeatureInfoMap -getFeatureInfoMap(Database::Session& session) -{ - auto transaction {session.createSharedTransaction()}; - - auto settings {Database::SimilaritySettings::get(session)}; - - std::map featuresInfo; - for (auto feature : settings->getFeatures()) + static FeatureSettingsMap defaultTrainFeatureSettings { - LMS_LOG(SIMILARITY, DEBUG) << "Feature '" << feature->getName() << "', nbDimns = " << feature->getNbDimensions() << ", weight = " << feature->getWeight() ; - featuresInfo[feature->getName()] = { feature->getNbDimensions(), feature->getWeight() }; - } + { "lowlevel.spectral_energyband_high.mean", {1}}, + { "lowlevel.spectral_rolloff.median", {1}}, + { "lowlevel.spectral_contrast_valleys.var", {1}}, + { "lowlevel.erbbands.mean", {1}}, + { "lowlevel.gfcc.mean", {1}}, + }; - return featuresInfo; + return defaultTrainFeatureSettings; } static -std::size_t -getFeatureInfoMapNbDimensions(const FeatureInfoMap& featureInfoMap) +std::optional +getTrackFeatureValues(FeaturesSearcher::FeaturesFetchFunc func, Database::IdType trackId, const std::unordered_set& featureNames) { - return std::accumulate(featureInfoMap.begin(), featureInfoMap.end(), 0, [](std::size_t sum, auto it) { return sum + it.second.nbDimensions; }); + return func(trackId, featureNames); +} + +static +std::optional +getTrackFeatureValuesFromDb(Database::Session& session, Database::IdType trackId, const std::unordered_set& featureNames) +{ + auto func = [&](Database::IdType trackId, const std::unordered_set& featureNames) + { + std::optional res; + + auto transaction {session.createSharedTransaction()}; + + Database::Track::pointer track {Database::Track::getById(session, trackId)}; + if (!track) + return res; + + res = track->getTrackFeatures()->getFeatureValuesMap(featureNames); + if (res->empty()) + res.reset(); + + return res; + }; + + return getTrackFeatureValues(func, trackId, featureNames); } static std::optional -getInputVectorFromTrack(Database::Session& session, Database::IdType trackId, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions) +convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions) { - std::optional res {SOM::InputVector {nbDimensions}}; - - std::map> features; - for (auto itFeatureInfo : featuresInfo) - features[itFeatureInfo.first] = {}; - - auto transaction {session.createSharedTransaction()}; - - Database::Track::pointer track {Database::Track::getById(session, trackId)}; - if (!track) - return res; - - if (!track->getTrackFeatures()->getFeatures(features)) - return res; - std::size_t i {}; - for (const auto& feature : features) + std::optional res {SOM::InputVector {nbDimensions}}; + for (const auto& [featureName, values] : featureValuesMap) { - // Check dimensions for each feature - auto it {featuresInfo.find(feature.first)}; - if (it == featuresInfo.end() || it->second.nbDimensions != feature.second.size()) + if (values.size() != getFeatureDef(featureName).nbDimensions) { - LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << feature.first << "'. Expected " << it->second.nbDimensions << ", got " << feature.second.size(); + LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size(); res.reset(); break; } - for (double val : feature.second) + for (double val : values) (*res)[i++] = val; } @@ -107,25 +104,35 @@ getInputVectorFromTrack(Database::Session& session, Database::IdType trackId, co static SOM::InputVector -getInputVectorWeights(const FeatureInfoMap& featuresInfo, std::size_t nbDimensions) +getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions) { SOM::InputVector weights {nbDimensions}; std::size_t index {}; - for (const auto& featureInfo : featuresInfo) + for (const auto& [featureName, featureSettings] : featureSettingsMap) { - for (std::size_t i {}; i < featureInfo.second.nbDimensions; ++i) - weights[index++] = (1. / featureInfo.second.nbDimensions * featureInfo.second.weight); + const std::size_t featureNbDimensions {getFeatureDef(featureName).nbDimensions}; + + for (std::size_t i {}; i < featureNbDimensions; ++i) + weights[index++] = (1. / featureNbDimensions * featureSettings.weight); } + assert(index == nbDimensions); + return weights; } -FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function stopRequested) +FeaturesSearcher::FeaturesSearcher(Database::Session& session, + const TrainSettings& trainSettings, + StopRequestedFunction stopRequested) { LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher..."; - const FeatureInfoMap featuresInfo {getFeatureInfoMap(session)}; - const std::size_t nbDimensions {getFeatureInfoMapNbDimensions(featuresInfo)}; + std::unordered_set featureNames; + std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)), + [](const auto& itFeatureSetting) { return itFeatureSetting.first; }); + + const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0}, + [](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; })}; LMS_LOG(SIMILARITY, DEBUG) << "Features dimension = " << nbDimensions; @@ -135,7 +142,7 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function samples; @@ -147,10 +154,20 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function inputVector {getInputVectorFromTrack(session, trackId, featuresInfo, nbDimensions)}; + std::optional featureValuesMap; + + if (_featuresFetchFunc) + featureValuesMap = getTrackFeatureValues(_featuresFetchFunc, trackId, featureNames); + else + featureValuesMap = getTrackFeatureValuesFromDb(session, trackId, featureNames); + + if (!featureValuesMap) + continue; + + std::optional inputVector {convertFeatureValuesMapToInputVector(*featureValuesMap, nbDimensions)}; if (!inputVector) continue; @@ -172,12 +189,12 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function(std::sqrt(samples.size() / 4))}; + SOM::Coordinate size {static_cast(std::sqrt(samples.size() / trainSettings.sampleCountPerNeuron))}; LMS_LOG(SIMILARITY, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network"; SOM::Network network {size, size, nbDimensions}; - SOM::InputVector weights {getInputVectorWeights(featuresInfo, nbDimensions)}; + SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)}; network.setDataWeights(weights); auto progressIndicator{[](const auto& iter) @@ -186,17 +203,17 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function> trackPositions; for (std::size_t i {}; i < samples.size(); ++i) { - if (stopRequested()) + if (stopRequested && stopRequested()) return; const SOM::Position position {network.getClosestRefVectorPosition(samples[i])}; @@ -211,7 +228,7 @@ FeaturesSearcher::FeaturesSearcher(Database::Session& session, std::function stopRequested) +FeaturesSearcher::FeaturesSearcher(Database::Session& session, FeaturesCache cache, StopRequestedFunction stopRequested) { LMS_LOG(SIMILARITY, INFO) << "Constructing features searcher from cache..."; @@ -341,7 +358,7 @@ FeaturesSearcher::init(Database::Session& session, for (auto itTrackCoord : tracksPosition) { - if (stopRequested()) + if (stopRequested && stopRequested()) return; auto transaction {session.createSharedTransaction()}; diff --git a/src/similarity/features/SimilarityFeaturesSearcher.hpp b/src/similarity/features/SimilarityFeaturesSearcher.hpp index 6ed68273..625877c1 100644 --- a/src/similarity/features/SimilarityFeaturesSearcher.hpp +++ b/src/similarity/features/SimilarityFeaturesSearcher.hpp @@ -20,12 +20,15 @@ #pragma once #include +#include #include +#include #include "database/Types.hpp" #include "som/DataNormalizer.hpp" #include "som/Network.hpp" #include "SimilarityFeaturesCache.hpp" +#include "SimilarityFeaturesDefs.hpp" namespace Database { @@ -34,15 +37,27 @@ namespace Database namespace Similarity { +using FeatureWeight = double; + class FeaturesSearcher { public: + using StopRequestedFunction = std::function; // return true if stop requested + // Use cache - FeaturesSearcher(Database::Session& session, FeaturesCache cache, std::function stopRequested); + FeaturesSearcher(Database::Session& session, FeaturesCache cache, StopRequestedFunction stopRequested); // Use training (may be very slow) - FeaturesSearcher(Database::Session& session, std::function stopRequested); + struct TrainSettings + { + std::size_t iterationCount {10}; + float sampleCountPerNeuron {4}; + FeatureSettingsMap featureSettingsMap; + }; + FeaturesSearcher(Database::Session& session, const TrainSettings& trainSettings, StopRequestedFunction stopRequested = {}); + + static const FeatureSettingsMap& getDefaultTrainFeatureSettings(); bool isValid() const; @@ -58,6 +73,11 @@ class FeaturesSearcher FeaturesCache toCache() const; + using FeaturesFetchFunc = std::function>>(Database::IdType /*trackId*/, const std::unordered_set& /*features*/)>; + // Default is to retrieve the features from the database (may be slow). + // Use this only if you want to train different searchers with the same data + static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; } + private: using ObjectPositions = std::map>; @@ -65,7 +85,7 @@ class FeaturesSearcher void init(Database::Session& session, SOM::Network network, ObjectPositions tracksPosition, - std::function stopRequested); + StopRequestedFunction stopRequested); std::vector getSimilarObjects(const std::set& ids, const SOM::Matrix>& objectsMap, @@ -84,6 +104,7 @@ class FeaturesSearcher SOM::Matrix> _tracksMap; ObjectPositions _trackPositions; + static inline FeaturesFetchFunc _featuresFetchFunc; }; } // ns Similarity diff --git a/src/similarity/features/som/Network.cpp b/src/similarity/features/som/Network.cpp index 7cb492af..0431c966 100644 --- a/src/similarity/features/som/Network.cpp +++ b/src/similarity/features/som/Network.cpp @@ -279,7 +279,7 @@ Network::updateRefVectors(const Position& closestRefVectorPosition, const InputV InputVector delta {input - refVector}; delta *= (learningFactor * _neighbourhoodFunc(norm, iteration)); - refVector += delta; // * (learningFactor * _neighbourhoodFunc(norm, iteration)); + refVector += delta; } } } diff --git a/src/ui/Auth.cpp b/src/ui/Auth.cpp index 3758967b..f1f37124 100644 --- a/src/ui/Auth.cpp +++ b/src/ui/Auth.cpp @@ -43,7 +43,7 @@ static void createAuthToken(Database::IdType userId, const Wt::WDateTime& expiry) { - const std::string secret {getService<::Auth::AuthTokenService>()->createAuthToken(LmsApp->getDbSession(), userId, expiry)}; + const std::string secret {ServiceProvider<::Auth::AuthTokenService>::get()->createAuthToken(LmsApp->getDbSession(), userId, expiry)}; LmsApp->setCookie(authCookieName, secret, @@ -61,7 +61,7 @@ processAuthToken(const Wt::WEnvironment& env) if (!authCookie) return std::nullopt; - const auto res {getService<::Auth::AuthTokenService>()->processAuthToken(LmsApp->getDbSession(), boost::asio::ip::address::from_string(env.clientAddress()), *authCookie)}; + const auto res {ServiceProvider<::Auth::AuthTokenService>::get()->processAuthToken(LmsApp->getDbSession(), boost::asio::ip::address::from_string(env.clientAddress()), *authCookie)}; switch (res.state) { case ::Auth::AuthTokenService::AuthTokenProcessResult::State::NotFound: @@ -124,7 +124,7 @@ class AuthModel : public Wt::WFormModel if (field == PasswordField) { - switch (getService<::Auth::PasswordService>()->checkUserPassword( + switch (ServiceProvider<::Auth::PasswordService>::get()->checkUserPassword( LmsApp->getDbSession(), boost::asio::ip::address::from_string(LmsApp->environment().clientAddress()), valueText(LoginNameField).toUTF8(), diff --git a/src/ui/LmsApplication.cpp b/src/ui/LmsApplication.cpp index a76ea3ee..ef746020 100644 --- a/src/ui/LmsApplication.cpp +++ b/src/ui/LmsApplication.cpp @@ -60,7 +60,7 @@ namespace UserInterface { std::unique_ptr LmsApplication::create(const Wt::WEnvironment& env, Database::Db& db, LmsApplicationGroupContainer& appGroups) { - return std::make_unique(env, db.createSession(), appGroups); + return std::make_unique(env, db, appGroups); } LmsApplication* @@ -70,12 +70,12 @@ LmsApplication::instance() } Wt::Dbo::ptr -LmsApplication::getUser() const +LmsApplication::getUser() { if (!_userId) return {}; - return Database::User::getById(*_dbSession, *_userId); + return Database::User::getById(_dbSession, *_userId); } bool @@ -85,34 +85,34 @@ LmsApplication::isUserAuthStrong() const } bool -LmsApplication::isUserAdmin() const +LmsApplication::isUserAdmin() { - auto transaction {_dbSession->createSharedTransaction()}; + auto transaction {_dbSession.createSharedTransaction()}; return getUser()->isAdmin(); } bool -LmsApplication::isUserDemo() const +LmsApplication::isUserDemo() { - auto transaction {_dbSession->createSharedTransaction()}; + auto transaction {_dbSession.createSharedTransaction()}; return getUser()->isDemo(); } std::string -LmsApplication::getUserLoginName() const +LmsApplication::getUserLoginName() { - auto transaction {_dbSession->createSharedTransaction()}; + auto transaction {_dbSession.createSharedTransaction()}; return getUser()->getLoginName(); } LmsApplication::LmsApplication(const Wt::WEnvironment& env, - std::unique_ptr dbSession, + Database::Db& db, LmsApplicationGroupContainer& appGroups) : Wt::WApplication {env}, - _dbSession {std::move(dbSession)}, + _dbSession {db}, _appGroups {appGroups} { auto bootstrapTheme = std::make_unique(); @@ -165,8 +165,8 @@ LmsApplication::LmsApplication(const Wt::WEnvironment& env, // If here is no account in the database, launch the first connection wizard bool firstConnection {}; { - auto transaction {_dbSession->createSharedTransaction()}; - firstConnection = Database::User::getAll(*_dbSession).empty(); + auto transaction {_dbSession.createSharedTransaction()}; + firstConnection = Database::User::getAll(_dbSession).empty(); } LMS_LOG(UI, DEBUG) << "Creating root widget. First connection = " << firstConnection; @@ -368,7 +368,7 @@ LmsApplication::handleUserLoggedOut() LMS_LOG(UI, INFO) << "User '" << getUserLoginName() << " 'logged out"; { - auto transaction {_dbSession->createUniqueTransaction()}; + auto transaction {_dbSession.createUniqueTransaction()}; getUser().modify()->clearAuthTokens(); } @@ -529,7 +529,7 @@ LmsApplication::createHome() // Events from MediaScanner { const std::string sessionId {LmsApp->sessionId()}; - getService()->scanComplete().connect(this, [=] () + ServiceProvider::get()->scanComplete().connect(this, [=] () { Wt::WServer::instance()->post(sessionId, [=] { @@ -538,7 +538,7 @@ LmsApplication::createHome() }); }); - getService()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats) + ServiceProvider::get()->scanInProgress().connect(this, [=] (Scanner::ScanProgressStats stats) { Wt::WServer::instance()->post(sessionId, [=] { @@ -547,7 +547,7 @@ LmsApplication::createHome() }); }); - getService()->scheduled().connect(this, [=] (Wt::WDateTime dateTime) + ServiceProvider::get()->scheduled().connect(this, [=] (Wt::WDateTime dateTime) { Wt::WServer::instance()->post(sessionId, [=] { @@ -562,7 +562,7 @@ LmsApplication::createHome() { if (isUserAdmin()) { - const auto& stats {*getService()->getStatus().lastCompleteScanStats}; + const auto& stats {*ServiceProvider::get()->getStatus().lastCompleteScanStats}; notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-complete") .arg(static_cast(stats.nbFiles())) diff --git a/src/ui/LmsApplication.hpp b/src/ui/LmsApplication.hpp index 39c4744c..9e6eb57d 100644 --- a/src/ui/LmsApplication.hpp +++ b/src/ui/LmsApplication.hpp @@ -23,6 +23,8 @@ #include +#include "database/Db.hpp" +#include "database/Session.hpp" #include "scanner/MediaScanner.hpp" #include "LmsApplicationGroup.hpp" @@ -73,7 +75,7 @@ enum class MsgType class LmsApplication : public Wt::WApplication { public: - LmsApplication(const Wt::WEnvironment& env, std::unique_ptr dbSession, LmsApplicationGroupContainer& appGroups); + LmsApplication(const Wt::WEnvironment& env, Database::Db& db, LmsApplicationGroupContainer& appGroups); static std::unique_ptr create(const Wt::WEnvironment& env, Database::Db& db, LmsApplicationGroupContainer& appGroups); static LmsApplication* instance(); @@ -81,13 +83,13 @@ class LmsApplication : public Wt::WApplication // Session application data std::shared_ptr getImageResource() { return _imageResource; } std::shared_ptr getAudioResource() { return _audioResource; } - Database::Session& getDbSession() { return *_dbSession.get();} + Database::Session& getDbSession() { return _dbSession;} - Wt::Dbo::ptr getUser() const; + Wt::Dbo::ptr getUser(); bool isUserAuthStrong() const; // user must be logged in prior this call - bool isUserAdmin() const; // user must be logged in prior this call - bool isUserDemo() const; // user must be logged in prior this call - std::string getUserLoginName() const; // user must be logged in prior this call + bool isUserAdmin(); // user must be logged in prior this call + bool isUserDemo(); // user must be logged in prior this call + std::string getUserLoginName(); // user must be logged in prior this call Events& getEvents() { return _events; } @@ -121,7 +123,7 @@ class LmsApplication : public Wt::WApplication void createHome(); Wt::Signal<> _preQuit; - std::unique_ptr _dbSession; + Database::Session _dbSession; LmsApplicationGroupContainer& _appGroups; Events _events; std::optional _userId; diff --git a/src/ui/PlayQueueView.cpp b/src/ui/PlayQueueView.cpp index 6f0dd5a9..34b4305b 100644 --- a/src/ui/PlayQueueView.cpp +++ b/src/ui/PlayQueueView.cpp @@ -415,7 +415,7 @@ PlayQueue::addSome() void PlayQueue::enqueueRadioTrack() { - const std::vector trackToAddIds {getService()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 1)}; + const std::vector trackToAddIds {ServiceProvider::get()->getSimilarTracksFromTrackList(LmsApp->getDbSession(), _tracklistId, 1)}; enqueueTracks(trackToAddIds); } diff --git a/src/ui/SettingsView.cpp b/src/ui/SettingsView.cpp index 3b983866..c1b430b6 100644 --- a/src/ui/SettingsView.cpp +++ b/src/ui/SettingsView.cpp @@ -80,7 +80,7 @@ class SettingsModel : public Wt::WFormModel Database::User::PasswordHash passwordHash; if (!valueText(PasswordField).empty()) - passwordHash = getService<::Auth::PasswordService>()->hashPassword(valueText(PasswordField).toUTF8()); + passwordHash = ServiceProvider<::Auth::PasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); auto transaction {LmsApp->getDbSession().createUniqueTransaction()}; @@ -133,7 +133,7 @@ class SettingsModel : public Wt::WFormModel { if (!valueText(PasswordOldField).empty()) { - switch (getService<::Auth::PasswordService>()->checkUserPassword( + switch (ServiceProvider<::Auth::PasswordService>::get()->checkUserPassword( LmsApp->getDbSession(), boost::asio::ip::address::from_string(LmsApp->environment().clientAddress()), LmsApp->getUserLoginName(), @@ -161,7 +161,7 @@ class SettingsModel : public Wt::WFormModel { if (!valueText(PasswordField).empty()) { - if (!getService<::Auth::PasswordService>()->evaluatePasswordStrength(LmsApp->getUserLoginName(), valueText(PasswordField).toUTF8())) + if (!ServiceProvider<::Auth::PasswordService>::get()->evaluatePasswordStrength(LmsApp->getUserLoginName(), valueText(PasswordField).toUTF8())) error = Wt::WString::tr("Lms.password-too-weak"); } else diff --git a/src/ui/admin/DatabaseSettingsView.cpp b/src/ui/admin/DatabaseSettingsView.cpp index b28dcefc..d6e44833 100644 --- a/src/ui/admin/DatabaseSettingsView.cpp +++ b/src/ui/admin/DatabaseSettingsView.cpp @@ -27,7 +27,6 @@ #include #include "database/Cluster.hpp" -#include "database/SimilaritySettings.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" #include "utils/Utils.hpp" @@ -85,7 +84,6 @@ class DatabaseSettingsModel : public Wt::WFormModel auto transaction {LmsApp->getDbSession().createSharedTransaction()}; const ScanSettings::pointer scanSettings {ScanSettings::get(LmsApp->getDbSession())}; - const SimilaritySettings::pointer similaritySettings {SimilaritySettings::get(LmsApp->getDbSession())}; setValue(MediaDirectoryField, scanSettings->getMediaDirectory().string()); @@ -97,7 +95,7 @@ class DatabaseSettingsModel : public Wt::WFormModel if (startTimeRow) setValue(UpdateStartTimeField, _updateStartTimeModel->getString(*startTimeRow)); - auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromValue(similaritySettings->getEngineType())}; + auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromValue(scanSettings->getSimilarityEngineType())}; if (similarityEngineTypeRow) setValue(SimilarityEngineTypeField, _similarityEngineTypeModel->getString(*similarityEngineTypeRow)); @@ -115,7 +113,6 @@ class DatabaseSettingsModel : public Wt::WFormModel auto transaction {LmsApp->getDbSession().createUniqueTransaction()}; ScanSettings::pointer scanSettings {ScanSettings::get(LmsApp->getDbSession())}; - SimilaritySettings::pointer similaritySettings {SimilaritySettings::get(LmsApp->getDbSession())}; scanSettings.modify()->setMediaDirectory(valueText(MediaDirectoryField).toUTF8()); @@ -129,7 +126,7 @@ class DatabaseSettingsModel : public Wt::WFormModel auto similarityEngineTypeRow {_similarityEngineTypeModel->getRowFromString(valueText(SimilarityEngineTypeField))}; if (similarityEngineTypeRow) - similaritySettings.modify()->setEngineType(_similarityEngineTypeModel->getValue(*similarityEngineTypeRow)); + scanSettings.modify()->setSimilarityEngineType(_similarityEngineTypeModel->getValue(*similarityEngineTypeRow)); auto clusterTypes {splitString(valueText(TagsField).toUTF8(), " ")}; scanSettings.modify()->setClusterTypes(LmsApp->getDbSession(), std::set(clusterTypes.begin(), clusterTypes.end())); @@ -158,14 +155,14 @@ class DatabaseSettingsModel : public Wt::WFormModel _updateStartTimeModel->add(time.toString(), time); } - _similarityEngineTypeModel = std::make_shared>(); - _similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.clusters"), SimilaritySettings::EngineType::Clusters); - _similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.features"), SimilaritySettings::EngineType::Features); + _similarityEngineTypeModel = std::make_shared>(); + _similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.clusters"), ScanSettings::SimilarityEngineType::Clusters); + _similarityEngineTypeModel->add(Wt::WString::tr("Lms.Admin.Database.similarity-engine-type.features"), ScanSettings::SimilarityEngineType::Features); } std::shared_ptr> _updatePeriodModel; std::shared_ptr> _updateStartTimeModel; - std::shared_ptr> _similarityEngineTypeModel; + std::shared_ptr> _similarityEngineTypeModel; }; @@ -232,7 +229,7 @@ DatabaseSettingsView::refreshView() { model->saveData(); - getService()->requestReschedule(); + ServiceProvider::get()->requestReschedule(); LmsApp->notifyMsg(MsgType::Success, Wt::WString::tr("Lms.Admin.Database.settings-saved")); } @@ -249,7 +246,7 @@ DatabaseSettingsView::refreshView() immScanBtn->clicked().connect([=] () { - getService()->requestImmediateScan(); + ServiceProvider::get()->requestImmediateScan(); LmsApp->notifyMsg(MsgType::Info, Wt::WString::tr("Lms.Admin.Database.scan-launched")); }); diff --git a/src/ui/admin/DatabaseStatus.cpp b/src/ui/admin/DatabaseStatus.cpp index 4d758eb6..23577bfa 100644 --- a/src/ui/admin/DatabaseStatus.cpp +++ b/src/ui/admin/DatabaseStatus.cpp @@ -136,7 +136,7 @@ DatabaseStatus::refreshContents() Wt::WPushButton* reportBtn {bindNew("btn-report", Wt::WString::tr("Lms.Admin.Database.Status.get-report"))}; - const MediaScanner::Status status {getService()->getStatus()}; + const MediaScanner::Status status {ServiceProvider::get()->getStatus()}; if (status.lastCompleteScanStats) { bindString("last-scan", Wt::WString::tr("Lms.Admin.Database.Status.last-scan-status") diff --git a/src/ui/admin/InitWizardView.cpp b/src/ui/admin/InitWizardView.cpp index 28c7d350..2eeba3e5 100644 --- a/src/ui/admin/InitWizardView.cpp +++ b/src/ui/admin/InitWizardView.cpp @@ -55,7 +55,7 @@ class InitWizardModel : public Wt::WFormModel void saveData() { - const Database::User::PasswordHash passwordHash {getService<::Auth::PasswordService>()->hashPassword(valueText(PasswordField).toUTF8())}; + const Database::User::PasswordHash passwordHash {ServiceProvider<::Auth::PasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8())}; auto transaction(LmsApp->getDbSession().createUniqueTransaction()); @@ -77,7 +77,7 @@ class InitWizardModel : public Wt::WFormModel if (!valueText(PasswordField).empty()) { // Evaluate the strength of the password - if (!getService<::Auth::PasswordService>()->evaluatePasswordStrength(valueText(AdminLoginField).toUTF8(), valueText(PasswordField).toUTF8())) + if (!ServiceProvider<::Auth::PasswordService>::get()->evaluatePasswordStrength(valueText(AdminLoginField).toUTF8(), valueText(PasswordField).toUTF8())) error = Wt::WString::tr("Lms.password-too-weak"); } else diff --git a/src/ui/admin/UserView.cpp b/src/ui/admin/UserView.cpp index a4bec9cc..dd438f88 100644 --- a/src/ui/admin/UserView.cpp +++ b/src/ui/admin/UserView.cpp @@ -82,7 +82,7 @@ class UserModel : public Wt::WFormModel { std::optional passwordHash; if (!valueText(PasswordField).empty()) - passwordHash = getService<::Auth::PasswordService>()->hashPassword(valueText(PasswordField).toUTF8()); + passwordHash = ServiceProvider<::Auth::PasswordService>::get()->hashPassword(valueText(PasswordField).toUTF8()); auto transaction {LmsApp->getDbSession().createUniqueTransaction()}; @@ -174,7 +174,7 @@ class UserModel : public Wt::WFormModel else { // Evaluate the strength of the password for non demo accounts - if (!getService<::Auth::PasswordService>()->evaluatePasswordStrength(getLoginName(), valueText(PasswordField).toUTF8())) + if (!ServiceProvider<::Auth::PasswordService>::get()->evaluatePasswordStrength(getLoginName(), valueText(PasswordField).toUTF8())) error = Wt::WString::tr("Lms.password-too-weak"); } } @@ -270,7 +270,7 @@ UserView::refreshView() // Demo account t->setFormWidget(UserModel::DemoField, std::make_unique()); - if (!userId && getService()->getBool("demo", false)) + if (!userId && ServiceProvider::get()->getBool("demo", false)) t->setCondition("if-demo", true); Wt::WPushButton* saveBtn = t->bindNew("save-btn", Wt::WString::tr(userId ? "Lms.save" : "Lms.create")); diff --git a/src/ui/explore/ArtistInfoView.cpp b/src/ui/explore/ArtistInfoView.cpp index 3c232a7c..5b260cc2 100644 --- a/src/ui/explore/ArtistInfoView.cpp +++ b/src/ui/explore/ArtistInfoView.cpp @@ -63,7 +63,7 @@ ArtistInfo::refresh() if (!artistId) return; - const std::vector artistsIds {getService()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)}; + const std::vector artistsIds {ServiceProvider::get()->getSimilarArtists(LmsApp->getDbSession(), *artistId, 5)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; diff --git a/src/ui/explore/ReleaseInfoView.cpp b/src/ui/explore/ReleaseInfoView.cpp index d3e7c238..16d14969 100644 --- a/src/ui/explore/ReleaseInfoView.cpp +++ b/src/ui/explore/ReleaseInfoView.cpp @@ -68,7 +68,7 @@ ReleaseInfo::refresh() if (!releaseId) return; - const std::vector releasesIds {getService()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 5)}; + const std::vector releasesIds {ServiceProvider::get()->getSimilarReleases(LmsApp->getDbSession(), *releaseId, 5)}; auto transaction {LmsApp->getDbSession().createSharedTransaction()}; diff --git a/src/ui/resource/ImageResource.cpp b/src/ui/resource/ImageResource.cpp index f11a7301..114ce5b5 100644 --- a/src/ui/resource/ImageResource.cpp +++ b/src/ui/resource/ImageResource.cpp @@ -80,7 +80,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons // DbSession are not thread safe { Wt::WApplication::UpdateLock lock {LmsApp}; - cover = getService()->getFromTrack(LmsApp->getDbSession(), *trackId, Image::Format::JPEG, *size); + cover = ServiceProvider::get()->getFromTrack(LmsApp->getDbSession(), *trackId, Image::Format::JPEG, *size); } } else if (releaseIdStr) @@ -92,7 +92,7 @@ ImageResource::handleRequest(const Wt::Http::Request& request, Wt::Http::Respons // DbSession are not thread safe { Wt::WApplication::UpdateLock lock {LmsApp}; - cover = getService()->getFromRelease(LmsApp->getDbSession(), *releaseId, Image::Format::JPEG, *size); + cover = ServiceProvider::get()->getFromRelease(LmsApp->getDbSession(), *releaseId, Image::Format::JPEG, *size); } } else diff --git a/src/utils/Logger.cpp b/src/utils/Logger.cpp index c25c1b40..79bfafe9 100644 --- a/src/utils/Logger.cpp +++ b/src/utils/Logger.cpp @@ -19,7 +19,7 @@ #include "Logger.hpp" -std::string getModuleName(Module mod) +const char* getModuleName(Module mod) { switch (mod) { @@ -41,7 +41,7 @@ std::string getModuleName(Module mod) return ""; } -std::string getSeverityName(Severity sev) +const char* getSeverityName(Severity sev) { switch (sev) { @@ -54,3 +54,21 @@ std::string getSeverityName(Severity sev) return ""; } +Log::Log(Logger* logger, Module module, Severity severity) + : _module {module}, + _severity {severity}, + _logger {logger} +{} + +Log::~Log() +{ + if (_logger) + _logger->processLog(*this); +} + +std::string +Log::getMessage() const +{ + return _oss.str(); +} + diff --git a/src/utils/Logger.hpp b/src/utils/Logger.hpp index a2ce9d4a..9597aa5c 100644 --- a/src/utils/Logger.hpp +++ b/src/utils/Logger.hpp @@ -20,9 +20,9 @@ #pragma once #include +#include -#include -#include +#include "Service.hpp" enum class Severity { @@ -51,8 +51,34 @@ enum class Module UI, }; -std::string getModuleName(Module mod); -std::string getSeverityName(Severity sev); +const char* getModuleName(Module mod); +const char* getSeverityName(Severity sev); -#define LMS_LOG(module, level) Wt::log(getSeverityName(Severity::level)) << Wt::WLogger::sep << "[" << getModuleName(Module::module) << "]" << Wt::WLogger::sep +class Logger; +class Log +{ + public: + Log(Logger* logger, Module module, Severity severity); + ~Log(); + + Module getModule() const { return _module; } + Severity getSeverity() const { return _severity; } + std::string getMessage() const; + + std::ostringstream& getOstream() { return _oss; } + + private: + Module _module; + Severity _severity; + std::ostringstream _oss; + Logger* _logger {}; +}; + +class Logger +{ + public: + virtual void processLog(const Log& log) = 0; +}; + +#define LMS_LOG(module, severity) Log(ServiceProvider::get(), Module::module, Severity::severity).getOstream() diff --git a/src/utils/Service.hpp b/src/utils/Service.hpp index e72ef990..a18363ce 100644 --- a/src/utils/Service.hpp +++ b/src/utils/Service.hpp @@ -17,38 +17,41 @@ * along with LMS. If not, see . */ -#include +#pragma once -template +#include +#include + +template class ServiceProvider { public: + template + static + Class& + create(Args&&... args) + { + static_assert(std::is_base_of::value); + + assign(std::make_unique(std::forward(args)...)); + return *get(); + } template static - T& + Class& create(Args&&... args) { - assign(std::make_unique(std::forward(args)...)); + assign(std::make_unique(std::forward(args)...)); return *get(); } - static void assign(std::unique_ptr service) { _service = std::move(service); } + + static void assign(std::unique_ptr service) { _service = std::move(service); } static void clear() { _service.reset(); } - static T* get() { return _service.get(); } + static Class* get() { return _service.get(); } private: - static std::unique_ptr _service; + static inline std::unique_ptr _service; }; -template -std::unique_ptr ServiceProvider::_service = {}; - -template -T* -getService() -{ - return ServiceProvider::get(); -} - - diff --git a/src/utils/StreamLogger.cpp b/src/utils/StreamLogger.cpp new file mode 100644 index 00000000..f4fc9cf5 --- /dev/null +++ b/src/utils/StreamLogger.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "StreamLogger.hpp" + +StreamLogger::StreamLogger(std::ostream& os) +: _os {os} +{ +} + +void +StreamLogger::processLog(const Log& log) +{ + _os << "[" << getSeverityName(log.getSeverity()) << "] [" << getModuleName(log.getModule()) << "] " << log.getMessage() << std::endl; +} + diff --git a/src/utils/StreamLogger.hpp b/src/utils/StreamLogger.hpp new file mode 100644 index 00000000..8e1af39a --- /dev/null +++ b/src/utils/StreamLogger.hpp @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "Logger.hpp" + +class StreamLogger final : public Logger +{ + public: + StreamLogger(std::ostream& oss); + + void processLog(const Log& log); + + private: + std::ostream& _os; +}; + diff --git a/src/utils/Utils.cpp b/src/utils/Utils.cpp index 50076276..c199386b 100644 --- a/src/utils/Utils.cpp +++ b/src/utils/Utils.cpp @@ -166,6 +166,13 @@ stringFromHex(const std::string& str) } return res; - +} + +RandGenerator& getRandGenerator() +{ + static thread_local std::random_device rd; + static thread_local std::mt19937 randGenerator(rd()); + + return randGenerator; } diff --git a/src/utils/Utils.hpp b/src/utils/Utils.hpp index 39336000..17aacb83 100644 --- a/src/utils/Utils.hpp +++ b/src/utils/Utils.hpp @@ -110,24 +110,40 @@ constexpr T clamp(T v, T lo, T hi, Compare comp = {}) return comp(v, lo) ? lo : comp(hi, v) ? hi : v; } +using RandGenerator = std::mt19937; +RandGenerator& getRandGenerator(); + +template +T +getRandom(T min, T max) +{ + std::uniform_int_distribution<> dist {min, max}; + return dist (getRandGenerator()); +} + +template +T +getRealRandom(T min, T max) +{ + std::uniform_real_distribution<> dist {min, max}; + return dist (getRandGenerator()); +} + template void shuffleContainer(Container& container) { - auto now {std::chrono::system_clock::now()}; - std::mt19937 randGenerator(std::chrono::duration_cast(now.time_since_epoch()).count()); - std::shuffle(std::begin(container), std::end(container), randGenerator); + std::shuffle(std::begin(container), std::end(container), getRandGenerator()); } template -typename Container::iterator -pickRandom(Container& container) +typename Container::const_iterator +pickRandom(const Container& container) { - auto now {std::chrono::system_clock::now()}; - std::mt19937 randGenerator (std::chrono::duration_cast(now.time_since_epoch()).count()); - std::uniform_int_distribution<> dist {0, static_cast(container.size())}; + if (container.empty()) + return std::end(container); - return std::next(std::begin(container), dist(randGenerator )); + return std::next(std::begin(container), getRandom(0, static_cast(container.size() - 1))); } diff --git a/src/utils/WtLogger.cpp b/src/utils/WtLogger.cpp new file mode 100644 index 00000000..393ed4df --- /dev/null +++ b/src/utils/WtLogger.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "WtLogger.hpp" + +#include +#include + +#include "Logger.hpp" + +void +WtLogger::processLog(const Log& log) +{ + Wt::log(getSeverityName(log.getSeverity())) << Wt::WLogger::sep << "[" << getModuleName(log.getModule()) << "]" << log.getMessage(); +} + diff --git a/src/utils/WtLogger.hpp b/src/utils/WtLogger.hpp new file mode 100644 index 00000000..65b94018 --- /dev/null +++ b/src/utils/WtLogger.hpp @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "Logger.hpp" + +class WtLogger final : public Logger +{ + public: + void processLog(const Log& log) override; +}; + diff --git a/test/Makefile.am b/test/Makefile.am index 15efaba5..eefde4b8 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -22,11 +22,11 @@ test_database_SOURCES = \ $(top_srcdir)/src/database/Release.cpp \ $(top_srcdir)/src/database/ScanSettings.cpp \ $(top_srcdir)/src/database/Session.cpp \ - $(top_srcdir)/src/database/SimilaritySettings.cpp \ $(top_srcdir)/src/database/SqlQuery.cpp \ $(top_srcdir)/src/database/Track.cpp \ $(top_srcdir)/src/database/User.cpp \ $(top_srcdir)/src/utils/Logger.cpp \ + $(top_srcdir)/src/utils/StreamLogger.cpp \ $(top_srcdir)/src/utils/Utils.cpp test_database_CXXFLAGS=-std=c++17 -I${top_srcdir}/src/ diff --git a/test/database/DatabaseTest.cpp b/test/database/DatabaseTest.cpp index e53dc774..26ecf7a5 100644 --- a/test/database/DatabaseTest.cpp +++ b/test/database/DatabaseTest.cpp @@ -25,11 +25,14 @@ #include "database/Artist.hpp" #include "database/Cluster.hpp" #include "database/Db.hpp" -#include "database/TrackList.hpp" #include "database/Release.hpp" +#include "database/Session.hpp" #include "database/Track.hpp" +#include "database/TrackList.hpp" #include "database/User.hpp" +#include "utils/StreamLogger.hpp" + using namespace Database; #define CHECK(PRED) \ @@ -434,6 +437,8 @@ testSingleTrackSingleCluster(Session& session) auto transaction {session.createSharedTransaction()}; auto clusters {Cluster::getAllOrphans(session)}; CHECK(clusters.size() == 2); + CHECK(track->getClusters().empty()); + CHECK(track->getClusterIds().empty()); } { @@ -461,6 +466,18 @@ testSingleTrackSingleCluster(Session& session) tracks = Track::getByClusters(session, {cluster2.getId()}); CHECK(tracks.empty()); } + + { + auto transaction {session.createSharedTransaction()}; + + auto clusters {track->getClusters()}; + CHECK(clusters.size() == 1); + CHECK(clusters.front().id() == cluster1.getId()); + + auto clusterIds {track->getClusterIds()}; + CHECK(clusterIds.size() == 1); + CHECK(clusterIds.front() == cluster1.getId()); + } } static @@ -637,6 +654,12 @@ testSingleTrackSingleArtistMultiClusters(Session& session) CHECK(Artist::getAllOrphans(session).empty()); } + { + auto transaction {session.createSharedTransaction()}; + CHECK(track->getClusters().size() == 1); + CHECK(track->getClusterIds().size() == 1); + } + { auto transaction {session.createSharedTransaction()}; @@ -1253,6 +1276,9 @@ int main() try { + // log to stdout + ServiceProvider::create(std::cout); + const std::filesystem::path tmpFile {std::tmpnam(nullptr)}; ScopedFileDeleter tmpFileDeleter {tmpFile}; @@ -1261,13 +1287,14 @@ int main() for (std::size_t i = 0; i < 2; ++i) { Database::Db db {tmpFile}; - std::unique_ptr session {db.createSession()}; + Database::Session session {db}; + session.prepareTables(); auto runTest = [&session](const std::string& name, std::function testFunc) { std::cout << "Running test '" << name << "'..." << std::endl; - testFunc(*session); - testDatabaseEmpty(*session); + testFunc(session); + testDatabaseEmpty(session); std::cout << "Running test '" << name << "': SUCCESS" << std::endl; }; diff --git a/tools/Makefile.am b/tools/Makefile.am index ce4f09cc..c382f8bc 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -1,4 +1,4 @@ if BUILD_TOOLS -SUBDIRS = similarity metadata +SUBDIRS = similarity similarity-parameters metadata endif diff --git a/tools/metadata/LmsMetadata.cpp b/tools/metadata/LmsMetadata.cpp index a32eb64b..a3019e5f 100644 --- a/tools/metadata/LmsMetadata.cpp +++ b/tools/metadata/LmsMetadata.cpp @@ -1,3 +1,22 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + #include #include #include @@ -9,6 +28,7 @@ #include "av/AvInfo.hpp" #include "metadata/AvFormat.hpp" #include "metadata/TagLibParser.hpp" +#include "utils/StreamLogger.hpp" std::ostream& operator<<(std::ostream& os, const MetaData::Artist& artist) { @@ -124,6 +144,9 @@ int main(int argc, char *argv[]) try { + // log to stdout + ServiceProvider::create(std::cout); + for (std::size_t i {}; i < static_cast(argc - 1); ++i) { std::filesystem::path file {argv[i + 1]}; diff --git a/tools/metadata/Makefile.am b/tools/metadata/Makefile.am index d6fe053c..027773ec 100644 --- a/tools/metadata/Makefile.am +++ b/tools/metadata/Makefile.am @@ -6,6 +6,7 @@ lms_metadata_SOURCES = \ $(top_srcdir)/src/metadata/AvFormat.cpp \ $(top_srcdir)/src/metadata/TagLibParser.cpp \ $(top_srcdir)/src/utils/Logger.cpp \ + $(top_srcdir)/src/utils/StreamLogger.cpp \ $(top_srcdir)/src/utils/Utils.cpp lms_metadata_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT diff --git a/tools/similarity-parameters/GeneticAlgorithm.hpp b/tools/similarity-parameters/GeneticAlgorithm.hpp new file mode 100644 index 00000000..d4e0e402 --- /dev/null +++ b/tools/similarity-parameters/GeneticAlgorithm.hpp @@ -0,0 +1,173 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include + +#include "utils/Utils.hpp" +#include "ParallelFor.hpp" + +template +class GeneticAlgorithm +{ + public: + using Score = float; + + using BreedFunction = std::function; + using MutateFunction = std::function; + using ScoreFunction = std::function; + + struct Params + { + std::size_t nbWorkers {1}; + std::size_t nbGenerations; + float crossoverRatio {0.5}; + float mutationProbability {0.05}; + BreedFunction breedFunction; + MutateFunction mutateFunction; + ScoreFunction scoreFunction; + }; + + GeneticAlgorithm(const Params& params); + + // Returns the individual that has the maximum score after processing the requested generations + Individual simulate(const std::vector& initialPopulation); + + private: + + struct ScoredIndividual + { + Individual individual; + std::optional score {}; + }; + + void scoreAndSortPopulation(std::vector& population); + Score getTotalScore(const std::vector& population) const; + typename std::vector::const_iterator pickRandomRouletteWheel(const std::vector& population, Score totalScore); + + Params _params; +}; + +template +GeneticAlgorithm::GeneticAlgorithm(const Params& params) +: _params {params} +{ +} + + +template +Individual +GeneticAlgorithm::simulate(const std::vector& initialPopulation) +{ + const std::size_t childrenCountPerGeneration {static_cast(initialPopulation.size() * _params.crossoverRatio)}; + if (initialPopulation.size() < 10) + throw std::runtime_error("Initial population must has at least 10 elements"); + + std::vector scoredPopulation; + scoredPopulation.reserve(initialPopulation.size()); + + std::transform(std::cbegin(initialPopulation), std::cend(initialPopulation), std::back_inserter(scoredPopulation ), + [](const Individual& individual) { return ScoredIndividual {individual};}); + + scoreAndSortPopulation(scoredPopulation); + + for (std::size_t currentGeneration {}; currentGeneration < _params.nbGenerations; ++currentGeneration) + { + assert(scoredPopulation.size() == initialPopulation.size()); + std::cout << "Processing generation " << currentGeneration << "..." << std::endl; + std::cout << "Need to create " << childrenCountPerGeneration << " new children" << std::endl; + + // breed + const Score populationTotalScore {getTotalScore(scoredPopulation)}; + std::vector children; + children.reserve(childrenCountPerGeneration); + + while (children.size() < childrenCountPerGeneration) + { + // Select two random parents using their score as weight + const auto itParent1 {pickRandomRouletteWheel(scoredPopulation, populationTotalScore)}; + const auto itParent2 {pickRandomRouletteWheel(scoredPopulation, populationTotalScore)}; + + if (itParent1 == itParent2) + continue; + + ScoredIndividual child {_params.breedFunction(itParent1->individual, itParent2->individual)}; + + if (getRealRandom(float {}, float {1}) <= _params.mutationProbability) + _params.mutateFunction(child.individual); + + children.emplace_back(std::move(child)); + } + + // Elitist selection + scoredPopulation.resize(initialPopulation.size() - childrenCountPerGeneration); + + scoredPopulation.insert(std::end(scoredPopulation), std::make_move_iterator(std::begin(children)), std::make_move_iterator(std::end(children))); + assert(scoredPopulation.size() == initialPopulation.size()); + + scoreAndSortPopulation(scoredPopulation); + + std::cout << "Mean score = " << getTotalScore(scoredPopulation) / scoredPopulation.size() << std::endl; + std::cout << "Current best score = " << *scoredPopulation.front().score << std::endl; + } + + std::cout << "Best score = " << *scoredPopulation.front().score << std::endl; + return scoredPopulation.front().individual; +} + + +template +void +GeneticAlgorithm::scoreAndSortPopulation(std::vector& scoredPopulation) +{ + parallel_foreach(_params.nbWorkers, std::begin(scoredPopulation), std::end(scoredPopulation), + [&](ScoredIndividual& scoredIndividual) + { + if (!scoredIndividual.score) + scoredIndividual.score = _params.scoreFunction(scoredIndividual.individual); + }); + + std::sort(std::begin(scoredPopulation), std::end(scoredPopulation), [](const ScoredIndividual& a, const ScoredIndividual& b) { return a.score > b.score; }); +} + +template +typename GeneticAlgorithm::Score +GeneticAlgorithm::getTotalScore(const std::vector& scoredPopulation) const +{ + return std::accumulate(std::cbegin(scoredPopulation), std::cend(scoredPopulation), Score {}, [](Score score, const ScoredIndividual& individual) { return score + *individual.score; }); +} + +template +typename std::vector::ScoredIndividual>::const_iterator +GeneticAlgorithm::pickRandomRouletteWheel(const std::vector& population, Score totalScore) +{ + const Score randomScore {getRealRandom(Score {}, totalScore)}; + + Score curScore{}; + for (auto itScoredIndividual {std::cbegin(population)}; itScoredIndividual != std::cend(population); ++itScoredIndividual ) + { + if (curScore + *itScoredIndividual->score > randomScore) + return itScoredIndividual; + + curScore += *itScoredIndividual->score; + } + + throw std::runtime_error("bad random or empty population"); +} + + diff --git a/tools/similarity-parameters/LmsSimilarityParameters.cpp b/tools/similarity-parameters/LmsSimilarityParameters.cpp new file mode 100644 index 00000000..ed5b1f66 --- /dev/null +++ b/tools/similarity-parameters/LmsSimilarityParameters.cpp @@ -0,0 +1,489 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include +#include +#include + +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Db.hpp" +#include "database/Release.hpp" +#include "database/SessionPool.hpp" +#include "database/Track.hpp" +#include "database/TrackFeatures.hpp" +#include "similarity/features/SimilarityFeaturesSearcher.hpp" +#include "utils/Config.hpp" +#include "utils/Service.hpp" +#include "utils/StreamLogger.hpp" + +#include "GeneticAlgorithm.hpp" + +using namespace Similarity; +using SimilarityScore = GeneticAlgorithm::Score; + +// An individual is just a FeatureSettingsMap +// The goal is to get the FeatureSettingsMap that maximize the score +const FeatureSettingsMap featuresSettings +{ + { "lowlevel.average_loudness", {1}}, + { "lowlevel.barkbands.mean", {1}}, + { "lowlevel.barkbands.median", {1}}, + { "lowlevel.barkbands.var", {1}}, + { "lowlevel.barkbands_crest.mean", {1}}, + { "lowlevel.barkbands_crest.median", {1}}, + { "lowlevel.barkbands_crest.var", {1}}, + { "lowlevel.barkbands_flatness_db.mean", {1}}, + { "lowlevel.barkbands_flatness_db.median", {1}}, + { "lowlevel.barkbands_flatness_db.var", {1}}, + { "lowlevel.barkbands_kurtosis.mean", {1}}, + { "lowlevel.barkbands_kurtosis.median", {1}}, + { "lowlevel.barkbands_kurtosis.var", {1}}, + { "lowlevel.barkbands_skewness.mean", {1}}, + { "lowlevel.barkbands_skewness.median", {1}}, + { "lowlevel.barkbands_skewness.var", {1}}, + { "lowlevel.barkbands_spread.mean", {1}}, + { "lowlevel.barkbands_spread.median", {1}}, + { "lowlevel.barkbands_spread.var", {1}}, + { "lowlevel.dissonance.mean", {1}}, + { "lowlevel.dissonance.median", {1}}, + { "lowlevel.dissonance.var", {1}}, + { "lowlevel.dynamic_complexity", {1}}, + { "lowlevel.spectral_contrast_coeffs.mean", {1}}, + { "lowlevel.spectral_contrast_coeffs.median", {1}}, + { "lowlevel.spectral_contrast_coeffs.var", {1}}, + { "lowlevel.erbbands.mean", {1}}, + { "lowlevel.erbbands.median", {1}}, + { "lowlevel.erbbands.var", {1}}, + { "lowlevel.gfcc.mean", {1}}, + { "lowlevel.hfc.mean", {1}}, + { "lowlevel.hfc.median", {1}}, + { "lowlevel.hfc.var", {1}}, + { "tonal.hpcp.median", {1}}, + { "lowlevel.melbands.mean", {1}}, + { "lowlevel.melbands.median", {1}}, + { "lowlevel.melbands.var", {1}}, + { "lowlevel.melbands_crest.mean", {1}}, + { "lowlevel.melbands_crest.median", {1}}, + { "lowlevel.melbands_crest.var", {1}}, + { "lowlevel.melbands_flatness_db.mean", {1}}, + { "lowlevel.melbands_flatness_db.median", {1}}, + { "lowlevel.melbands_flatness_db.var", {1}}, + { "lowlevel.melbands_kurtosis.mean", {1}}, + { "lowlevel.melbands_kurtosis.median", {1}}, + { "lowlevel.melbands_kurtosis.var", {1}}, + { "lowlevel.melbands_skewness.mean", {1}}, + { "lowlevel.melbands_skewness.median", {1}}, + { "lowlevel.melbands_skewness.var", {1}}, + { "lowlevel.melbands_spread.mean", {1}}, + { "lowlevel.melbands_spread.median", {1}}, + { "lowlevel.melbands_spread.var", {1}}, + { "lowlevel.mfcc.mean", {1}}, + { "lowlevel.pitch_salience.mean", {1}}, + { "lowlevel.pitch_salience.median", {1}}, + { "lowlevel.pitch_salience.var", {1}}, + { "lowlevel.silence_rate_30dB.mean", {1}}, + { "lowlevel.silence_rate_30dB.median", {1}}, + { "lowlevel.silence_rate_30dB.var", {1}}, + { "lowlevel.silence_rate_60dB.mean", {1}}, + { "lowlevel.silence_rate_60dB.median", {1}}, + { "lowlevel.silence_rate_60dB.var", {1}}, + { "lowlevel.spectral_centroid.mean", {1}}, + { "lowlevel.spectral_centroid.median", {1}}, + { "lowlevel.spectral_centroid.var", {1}}, + { "lowlevel.spectral_complexity.mean", {1}}, + { "lowlevel.spectral_complexity.median", {1}}, + { "lowlevel.spectral_complexity.var", {1}}, + { "lowlevel.spectral_contrast_coeffs.mean", {1}}, + { "lowlevel.spectral_contrast_coeffs.median", {1}}, + { "lowlevel.spectral_contrast_coeffs.var", {1}}, + { "lowlevel.spectral_contrast_valleys.mean", {1}}, + { "lowlevel.spectral_contrast_valleys.median", {1}}, + { "lowlevel.spectral_contrast_valleys.var", {1}}, + { "lowlevel.spectral_decrease.mean", {1}}, + { "lowlevel.spectral_decrease.median", {1}}, + { "lowlevel.spectral_decrease.var", {1}}, + { "lowlevel.spectral_energy.mean", {1}}, + { "lowlevel.spectral_energy.median", {1}}, + { "lowlevel.spectral_energy.var", {1}}, + { "lowlevel.spectral_energyband_high.mean", {1}}, + { "lowlevel.spectral_energyband_high.median", {1}}, + { "lowlevel.spectral_energyband_high.var", {1}}, + { "lowlevel.spectral_energyband_low.mean", {1}}, + { "lowlevel.spectral_energyband_low.median", {1}}, + { "lowlevel.spectral_energyband_low.var", {1}}, + { "lowlevel.spectral_energyband_middle_high.mean", {1}}, + { "lowlevel.spectral_energyband_middle_high.median", {1}}, + { "lowlevel.spectral_energyband_middle_high.var", {1}}, + { "lowlevel.spectral_energyband_middle_low.mean", {1}}, + { "lowlevel.spectral_energyband_middle_low.median", {1}}, + { "lowlevel.spectral_energyband_middle_low.var", {1}}, + { "lowlevel.spectral_entropy.mean", {1}}, + { "lowlevel.spectral_entropy.median", {1}}, + { "lowlevel.spectral_entropy.var", {1}}, + { "lowlevel.spectral_flux.mean", {1}}, + { "lowlevel.spectral_flux.median", {1}}, + { "lowlevel.spectral_flux.var", {1}}, + { "lowlevel.spectral_kurtosis.mean", {1}}, + { "lowlevel.spectral_kurtosis.median", {1}}, + { "lowlevel.spectral_kurtosis.var", {1}}, + { "lowlevel.spectral_rms.mean", {1}}, + { "lowlevel.spectral_rms.median", {1}}, + { "lowlevel.spectral_rms.var", {1}}, + { "lowlevel.spectral_rolloff.mean", {1}}, + { "lowlevel.spectral_rolloff.median", {1}}, + { "lowlevel.spectral_rolloff.var", {1}}, + { "lowlevel.spectral_skewness.mean", {1}}, + { "lowlevel.spectral_skewness.median", {1}}, + { "lowlevel.spectral_skewness.var", {1}}, + { "lowlevel.spectral_spread.mean", {1}}, + { "lowlevel.spectral_spread.median", {1}}, + { "lowlevel.spectral_spread.var", {1}}, + { "lowlevel.zerocrossingrate.mean", {1}}, + { "lowlevel.zerocrossingrate.median", {1}}, + { "lowlevel.zerocrossingrate.var", {1}}, +}; + +static +std::unordered_map +constructFeaturesCache(Database::Session& session, const FeatureSettingsMap& featureSettings) +{ + std::unordered_map cache; + + std::unordered_set names; + std::transform(std::cbegin(featureSettings), std::cend(featureSettings), std::inserter(names, std::begin(names)), + [](const auto& itFeature) { return itFeature.first; }); + + auto transaction {session.createSharedTransaction()}; + + for (auto trackId : Database::Track::getAllIdsWithFeatures(session)) + { + const Database::Track::pointer track {Database::Track::getById(session, trackId)}; + const Database::TrackFeatures::pointer trackFeatures {track->getTrackFeatures()}; + + cache[trackId] = trackFeatures->getFeatureValuesMap(names); + } + + return cache; +} + +static +std::optional +getFeaturesFromCache(const std::unordered_map& cache, Database::IdType trackId, const FeatureNames& names) +{ + std::optional res; + + auto it {cache.find(trackId)}; + if (it == std::cend(cache)) + return res; + + res = FeatureValuesMap{}; + + const FeatureValuesMap& trackFeatures {it->second}; + for (const FeatureName& name : names) + { + auto itFeatures {trackFeatures.find(name)}; + if (itFeatures == std::cend(trackFeatures)) + { + res.reset(); + break; + } + + res->emplace(name, itFeatures ->second); + } + + return res; +} + +static +void +printFeatureSettingsMap(const FeatureSettingsMap& featureSettings) +{ + std::cout << "FeatureSettingsMap: (" << featureSettings.size() << " features)" << std::endl; + for (const auto& [name, settings] : featureSettings) + std::cout << "\t" << name << std::endl; +} + +static +std::string +trackToString(Database::Session& session, Database::IdType trackId) +{ + std::string res; + auto transaction {session.createSharedTransaction()}; + Database::Track::pointer track {Database::Track::getById(session, trackId)}; + + res += track->getName(); + if (track->getRelease()) + res += " [" + track->getRelease()->getName() + "]"; + for (auto artist : track->getArtists()) + res += " - " + artist->getName(); + for (auto cluster : track->getClusters()) + res += " {" + cluster->getType()->getName() + "-"+ cluster->getName() + "}"; + + return res; +} + +static +SimilarityScore +computeTrackScore(Database::Session& session, Database::IdType track1Id, Database::IdType track2Id) +{ + SimilarityScore score {}; + + auto transaction {session.createSharedTransaction()}; + + auto track1 {Database::Track::getById(session, track1Id)}; + auto track2 {Database::Track::getById(session, track2Id)}; + + if (track1->getRelease() == track2->getRelease()) + score += 1; + + // Artists in common + { + auto track1ArtistIds {track1->getArtistIds()}; + auto track2ArtistIds {track2->getArtistIds()}; + + std::vector commonArtistIds; + std::set_intersection(std::cbegin(track1ArtistIds), std::cend(track1ArtistIds), + std::cbegin(track2ArtistIds), std::cend(track2ArtistIds), + std::back_inserter(commonArtistIds)); + + score += commonArtistIds.size(); + } + + // Clusters in common + { + auto track1ClusterIds {track1->getClusterIds()}; + auto track2ClusterIds {track2->getClusterIds()}; + + std::vector commonClusterIds; + std::set_intersection(std::cbegin(track1ClusterIds), std::cend(track1ClusterIds), + std::cbegin(track2ClusterIds), std::cend(track2ClusterIds), + std::back_inserter(commonClusterIds)); + + score += commonClusterIds.size(); + } + + return score; +} + +static +SimilarityScore +computeSimilarityScore(Database::Session& session, FeaturesSearcher::TrainSettings trainSettings) +{ + std::cout << "Compute score of: "; + printFeatureSettingsMap(trainSettings.featureSettingsMap); + std::cout << std::endl; + + FeaturesSearcher searcher {session, trainSettings}; + + const std::vector trackIds = std::invoke([&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Track::getAllIdsWithFeatures(session); + }); + + SimilarityScore score {}; + for (Database::IdType trackId : trackIds) + { + constexpr std::size_t nbSimilarTracks {3}; +// std::cout << "Processing track '" << trackToString(session, trackId) << "'" << std::endl; + SimilarityScore factor {1}; + for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, nbSimilarTracks)) + { + SimilarityScore trackScore {computeTrackScore(session, trackId, similarTrackId)}; +// std::cout << "\tScore = " << trackScore << " (*" << factor << ") with track '" << trackToString(session, similarTrackId) << "'" << std::endl; + trackScore *= factor; + score += trackScore; + + factor -= (SimilarityScore {1}/nbSimilarTracks ); + } + } + + std::cout << "Total score = " << score << std::endl; + + return score; +} + +static +void +printBadlyClassifiedTracks(Database::Session& session, FeaturesSearcher::TrainSettings trainSettings) +{ + + FeaturesSearcher searcher {session, trainSettings}; + + const std::vector trackIds = std::invoke([&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Track::getAllIdsWithFeatures(session); + }); + + for (Database::IdType trackId : trackIds) + { + constexpr std::size_t nbSimilarTracks {3}; + for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, nbSimilarTracks)) + { + SimilarityScore trackScore {computeTrackScore(session, trackId, similarTrackId)}; + if (trackScore == 0) + std::cout << "Badly classified tracks: '" << trackToString(session, trackId) << "'\n\twith track '" << trackToString(session, similarTrackId) << "'" < a.size()) + { + const auto itFeature {pickRandom(res)}; + res.erase(itFeature); + } + + return res; +} + +static +void +mutateFeatureSettingsMap(FeatureSettingsMap& a) +{ + const std::size_t size {a.size()}; + // Replace one of the feature with another one, random + a.erase(pickRandom(a)); + + while (a.size() != size) + { + const auto itFeatureSetting {pickRandom(featuresSettings)}; + a.emplace(itFeatureSetting->first, itFeatureSetting->second); + } +} + +int main(int argc, char *argv[]) +{ + try + { + + // log to stdout +// ServiceProvider::create(std::cout); + + if (argc != 3) + { + std::cerr << "usage: " << std::endl; + return EXIT_FAILURE; + } + + const std::filesystem::path configFilePath {std::string(argv[1], 0, 256)}; + const std::size_t nbWorkers = atoi(argv[2]); + + ServiceProvider::create(configFilePath); + + Database::Db db {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; + Database::SessionPool sessionPool {db, nbWorkers}; + + std::cout << "Caching all features..." << std::endl; + // Cache all the features of all the music in order to speed up the multiple trainings + const auto cachedFeatures { constructFeaturesCache(Database::SessionPool::ScopedSession {sessionPool}.get(), featuresSettings) }; + std::cout << "Caching all features DONE" << std::endl; + + FeaturesSearcher::setFeaturesFetchFunc( + [&](Database::IdType trackId, const FeatureNames& featureNames) + { + return getFeaturesFromCache(cachedFeatures, trackId, featureNames); + }); + + // Create some random settings (i.e random population) + std::vector initialPopulation; + + constexpr std::size_t populationSize {200}; + constexpr std::size_t nbFeatures {5}; + + for (std::size_t i {}; i < populationSize; ++i) + { + FeatureSettingsMap settings; + + while (settings.size() < nbFeatures) + { + const auto itFeatureSetting {pickRandom(featuresSettings)}; + settings.emplace(itFeatureSetting->first, itFeatureSetting->second); + } + + initialPopulation.emplace_back(std::move(settings)); + } + + FeaturesSearcher::TrainSettings trainSettings; + trainSettings.iterationCount = 8; + trainSettings.sampleCountPerNeuron = 1.5; + + GeneticAlgorithm::Params params; + params.nbWorkers = nbWorkers; + params.nbGenerations = 1; + params.crossoverRatio = 0.78; + params.mutationProbability = 0.2; + params.breedFunction = breedFeatureSettingsMap; + params.mutateFunction = mutateFeatureSettingsMap; + params.scoreFunction = + [&](const FeatureSettingsMap& featureSettings) + { + FeaturesSearcher::TrainSettings settings {trainSettings}; + settings.featureSettingsMap = featureSettings; + + Database::SessionPool::ScopedSession scopedSession {sessionPool}; + return computeSimilarityScore(scopedSession.get(), settings); + }; + + GeneticAlgorithm geneticAlgorithm {params}; + + std::cout << "Parameters:\n" + << "\tnb total settings = "<< featuresSettings.size() << "\n" + << "\tnb generations = " << params.nbGenerations << "\n" + << "\tpopulationSize = " << populationSize << "\n" + << "\tnbFeatures = " << nbFeatures << "\n" + << "\tcrossoverRatio = " << params.crossoverRatio << "\n" + << "\tmutationProbability = " << params.mutationProbability << "\n" + << std::endl; + + std::cout << "Starting simulation..." << std::endl; + const FeatureSettingsMap selectedSettings {geneticAlgorithm.simulate(initialPopulation)}; + std::cout << "Simulation complete! Best result:" << std::endl; + printFeatureSettingsMap(selectedSettings); + + // print all badly classified tracks + { + FeaturesSearcher::TrainSettings settings {trainSettings}; + settings.featureSettingsMap = selectedSettings; + + Database::SessionPool::ScopedSession scopedSession {sessionPool}; + printBadlyClassifiedTracks(scopedSession.get(), settings); + } + } + catch (std::exception& e) + { + std::cerr << "Caught exception: " << e.what() << std::endl; + } + + return EXIT_SUCCESS; +} + + diff --git a/tools/similarity-parameters/Makefile.am b/tools/similarity-parameters/Makefile.am new file mode 100644 index 00000000..1d47f224 --- /dev/null +++ b/tools/similarity-parameters/Makefile.am @@ -0,0 +1,28 @@ +noinst_PROGRAMS = lms-similarity-parameters + +lms_similarity_parameters_SOURCES = \ + $(srcdir)/LmsSimilarityParameters.cpp \ + $(top_srcdir)/src/database/Artist.cpp \ + $(top_srcdir)/src/database/Cluster.cpp \ + $(top_srcdir)/src/database/Db.cpp \ + $(top_srcdir)/src/database/TrackFeatures.cpp \ + $(top_srcdir)/src/database/TrackList.cpp \ + $(top_srcdir)/src/database/Release.cpp \ + $(top_srcdir)/src/database/ScanSettings.cpp \ + $(top_srcdir)/src/database/Session.cpp \ + $(top_srcdir)/src/database/SessionPool.cpp \ + $(top_srcdir)/src/database/SqlQuery.cpp \ + $(top_srcdir)/src/database/Track.cpp \ + $(top_srcdir)/src/database/User.cpp \ + $(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp \ + $(top_srcdir)/src/similarity/features/som/Network.cpp \ + $(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp \ + $(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp \ + $(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp \ + $(top_srcdir)/src/utils/Config.cpp \ + $(top_srcdir)/src/utils/Logger.cpp \ + $(top_srcdir)/src/utils/StreamLogger.cpp \ + $(top_srcdir)/src/utils/Utils.cpp + +lms_similarity_parameters_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT + diff --git a/tools/similarity-parameters/ParallelFor.hpp b/tools/similarity-parameters/ParallelFor.hpp new file mode 100644 index 00000000..141c7a69 --- /dev/null +++ b/tools/similarity-parameters/ParallelFor.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include +#include +#include + +template +void parallel_foreach(std::size_t nbWorkers, It begin, It end, Func&& func) +{ + if (nbWorkers == 0) + throw std::runtime_error("Invalid worker count"); + + boost::asio::io_context ioContext; + + for (It it {begin}; it != end; ++it) + { + auto refValue {std::ref(*it)}; + ioContext.post([refValue, &func]() { std::cout << "EXEC FROM WORKER" << std::endl; func(refValue); std::cout << "END EXEC FROM WORKER" << std::endl; }); + } + + std::vector threads; + for (std::size_t i {}; i < nbWorkers - 1; ++i) + threads.emplace_back([&]() { ioContext.run(); }); + + ioContext.run(); + + for (std::thread& t : threads) + t.join(); +} + diff --git a/tools/similarity/LmsSimilarity.cpp b/tools/similarity/LmsSimilarity.cpp index 618356c4..522b6558 100644 --- a/tools/similarity/LmsSimilarity.cpp +++ b/tools/similarity/LmsSimilarity.cpp @@ -1,88 +1,46 @@ -#include +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + #include #include #include #include -#include -#include "database/Db.hpp" -#include "database/Session.hpp" -#include "database/Track.hpp" #include "database/Artist.hpp" #include "database/Cluster.hpp" +#include "database/Db.hpp" #include "database/Release.hpp" -#include "database/TrackFeatures.hpp" +#include "database/Session.hpp" +#include "database/Track.hpp" #include "utils/Config.hpp" #include "utils/Service.hpp" -#include "similarity/features/som/DataNormalizer.hpp" -#include "similarity/features/som/Network.hpp" - -static -std::ostream& operator<<(std::ostream& os, const Database::Track::pointer& track) -{ - os << "["; - for (auto artist : track->getArtists()) - os << artist->getName() << " - "; - if (track->getRelease()) - os << track->getRelease()->getName() << " - "; - os << track->getName() << "]"; - - return os; -} - -static -bool -getTrackFeatures(Database::Session&, const Database::Track::pointer& track, const std::map& featuresSettings, SOM::InputVector& res) -{ - std::map> features; - for (const auto& featureSettings : featuresSettings) - features[featureSettings.first] = {}; - - if (!track->getTrackFeatures()->getFeatures(features)) - { - std::cout << "Skipping track '" << track->getMBID() << "': missing item" << std::endl; - return false; - }; - - std::size_t index {}; - for (const auto& feature : features) - { - auto it = featuresSettings.find(feature.first); - if (it == featuresSettings.end() || (feature.second.size() != it->second)) - return false; - - for (double value : feature.second) - res[index++] = value; - } - - return true; -} - +#include "utils/StreamLogger.hpp" +#include "similarity/features/SimilarityFeaturesSearcher.hpp" int main(int argc, char *argv[]) { try { - const std::size_t width = 5; - const std::size_t height = 5; - const std::size_t nbIterations = 10; - std::size_t nbTracks = 5000; + using namespace Similarity; - const std::map featuresSettings = - { -// { "lowlevel.average_loudness", 1 }, -// { "lowlevel.dynamic_complexity", 1 }, - { "lowlevel.spectral_contrast_coeffs.median", 6 }, - { "lowlevel.erbbands.median", 40 }, - { "tonal.hpcp.median", 36 }, - { "lowlevel.melbands.median", 40 }, - { "lowlevel.barkbands.median", 27 }, - { "lowlevel.mfcc.mean", 13 }, - { "lowlevel.gfcc.mean", 13 }, - }; - std::size_t nbDims = 0; - for (const auto& featureSettings : featuresSettings) - nbDims += featureSettings.second; + // log to stdout + ServiceProvider::create(std::cout); std::filesystem::path configFilePath {"/etc/lms.conf"}; if (argc >= 2) @@ -90,151 +48,96 @@ int main(int argc, char *argv[]) ServiceProvider::create(configFilePath); - Database::Db db {getService()->getPath("working-dir") / "lms.db"}; - auto session {db.createSession()}; - - std::cout << "Getting all features..." << std::endl; - auto transaction {session->createUniqueTransaction()}; - - std::vector trackIds {Database::Track::getAllIdsWithFeatures(*session, nbTracks)}; - - nbTracks = trackIds.size(); - std::cout << "Getting features DONE (" << nbTracks << " tracks)" << std::endl; - - std::cout << "Reading features..." << std::endl; - std::vector tracksFeatures; - - for (Database::IdType trackId : trackIds) - { - Database::Track::pointer track {Database::Track::getById(*session, trackId)}; - if (!track) - continue; - - SOM::InputVector features {nbDims}; - if (!getTrackFeatures(*session, track, featuresSettings, features)) - continue; - - tracksFeatures.emplace_back(std::move(features)); - } - std::cout << "Reading features DONE" << std::endl; - - SOM::Network network {width, height, nbDims}; - SOM::DataNormalizer normalizer {nbDims}; - - SOM::InputVector weights {nbDims}; - { - std::size_t index {}; - for (const auto& featureSettings : featuresSettings) - { - for (std::size_t i {}; i < featureSettings.second; ++i) - weights[index++] = SOM::InputVector::value_type{1. / featureSettings.second}; - } - } - - network.setDataWeights(weights); - - std::cout << "Weights: " << weights << std::endl; - - std::cout << "Normalizing..." << std::endl; - normalizer.computeNormalizationFactors(tracksFeatures); - - std::cout << "Dumping normalizer: " << std::endl; - normalizer.dump(std::cout); - std::cout << "Dumping normalizer DONE" << std::endl; - - for (SOM::InputVector& features : tracksFeatures) - normalizer.normalizeData(features); - std::cout << "Normalizing DONE" << std::endl; - - auto progress {[](const SOM::Network::CurrentIteration& iteration) - { - std::cout << "Iteration " << iteration.idIteration + 1 << " of " << iteration.iterationCount << std::endl;; - }}; - - std::cout << "Training..." << std::endl; - network.train(tracksFeatures, nbIterations, progress); - std::cout << "Training DONE" << std::endl; - - auto meanDistance = network.computeRefVectorsDistanceMean(); - std::cout << "MEAN distance = " << meanDistance << std::endl; - auto medianDistance = network.computeRefVectorsDistanceMedian(); - std::cout << "MEDIAN distance = " << medianDistance << std::endl; + Database::Db db {ServiceProvider::get()->getPath("working-dir") / "lms.db"}; + Database::Session session {db}; std::cout << "Classifying tracks..." << std::endl; - - SOM::Matrix< std::vector > tracksMap(width, height); - for (Database::IdType trackId : trackIds) - { - Database::Track::pointer track {Database::Track::getById(*session, trackId)}; - if (!track) - continue; - - SOM::InputVector features {nbDims}; - if (!getTrackFeatures(*session, track, featuresSettings, features)) - continue; - - normalizer.normalizeData(features); - - SOM::Position position = network.getClosestRefVectorPosition(features); - tracksMap[position].push_back(track); - } - + // may be long... + struct FeaturesSearcher::TrainSettings trainSettings; + trainSettings.featureSettingsMap = FeaturesSearcher::getDefaultTrainFeatureSettings(); + FeaturesSearcher searcher {session, trainSettings}; std::cout << "Classifying tracks DONE" << std::endl; - // Dump tracks + const std::vector trackIds = std::invoke([&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Track::getAllIdsWithFeatures(session); + }); - for (SOM::Coordinate y = 0; y < tracksMap.getHeight(); ++y) - { - for (SOM::Coordinate x = 0; x < tracksMap.getWidth(); ++x) - { - std::cout << "{" << x << ", " << y << "}" << std::endl; - const auto& tracks = tracksMap[{x, y}]; - - for (const auto& track : tracks) - { - std::cout << " - " << track << std::endl; - } - } - } - - // For each track, get the nearest tracks + std::cout << "*** Tracks (" << trackIds.size() << ") ***" << std::endl; for (Database::IdType trackId : trackIds) { - Database::Track::pointer track {Database::Track::getById(*session, trackId)}; - if (!track) - continue; - - SOM::InputVector features {nbDims}; - if (!getTrackFeatures(*session, track, featuresSettings, features)) - continue; - - normalizer.normalizeData(features); - - SOM::Position refVectorPosition {network.getClosestRefVectorPosition(features)}; - - std::cout << "Getting nearest songs for track " << track << " in {" << refVectorPosition.x << ", " << refVectorPosition.y << "}:" << std::endl; - for (auto similarTrack : tracksMap[refVectorPosition]) - std::cout << " - " << similarTrack << std::endl; - - std::set neighbourPosition {refVectorPosition}; - for (std::size_t i {}; i < 3; ++i) + auto trackToString = [&](Database::IdType trackId) { - auto position = network.getClosestRefVectorPosition(neighbourPosition, medianDistance); - if (!position) - break; + std::string res; + auto transaction {session.createSharedTransaction()}; + Database::Track::pointer track {Database::Track::getById(session, trackId)}; - std::cout << " - in {" << position->x << ", " << position->y << "}, dist = " << network.getRefVectorsDistance(*position, refVectorPosition) << std::endl; - for (const auto& similarTrack : tracksMap[*position]) - std::cout << " - " << similarTrack << std::endl; + res += track->getName(); + if (track->getRelease()) + res += " [" + track->getRelease()->getName() + "]"; + for (auto artist : track->getArtists()) + res += " - " + artist->getName(); + for (auto cluster : track->getClusters()) + res += " {" + cluster->getType()->getName() + "-"+ cluster->getName() + "}"; - neighbourPosition.insert(*position); - } + return res; + }; + std::cout << "Processing track '" << trackToString(trackId) << std::endl; + for (Database::IdType similarTrackId : searcher.getSimilarTracks({trackId}, 3)) + std::cout << "\t- Similar track '" << trackToString(similarTrackId) << std::endl; } + + const std::vector releaseIds = std::invoke([&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Release::getAllIds(session); + }); + + std::cout << "*** Releases ***" << std::endl; + for (Database::IdType releaseId : releaseIds) + { + auto releaseToString = [&](Database::IdType releaseId) + { + auto transaction {session.createSharedTransaction()}; + + Database::Release::pointer release {Database::Release::getById(session, releaseId)}; + return release->getName(); + }; + + std::cout << "Processing release '" << releaseToString(releaseId) << "'" << std::endl; + for (Database::IdType similarReleaseId : searcher.getSimilarReleases({releaseId}, 3)) + std::cout << "\t- Similar release '" << releaseToString(similarReleaseId) << "'" << std::endl; + } + + const std::vector artistIds = std::invoke([&]() + { + auto transaction {session.createSharedTransaction()}; + return Database::Artist::getAllIds(session); + }); + + std::cout << "*** Artists ***" << std::endl; + for (Database::IdType artistId : artistIds) + { + auto artistToString = [&](Database::IdType artistId) + { + auto transaction {session.createSharedTransaction()}; + + Database::Artist::pointer artist {Database::Artist::getById(session, artistId)}; + return artist->getName(); + }; + + std::cout << "Processing artist '" << artistToString(artistId) << "'" << std::endl; + for (Database::IdType similarArtistId : searcher.getSimilarArtists({artistId}, 3)) + std::cout << "\t- Similar artist '" << artistToString(similarArtistId) << "'" << std::endl; + } + } catch( std::exception& e) { std::cerr << "Caught exception: " << e.what() << std::endl; + return EXIT_FAILURE; } return EXIT_SUCCESS; diff --git a/tools/similarity/Makefile.am b/tools/similarity/Makefile.am index 441a3be2..11657a5e 100644 --- a/tools/similarity/Makefile.am +++ b/tools/similarity/Makefile.am @@ -10,14 +10,17 @@ lms_similarity_SOURCES = \ $(top_srcdir)/src/database/Release.cpp \ $(top_srcdir)/src/database/ScanSettings.cpp \ $(top_srcdir)/src/database/Session.cpp \ - $(top_srcdir)/src/database/SimilaritySettings.cpp \ $(top_srcdir)/src/database/SqlQuery.cpp \ $(top_srcdir)/src/database/Track.cpp \ $(top_srcdir)/src/database/User.cpp \ $(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp \ $(top_srcdir)/src/similarity/features/som/Network.cpp \ + $(top_srcdir)/src/similarity/features/SimilarityFeaturesCache.cpp \ + $(top_srcdir)/src/similarity/features/SimilarityFeaturesSearcher.cpp \ + $(top_srcdir)/src/similarity/features/SimilarityFeaturesDefs.cpp \ $(top_srcdir)/src/utils/Config.cpp \ $(top_srcdir)/src/utils/Logger.cpp \ + $(top_srcdir)/src/utils/StreamLogger.cpp \ $(top_srcdir)/src/utils/Utils.cpp lms_similarity_CXXFLAGS=-std=c++17 -I$(top_srcdir)/src -D_REENTRANT