From 6fcb69326148dcc4e076e1788a42ff64959fd74e Mon Sep 17 00:00:00 2001 From: emeric Date: Tue, 5 Feb 2019 14:08:46 +0100 Subject: [PATCH] Reorganized sources, better accuracy for similarities based on features --- src/Makefile.am | 12 +- src/database/DatabaseHandler.cpp | 11 +- src/database/Release.hpp | 1 + src/database/SimilaritySettings.cpp | 123 +++---- src/database/SimilaritySettings.hpp | 77 +++-- src/database/Track.cpp | 20 +- src/database/Track.hpp | 16 +- src/database/TrackFeature.cpp | 58 ---- src/database/TrackFeature.hpp | 97 ------ src/database/TrackFeatures.cpp | 93 ++++++ src/database/TrackFeatures.hpp | 63 ++++ src/main/main.cpp | 8 +- src/scanner/MediaScanner.cpp | 32 +- src/similarity/SimilaritySearcher.cpp | 54 +++- src/similarity/SimilaritySearcher.hpp | 11 +- .../cluster/SimilarityClusterSearcher.cpp | 125 +++++-- .../cluster/SimilarityClusterSearcher.hpp | 11 +- .../SimilarityFeaturesScannerAddon.cpp | 141 ++++++++ .../SimilarityFeaturesScannerAddon.hpp} | 24 +- .../features/SimilarityFeaturesSearcher.cpp | 305 ++++++++++++++++++ .../features/SimilarityFeaturesSearcher.hpp | 64 ++++ .../som/AcousticBrainzUtils.cpp | 35 +- .../som/AcousticBrainzUtils.hpp | 3 +- .../{ => features}/som/DataNormalizer.cpp | 33 -- .../{ => features}/som/DataNormalizer.hpp | 1 - src/similarity/{ => features}/som/Matrix.hpp | 19 +- src/similarity/{ => features}/som/Network.cpp | 217 +++++++------ src/similarity/{ => features}/som/Network.hpp | 37 +-- .../som/SimilaritySOMScannerAddon.cpp | 272 ---------------- src/similarity/som/SimilaritySOMSearcher.cpp | 259 --------------- src/similarity/som/SimilaritySOMSearcher.hpp | 65 ---- src/ui/PlayQueueView.cpp | 2 +- .../feature-extractor/LmsFeatureExtractor.cpp | 248 +++++++++++--- tools/feature-extractor/Makefile.am | 5 +- 34 files changed, 1324 insertions(+), 1218 deletions(-) delete mode 100644 src/database/TrackFeature.cpp delete mode 100644 src/database/TrackFeature.hpp create mode 100644 src/database/TrackFeatures.cpp create mode 100644 src/database/TrackFeatures.hpp create mode 100644 src/similarity/features/SimilarityFeaturesScannerAddon.cpp rename src/similarity/{som/SimilaritySOMScannerAddon.hpp => features/SimilarityFeaturesScannerAddon.hpp} (70%) create mode 100644 src/similarity/features/SimilarityFeaturesSearcher.cpp create mode 100644 src/similarity/features/SimilarityFeaturesSearcher.hpp rename src/similarity/{ => features}/som/AcousticBrainzUtils.cpp (68%) rename src/similarity/{ => features}/som/AcousticBrainzUtils.hpp (85%) rename src/similarity/{ => features}/som/DataNormalizer.cpp (82%) rename src/similarity/{ => features}/som/DataNormalizer.hpp (97%) rename src/similarity/{ => features}/som/Matrix.hpp (89%) rename src/similarity/{ => features}/som/Network.cpp (58%) rename src/similarity/{ => features}/som/Network.hpp (74%) delete mode 100644 src/similarity/som/SimilaritySOMScannerAddon.cpp delete mode 100644 src/similarity/som/SimilaritySOMSearcher.cpp delete mode 100644 src/similarity/som/SimilaritySOMSearcher.hpp diff --git a/src/Makefile.am b/src/Makefile.am index 5e27ba28..76aea669 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ lms_SOURCES = \ $(srcdir)/database/Artist.cpp \ $(srcdir)/database/Cluster.cpp \ $(srcdir)/database/DatabaseHandler.cpp \ - $(srcdir)/database/TrackFeature.cpp \ + $(srcdir)/database/TrackFeatures.cpp \ $(srcdir)/database/TrackList.cpp \ $(srcdir)/database/Release.cpp \ $(srcdir)/database/ScanSettings.cpp \ @@ -23,11 +23,11 @@ lms_SOURCES = \ $(srcdir)/scanner/MediaScanner.cpp \ $(srcdir)/similarity/SimilaritySearcher.cpp \ $(srcdir)/similarity/cluster/SimilarityClusterSearcher.cpp \ - $(srcdir)/similarity/som/AcousticBrainzUtils.cpp \ - $(srcdir)/similarity/som/DataNormalizer.cpp \ - $(srcdir)/similarity/som/Network.cpp \ - $(srcdir)/similarity/som/SimilaritySOMScannerAddon.cpp \ - $(srcdir)/similarity/som/SimilaritySOMSearcher.cpp \ + $(srcdir)/similarity/features/SimilarityFeaturesScannerAddon.cpp \ + $(srcdir)/similarity/features/SimilarityFeaturesSearcher.cpp \ + $(srcdir)/similarity/features/som/AcousticBrainzUtils.cpp \ + $(srcdir)/similarity/features/som/DataNormalizer.cpp \ + $(srcdir)/similarity/features/som/Network.cpp \ $(srcdir)/ui/Auth.cpp \ $(srcdir)/ui/LmsApplication.cpp \ $(srcdir)/ui/LmsApplicationGroup.cpp \ diff --git a/src/database/DatabaseHandler.cpp b/src/database/DatabaseHandler.cpp index 8be91b9b..c9165e4b 100644 --- a/src/database/DatabaseHandler.cpp +++ b/src/database/DatabaseHandler.cpp @@ -40,7 +40,7 @@ #include "SimilaritySettings.hpp" #include "Track.hpp" #include "TrackList.hpp" -#include "TrackFeature.hpp" +#include "TrackFeatures.hpp" namespace Database { @@ -106,11 +106,11 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool) _session.mapClass("tracklist_entry"); _session.mapClass("release"); _session.mapClass("track"); - _session.mapClass("track_feature"); - _session.mapClass("track_feature_type"); + _session.mapClass("track_features"); _session.mapClass("scan_settings"); _session.mapClass("similarity_settings"); + _session.mapClass("similarity_settings_feature"); _session.mapClass("auth_info"); _session.mapClass("auth_identity"); @@ -142,10 +142,7 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool) _session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)"); _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)"); _session.execute("CREATE INDEX IF NOT EXISTS tracklist_name_idx ON tracklist(name)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_feature_type_name_idx ON track_feature_type(name)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_feature_type_idx ON track_feature(type_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_feature_track_idx ON track_feature(track_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS track_feature_type_track_idx ON track_feature(type_id, track_id)"); + _session.execute("CREATE INDEX IF NOT EXISTS track_features_track_idx ON track_features(track_id)"); } _users = new UserDatabase(_session); diff --git a/src/database/Release.hpp b/src/database/Release.hpp index 86584919..2e24bbdd 100644 --- a/src/database/Release.hpp +++ b/src/database/Release.hpp @@ -59,6 +59,7 @@ class Release : public Wt::Dbo::Dbo bool& moreExpected); std::vector> getTracks(const std::set& clusters = std::set()) const; + // Get the cluster of the tracks that belong to this release // Each clusters are grouped by cluster type, sorted by the number of occurence // size is the max number of cluster per cluster type diff --git a/src/database/SimilaritySettings.cpp b/src/database/SimilaritySettings.cpp index 923c96d1..b104eb16 100644 --- a/src/database/SimilaritySettings.cpp +++ b/src/database/SimilaritySettings.cpp @@ -22,53 +22,42 @@ #include "utils/Logger.hpp" #include "utils/Utils.hpp" -#include "TrackFeature.hpp" - -namespace { - -std::set defaultTrackFeaturesNames = -{ - "lowlevel.average_loudness", - "lowlevel.barkbands_flatness_db.mean", - "lowlevel.dissonance.mean", - "lowlevel.dynamic_complexity", - "lowlevel.hfc.mean", // GOOD - "lowlevel.melbands_crest.mean", - "lowlevel.melbands_kurtosis.mean", - "lowlevel.melbands_skewness.mean", - "lowlevel.melbands_spread.mean", - "lowlevel.pitch_salience.mean", - "lowlevel.pitch_salience.var", - "lowlevel.silence_rate_30dB.mean", - "lowlevel.silence_rate_60dB.mean", - "lowlevel.spectral_centroid.mean", - "lowlevel.spectral_complexity.mean", - "lowlevel.spectral_decrease.mean", - "lowlevel.spectral_energy.mean", - "lowlevel.spectral_energyband_high.mean", - "lowlevel.spectral_energyband_low.mean", - "lowlevel.spectral_energyband_middle_high.mean", - "lowlevel.spectral_energyband_middle_low.mean", - "lowlevel.spectral_entropy.mean", - "lowlevel.spectral_flux.mean", - "lowlevel.spectral_kurtosis.mean", - "lowlevel.spectral_rms.mean", - "lowlevel.spectral_skewness.mean", - "lowlevel.spectral_spread.mean", - "lowlevel.spectral_strongpeak.mean", - "lowlevel.zerocrossingrate.mean", - "rhythm.beats_loudness.mean", // BAD - "rhythm.bpm", - "tonal.chords_changes_rate", // OK - "tonal.chords_number_rate", // BAD - "tonal.chords_strength.mean", // OK - "tonal.hpcp_entropy.mean", // GOOD -}; - -} // namespace +#include "TrackFeatures.hpp" namespace Database { +struct TrackFeatureInfo +{ + std::string name; + std::size_t nbDimensions; + double weight; +}; + +static std::vector defaultFeatures = +{ + { "lowlevel.spectral_contrast_coeffs.median", 6, 1. }, + { "lowlevel.erbbands.median", 40, 1. }, + { "tonal.hpcp.median", 36, 1. }, + { "lowlevel.melbands.median", 40, 1. }, + { "lowlevel.barkbands.median", 27, 1. }, + { "lowlevel.mfcc.mean", 13, 1. }, + { "lowlevel.gfcc.mean", 13, 1. }, +}; + + +SimilaritySettingsFeature::SimilaritySettingsFeature(Wt::Dbo::ptr settings, const std::string& name, std::size_t nbDimensions, double weight) +: _name(name), +_nbDimensions(nbDimensions), +_weight(weight), +_settings(settings) +{ +} + +SimilaritySettingsFeature::pointer +SimilaritySettingsFeature::create(Wt::Dbo::Session& session, Wt::Dbo::ptr settings, const std::string& name, std::size_t nbDimensions, double weight) +{ + return session.add(std::make_unique(settings, name, nbDimensions, weight)); +} SimilaritySettings::pointer SimilaritySettings::get(Wt::Dbo::Session& session) @@ -77,54 +66,20 @@ SimilaritySettings::get(Wt::Dbo::Session& session) if (!settings) { settings = session.add(std::make_unique()); - settings.modify()->setTrackFeatureTypes(defaultTrackFeaturesNames); + + for (const auto& feature : defaultFeatures) + SimilaritySettingsFeature::create(session, settings, feature.name, feature.nbDimensions, feature.weight); } return settings; } -std::vector> -SimilaritySettings::getTrackFeatureTypes() const +std::vector> +SimilaritySettings::getFeatures() const { - return std::vector>(_trackFeatureTypes.begin(), _trackFeatureTypes.end()); + return std::vector>(_features.begin(), _features.end()); } -void -SimilaritySettings::setTrackFeatureTypes(const std::set& featuresNames) -{ - bool needRescan = false; - assert(session()); - - // Create any missing feature type - for (const auto& featureName : featuresNames) - { - auto featureType = TrackFeatureType::getByName(*session(), featureName); - if (!featureType) - { - LMS_LOG(DB, INFO) << "Creating feature type " << featureName; - featureType = TrackFeatureType::create(*session(), featureName); - _trackFeatureTypes.insert(featureType); - - needRescan = true; - } - } - - // Delete no longer existing feature type - for (auto trackFeatureType : _trackFeatureTypes) - { - if (std::none_of(featuresNames.begin(), featuresNames.end(), - [trackFeatureType](const std::string& name) { return name == trackFeatureType->getName(); })) - { - LMS_LOG(DB, INFO) << "Deleting track feature type " << trackFeatureType->getName(); - trackFeatureType.remove(); - } - } - - if (needRescan) - _scanVersion += 1; -} - - } // namespace Database diff --git a/src/database/SimilaritySettings.hpp b/src/database/SimilaritySettings.hpp index e8e855bd..483e0f51 100644 --- a/src/database/SimilaritySettings.hpp +++ b/src/database/SimilaritySettings.hpp @@ -23,42 +23,75 @@ namespace Database { -class TrackFeature; -class TrackFeatureType; +class SimilaritySettings; -class SimilaritySettings : public Wt::Dbo::Dbo +class SimilaritySettingsFeature : public Wt::Dbo::Dbo { public: - using pointer = Wt::Dbo::ptr; + using pointer = Wt::Dbo::ptr; - static pointer get(Wt::Dbo::Session& session); + SimilaritySettingsFeature() = default; + SimilaritySettingsFeature(Wt::Dbo::ptr settings, const std::string& name, std::size_t nbDimensions, double weight); - std::size_t getVersion() const { return _scanVersion; } + static pointer create(Wt::Dbo::Session& session, Wt::Dbo::ptr settings, const std::string& name, std::size_t nbDimensions, double weight = 1); - std::vector> getTrackFeatureTypes() const; - void setTrackFeatureTypes(const std::set& featureTypeNames); - - void setNetworkData(std::string data) { _refFeaturesData = data; } - const std::string& getNetworkData() const { return _refFeaturesData; } - - void setNormalizationData(std::string data) { _normalizationData = data; } - const std::string& getNormalizationData() const { return _normalizationData; } + const std::string& getName() const { return _name; } ; + std::size_t getNbDimensions() const { return static_cast(_nbDimensions); } + double getWeight() const { return _weight; } template void persist(Action& a) { - Wt::Dbo::field(a, _scanVersion, "settings_version"); - Wt::Dbo::field(a, _normalizationData, "normalization_data"); - Wt::Dbo::field(a, _refFeaturesData, "ref_features_data"); - Wt::Dbo::hasMany(a, _trackFeatureTypes, Wt::Dbo::ManyToOne, "similarity_settings"); + Wt::Dbo::field(a, _name, "name"); + Wt::Dbo::field(a, _nbDimensions, "dimension_count"); + Wt::Dbo::field(a, _weight, "weight"); + + Wt::Dbo::belongsTo(a, _settings, "similarity_settings", Wt::Dbo::OnDeleteCascade); + } + + private: + std::string _name; + int _nbDimensions; + double _weight; + + Wt::Dbo::ptr _settings; +}; + +class SimilaritySettings : public Wt::Dbo::Dbo +{ + public: + + enum class PreferredMethod + { + Auto, + Features, + Clusters, + }; + + using pointer = Wt::Dbo::ptr; + + // Utils + static pointer get(Wt::Dbo::Session& session); + + // Accessors + std::size_t getVersion() const { return _settingsVersion; } + PreferredMethod getPreferredMethod() const { return _preferredMethod; } + std::vector> getFeatures() const; + + template + void persist(Action& a) + { + Wt::Dbo::field(a, _settingsVersion, "settings_version"); + Wt::Dbo::field(a, _preferredMethod, "preferred_method"); + + Wt::Dbo::hasMany(a, _features, Wt::Dbo::ManyToOne, "similarity_settings"); } private: - int _scanVersion = 0; - std::string _normalizationData; - std::string _refFeaturesData; - Wt::Dbo::collection> _trackFeatureTypes; + int _settingsVersion = 0; + PreferredMethod _preferredMethod = PreferredMethod::Auto; + Wt::Dbo::collection> _features; }; diff --git a/src/database/Track.cpp b/src/database/Track.cpp index 2fa2e508..df44a28a 100644 --- a/src/database/Track.cpp +++ b/src/database/Track.cpp @@ -26,7 +26,7 @@ #include "Artist.hpp" #include "Cluster.hpp" #include "Release.hpp" -#include "TrackFeature.hpp" +#include "TrackFeatures.hpp" #include "SqlQuery.hpp" namespace Database { @@ -114,7 +114,7 @@ Track::getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session) Wt::Dbo::collection res = session.query ("SELECT t FROM track t") .where("LENGTH(t.mbid) > 0") - .where("NOT EXISTS (SELECT * FROM track_feature t_f WHERE t_f.track_id = t.id)"); + .where("NOT EXISTS (SELECT * FROM track_features t_f WHERE t_f.track_id = t.id)"); return std::vector(res.begin(), res.end()); } @@ -123,7 +123,7 @@ Track::getAllWithFeatures(Wt::Dbo::Session& session) { Wt::Dbo::collection res = session.query ("SELECT t FROM track t") - .where("EXISTS (SELECT * from track_feature t_f WHERE t_f.track_id = t.id)"); + .where("EXISTS (SELECT * from track_features t_f WHERE t_f.track_id = t.id)"); return std::vector(res.begin(), res.end()); } @@ -138,7 +138,7 @@ Track::getClusters(void) const bool Track::hasTrackFeatures() const { - return !_trackFeatures.empty(); + return (_trackFeatures.lock() != Database::TrackFeatures::pointer()); } static @@ -262,16 +262,10 @@ Track::getCopyrightURL() const return _copyrightURL != "" ? boost::make_optional(_copyrightURL) : boost::none; } -Wt::Dbo::ptr -Track::getTrackFeature(Wt::Dbo::ptr type) const +Wt::Dbo::ptr +Track::getTrackFeatures() const { - assert(self()); - assert(IdIsValid(self()->id())); - assert(session()); - - return session()->find() - .where("type_id = ?").bind(type.id()) - .where("track_id = ?").bind(self()->id()); + return _trackFeatures.lock(); } std::vector> diff --git a/src/database/Track.hpp b/src/database/Track.hpp index 06d6d587..9ad21ddc 100644 --- a/src/database/Track.hpp +++ b/src/database/Track.hpp @@ -37,8 +37,7 @@ class Artist; class Cluster; class ClusterType; class Release; -class TrackFeature; -class TrackFeatureType; +class TrackFeatures; class TrackListEntry; class TrackStats; @@ -81,7 +80,6 @@ class Track : public Wt::Dbo::Dbo // Accessors void setScanVersion(std::size_t version) { _scanVersion = version; } - void setSimilarityScanVersion(std::size_t version) { _similarityScanVersion = version; } void setTrackNumber(int num) { _trackNumber = num; } void setTotalTrackNumber(int num) { _totalTrackNumber = num; } void setDiscNumber(int num) { _discNumber = num; } @@ -101,10 +99,9 @@ class Track : public Wt::Dbo::Dbo void setArtist(Wt::Dbo::ptr artist) { _artist = artist; } void setRelease(Wt::Dbo::ptr release) { _release = release; } void eraseClusters() { _clusters.clear(); } - void eraseFeatures() { _trackFeatures.clear(); } + void eraseFeatures() { /*_trackFeatures.reset();*/ } std::size_t getScanVersion() const { return _scanVersion; } - std::size_t getSimilarityScanVersion() const { return _similarityScanVersion; } boost::optional getTrackNumber() const; boost::optional getTotalTrackNumber() const; boost::optional getDiscNumber() const; @@ -124,9 +121,8 @@ class Track : public Wt::Dbo::Dbo Wt::Dbo::ptr getArtist() const { return _artist; } Wt::Dbo::ptr getRelease() const { return _release; } std::vector> getClusters() const; - std::vector> getTrackFeatures() const; // ordered by feature's name bool hasTrackFeatures() const; - Wt::Dbo::ptr getTrackFeature(Wt::Dbo::ptr type) const; + Wt::Dbo::ptr getTrackFeatures() const; std::vector>> getClusterGroups(std::vector> clusterTypes, std::size_t size) const; @@ -134,7 +130,6 @@ class Track : public Wt::Dbo::Dbo void persist(Action& a) { Wt::Dbo::field(a, _scanVersion, "scan_version"); - Wt::Dbo::field(a, _similarityScanVersion, "similarity_version"); Wt::Dbo::field(a, _trackNumber, "track_number"); Wt::Dbo::field(a, _totalTrackNumber, "total_track_number"); Wt::Dbo::field(a, _discNumber, "disc_number"); @@ -156,7 +151,7 @@ class Track : public Wt::Dbo::Dbo Wt::Dbo::belongsTo(a, _artist, "artist", Wt::Dbo::OnDeleteCascade); Wt::Dbo::hasMany(a, _clusters, Wt::Dbo::ManyToMany, "track_cluster", "", Wt::Dbo::OnDeleteCascade); Wt::Dbo::hasMany(a, _playlistEntries, Wt::Dbo::ManyToOne, "track"); - Wt::Dbo::hasMany(a, _trackFeatures, Wt::Dbo::ManyToOne, "track"); + Wt::Dbo::hasOne(a, _trackFeatures); } private: @@ -166,7 +161,6 @@ class Track : public Wt::Dbo::Dbo static const std::size_t _maxCopyrightURLLength = 128; int _scanVersion = 0; - int _similarityScanVersion = 0; int _trackNumber = 0; int _totalTrackNumber = 0; int _discNumber = 0; @@ -191,7 +185,7 @@ class Track : public Wt::Dbo::Dbo Wt::Dbo::ptr _release; Wt::Dbo::collection> _clusters; Wt::Dbo::collection> _playlistEntries; - Wt::Dbo::collection> _trackFeatures; + Wt::Dbo::weak_ptr _trackFeatures; }; diff --git a/src/database/TrackFeature.cpp b/src/database/TrackFeature.cpp deleted file mode 100644 index 90671241..00000000 --- a/src/database/TrackFeature.cpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "TrackFeature.hpp" - -#include "SimilaritySettings.hpp" -#include "Track.hpp" - -namespace Database { - -TrackFeatureType::TrackFeatureType(std::string name) - : _name(name) -{ -} - -TrackFeatureType::pointer -TrackFeatureType::getByName(Wt::Dbo::Session& session, std::string name) -{ - return session.find().where("name = ?").bind(name); -} - -TrackFeatureType::pointer -TrackFeatureType::create(Wt::Dbo::Session& session, std::string name) -{ - return session.add(std::make_unique(name)); -} - - -TrackFeature::TrackFeature(Wt::Dbo::ptr type, Wt::Dbo::ptr track, double value) -: _type(type), -_track(track), -_value(value) -{ -} - -TrackFeature::pointer -TrackFeature::create(Wt::Dbo::Session& session, Wt::Dbo::ptr type, Wt::Dbo::ptr track, double value) -{ - return session.add(std::make_unique(type, track, value)); -} - -} // namespace Database diff --git a/src/database/TrackFeature.hpp b/src/database/TrackFeature.hpp deleted file mode 100644 index 0ce01a47..00000000 --- a/src/database/TrackFeature.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include - -#include - -#include "Types.hpp" - -namespace Database { - -class Track; -class SimilaritySettings; - -class TrackFeatureType : public Wt::Dbo::Dbo -{ - public: - using pointer = Wt::Dbo::ptr; - - TrackFeatureType() = default; - TrackFeatureType(std::string name); - - // Find utility - static pointer getByName(Wt::Dbo::Session& session, std::string name); - - // Create utility - static pointer create(Wt::Dbo::Session& session, std::string name); - - // Accessors - const std::string& getName() const { return _name; } - - template - void persist(Action& a) - { - Wt::Dbo::field(a, _name, "name"); - Wt::Dbo::belongsTo(a, _similaritySettings, "similarity_settings", Wt::Dbo::OnDeleteCascade); - } - - private: - - std::string _name; - - Wt::Dbo::ptr _similaritySettings; -}; - -class TrackFeature : public Wt::Dbo::Dbo -{ - public: - - using pointer = Wt::Dbo::ptr; - - TrackFeature() = default; - TrackFeature(Wt::Dbo::ptr type, Wt::Dbo::ptr track, double value); - - // Create utility - static pointer create(Wt::Dbo::Session& session, Wt::Dbo::ptr type, Wt::Dbo::ptr track, double value); - - Wt::Dbo::ptr getType() const { return _type; } - double getValue() const { return _value; } - - template - void persist(Action& a) - { - Wt::Dbo::field(a, _value, "value"); - Wt::Dbo::belongsTo(a, _type, "type", Wt::Dbo::OnDeleteCascade); - Wt::Dbo::belongsTo(a, _track, "track", Wt::Dbo::OnDeleteCascade); - } - - private: - - Wt::Dbo::ptr _type; - Wt::Dbo::ptr _track; - double _value = 0.; -}; - - -} // namespace database - - diff --git a/src/database/TrackFeatures.cpp b/src/database/TrackFeatures.cpp new file mode 100644 index 00000000..a7b4e381 --- /dev/null +++ b/src/database/TrackFeatures.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "TrackFeatures.hpp" + +#include +#include + +#include "utils/Logger.hpp" +#include "Track.hpp" + +namespace Database { + +TrackFeatures::TrackFeatures(Wt::Dbo::ptr track, const std::string& jsonEncodedFeatures) +: _data(jsonEncodedFeatures), +_track(track) +{ +} + +TrackFeatures::pointer +TrackFeatures::create(Wt::Dbo::Session& session, Wt::Dbo::ptr track, const std::string& jsonEncodedFeatures) +{ + return session.add(std::make_unique(track, jsonEncodedFeatures)); +} + +std::vector +TrackFeatures::getFeatures(const std::string& featureNode) const +{ + std::vector res; + + std::map> features = { {featureNode, {}} }; + if (!getFeatures( features )) + return res; + + res = std::move(features[featureNode]); + + return res; +} + +bool +TrackFeatures::getFeatures(std::map /*values*/>& features) const +{ + try + { + boost::property_tree::ptree root; + + std::istringstream iss(_data); + boost::property_tree::read_json(iss, root); + + for (auto& featureNode : features) + { + auto node = root.get_child(featureNode.first); + + bool hasChildren = false; + for (const auto& child : node.get_child("")) + { + hasChildren = true; + featureNode.second.push_back(child.second.get_value()); + } + + if (!hasChildren) + { + featureNode.second.push_back(node.get_value()); + } + } + + return true; + } + catch (boost::property_tree::ptree_error& error) + { + LMS_LOG(DB, ERROR) << "ptree exception: " << error.what(); + std::cout << "ptree exception: " << error.what() << std::endl; + return false; + } +} + +} // namespace Database diff --git a/src/database/TrackFeatures.hpp b/src/database/TrackFeatures.hpp new file mode 100644 index 00000000..cd884d26 --- /dev/null +++ b/src/database/TrackFeatures.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include + +#include + +#include "Types.hpp" + +namespace Database { + +class Track; + +class TrackFeatures : public Wt::Dbo::Dbo +{ + public: + + using pointer = Wt::Dbo::ptr; + + TrackFeatures() = default; + TrackFeatures(Wt::Dbo::ptr track, const std::string& jsonEncodedFeatures); + + // Create utility + static pointer create(Wt::Dbo::Session& session, Wt::Dbo::ptr track, const std::string& jsonEncodedFeatures); + + std::vector getFeatures(const std::string& featureNode) const; + bool getFeatures(std::map /*values*/>& featureNodes) const; + + template + void persist(Action& a) + { + Wt::Dbo::field(a, _data, "data"); + Wt::Dbo::belongsTo(a, _track, "track", Wt::Dbo::OnDeleteCascade); + } + + private: + + std::string _data; + Wt::Dbo::ptr _track; +}; + + +} // namespace database + + diff --git a/src/main/main.cpp b/src/main/main.cpp index 8a4a4265..cb0c305b 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -28,7 +28,7 @@ #include "cover/CoverArtGrabber.hpp" #include "image/Image.hpp" #include "scanner/MediaScanner.hpp" -#include "similarity/som/SimilaritySOMScannerAddon.hpp" +#include "similarity/features/SimilarityFeaturesScannerAddon.hpp" #include "similarity/SimilaritySearcher.hpp" #include "ui/LmsApplication.hpp" #include "utils/Config.hpp" @@ -129,11 +129,11 @@ int main(int argc, char* argv[]) // Service initialization order is important getServices().mediaScanner = std::make_unique(*connectionPool); - Similarity::SOMScannerAddon similaritySOMScannerAddon(*connectionPool); + Similarity::FeaturesScannerAddon similarityFeaturesScannerAddon(*connectionPool); - getServices().mediaScanner->setAddon(similaritySOMScannerAddon); + getServices().mediaScanner->setAddon(similarityFeaturesScannerAddon); getServices().coverArtGrabber = std::make_unique(); - getServices().similaritySearcher = std::make_unique(similaritySOMScannerAddon); + getServices().similaritySearcher = std::make_unique(similarityFeaturesScannerAddon); // bind entry point server.addEntryPoint(Wt::EntryPointType::Application, diff --git a/src/scanner/MediaScanner.cpp b/src/scanner/MediaScanner.cpp index 7a8e03f7..43429ea9 100644 --- a/src/scanner/MediaScanner.cpp +++ b/src/scanner/MediaScanner.cpp @@ -642,32 +642,32 @@ MediaScanner::scanMediaDirectory(boost::filesystem::path mediaDirectory, bool fo // Check if a file exists and is still in a media directory static bool -checkFile(const boost::filesystem::path& p, boost::filesystem::path mediaDirectory, const std::set& extensions) +checkFile(const boost::filesystem::path& p, const boost::filesystem::path& mediaDirectory, const std::set& extensions) { try { - bool status = true; - // For each track, make sure the the file still exists // and still belongs to a media directory if (!boost::filesystem::exists( p ) || !boost::filesystem::is_regular( p ) ) { LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing"; - status = false; - } - else if (!isPathInParentPath(p, mediaDirectory)) - { - LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory"; - status = false; - } - else if (!isFileSupported(p, extensions)) - { - LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled"; - status = false; + return false; } - return status; + if (!isPathInParentPath(p, mediaDirectory)) + { + LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory"; + return false; + } + + if (!isFileSupported(p, extensions)) + { + LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled"; + return false; + } + + return true; } catch (boost::filesystem::filesystem_error& e) @@ -683,7 +683,7 @@ MediaScanner::removeMissingTracks(Stats& stats) std::vector trackPaths = Track::getAllPaths(_db.getSession());; LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks..."; - for (auto& trackPath : trackPaths) + for (const auto& trackPath : trackPaths) { if (!_running) return; diff --git a/src/similarity/SimilaritySearcher.cpp b/src/similarity/SimilaritySearcher.cpp index 4f504cf8..42fcf98b 100644 --- a/src/similarity/SimilaritySearcher.cpp +++ b/src/similarity/SimilaritySearcher.cpp @@ -19,40 +19,68 @@ #include "SimilaritySearcher.hpp" +#include "features/SimilarityFeaturesScannerAddon.hpp" +#include "cluster/SimilarityClusterSearcher.hpp" + +#include "database/SimilaritySettings.hpp" + namespace Similarity { -Searcher::Searcher(SOMScannerAddon& somAddon) +Searcher::Searcher(FeaturesScannerAddon& somAddon) : _somAddon(somAddon) {} -std::vector -Searcher::getSimilarTracks(const std::vector& tracksId, std::size_t maxCount) +static +Database::SimilaritySettings::PreferredMethod getPreferredMethod(Wt::Dbo::Session& session) { - auto somSearcher = _somAddon.getSearcher(); - if (!somSearcher) - return {}; + Wt::Dbo::Transaction transaction(session); + return Database::SimilaritySettings::get(session)->getPreferredMethod(); +} - return somSearcher->getSimilarTracks(tracksId, maxCount); +std::vector +Searcher::getSimilarTracks(Wt::Dbo::Session& session, const std::set& trackIds, std::size_t maxCount) +{ + + auto method = getPreferredMethod(session); + auto somSearcher = _somAddon.getSearcher(); + + if (method == Database::SimilaritySettings::PreferredMethod::Features + || (method == Database::SimilaritySettings::PreferredMethod::Auto && somSearcher)) + { + return somSearcher->getSimilarTracks(trackIds, maxCount); + } + else + return ClusterSearcher::getSimilarTracks(session, trackIds, maxCount); } std::vector Searcher::getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount) { + auto method = getPreferredMethod(session); auto somSearcher = _somAddon.getSearcher(); - if (!somSearcher) - return {}; - return somSearcher->getSimilarReleases(session, releaseId, maxCount); + if (method == Database::SimilaritySettings::PreferredMethod::Features + || (method == Database::SimilaritySettings::PreferredMethod::Auto && somSearcher)) + { + return somSearcher->getSimilarReleases(releaseId, maxCount); + } + else + return ClusterSearcher::getSimilarReleases(session, releaseId, maxCount); } std::vector Searcher::getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount) { + auto method = getPreferredMethod(session); auto somSearcher = _somAddon.getSearcher(); - if (!somSearcher) - return {}; - return somSearcher->getSimilarArtists(session, artistId, maxCount); + if (method == Database::SimilaritySettings::PreferredMethod::Features + || (method == Database::SimilaritySettings::PreferredMethod::Auto && somSearcher)) + { + return somSearcher->getSimilarArtists(artistId, maxCount); + } + else + return ClusterSearcher::getSimilarArtists(session, artistId, maxCount); } } // ns Similarity diff --git a/src/similarity/SimilaritySearcher.hpp b/src/similarity/SimilaritySearcher.hpp index 1ebd0839..80cf05e2 100644 --- a/src/similarity/SimilaritySearcher.hpp +++ b/src/similarity/SimilaritySearcher.hpp @@ -19,25 +19,28 @@ #pragma once +#include #include #include "database/Types.hpp" -#include "som/SimilaritySOMScannerAddon.hpp" namespace Similarity { +class FeaturesScannerAddon; + class Searcher { public: - Searcher(SOMScannerAddon& somAddon); + Searcher(FeaturesScannerAddon& somAddon); - std::vector getSimilarTracks(const std::vector& tracksId, std::size_t maxCount); + // Closest results first + std::vector getSimilarTracks(Wt::Dbo::Session& session, const std::set& tracksId, std::size_t maxCount); std::vector getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount); std::vector getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount); private: - SOMScannerAddon& _somAddon; + FeaturesScannerAddon& _somAddon; }; } // ns Similarity diff --git a/src/similarity/cluster/SimilarityClusterSearcher.cpp b/src/similarity/cluster/SimilarityClusterSearcher.cpp index 312fbe77..fe519758 100644 --- a/src/similarity/cluster/SimilarityClusterSearcher.cpp +++ b/src/similarity/cluster/SimilarityClusterSearcher.cpp @@ -22,19 +22,24 @@ #include #include +#include "database/Artist.hpp" #include "database/Cluster.hpp" +#include "database/Release.hpp" #include "database/Track.hpp" #include "utils/Utils.hpp" namespace Similarity { +namespace ClusterSearcher { std::vector -ClusterSearcher::getSimilarTracks(Wt::Dbo::Session& session, const std::vector& tracksId, std::size_t maxCount) +getSimilarTracks(Wt::Dbo::Session& session, const std::set& trackIds, std::size_t maxCount) { + std::vector res; + Wt::Dbo::Transaction transaction(session); std::vector clusterIds; - for (auto trackId : tracksId) + for (auto trackId : trackIds) { auto track = Database::Track::getById(session, trackId); if (!track) @@ -51,26 +56,12 @@ ClusterSearcher::getSimilarTracks(Wt::Dbo::Session& session, const std::vector sortedClusterIds; uniqueAndSortedByOccurence(clusterIds.begin(), clusterIds.end(), std::back_inserter(sortedClusterIds)); - -#if 0 - auto now = std::chrono::system_clock::now(); - std::mt19937 randGenerator(std::chrono::duration_cast(now.time_since_epoch()).count()); - - - std::set trackIds; + for (auto clusterId : clusterIds) { - auto ids = tracklist->getTrackIds(); - trackIds = std::set(ids.begin(), ids.end()); - } + auto cluster = Database::Cluster::getById(session, clusterId); + if (!cluster) + continue; - // Get all the tracks of the tracklist, get the cluster that is mostly used - // and reuse it to get the next track - auto clusters = tracklist->getClusters(); - if (clusters.empty()) - return; - - for (auto cluster : clusters) - { std::set clusterTrackIds = cluster->getTrackIds(); std::set candidateTrackIds; @@ -81,30 +72,102 @@ ClusterSearcher::getSimilarTracks(Wt::Dbo::Session& session, const std::vector dist(0, candidateTrackIds.size() - 1); + for (auto trackId : candidateTrackIds) + { + if (res.size() >= maxCount) + break; - auto trackToAdd = Database::Track::getById(LmsApp->getDboSession(), *std::next(candidateTrackIds.begin(), dist(randGenerator))); - enqueueTrack(trackToAdd); + res.push_back(trackId); + } - return; + if (res.size() >= maxCount) + break; } - LMS_LOG(UI, INFO) << "No more track to be added!"; -#endif - return {}; + return res; } std::vector -ClusterSearcher::getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount) +getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount) { + std::vector res; - return {}; + Wt::Dbo::Transaction transaction(session); + + auto release = Database::Release::getById(session, releaseId); + if (!release) + return res; + + auto releaseTracks = release->getTracks(); + std::set releaseTrackIds; + + for (const auto& releaseTrack : releaseTracks) + releaseTrackIds.insert(releaseTrack.id()); + + auto trackIds = getSimilarTracks(session, releaseTrackIds, maxCount * 5); + + for (auto trackId : trackIds) + { + auto track = Database::Track::getById(session, trackId); + if (!track) + continue; + + auto trackRelease = track->getRelease(); + if (!trackRelease || trackRelease.id() == releaseId) + continue; + + if (std::find(res.begin(), res.end(), trackRelease.id()) != res.end()) + continue; + + res.push_back(trackRelease.id()); + + if (res.size() == maxCount) + break; + } + + return res; } std::vector -ClusterSearcher::getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount) +getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount) { - return {}; + std::vector res; + + Wt::Dbo::Transaction transaction(session); + + auto artist = Database::Artist::getById(session, artistId); + if (!artist) + return res; + + auto artistTracks = artist->getTracks(); + std::set artistTrackIds; + + for (const auto& artistTrack : artistTracks) + artistTrackIds.insert(artistTrack.id()); + + auto trackIds = getSimilarTracks(session, artistTrackIds, maxCount * 5); + + for (auto trackId : trackIds) + { + auto track = Database::Track::getById(session, trackId); + if (!track) + continue; + + auto trackArtist = track->getArtist(); + if (!trackArtist || trackArtist.id() == artistId) + continue; + + if (std::find(res.begin(), res.end(), trackArtist.id()) != res.end()) + continue; + + res.push_back(trackArtist.id()); + + if (res.size() == maxCount) + break; + } + + return res; } +} // namespace ClusterSearcher } // namespace Similarity diff --git a/src/similarity/cluster/SimilarityClusterSearcher.hpp b/src/similarity/cluster/SimilarityClusterSearcher.hpp index 94aab4e7..b1fcb05d 100644 --- a/src/similarity/cluster/SimilarityClusterSearcher.hpp +++ b/src/similarity/cluster/SimilarityClusterSearcher.hpp @@ -19,18 +19,17 @@ #pragma once -#include +#include #include "database/Types.hpp" namespace Similarity { -class ClusterSearcher +namespace ClusterSearcher { - public: - std::vector getSimilarTracks(Wt::Dbo::Session& session, const std::vector& tracksId, std::size_t maxCount); - std::vector getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount); - std::vector getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount); + std::vector getSimilarTracks(Wt::Dbo::Session& session, const std::set& tracksId, std::size_t maxCount); + std::vector getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount); + std::vector getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount); }; } // namespace Similarity diff --git a/src/similarity/features/SimilarityFeaturesScannerAddon.cpp b/src/similarity/features/SimilarityFeaturesScannerAddon.cpp new file mode 100644 index 00000000..d22a85e2 --- /dev/null +++ b/src/similarity/features/SimilarityFeaturesScannerAddon.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2019 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "SimilarityFeaturesScannerAddon.hpp" + + +#include "database/Track.hpp" +#include "database/TrackFeatures.hpp" +#include "som/AcousticBrainzUtils.hpp" +#include "utils/Logger.hpp" + + +namespace Similarity { + +namespace { + +struct TrackInfo +{ + Database::IdType id; + std::string mbid; +}; + +std::vector +getTracksWithMBIDAndMissingFeatures(Wt::Dbo::Session& session) +{ + std::vector res; + + Wt::Dbo::Transaction transaction(session); + + auto tracks = Database::Track::getAllWithMBIDAndMissingFeatures(session); + for (auto track : tracks) + res.push_back({track.id(), track->getMBID()}); + + return res; +} + +} // namespace + +FeaturesScannerAddon::FeaturesScannerAddon(Wt::Dbo::SqlConnectionPool& connectionPool) +: _db(connectionPool) +{ +} + +std::shared_ptr +FeaturesScannerAddon::getSearcher() +{ + return std::atomic_load(&_searcher); +} + +void +FeaturesScannerAddon::trackUpdated(Database::IdType trackId) +{ + Wt::Dbo::Transaction transaction(_db.getSession()); + + auto track = Database::Track::getById(_db.getSession(), trackId); + if (!track) + return; + + track.modify()->eraseFeatures(); +} + +void +FeaturesScannerAddon::preScanComplete() +{ + LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features..."; + auto tracksInfo = getTracksWithMBIDAndMissingFeatures(_db.getSession()); + LMS_LOG(DBUPDATER, DEBUG) << "Getting tracks with missing Features DONE (found " << tracksInfo.size() << ")"; + + for (const auto& trackInfo : tracksInfo) + fetchFeatures(trackInfo.id, trackInfo.mbid); + + updateSearcher(); +} + +void +FeaturesScannerAddon::updateSearcher() +{ + Wt::Dbo::Transaction transaction(_db.getSession()); + auto tracks = Database::Track::getAllWithFeatures(_db.getSession()); + transaction.commit(); + + if (tracks.empty()) + { + LMS_LOG(DBUPDATER, INFO) << "No track suitable for features similarity clustering"; + std::atomic_store(&_searcher, std::shared_ptr()); + return; + } + + auto searcher = std::make_shared(_db.getSession()); + + std::atomic_store(&_searcher, searcher); + LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated"; +} + +bool +FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const std::string& MBID) +{ + std::map features; + + LMS_LOG(DBUPDATER, DEBUG) << "Fetching low level features for track '" << MBID << "'"; + std::string data = AcousticBrainz::extractLowLevelFeatures(MBID); + + if (data.empty()) + { + LMS_LOG(DBUPDATER, ERROR) << "Cannot extract features using AcousticBrainz!"; + return false; + } + + // TODO check if the expected features are here + + Wt::Dbo::Transaction transaction(_db.getSession()); + + Wt::Dbo::ptr track = Database::Track::getById(_db.getSession(), trackId); + if (!track) + return false; + + LMS_LOG(DBUPDATER, DEBUG) << "Successfully extracted AcousticBrainz lowlevel features for track '" << track->getPath().string() << "'"; + + Database::TrackFeatures::create(_db.getSession(), track, data); + + return true; +} + +} // namespace Similarity + diff --git a/src/similarity/som/SimilaritySOMScannerAddon.hpp b/src/similarity/features/SimilarityFeaturesScannerAddon.hpp similarity index 70% rename from src/similarity/som/SimilaritySOMScannerAddon.hpp rename to src/similarity/features/SimilarityFeaturesScannerAddon.hpp index c3ec01c7..8895961a 100644 --- a/src/similarity/som/SimilaritySOMScannerAddon.hpp +++ b/src/similarity/features/SimilarityFeaturesScannerAddon.hpp @@ -24,20 +24,21 @@ #include "database/DatabaseHandler.hpp" #include "scanner/MediaScannerAddon.hpp" -#include "SimilaritySOMSearcher.hpp" +#include "SimilarityFeaturesSearcher.hpp" namespace Similarity { -class SOMScannerAddon : public Scanner::MediaScannerAddon +class FeaturesScannerAddon final : public Scanner::MediaScannerAddon { public: - SOMScannerAddon(Wt::Dbo::SqlConnectionPool& connectionPool); + FeaturesScannerAddon(Wt::Dbo::SqlConnectionPool& connectionPool); - std::shared_ptr getSearcher(); + std::shared_ptr getSearcher(); private: - void refreshSettings() override; + + void refreshSettings() override {} void trackAdded(Database::IdType trackId) override {} void trackToRemove(Database::IdType trackId) override {} void trackUpdated(Database::IdType trackId) override; @@ -45,17 +46,14 @@ class SOMScannerAddon : public Scanner::MediaScannerAddon bool fetchFeatures(Database::IdType trackId, const std::string& MBID); - void clusterize(); + void updateSearcher(); - std::size_t _settingsVersion; - std::set _featuresName; - Database::Handler _db; - - std::shared_ptr _finder; + Database::Handler _db; + std::shared_ptr _searcher; }; -SOMScannerAddon* setSOMScannerAddon(SOMScannerAddon addon); -SOMScannerAddon* getSOMScannerAddon(); +FeaturesScannerAddon* setFeaturesScannerAddon(FeaturesScannerAddon addon); +FeaturesScannerAddon* getFeaturesScannerAddon(); } // namespace Similarity diff --git a/src/similarity/features/SimilarityFeaturesSearcher.cpp b/src/similarity/features/SimilarityFeaturesSearcher.cpp new file mode 100644 index 00000000..f0fa01dc --- /dev/null +++ b/src/similarity/features/SimilarityFeaturesSearcher.cpp @@ -0,0 +1,305 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "SimilarityFeaturesSearcher.hpp" + +#include + +#include "database/Artist.hpp" +#include "database/SimilaritySettings.hpp" +#include "database/Release.hpp" +#include "database/Track.hpp" +#include "database/TrackFeatures.hpp" +#include "som/DataNormalizer.hpp" +#include "utils/Logger.hpp" +#include "utils/Utils.hpp" + + +namespace Similarity { + + +FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session) +{ + Wt::Dbo::Transaction transaction(session); + + auto settings = Database::SimilaritySettings::get(session); + + struct FeatureInfo + { + std::size_t nbDimensions; + double weight; + }; + + std::map featuresInfo; + std::size_t nbDimensions = 0; + for (auto feature : settings->getFeatures()) + { + featuresInfo[feature->getName()] = { feature->getNbDimensions(), feature->getWeight() }; + nbDimensions += feature->getNbDimensions(); + } + + LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features..."; + auto tracks = Database::Track::getAllWithFeatures(session); + LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features DONE"; + + std::vector samples; + std::vector tracksIds; + + LMS_LOG(SIMILARITY, DEBUG) << "Extracting features..."; + for (auto track : tracks) + { + SOM::InputVector sample; + + std::map> features; + for (const auto& featureInfo : featuresInfo) + features[featureInfo.first] = {}; + + if (!track->getTrackFeatures()->getFeatures(features)) + continue; + + // Check dimensions for each feature + bool ok = true; + for (const auto& feature : features) + { + auto it = featuresInfo.find(feature.first); + if (it == featuresInfo.end() || it->second.nbDimensions != feature.second.size()) + { + LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << feature.first << "'. Expected " << it->second.nbDimensions << ", got " << feature.second.size(); + ok = false; + break; + } + + sample.insert( sample.end(), feature.second.begin(), feature.second.end() ); + } + + if (!ok) + continue; + + samples.emplace_back(std::move(sample)); + tracksIds.emplace_back(track.id()); + } + LMS_LOG(SIMILARITY, DEBUG) << "Extracting features DONE"; + + transaction.commit(); + + if (tracksIds.empty()) + { + LMS_LOG(SIMILARITY, INFO) << "Nothing to classify!"; + return; + } + + LMS_LOG(SIMILARITY, DEBUG) << "Normalizing data..."; + SOM::DataNormalizer normalizer(nbDimensions); + + normalizer.computeNormalizationFactors(samples); + for (auto& sample : samples) + normalizer.normalizeData(sample); + + std::size_t size = std::sqrt(samples.size()/2); + LMS_LOG(SIMILARITY, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network"; + + _network = std::make_unique(size, size, nbDimensions); + _artistsMap = SOM::Matrix>(size, size); + _releasesMap = SOM::Matrix>(size, size); + _tracksMap = SOM::Matrix>(size, size); + + std::vector weights; + for (const auto& featureInfo : featuresInfo) + { + for (std::size_t i = 0; i < featureInfo.second.nbDimensions; ++i) + weights.push_back(1. / featureInfo.second.nbDimensions * featureInfo.second.weight); + } + + LMS_LOG(SIMILARITY, DEBUG) << "Training network..."; + _network->train(samples, 20); + LMS_LOG(SIMILARITY, DEBUG) << "Training network DONE"; + + LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks..."; + + for (std::size_t i = 0; i < samples.size(); ++i) + { + Wt::Dbo::Transaction transaction(session); + + const auto& sample = samples[i]; + auto trackId = tracksIds[i]; + + auto coords = _network->getClosestRefVectorCoords(sample); + + _trackCoords[trackId].insert(coords); + _tracksMap[coords].insert(trackId); + + auto track = Database::Track::getById(session, trackId); + if (track->getRelease()) + { + _releaseCoords[track->getRelease().id()].insert(coords); + _releasesMap[coords].insert(track->getRelease().id()); + } + if (track->getArtist()) + { + _artistCoords[track->getArtist().id()].insert(coords); + _artistsMap[coords].insert(track->getArtist().id()); + } + } + + LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE"; + +} + + +std::vector +FeaturesSearcher::getSimilarTracks(const std::set& tracksIds, std::size_t maxCount) const +{ + return getSimilarObjects(tracksIds, _tracksMap, _trackCoords, maxCount); +} + +std::vector +FeaturesSearcher::getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const +{ + return getSimilarObjects({releaseId}, _releasesMap, _releaseCoords, maxCount); +} + +std::vector +FeaturesSearcher::getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const +{ + return getSimilarObjects({artistId}, _artistsMap, _artistCoords, maxCount); +} +#if 0 +void +FeaturesSearcher::dump(Wt::Dbo::Session& session, std::ostream& os) const +{ + os << "Number of tracks classified: " << _trackIdsCoords.size() << std::endl; + os << "Network size: " << _network.getWidth() << " * " << _network.getHeight() << std::endl; + + Wt::Dbo::Transaction transaction(session); + + for (std::size_t y = 0; y < _network.getHeight(); ++y) + { + for (std::size_t x = 0; x < _network.getWidth(); ++x) + { + const auto& trackIds = _tracksMap[{x, y}]; + + for (auto trackId : trackIds) + { + auto track = Database::Track::getById(session, trackId); + if (!track) + continue; + + os << "{"; + if (track->getArtist()) + os << track->getArtist()->getName() << " "; + if (track->getRelease()) + os << track->getRelease()->getName(); + os << "} "; + } + + os << "; "; + } + os << std::endl; + } +} +#endif + + +static +std::set +getMatchingRefVectorsCoords(const std::set& ids, const std::map>& objectCoords) +{ + std::set res; + + if (ids.empty()) + return res; + + for (auto id : ids) + { + auto it = objectCoords.find(id); + if (it == objectCoords.end()) + continue; + + for (const auto& coords : it->second) + res.insert(coords); + } + + return res; +} + +static +std::set +getObjectsIds(const std::set& coordsSet, const SOM::Matrix>& objectsMap ) +{ + std::set res; + + for (const auto& coords : coordsSet) + { + for (auto id : objectsMap.get(coords)) + res.insert(id); + } + + return res; +} + +std::vector +FeaturesSearcher::getSimilarObjects(const std::set& ids, + const SOM::Matrix>& objectsMap, + const std::map>& objectCoords, + std::size_t maxCount) const +{ + std::vector res; + + auto now = std::chrono::system_clock::now(); + std::mt19937 randGenerator(std::chrono::duration_cast(now.time_since_epoch()).count()); + + std::set searchedRefVectorsCoords = getMatchingRefVectorsCoords(ids, objectCoords); + if (searchedRefVectorsCoords.empty()) + return res; + + while (1) + { + std::set closestObjectIds = getObjectsIds(searchedRefVectorsCoords, objectsMap); + + // Remove objects that are already in input + for (auto id : ids) + closestObjectIds.erase(id); + + { + std::vector objectIdsToAdd(closestObjectIds.begin(), closestObjectIds.end()); + + std::shuffle(objectIdsToAdd.begin(), objectIdsToAdd.end(), randGenerator); + std::copy(objectIdsToAdd.begin(), objectIdsToAdd.end(), std::back_inserter(res)); + } + + if (res.size() > maxCount) + res.resize(maxCount); + + if (res.size() == maxCount) + break; + + // If there is not enough objects, try again with closest neighbour until there is too much distance + auto closestRefVectorCoords = _network->getClosestRefVectorCoords(searchedRefVectorsCoords, _networkRefVectorsDistanceMedian * 0.75); + if (!closestRefVectorCoords) + break; + + searchedRefVectorsCoords.insert(*closestRefVectorCoords); + } + + return res; +} + + + +} // ns Similarity diff --git a/src/similarity/features/SimilarityFeaturesSearcher.hpp b/src/similarity/features/SimilarityFeaturesSearcher.hpp new file mode 100644 index 00000000..1cafe6d9 --- /dev/null +++ b/src/similarity/features/SimilarityFeaturesSearcher.hpp @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2018 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include "database/DatabaseHandler.hpp" +#include "database/Types.hpp" +#include "som/Network.hpp" + +namespace Similarity { + +class FeaturesSearcher +{ + public: + + FeaturesSearcher(Wt::Dbo::Session& session); + + std::vector getSimilarTracks(const std::set& tracksId, std::size_t maxCount) const; + std::vector getSimilarReleases(Database::IdType releaseId, std::size_t maxCount) const; + std::vector getSimilarArtists(Database::IdType artistId, std::size_t maxCount) const; + + void dump(Wt::Dbo::Session& session, std::ostream& os) const; + + private: + + std::vector getSimilarObjects(const std::set& ids, + const SOM::Matrix>& objectsMap, + const std::map>& objectCoords, + std::size_t maxCount) const; + + std::unique_ptr _network; + double _networkRefVectorsDistanceMedian = 0; + + SOM::Matrix> _artistsMap; + std::map> _artistCoords; + + SOM::Matrix> _releasesMap; + std::map> _releaseCoords; + + SOM::Matrix> _tracksMap; + std::map> _trackCoords; + +}; + +} // ns Similarity diff --git a/src/similarity/som/AcousticBrainzUtils.cpp b/src/similarity/features/som/AcousticBrainzUtils.cpp similarity index 68% rename from src/similarity/som/AcousticBrainzUtils.cpp rename to src/similarity/features/som/AcousticBrainzUtils.cpp index 3b558ce6..9580f163 100644 --- a/src/similarity/som/AcousticBrainzUtils.cpp +++ b/src/similarity/features/som/AcousticBrainzUtils.cpp @@ -40,31 +40,6 @@ static size_t writeToOStringStream(void *buffer, size_t size, size_t nmemb, void return size * nmemb; } -static bool -getFeaturesFromJsonData(const std::string& jsonData, const std::set& featuresName, std::map& features) -{ - try - { - boost::property_tree::ptree root; - - std::istringstream iss(jsonData); - - boost::property_tree::read_json(iss, root); - - for (const auto& featureName : featuresName) - { - features[featureName] = root.get(featureName); - } - - return true; - } - catch (std::exception& e) - { - LMS_LOG(DBUPDATER, ERROR) << "Cannot extract feature: " << e.what(); - return false; - } -} - static std::string getJsonData(const std::string& mbid) { @@ -79,7 +54,7 @@ getJsonData(const std::string& mbid) curl = curl_easy_init(); if (!curl) { - LMS_LOG(DBUPDATER, ERROR) << "CURL init failed"; + LMS_LOG(SIMILARITY, ERROR) << "CURL init failed"; return data; } @@ -92,7 +67,7 @@ getJsonData(const std::string& mbid) res = curl_easy_perform(curl); if (res != CURLE_OK) { - LMS_LOG(DBUPDATER, ERROR) << "CURL perform failed: " << curl_easy_strerror(res); + LMS_LOG(SIMILARITY, ERROR) << "CURL perform failed: " << curl_easy_strerror(res); return data; } @@ -103,10 +78,10 @@ getJsonData(const std::string& mbid) return data; } -bool -extractFeatures(const std::string& mbid, const std::set& featuresName, std::map& features) +std::string +extractLowLevelFeatures(const std::string& mbid) { - return getFeaturesFromJsonData(getJsonData(mbid), featuresName, features); + return getJsonData(mbid); } } // namespace Scanner::AcousticBrainz diff --git a/src/similarity/som/AcousticBrainzUtils.hpp b/src/similarity/features/som/AcousticBrainzUtils.hpp similarity index 85% rename from src/similarity/som/AcousticBrainzUtils.hpp rename to src/similarity/features/som/AcousticBrainzUtils.hpp index 079d8ef9..8be80f93 100644 --- a/src/similarity/som/AcousticBrainzUtils.hpp +++ b/src/similarity/features/som/AcousticBrainzUtils.hpp @@ -25,7 +25,6 @@ namespace AcousticBrainz { - - bool extractFeatures(const std::string& MBID, const std::set& featuresName, std::map& features); + std::string extractLowLevelFeatures(const std::string& MBID); } diff --git a/src/similarity/som/DataNormalizer.cpp b/src/similarity/features/som/DataNormalizer.cpp similarity index 82% rename from src/similarity/som/DataNormalizer.cpp rename to src/similarity/features/som/DataNormalizer.cpp index 33f0e810..ec1e27b5 100644 --- a/src/similarity/som/DataNormalizer.cpp +++ b/src/similarity/features/som/DataNormalizer.cpp @@ -50,11 +50,6 @@ DataNormalizer::DataNormalizer(std::size_t inputDimCount) { } -DataNormalizer::DataNormalizer(const std::string& data) -{ - serializeFrom(data); -} - void DataNormalizer::computeNormalizationFactors(const std::vector& inputVectors) { @@ -103,34 +98,6 @@ DataNormalizer::normalizeData(InputVector& a) const } } -std::string -DataNormalizer::serializeTo() const -{ - std::ostringstream oss; - - oss << _inputDimCount << " "; - for (std::size_t i = 0; i < _inputDimCount; ++i) - oss << _minmax[i].min << " " << _minmax[i].max; - - return oss.str(); -} - -void -DataNormalizer::serializeFrom(const std::string& data) -{ - std::istringstream iss(data); - - iss >> _inputDimCount; - _minmax.resize(_inputDimCount); - - for (std::size_t i = 0; i < _inputDimCount; ++i) - { - iss >> _minmax[i].min; - iss >> _minmax[i].max; - } - -} - void DataNormalizer::dump(std::ostream& os) const { diff --git a/src/similarity/som/DataNormalizer.hpp b/src/similarity/features/som/DataNormalizer.hpp similarity index 97% rename from src/similarity/som/DataNormalizer.hpp rename to src/similarity/features/som/DataNormalizer.hpp index cf69f8e9..75776226 100644 --- a/src/similarity/som/DataNormalizer.hpp +++ b/src/similarity/features/som/DataNormalizer.hpp @@ -32,7 +32,6 @@ class DataNormalizer public: DataNormalizer(std::size_t inputDimCount); - DataNormalizer(const std::string& data); void computeNormalizationFactors(const std::vector& dataSamples); diff --git a/src/similarity/som/Matrix.hpp b/src/similarity/features/som/Matrix.hpp similarity index 89% rename from src/similarity/som/Matrix.hpp rename to src/similarity/features/som/Matrix.hpp index f0899b79..6113538b 100644 --- a/src/similarity/som/Matrix.hpp +++ b/src/similarity/features/som/Matrix.hpp @@ -34,7 +34,15 @@ struct Coords bool operator<(const Coords& other) const { - return x < other.x && y < other.y; + if (x == other.x) + return y < other.y; + else + return x < other.x; + } + + bool operator==(const Coords& other) const + { + return x == other.x && y == other.y; } }; @@ -43,6 +51,8 @@ class Matrix { public: + Matrix() = default; + Matrix(std::size_t width, std::size_t height) : _width(width), _height(height) @@ -71,7 +81,6 @@ class Matrix { assert(coords.x < _width); assert(coords.y < _height); - return _values[coords.x + _width*coords.y]; } @@ -88,6 +97,8 @@ class Matrix template Coords getCoordsMinElement(Func func) const { + assert(!_values.empty()); + auto it = std::min_element(_values.begin(), _values.end(), func); auto index = std::distance(_values.begin(), it); @@ -96,8 +107,8 @@ class Matrix private: - std::size_t _width; - std::size_t _height; + std::size_t _width = 0; + std::size_t _height = 0; std::vector _values; }; diff --git a/src/similarity/som/Network.cpp b/src/similarity/features/som/Network.cpp similarity index 58% rename from src/similarity/som/Network.cpp rename to src/similarity/features/som/Network.cpp index b3bd0b5d..0e211c84 100644 --- a/src/similarity/som/Network.cpp +++ b/src/similarity/features/som/Network.cpp @@ -45,11 +45,11 @@ checkSameDimensions(const InputVector& a, std::size_t inputDimCount) } static InputVector::value_type -defaultLearningFactor(Network::Progress progress) +defaultLearningFactor(Network::CurrentIteration iteration) { constexpr InputVector::value_type initialValue = 1; - return initialValue * exp(-((progress.idIteration + 1) / static_cast(progress.iterationCount))); + return initialValue * exp(-((iteration.idIteration + 1) / static_cast(iteration.iterationCount))); } static InputVector::value_type @@ -70,18 +70,18 @@ euclidianSquareDistance(const InputVector& a, const InputVector& b, const InputV static InputVector::value_type -sigmaFunc(Network::Progress progress) +sigmaFunc(Network::CurrentIteration iteration) { constexpr InputVector::value_type sigma0 = 1; - return sigma0 * exp(- ((progress.idIteration + 1) / static_cast(progress.iterationCount))); + return sigma0 * exp(- ((iteration.idIteration + 1) / static_cast(iteration.iterationCount))); } static InputVector::value_type -defaultNeighborhoodFunc(InputVector::value_type norm, Network::Progress progress) +defaultNeighbourhoodFunc(InputVector::value_type norm, Network::CurrentIteration iteration) { - auto sigma = sigmaFunc(progress); + auto sigma = sigmaFunc(iteration); return exp(-norm / (2 * sigma * sigma)); } @@ -168,7 +168,7 @@ _weights(inputDimCount, static_cast(1)), _refVectors(width, height), _distanceFunc(euclidianSquareDistance), _learningFactorFunc(defaultLearningFactor), -_neighborhoodFunc(defaultNeighborhoodFunc) +_neighbourhoodFunc(defaultNeighbourhoodFunc) { auto now = std::chrono::system_clock::now(); std::mt19937 randGenerator(std::chrono::duration_cast(now.time_since_epoch()).count()); @@ -188,15 +188,6 @@ _neighborhoodFunc(defaultNeighborhoodFunc) } } -Network::Network(const std::string& data) -: _refVectors(0, 0), -_distanceFunc(euclidianSquareDistance), -_learningFactorFunc(defaultLearningFactor), -_neighborhoodFunc(defaultNeighborhoodFunc) -{ - serializeFrom(data); -} - void Network::setDataWeights(const InputVector& weights) { @@ -205,6 +196,50 @@ Network::setDataWeights(const InputVector& weights) _weights = weights; } +double +Network::getRefVectorsDistance(Coords coords1, Coords coords2) const +{ + return _distanceFunc(_refVectors.get(coords1), _refVectors.get(coords2), _weights); +} + +double +Network::computeRefVectorsDistanceMean() const +{ + std::vector values; + values.reserve(2*_refVectors.getHeight()*_refVectors.getWidth() - _refVectors.getWidth() - _refVectors.getHeight()); + for (std::size_t y = 0; y < _refVectors.getHeight(); ++y) + { + for (std::size_t x = 0; x < _refVectors.getWidth(); ++x) + { + if (x != _refVectors.getWidth() - 1) + values.push_back(getRefVectorsDistance( {x, y}, {x + 1, y})); + if (y != _refVectors.getHeight() - 1) + values.push_back(getRefVectorsDistance( {x, y}, {x, y + 1})); + } + } + + return std::accumulate(values.begin(), values.end(), 0.) / values.size(); +} + +double +Network::computeRefVectorsDistanceMedian() const +{ + std::vector values; + values.reserve(2*_refVectors.getHeight()*_refVectors.getWidth() - _refVectors.getWidth() - _refVectors.getHeight()); + for (std::size_t y = 0; y < _refVectors.getHeight(); ++y) + { + for (std::size_t x = 0; x < _refVectors.getWidth(); ++x) + { + if (x != _refVectors.getWidth() - 1) + values.push_back(getRefVectorsDistance( {x, y}, {x + 1, y})); + if (y != _refVectors.getHeight() - 1) + values.push_back(getRefVectorsDistance( {x, y}, {x, y + 1})); + } + } + + return values[values.size()/2 - 1]; +} + void Network::dump(std::ostream& os) const { @@ -223,7 +258,7 @@ Network::dump(std::ostream& os) const } Coords -Network::getClosestRefVector(const InputVector& data) const +Network::getClosestRefVectorCoords(const InputVector& data) const { return _refVectors.getCoordsMinElement([&](const auto& a, const auto& b) { @@ -231,48 +266,77 @@ Network::getClosestRefVector(const InputVector& data) const }); } -Coords -Network::classify(const InputVector& data) const +boost::optional +Network::getClosestRefVectorCoords(const InputVector& data, double maxDistance) const { - return getClosestRefVector(data); + Coords coords = _refVectors.getCoordsMinElement([&](const auto& a, const auto& b) + { + return (_distanceFunc(a, data, _weights) < _distanceFunc(b, data, _weights)); + }); + + if (_distanceFunc(data, _refVectors.get(coords), _weights) > maxDistance) + return boost::none; + + return coords; } -std::vector -Network::classify(const InputVector& data, std::size_t size) const +boost::optional +Network::getClosestRefVectorCoords(const std::set& refVectorsCoords, double maxDistance) const { - struct Entry + std::set neighboursCoords; + for (const Coords& refVectorCoords : refVectorsCoords) + { + if (refVectorCoords.y > 0) + neighboursCoords.insert({ refVectorCoords.x, refVectorCoords.y - 1 }); + if (refVectorCoords.y < _refVectors.getHeight() - 1) + neighboursCoords.insert({ refVectorCoords.x, refVectorCoords.y + 1 }); + if (refVectorCoords.x > 0) + neighboursCoords.insert({ refVectorCoords.x - 1, refVectorCoords.y }); + if (refVectorCoords.x < _refVectors.getWidth() - 1) + neighboursCoords.insert({ refVectorCoords.x + 1, refVectorCoords.y }); + } + + // remove coords that are in the input coords + for (const auto& refVectorCoords : refVectorsCoords) + neighboursCoords.erase(refVectorCoords); + + if (neighboursCoords.empty()) + return boost::none; + + // Now compute the distance for each neighbour + struct NeighbourInfo { Coords coords; - InputVector refVector; + double distance; }; - std::vector sortedEntries; - for (std::size_t x = 0; x < _refVectors.getWidth(); ++x) + std::vector neighboursInfo; + for (const Coords& neighbourCoords : neighboursCoords) { - for (std::size_t y = 0; y < _refVectors.getHeight(); ++y) - { - sortedEntries.push_back( Entry{{x, y}, _refVectors.get({x, y})} ); - } + auto min = std::min_element(refVectorsCoords.begin(), refVectorsCoords.end(), + [this, neighbourCoords](const auto& a, const auto& b) + { + return (this->getRefVectorsDistance(a, neighbourCoords) < this->getRefVectorsDistance(b, neighbourCoords)); + }); + + double distance = getRefVectorsDistance(neighbourCoords, *min); + if (distance > maxDistance) + continue; + + neighboursInfo.push_back({neighbourCoords, distance}); } - const InputVector& closestRefVector = _refVectors.get(getClosestRefVector(data)); + if (neighboursInfo.empty()) + return boost::none; - std::sort(sortedEntries.begin(), sortedEntries.end(), - [&](const Entry& a, const Entry& b) + auto min = std::min_element(neighboursInfo.begin(), neighboursInfo.end(), + [&](const auto& a, const auto& b) { - return _distanceFunc(a.refVector, closestRefVector, _weights) < _distanceFunc(b.refVector, closestRefVector, _weights); + return a.distance < b.distance; }); - std::vector res; - for (const Entry& entry : sortedEntries) - { - res.push_back(entry.coords); - if (res.size() == size) - break; - } - - return res; + return min->coords; } static InputVector::value_type @@ -286,7 +350,7 @@ computeCoordsNorm(Coords c1, Coords c2) void -Network::updateRefVectors(Coords closestRefVectorCoords, const InputVector& input, Progress progress) +Network::updateRefVectors(Coords closestRefVectorCoords, const InputVector& input, CurrentIteration iteration) { for (std::size_t y = 0; y < _refVectors.getHeight(); ++y) { @@ -298,7 +362,7 @@ Network::updateRefVectors(Coords closestRefVectorCoords, const InputVector& inpu auto n = computeCoordsNorm({x, y}, closestRefVectorCoords); auto oldRefVector = refVector; - refVector = refVector + delta * (_learningFactorFunc(progress) * _neighborhoodFunc(n, progress)); + refVector = refVector + delta * (_learningFactorFunc(iteration) * _neighbourhoodFunc(n, iteration)); } } } @@ -324,74 +388,13 @@ Network::train(const std::vector& inputData, std::size_t nbIteratio for (auto input : inputDataShuffled) { - Coords closestRefVectorCoords = getClosestRefVector(*input); + Coords closestRefVectorCoords = getClosestRefVectorCoords(*input); updateRefVectors(closestRefVectorCoords, *input, {i, nbIterations}); } } } -std::string -Network::serializeTo() const -{ - std::ostringstream oss; - - oss << _inputDimCount << " "; - - for (auto weight : _weights) - oss << weight << " "; - - // Matrix - oss << _refVectors.getWidth() << " " << _refVectors.getHeight() << " "; - for (std::size_t x = 0; x < _refVectors.getWidth(); ++x) - { - for (std::size_t y = 0; y < _refVectors.getHeight(); ++y) - { - for (auto val : _refVectors.get({x,y})) - oss << val << " "; - } - } - - return oss.str(); -} - -void -Network::serializeFrom(const std::string& data) -{ - std::istringstream iss(data); - - LMS_LOG(SIMILARITY, DEBUG) << "data = '" << data << "'"; - iss >> _inputDimCount; - LMS_LOG(SIMILARITY, DEBUG) << "Input dim count = " << _inputDimCount; - - for (std::size_t i = 0; i < _inputDimCount; ++i) - { - InputVector::value_type val; - iss >> val; - _weights.push_back(val); - } - - LMS_LOG(SIMILARITY, DEBUG) << "Reading matrix..."; - std::size_t width, height; - iss >> width >> height; - _refVectors = Matrix(width, height); - - for (std::size_t x = 0; x < _refVectors.getWidth(); ++x) - { - for (std::size_t y = 0; y < _refVectors.getHeight(); ++y) - { - InputVector refVector; - refVector.reserve(_inputDimCount); - for (std::size_t i = 0; i < _inputDimCount; ++i) - { - InputVector::value_type val; - iss >> val; - refVector.push_back(val); - } - _refVectors.get({x, y}) = refVector; - } - } -} } // namespace SOM diff --git a/src/similarity/som/Network.hpp b/src/similarity/features/som/Network.hpp similarity index 74% rename from src/similarity/som/Network.hpp rename to src/similarity/features/som/Network.hpp index c6a997d9..189fa4e2 100644 --- a/src/similarity/som/Network.hpp +++ b/src/similarity/features/som/Network.hpp @@ -20,9 +20,12 @@ #pragma once #include +#include #include #include +#include + #include "Matrix.hpp" #include "utils/Exception.hpp" @@ -58,45 +61,43 @@ class Network // Set weight for each dimension (default is 1 for each weight) void setDataWeights(const InputVector& weights); - // data must be normalized + // data must be normalized void train(const std::vector& dataSamples, std::size_t nbIterations); - // data must be normalized - Coords classify(const InputVector& data) const; + Coords getClosestRefVectorCoords(const InputVector& data) const; + boost::optional getClosestRefVectorCoords(const InputVector& data, double maxDistance) const; - // ordered from closest to farthest - std::vector classify(const InputVector& data, std::size_t size) const; + boost::optional getClosestRefVectorCoords(const std::set& refVectorsCoords, double maxDistance) const; + + double getRefVectorsDistance(Coords coords1, Coords coords2) const; + + double computeRefVectorsDistanceMean() const; + double computeRefVectorsDistanceMedian() const; void dump(std::ostream& os) const; // For each ref vector, update formula is: // i is the current iteration - // refVector(i+1) = refVector(i) + LearningFactor(i) * NeighborhoodFunc(i) * (MatchingRefVector - refVector) + // refVector(i+1) = refVector(i) + LearningFactor(i) * NeighbourhoodFunc(i) * (MatchingRefVector - refVector) using DistanceFunc = std::function; void setDistanceFunc(DistanceFunc distanceFunc); - struct Progress + struct CurrentIteration { std::size_t idIteration; std::size_t iterationCount; }; - using LearningFactorFunc = std::function; + using LearningFactorFunc = std::function; void setLearningFactorFunc(LearningFactorFunc learningFactorFunc); - using NeighborhoodFunc = std::function; - void setNeighborhoodFunc(NeighborhoodFunc neighborhoodFunc); - - std::string serializeTo() const; + using NeighbourhoodFunc = std::function; + void setNeighbourhoodFunc(NeighbourhoodFunc neighbourhoodFunc); private: - void serializeFrom(const std::string& data); - - Coords getClosestRefVector(const InputVector& data) const; - - void updateRefVectors(Coords closestRefVectorCoords, const InputVector& input, Progress progress); + void updateRefVectors(Coords closestRefVectorCoords, const InputVector& input, CurrentIteration iteration); std::size_t _inputDimCount; InputVector _weights; // weight for each dimension @@ -104,7 +105,7 @@ class Network DistanceFunc _distanceFunc; LearningFactorFunc _learningFactorFunc; - NeighborhoodFunc _neighborhoodFunc; + NeighbourhoodFunc _neighbourhoodFunc; }; } // namespace SOM diff --git a/src/similarity/som/SimilaritySOMScannerAddon.cpp b/src/similarity/som/SimilaritySOMScannerAddon.cpp deleted file mode 100644 index d3474095..00000000 --- a/src/similarity/som/SimilaritySOMScannerAddon.cpp +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright (C) 2019 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "SimilaritySOMScannerAddon.hpp" - -#include - -#include "database/Track.hpp" -#include "database/SimilaritySettings.hpp" -#include "database/TrackFeature.hpp" -#include "utils/Logger.hpp" - -#include "AcousticBrainzUtils.hpp" -#include "DataNormalizer.hpp" -#include "Network.hpp" - - -namespace Similarity { - -namespace { - -struct TrackInfo -{ - Database::IdType id; - std::string mbid; -}; - -std::vector -getTracksWithMBIDAndMissingFeatures(Wt::Dbo::Session& session) -{ - std::vector res; - - Wt::Dbo::Transaction transaction(session); - - auto tracks = Database::Track::getAllWithMBIDAndMissingFeatures(session); - for (auto track : tracks) - res.push_back({track.id(), track->getMBID()}); - - return res; -} - -std::vector -getTrackFeatureTypes(Wt::Dbo::Session& session, const std::set& featureNames) -{ - std::vector res; - - for (const auto& featureName : featureNames) - { - auto trackFeatureType = Database::TrackFeatureType::getByName(session, featureName); - if (!trackFeatureType) - { - LMS_LOG(DBUPDATER, ERROR) << "Missing feature type '" << featureName << "'"; - res.clear(); - return res; - } - - res.push_back(trackFeatureType); - } - - return res; -} - - -bool -extractFeatures(const Database::Track::pointer& track, const std::vector& trackFeatureTypes, std::vector& features) -{ - features.reserve(trackFeatureTypes.size()); - - for (const auto& trackFeatureType : trackFeatureTypes) - { - auto feature = track->getTrackFeature(trackFeatureType); - if (!feature) - { - LMS_LOG(DBUPDATER, ERROR) << "Missing feature " << trackFeatureType->getName() << " for track '" << track->getPath().string() << "'"; - return false; - } - - features.emplace_back(feature->getValue()); - } - - return true; -} - - -} // namespace - -SOMScannerAddon::SOMScannerAddon(Wt::Dbo::SqlConnectionPool& connectionPool) -: _db(connectionPool) -{ - refreshSettings(); - clusterize(); -} - -std::shared_ptr -SOMScannerAddon::getSearcher() -{ - return std::atomic_load(&_finder); -} - -void -SOMScannerAddon::trackUpdated(Database::IdType trackId) -{ - Wt::Dbo::Transaction transaction(_db.getSession()); - - auto track = Database::Track::getById(_db.getSession(), trackId); - if (!track) - return; - - track.modify()->eraseFeatures(); -} - -void -SOMScannerAddon::preScanComplete() -{ - auto tracksInfo = getTracksWithMBIDAndMissingFeatures(_db.getSession()); - for (const auto& trackInfo : tracksInfo) - fetchFeatures(trackInfo.id, trackInfo.mbid); - - LMS_LOG(DBUPDATER, INFO) << "Clustering tracks..."; - clusterize(); - LMS_LOG(DBUPDATER, INFO) << "Clusterization complete!"; -} - -void -SOMScannerAddon::clusterize() -{ - Wt::Dbo::Transaction transaction(_db.getSession()); - - auto trackFeatureTypes = getTrackFeatureTypes(_db.getSession(), _featuresName); - - LMS_LOG(DBUPDATER, DEBUG) << "Getting feature types DONE..."; - - LMS_LOG(DBUPDATER, DEBUG) << "Getting Tracks with features..."; - auto tracks = Database::Track::getAllWithFeatures(_db.getSession()); - LMS_LOG(DBUPDATER, DEBUG) << "Getting Tracks with features DONE"; - - std::vector samples; - std::vector tracksIds; - - LMS_LOG(DBUPDATER, DEBUG) << "Extracting features..."; - for (auto track : tracks) - { - SOM::InputVector sample; - - if (!extractFeatures(track, trackFeatureTypes, sample)) - continue; - - samples.emplace_back(std::move(sample)); - tracksIds.emplace_back(track.id()); - } - LMS_LOG(DBUPDATER, DEBUG) << "Extracting features DONE"; - - transaction.commit(); - - if (tracksIds.empty()) - { - LMS_LOG(DBUPDATER, INFO) << "Nothing to classify!"; - std::atomic_store(&_finder, std::shared_ptr()); - return; - } - - LMS_LOG(DBUPDATER, DEBUG) << "Normalizing data..."; - SOM::DataNormalizer normalizer(_featuresName.size()); - - normalizer.computeNormalizationFactors(samples); - for (auto& sample : samples) - normalizer.normalizeData(sample); - - std::size_t size = std::sqrt(samples.size()/5); - LMS_LOG(DBUPDATER, DEBUG) << "Found " << samples.size() << " tracks, Constructing a " << size << "*" << size << " network"; - SOM::Network network(size, size, _featuresName.size()); - - LMS_LOG(DBUPDATER, DEBUG) << "Training network..."; - network.train(samples, 20); - LMS_LOG(DBUPDATER, DEBUG) << "Training network DONE"; - - // Now classify all the tracks - LMS_LOG(DBUPDATER, DEBUG) << "Classifying tracks..."; - SOM::Matrix> tracksMap(network.getWidth(), network.getHeight()); - std::map trackIdsCoords; - - for (std::size_t i = 0; i < samples.size(); ++i) - { - const auto& sample = samples[i]; - auto trackId = tracksIds[i]; - - auto coords = network.classify(sample); - tracksMap[coords].push_back(trackId); - trackIdsCoords[trackId] = coords; - } - - Similarity::SOMSearcher::ConstructionParams params{std::move(network), std::move(normalizer), std::move(tracksMap), std::move(trackIdsCoords)}; - - auto finder = std::make_shared(std::move(params)); - - std::atomic_store(&_finder, finder); - - LMS_LOG(DBUPDATER, DEBUG) << "Classifying tracks DONE"; - - LMS_LOG(DBUPDATER, DEBUG) << "Dumping classifier:"; - - std::ofstream ofs("/tmp/output"); - finder->dump(_db.getSession(), ofs); - - LMS_LOG(DBUPDATER, DEBUG) << "Dumping classifier DONE"; - -} - -void -SOMScannerAddon::refreshSettings() -{ - Wt::Dbo::Transaction transaction(_db.getSession()); - - auto settings = Database::SimilaritySettings::get(_db.getSession()); - - _settingsVersion = settings->getVersion(); - - for (auto trackFeatureType : settings->getTrackFeatureTypes()) - { - _featuresName.insert(trackFeatureType->getName()); - } -} - -bool -SOMScannerAddon::fetchFeatures(Database::IdType trackId, const std::string& MBID) -{ - std::map features; - - if (!AcousticBrainz::extractFeatures(MBID, _featuresName, features)) - { - LMS_LOG(DBUPDATER, ERROR) << "Cannot extract features using AcousticBrainz!"; - return false; - } - - Wt::Dbo::Transaction transaction(_db.getSession()); - - Wt::Dbo::ptr track = Database::Track::getById(_db.getSession(), trackId); - if (!track) - return false; - - LMS_LOG(DBUPDATER, DEBUG) << "Successfully extracted AcousticBrainz lowlevel features for track '" << track->getPath().string() << "'"; - - for (const auto& feature : features) - { - auto featureType = Database::TrackFeatureType::getByName(_db.getSession(), feature.first); - if (!featureType) - return false; - - Database::TrackFeature::create(_db.getSession(), featureType, track, feature.second); - } - - return true; -} - -} // namespace Similarity - diff --git a/src/similarity/som/SimilaritySOMSearcher.cpp b/src/similarity/som/SimilaritySOMSearcher.cpp deleted file mode 100644 index 7b751b6f..00000000 --- a/src/similarity/som/SimilaritySOMSearcher.cpp +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#include "SimilaritySOMSearcher.hpp" - -#include - -#include "database/Artist.hpp" -#include "database/SimilaritySettings.hpp" -#include "database/Release.hpp" -#include "database/Track.hpp" -#include "utils/Logger.hpp" -#include "utils/Utils.hpp" - -namespace Similarity { - -SOMSearcher::SOMSearcher(ConstructionParams params) -: _network(std::move(params.network)), -_normalizer(std::move(params.normalizer)), -_tracksMap(std::move(params.tracksMap)), -_trackIdsCoords(std::move(params.trackIdsCoords)) -{ -} - -std::vector -SOMSearcher::getSimilarTracks(const std::vector& tracksIds, std::size_t maxCount) -{ - std::vector res; - - auto bestCoords = getBestMatchingCoords(tracksIds); - if (!bestCoords) - return res; - - auto tracks = _tracksMap[*bestCoords]; - - auto now = std::chrono::system_clock::now(); - std::mt19937 randGenerator(std::chrono::duration_cast(now.time_since_epoch()).count()); - - std::shuffle(tracks.begin(), tracks.end(), randGenerator); - - if (tracks.size() > maxCount) - tracks.resize(maxCount); - - return tracks; -} - -std::vector -SOMSearcher::getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount) -{ - std::vector res; - - Wt::Dbo::Transaction transaction(session); - - auto release = Database::Release::getById(session, releaseId); - if (!release) - return res; - - auto tracks = release->getTracks(); - - std::vector tracksIds; - for (auto track : tracks) - tracksIds.push_back(track.id()); - - auto matchingCoords = getMatchingCoords(tracksIds); - if (matchingCoords.empty()) - return res; - - auto releases = getReleases(session, matchingCoords); - uniqueAndSortedByOccurence(releases.begin(), releases.end(), std::back_inserter(res)); - - res.erase(std::remove_if(res.begin(), res.end(), [&](auto releaseId) { return releaseId == release.id(); }), res.end()); - - if (res.size() > maxCount) - res.resize(maxCount); - - LMS_LOG(SIMILARITY, DEBUG) << "*** SIMILARITY RESULT *** :"; - for (auto id : res) - LMS_LOG(SIMILARITY, DEBUG) << id; - - return res; -} - -std::vector -SOMSearcher::getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount) -{ - std::vector res; - - Wt::Dbo::Transaction transaction(session); - - auto artist = Database::Artist::getById(session, artistId); - if (!artist) - return res; - - auto tracks = artist->getTracks(); - - std::vector tracksIds; - for (auto track : tracks) - tracksIds.push_back(track.id()); - - auto matchingCoords = getMatchingCoords(tracksIds); - if (matchingCoords.empty()) - return res; - - auto artists = getArtists(session, matchingCoords); - uniqueAndSortedByOccurence(artists.begin(), artists.end(), std::back_inserter(res)); - - res.erase(std::remove_if(res.begin(), res.end(), [&](auto artistId) { return artistId == artist.id(); }), res.end()); - - if (res.size() > maxCount) - res.resize(maxCount); - - LMS_LOG(SIMILARITY, DEBUG) << "*** SIMILARITY RESULT *** :"; - for (auto id : res) - LMS_LOG(SIMILARITY, DEBUG) << id; - - return res; -} - -void -SOMSearcher::dump(Wt::Dbo::Session& session, std::ostream& os) const -{ - os << "Number of tracks classified: " << _trackIdsCoords.size() << std::endl; - os << "Network size: " << _network.getWidth() << " * " << _network.getHeight() << std::endl; - - Wt::Dbo::Transaction transaction(session); - - for (std::size_t y = 0; y < _network.getHeight(); ++y) - { - for (std::size_t x = 0; x < _network.getWidth(); ++x) - { - const auto& trackIds = _tracksMap[{x, y}]; - - for (auto trackId : trackIds) - { - auto track = Database::Track::getById(session, trackId); - if (!track) - continue; - - os << "{"; - if (track->getArtist()) - os << track->getArtist()->getName() << " "; - if (track->getRelease()) - os << track->getRelease()->getName(); - os << "} "; - } - - os << "; "; - } - os << std::endl; - } -} - -boost::optional -SOMSearcher::getBestMatchingCoords(const std::vector& tracksIds) const -{ - if (tracksIds.empty()) - return boost::none; - - std::map coordsCount; - - for (auto trackId : tracksIds) - { - auto it = _trackIdsCoords.find(trackId); - if (it == _trackIdsCoords.end()) - continue; - - if (coordsCount.find(it->second) == coordsCount.end()) - coordsCount[it->second] = 0; - - coordsCount[it->second]++; - } - - if (coordsCount.empty()) - return boost::none; - - auto bestCoords = std::max_element(std::begin(coordsCount), std::end(coordsCount), - [](const auto& a, const auto& b) - { - return a.second < b.second; - }); - - return bestCoords->first; -} - -std::vector -SOMSearcher::getMatchingCoords(const std::vector& tracksIds) const -{ - std::vector res; - - if (tracksIds.empty()) - return res; - - for (auto trackId : tracksIds) - { - auto it = _trackIdsCoords.find(trackId); - if (it == _trackIdsCoords.end()) - continue; - - res.push_back(it->second); - } - - return res; -} - -std::vector -SOMSearcher::getReleases(Wt::Dbo::Session& session, const std::vector& coords) const -{ - std::vector res; - for (const auto& c : coords) - { - for (auto trackId : _tracksMap[c]) - { - auto track = Database::Track::getById(session, trackId); - if (!track || !track->getRelease()) - continue; - - res.emplace_back(track->getRelease().id()); - } - } - - return res; -} - -std::vector -SOMSearcher::getArtists(Wt::Dbo::Session& session, const std::vector& coords) const -{ - std::vector res; - for (const auto& c : coords) - { - for (auto trackId : _tracksMap[c]) - { - auto track = Database::Track::getById(session, trackId); - if (!track || !track->getArtist()) - continue; - - res.emplace_back(track->getArtist().id()); - } - } - - return res; -} - -} // ns Similarity diff --git a/src/similarity/som/SimilaritySOMSearcher.hpp b/src/similarity/som/SimilaritySOMSearcher.hpp deleted file mode 100644 index 17d7be50..00000000 --- a/src/similarity/som/SimilaritySOMSearcher.hpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2018 Emeric Poupon - * - * This file is part of LMS. - * - * LMS is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * LMS is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with LMS. If not, see . - */ - -#pragma once - -#include -#include - -#include "database/DatabaseHandler.hpp" -#include "database/Types.hpp" -#include "DataNormalizer.hpp" -#include "Network.hpp" - -namespace Similarity { - -class SOMSearcher -{ - public: - - struct ConstructionParams - { - SOM::Network network; - SOM::DataNormalizer normalizer; - SOM::Matrix> tracksMap; - std::map trackIdsCoords; - }; - - SOMSearcher(ConstructionParams params); - - std::vector getSimilarTracks(const std::vector& tracksId, std::size_t maxCount); - std::vector getSimilarReleases(Wt::Dbo::Session& session, Database::IdType releaseId, std::size_t maxCount); - std::vector getSimilarArtists(Wt::Dbo::Session& session, Database::IdType artistId, std::size_t maxCount); - - void dump(Wt::Dbo::Session& session, std::ostream& os) const; - - private: - boost::optional getBestMatchingCoords(const std::vector& tracksIds) const; - std::vector getMatchingCoords(const std::vector& tracksIds) const; - - std::vector getReleases(Wt::Dbo::Session& session, const std::vector& coords) const; - std::vector getArtists(Wt::Dbo::Session& session, const std::vector& coords) const; - - SOM::Network _network; - SOM::DataNormalizer _normalizer; - SOM::Matrix> _tracksMap; - std::map _trackIdsCoords; -}; - -} // ns Similarity diff --git a/src/ui/PlayQueueView.cpp b/src/ui/PlayQueueView.cpp index 52168206..4605813e 100644 --- a/src/ui/PlayQueueView.cpp +++ b/src/ui/PlayQueueView.cpp @@ -388,7 +388,7 @@ PlayQueue::addRadioTrack() if (trackIds.empty()) return; - auto res = getServices().similaritySearcher->getSimilarTracks(trackIds, 1); + auto res = getServices().similaritySearcher->getSimilarTracks(LmsApp->getDboSession(), std::set(trackIds.begin(), trackIds.end()), 1); for (auto trackId : res) { auto trackToAdd = Database::Track::getById(LmsApp->getDboSession(), trackId); diff --git a/tools/feature-extractor/LmsFeatureExtractor.cpp b/tools/feature-extractor/LmsFeatureExtractor.cpp index aed394db..e6e9f71b 100644 --- a/tools/feature-extractor/LmsFeatureExtractor.cpp +++ b/tools/feature-extractor/LmsFeatureExtractor.cpp @@ -2,66 +2,101 @@ #include #include #include +#include +#include #include + #include "database/DatabaseHandler.hpp" #include "database/Track.hpp" +#include "database/Artist.hpp" +#include "database/Cluster.hpp" +#include "database/Release.hpp" +#include "database/TrackFeatures.hpp" #include "utils/Config.hpp" +#include "similarity/features/som/DataNormalizer.hpp" +#include "similarity/features/som/Network.hpp" +#include "similarity/features/som/AcousticBrainzUtils.hpp" -static size_t writeToOstream(char *ptr, size_t size, size_t nmemb, void *userdata) +static +std::ostream& operator<<(std::ostream& os, const Database::Track::pointer& track) { - std::ofstream& ofs = *reinterpret_cast(userdata); - ofs.write(ptr, size*nmemb); - return size*nmemb; + auto genreClusterType = Database::ClusterType::getByName(*track->session(), "GENRE"); + + os << "["; + auto genreClusters = track->getClusterGroups({genreClusterType}, 1); + for (auto genreCluster : genreClusters) + os << genreCluster.front()->getName() << " - "; + if (track->getArtist()) + os << track->getArtist()->getName() << " - "; + if (track->getRelease()) + os << track->getRelease()->getName() << " - "; + os << track->getName() << "]"; + + return os; } -static void acousticBrainzGetLowLevel(const std::string& mbid, boost::filesystem::path output) +static +std::vector +getTrackFeatures(Wt::Dbo::Session &session, Database::Track::pointer track, const std::map& featuresSettings) { - std::string url = "http://acousticbrainz.org/api/v1/" + mbid + "/low-level"; + std::vector res; - std::cout << "GET " << url << std::endl; + std::map> features; + for (const auto& featureSettings : featuresSettings) + features[featureSettings.first] = {}; - CURL *curl; - CURLcode res; - - curl = curl_easy_init(); - if (!curl) + if (!track->getTrackFeatures()->getFeatures(features)) { - return; + std::cout << "Skipping track '" << track->getMBID() << "': missing item" << std::endl; + return res; + }; + + for (const auto& feature : features) + { + auto it = featuresSettings.find(feature.first); + if (it == featuresSettings.end() || (feature.second.size() != it->second)) + { + res.clear(); + break; + } + + res.insert( res.end(), feature.second.begin(), feature.second.end() ); } - std::ofstream ofs(output.string().c_str()); - if (!ofs) - { - curl_easy_cleanup(curl); - std::cerr << "Cannot open " << output.string() << " for writing purpose" << std::endl; - return; - } - - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeToOstream); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ofs); - - res = curl_easy_perform(curl); - - if (res != CURLE_OK) - { - std::cerr << "perform failed: " << curl_easy_strerror(res) << std::endl; - } - - curl_easy_cleanup(curl); + return res; } -static boost::filesystem::path getLowLevelFeaturePath(const std::string& mbid) -{ - return boost::filesystem::path(Config::instance().getPath("working-dir") / "features" / mbid); -} int main(int argc, char *argv[]) { try { + constexpr std::size_t width = 10; + constexpr std::size_t height = 10; +// constexpr std::size_t nbTracks = 80; + constexpr std::size_t nbIterations = 100; + +// std::vector items = { "lowlevel.barkbands.median", "lowlevel.erbbands.median", "lowlevel.melbands.median"}; +// constexpr std::size_t nbDims = 27 + 40 + 40; + //std::vector items = { "tonal.hpcp.median"}; + const std::map featuresSettings = + { +// { "lowlevel.average_loudness", 1 }, +// { "lowlevel.dynamic_complexity", 1 }, + { "lowlevel.spectral_contrast_coeffs.median", 6 }, + { "lowlevel.erbbands.median", 40 }, + { "tonal.hpcp.median", 36 }, + { "lowlevel.melbands.median", 40 }, + { "lowlevel.barkbands.median", 27 }, + { "lowlevel.mfcc.mean", 13 }, + { "lowlevel.gfcc.mean", 13 }, + }; + std::size_t nbDims = 0; + for (const auto& featureSettings : featuresSettings) + nbDims += featureSettings.second; + boost::filesystem::path configFilePath = "/etc/lms.conf"; if (argc >= 2) @@ -73,20 +108,153 @@ int main(int argc, char *argv[]) auto connectionPool = Database::Handler::createConnectionPool(Config::instance().getPath("working-dir") / "lms.db"); Database::Handler db(*connectionPool); + std::cout << "Getting all features..." << std::endl; Wt::Dbo::Transaction transaction(db.getSession()); auto tracks = Database::Track::getAll(db.getSession()); + std::vector trainingTracks; for (auto track : tracks) { if (track->getMBID().empty()) continue; - auto path = getLowLevelFeaturePath(track->getMBID()); - if (!boost::filesystem::exists(path)) - acousticBrainzGetLowLevel(track->getMBID(), path); + if (!track->hasTrackFeatures()) + { + std::string features = AcousticBrainz::extractLowLevelFeatures(track->getMBID()); + + if (features.empty()) + continue; + + Database::TrackFeatures::create(db.getSession(), track, features); + } + + trainingTracks.push_back(track); } + std::cout << "Getting all features DONE" << std::endl; + +/* auto now = std::chrono::system_clock::now(); + std::mt19937 randGenerator(std::chrono::duration_cast(now.time_since_epoch()).count()); + std::shuffle(trainingTracks.begin(), trainingTracks.end(), randGenerator); + + trainingTracks.resize(nbTracks); +*/ + std::cout << "Getting all features DONE" << std::endl; + + std::cout << "Reading features..." << std::endl; + std::vector< std::vector > tracksFeatures; + + for (auto track : trainingTracks) + { + auto features = getTrackFeatures(db.getSession(), track, featuresSettings); + + if (features.empty()) + continue; + + tracksFeatures.emplace_back(std::move(features)); + } + std::cout << "Reading features DONE" << std::endl; + + SOM::Network network(width, height, nbDims); + SOM::DataNormalizer normalizer(nbDims); + + std::vector weights; + for (const auto& featureSettings : featuresSettings) + { + for (std::size_t i = 0; i < featureSettings.second; ++i) + weights.push_back(1. / featureSettings.second); + } + + network.setDataWeights(weights); + + std::cout << "Normalizing..." << std::endl; + normalizer.computeNormalizationFactors(tracksFeatures); + + std::cout << "Dumping normalizer: " << std::endl; + normalizer.dump(std::cout); + std::cout << "Dumping normalizer DONE" << std::endl; + + for (auto& features : tracksFeatures) + normalizer.normalizeData(features); + std::cout << "Normalizing DONE" << std::endl; + + std::cout << "Training..." << std::endl; + network.train(tracksFeatures, nbIterations); + std::cout << "Training DONE" << std::endl; + + auto meanDistance = network.computeRefVectorsDistanceMean(); + std::cout << "MEAN distance = " << meanDistance << std::endl; + auto medianDistance = network.computeRefVectorsDistanceMedian(); + std::cout << "MEDIAN distance = " << medianDistance << std::endl; + + std::cout << "Classifying tracks..." << std::endl; + + SOM::Matrix< std::vector > tracksMap(width, height); + for (auto track : trainingTracks) + { + auto features = getTrackFeatures(db.getSession(), track, featuresSettings); + + if (features.empty()) + continue; + + normalizer.normalizeData(features); + + auto coords = network.getClosestRefVectorCoords(features); + tracksMap[coords].push_back(track); + } + + std::cout << "Classifying tracks DONE" << std::endl; + + // Dump tracks + + for (std::size_t y = 0; y < tracksMap.getHeight(); ++y) + { + for (std::size_t x = 0; x < tracksMap.getWidth(); ++x) + { + std::cout << "{" << x << ", " << y << "}" << std::endl; + const auto& tracks = tracksMap[{x, y}]; + + for (auto track : tracks) + { + std::cout << " - " << track << std::endl; + } + } + } + + // For each track, get the nearest tracks + for (auto track : trainingTracks) + { + auto features = getTrackFeatures(db.getSession(), track, featuresSettings); + + if (features.empty()) + continue; + + normalizer.normalizeData(features); + + auto refVectorCoords = network.getClosestRefVectorCoords(features); + + std::cout << "Getting nearest songs for track " << track << " in {" << refVectorCoords.x << ", " << refVectorCoords.y << "}:" << std::endl; + for (auto similarTrack : tracksMap[refVectorCoords]) + std::cout << " - " << similarTrack << std::endl; + + std::set neighbourCoords = {refVectorCoords}; + for (std::size_t i = 0; i < 5; ++i) + { + auto coords = network.getClosestRefVectorCoords(neighbourCoords, medianDistance); + if (!coords) + break; + + std::cout << " - in {" << coords->x << ", " << coords->y << "}, dist = " << network.getRefVectorsDistance(*coords, refVectorCoords) << std::endl; + for (auto similarTrack : tracksMap[*coords]) + std::cout << " - " << similarTrack << std::endl; + + neighbourCoords.insert(*coords); + } + + } + + std::cout << "Classifying tracks DONE" << std::endl; } catch( std::exception& e) { diff --git a/tools/feature-extractor/Makefile.am b/tools/feature-extractor/Makefile.am index b886b16e..6aea162a 100644 --- a/tools/feature-extractor/Makefile.am +++ b/tools/feature-extractor/Makefile.am @@ -5,13 +5,16 @@ lms_feature_extractor_SOURCES = \ $(top_srcdir)/src/database/Artist.cpp \ $(top_srcdir)/src/database/Cluster.cpp \ $(top_srcdir)/src/database/DatabaseHandler.cpp \ - $(top_srcdir)/src/database/TrackFeature.cpp \ + $(top_srcdir)/src/database/TrackFeatures.cpp \ $(top_srcdir)/src/database/TrackList.cpp \ $(top_srcdir)/src/database/Release.cpp \ $(top_srcdir)/src/database/ScanSettings.cpp \ $(top_srcdir)/src/database/SqlQuery.cpp \ $(top_srcdir)/src/database/Track.cpp \ $(top_srcdir)/src/database/User.cpp \ + $(top_srcdir)/src/similarity/features/som/AcousticBrainzUtils.cpp \ + $(top_srcdir)/src/similarity/features/som/DataNormalizer.cpp \ + $(top_srcdir)/src/similarity/features/som/Network.cpp \ $(top_srcdir)/src/utils/Config.cpp \ $(top_srcdir)/src/utils/Logger.cpp \ $(top_srcdir)/src/utils/Utils.cpp