owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on the clusterer

Browse Source
This commit is contained in:
emeric
2018-12-14 13:41:22 +01:00
parent de712936eb
commit b76c60f437
12 changed files with 372 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/Makefile.am
+1 -1
View File
@@ -1,2 +1,2 @@
SUBDIRS = metadata classifier
SUBDIRS = metadata clusterer
tools/classifier/LmsClassifier.cpp
-44
View File
@@ -1,44 +0,0 @@
#include <stdlib.h>
#include <stdexcept>
#include <iostream>
#include <string>
#include "classifier/SOM.hpp"
#include "classifier/DataNormalizer.hpp"
#include "classifier/Clusterer.hpp"
int main(int argc, char *argv[])
{
if (argc != 2)
{
std::cerr << "Usage: <file>" << std::endl;
return EXIT_FAILURE;
}
auto iterationCount = std::stoul(argv[1]);
std::vector< std::pair<std::vector<SOM::InputVector::value_type>, std::string> > inputValues =
{
{{ 160, 1 }, { "banane" }},
{{ 80, -1 }, { "poire" }},
{{ 80, -0.75 }, {"pocolat"}},
{{ 240, 0.5 }, {"abricot"}},
{{ 240, -0.5 }, {"peche"}},
{{ 120, -0.5 }, {"fraise"}},
{{ 140, -0.5 }, {"myrtille"}},
};
Clusterer<std::string> classifier(inputValues, 2, iterationCount);
std::cout << "Clusterer :" << std::endl;
classifier.dump(std::cout);
std::cout << "Classify 195, 0.35 = " << std::endl;
for (const auto& val : classifier.getClusterValues({195, 0.35}))
std::cout << val << " " << std::endl;
return EXIT_SUCCESS;
}
tools/classifier/Makefile.am
-11
View File
@@ -1,11 +0,0 @@
bin_PROGRAMS = lms-classifier
lms_classifier_SOURCES = \
$(srcdir)/LmsClassifier.cpp \
$(top_srcdir)/src/classifier/DataNormalizer.cpp \
$(top_srcdir)/src/classifier/SOM.cpp \
$(top_srcdir)/src/utils/Logger.cpp \
$(top_srcdir)/src/utils/Utils.cpp
lms_classifier_CXXFLAGS=-std=c++14 -Wall -I$(top_srcdir)/src -D_REENTRANT
tools/clusterer/LmsClusterer.cpp
+209
View File
@@ -0,0 +1,209 @@
#include <stdlib.h>
#include <stdexcept>
#include <iostream>
#include <string>
#include <boost/filesystem.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <curl/curl.h>
#include "clusterer/SOM.hpp"
#include "clusterer/DataNormalizer.hpp"
#include "clusterer/Clusterer.hpp"
#include "database/DatabaseHandler.hpp"
#include "database/Track.hpp"
#include "database/Artist.hpp"
#include "database/Release.hpp"
#include "utils/Config.hpp"
static std::vector<std::string> features =
{
"lowlevel.average_loudness",
"lowlevel.barkbands_flatness_db.mean",
"lowlevel.dissonance.mean",
"lowlevel.dynamic_complexity",
"lowlevel.hfc.mean", // GOOD
"lowlevel.melbands_crest.mean",
"lowlevel.melbands_kurtosis.mean",
"lowlevel.melbands_skewness.mean",
"lowlevel.melbands_spread.mean",
"lowlevel.pitch_salience.mean",
"lowlevel.pitch_salience.var",
"lowlevel.silence_rate_30dB.mean",
"lowlevel.silence_rate_60dB.mean",
"lowlevel.spectral_centroid.mean",
"lowlevel.spectral_complexity.mean",
"lowlevel.spectral_decrease.mean",
"lowlevel.spectral_energy.mean",
"lowlevel.spectral_energyband_high.mean",
"lowlevel.spectral_energyband_low.mean",
"lowlevel.spectral_energyband_middle_high.mean",
"lowlevel.spectral_energyband_middle_low.mean",
"lowlevel.spectral_entropy.mean",
"lowlevel.spectral_flux.mean",
"lowlevel.spectral_kurtosis.mean",
"lowlevel.spectral_rms.mean",
"lowlevel.spectral_skewness.mean",
"lowlevel.spectral_spread.mean",
"lowlevel.spectral_strongpeak.mean",
"lowlevel.zerocrossingrate.mean",
"rhythm.beats_loudness.mean", // BAD
"rhythm.bpm",
"tonal.chords_changes_rate", // OK
// "tonal.chords_number_rate", // BAD
"tonal.chords_strength.mean", // OK
"tonal.hpcp_entropy.mean", // GOOD
};
static size_t writeToFile(void *buffer, size_t size, size_t nmemb, void* ctx)
{
std::ofstream& ofs = *reinterpret_cast<std::ofstream*>(ctx);
ofs.write(reinterpret_cast<char*>(buffer), size * nmemb);
return size * nmemb;
}
static void acousticBrainzGetLowLevel(const std::string& mbid, boost::filesystem::path output)
{
std::string url = "http://acousticbrainz.org/api/v1/" + mbid + "/low-level";
std::cout << "GET " << url << std::endl;
CURL *curl;
CURLcode res;
curl = curl_easy_init();
if (!curl)
{
return;
}
std::ofstream ofs(output.string().c_str());
if (!ofs)
{
curl_easy_cleanup(curl);
std::cerr << "Cannot open " << output.string() << " for writing purpose" << std::endl;
return;
}
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeToFile);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ofs);
res = curl_easy_perform(curl);
if (res != CURLE_OK)
{
std::cerr << "perform failed: " << curl_easy_strerror(res) << std::endl;
}
curl_easy_cleanup(curl);
}
static boost::filesystem::path getLowLevelFeaturePath(const std::string& mbid)
{
return boost::filesystem::path(Config::instance().getPath("working-dir") / "features" / mbid);
}
std::vector<double> getFeatures(const std::string& mbid)
{
std::vector<double> res;
try
{
boost::property_tree::ptree root;
boost::property_tree::read_json(getLowLevelFeaturePath(mbid).string(), root);
for (const auto& feature : features)
{
res.push_back(root.get<double>(feature));
}
}
catch (std::exception& e)
{
std::cerr << "Caught exception during processing " << mbid << std::endl;
}
return res;
}
int main(int argc, char *argv[])
{
try
{
boost::filesystem::path configFilePath = "/etc/lms.conf";
if (argc >= 2)
configFilePath = std::string(argv[1], 0, 256);
Config::instance().setFile(configFilePath);
Database::Handler::configureAuth();
auto connectionPool = Database::Handler::createConnectionPool(Config::instance().getPath("working-dir") / "lms.db");
Database::Handler db(*connectionPool);
Wt::Dbo::Transaction transaction(db.getSession());
auto tracks = Database::Track::getAll(db.getSession());
std::vector<std::pair<std::vector<double>, Database::IdType>> entries;
std::cout << "Constructing input vectors..." << std::endl;
for (auto track : tracks)
{
if (track->getMBID().empty())
continue;
auto path = getLowLevelFeaturePath(track->getMBID());
if (!boost::filesystem::exists(path))
acousticBrainzGetLowLevel(track->getMBID(), path);
if (!boost::filesystem::exists(path))
continue;
std::pair<std::vector<double>, Database::IdType> entry;
entry.first = getFeatures(track->getMBID());
entry.second = track.id();
if (entry.first.size() == features.size())
entries.push_back(std::move(entry));
}
std::cout << "Constructing input vectors... DONE" << std::endl;
std::cout << "Clutering..." << std::endl;
Clusterer<Database::IdType> clusterer(entries, features.size(), 500);
std::cout << "Clusterer :" << std::endl;
clusterer.dump(std::cout);
std::cout << std::endl;
for (const auto& cluster : clusterer.getAllClusters())
{
std::cout << "******************" << std::endl;
for (const auto& value : cluster)
{
auto track = Database::Track::getById(db.getSession(), value);
auto artist = track->getArtist();
auto release = track->getRelease();
std::cout << "\t" << value << " - " << (artist ? artist->getName() : "") << " - " << (release ? release->getName() : "" ) << " - " << track->getName() << std::endl;
}
std::cout << std::endl;
}
}
catch( std::exception& e)
{
std::cerr << "Caught exception: " << e.what() << std::endl;
}
return EXIT_SUCCESS;
}
tools/clusterer/Makefile.am
+21
View File
@@ -0,0 +1,21 @@
bin_PROGRAMS = lms-clusterer
lms_clusterer_SOURCES = \
$(srcdir)/LmsClusterer.cpp \
$(top_srcdir)/src/clusterer/DataNormalizer.cpp \
$(top_srcdir)/src/clusterer/SOM.cpp \
$(top_srcdir)/src/database/Artist.cpp \
$(top_srcdir)/src/database/Cluster.cpp \
$(top_srcdir)/src/database/DatabaseHandler.cpp \
$(top_srcdir)/src/database/TrackList.cpp \
$(top_srcdir)/src/database/Release.cpp \
$(top_srcdir)/src/database/ScanSettings.cpp \
$(top_srcdir)/src/database/SqlQuery.cpp \
$(top_srcdir)/src/database/Track.cpp \
$(top_srcdir)/src/database/User.cpp \
$(top_srcdir)/src/utils/Config.cpp \
$(top_srcdir)/src/utils/Logger.cpp \
$(top_srcdir)/src/utils/Utils.cpp
lms_clusterer_CXXFLAGS=-std=c++14 -Wall -I$(top_srcdir)/src -D_REENTRANT