Migrated scrobbling stuff
This commit is contained in:
emeric
@@ -0,0 +1,393 @@
|
||||
/*
|
||||
* Copyright (C) 2019 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "FeaturesDefs.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
|
||||
#include "utils/Exception.hpp"
|
||||
|
||||
namespace Recommendation {
|
||||
|
||||
static const std::unordered_map<FeatureName, FeatureDef> featureDefinitions
|
||||
{
|
||||
{ "lowlevel.average_loudness", {1}},
|
||||
{ "lowlevel.barkbands.dmean", {27}},
|
||||
{ "lowlevel.barkbands.dmean2", {27}},
|
||||
{ "lowlevel.barkbands.dvar", {27}},
|
||||
{ "lowlevel.barkbands.dvar2", {27}},
|
||||
{ "lowlevel.barkbands.max", {27}},
|
||||
{ "lowlevel.barkbands.mean", {27}},
|
||||
{ "lowlevel.barkbands.median", {27}},
|
||||
{ "lowlevel.barkbands.min", {27}},
|
||||
{ "lowlevel.barkbands.var", {27}},
|
||||
{ "lowlevel.barkbands_crest.dmean", {1}},
|
||||
{ "lowlevel.barkbands_crest.dmean2", {1}},
|
||||
{ "lowlevel.barkbands_crest.dvar", {1}},
|
||||
{ "lowlevel.barkbands_crest.dvar2", {1}},
|
||||
{ "lowlevel.barkbands_crest.max", {1}},
|
||||
{ "lowlevel.barkbands_crest.mean", {1}},
|
||||
{ "lowlevel.barkbands_crest.median", {1}},
|
||||
{ "lowlevel.barkbands_crest.min", {1}},
|
||||
{ "lowlevel.barkbands_crest.var", {1}},
|
||||
{ "lowlevel.barkbands_flatness_db.dmean", {1}},
|
||||
{ "lowlevel.barkbands_flatness_db.dmean2", {1}},
|
||||
{ "lowlevel.barkbands_flatness_db.dvar", {1}},
|
||||
{ "lowlevel.barkbands_flatness_db.dvar2", {1}},
|
||||
{ "lowlevel.barkbands_flatness_db.max", {1}},
|
||||
{ "lowlevel.barkbands_flatness_db.mean", {1}},
|
||||
{ "lowlevel.barkbands_flatness_db.median", {1}},
|
||||
{ "lowlevel.barkbands_flatness_db.min", {1}},
|
||||
{ "lowlevel.barkbands_flatness_db.var", {1}},
|
||||
{ "lowlevel.barkbands_kurtosis.dmean", {1}},
|
||||
{ "lowlevel.barkbands_kurtosis.dmean2", {1}},
|
||||
{ "lowlevel.barkbands_kurtosis.dvar", {1}},
|
||||
{ "lowlevel.barkbands_kurtosis.dvar2", {1}},
|
||||
{ "lowlevel.barkbands_kurtosis.max", {1}},
|
||||
{ "lowlevel.barkbands_kurtosis.mean", {1}},
|
||||
{ "lowlevel.barkbands_kurtosis.median", {1}},
|
||||
{ "lowlevel.barkbands_kurtosis.min", {1}},
|
||||
{ "lowlevel.barkbands_kurtosis.var", {1}},
|
||||
{ "lowlevel.barkbands_skewness.dmean", {1}},
|
||||
{ "lowlevel.barkbands_skewness.dmean2", {1}},
|
||||
{ "lowlevel.barkbands_skewness.dvar", {1}},
|
||||
{ "lowlevel.barkbands_skewness.dvar2", {1}},
|
||||
{ "lowlevel.barkbands_skewness.max", {1}},
|
||||
{ "lowlevel.barkbands_skewness.mean", {1}},
|
||||
{ "lowlevel.barkbands_skewness.median", {1}},
|
||||
{ "lowlevel.barkbands_skewness.min", {1}},
|
||||
{ "lowlevel.barkbands_skewness.var", {1}},
|
||||
{ "lowlevel.barkbands_spread.dmean", {1}},
|
||||
{ "lowlevel.barkbands_spread.dmean2", {1}},
|
||||
{ "lowlevel.barkbands_spread.dvar", {1}},
|
||||
{ "lowlevel.barkbands_spread.dvar2", {1}},
|
||||
{ "lowlevel.barkbands_spread.max", {1}},
|
||||
{ "lowlevel.barkbands_spread.mean", {1}},
|
||||
{ "lowlevel.barkbands_spread.median", {1}},
|
||||
{ "lowlevel.barkbands_spread.min", {1}},
|
||||
{ "lowlevel.barkbands_spread.var", {1}},
|
||||
{ "lowlevel.dissonance.dmean", {1}},
|
||||
{ "lowlevel.dissonance.dmean2", {1}},
|
||||
{ "lowlevel.dissonance.dvar", {1}},
|
||||
{ "lowlevel.dissonance.dvar2", {1}},
|
||||
{ "lowlevel.dissonance.max", {1}},
|
||||
{ "lowlevel.dissonance.mean", {1}},
|
||||
{ "lowlevel.dissonance.median", {1}},
|
||||
{ "lowlevel.dissonance.min", {1}},
|
||||
{ "lowlevel.dissonance.var", {1}},
|
||||
{ "lowlevel.dynamic_complexity", {1}},
|
||||
{ "lowlevel.erbbands.dmean", {40}},
|
||||
{ "lowlevel.erbbands.dmean2", {40}},
|
||||
{ "lowlevel.erbbands.dvar", {40}},
|
||||
{ "lowlevel.erbbands.dvar2", {40}},
|
||||
{ "lowlevel.erbbands.max", {40}},
|
||||
{ "lowlevel.erbbands.mean", {40}},
|
||||
{ "lowlevel.erbbands.median", {40}},
|
||||
{ "lowlevel.erbbands.min", {40}},
|
||||
{ "lowlevel.erbbands.var", {40}},
|
||||
{ "lowlevel.gfcc.mean", {13}},
|
||||
{ "lowlevel.hfc.dmean", {1}},
|
||||
{ "lowlevel.hfc.dmean2", {1}},
|
||||
{ "lowlevel.hfc.dvar", {1}},
|
||||
{ "lowlevel.hfc.dvar2", {1}},
|
||||
{ "lowlevel.hfc.max", {1}},
|
||||
{ "lowlevel.hfc.mean", {1}},
|
||||
{ "lowlevel.hfc.median", {1}},
|
||||
{ "lowlevel.hfc.min", {1}},
|
||||
{ "lowlevel.hfc.var", {1}},
|
||||
{ "tonal.hpcp.median", {36}},
|
||||
{ "lowlevel.melbands.dmean", {40}},
|
||||
{ "lowlevel.melbands.dmean2", {40}},
|
||||
{ "lowlevel.melbands.dvar", {40}},
|
||||
{ "lowlevel.melbands.dvar2", {40}},
|
||||
{ "lowlevel.melbands.max", {40}},
|
||||
{ "lowlevel.melbands.mean", {40}},
|
||||
{ "lowlevel.melbands.median", {40}},
|
||||
{ "lowlevel.melbands.min", {40}},
|
||||
{ "lowlevel.melbands.var", {40}},
|
||||
{ "lowlevel.melbands_crest.dmean", {1}},
|
||||
{ "lowlevel.melbands_crest.dmean2", {1}},
|
||||
{ "lowlevel.melbands_crest.dvar", {1}},
|
||||
{ "lowlevel.melbands_crest.dvar2", {1}},
|
||||
{ "lowlevel.melbands_crest.max", {1}},
|
||||
{ "lowlevel.melbands_crest.mean", {1}},
|
||||
{ "lowlevel.melbands_crest.median", {1}},
|
||||
{ "lowlevel.melbands_crest.min", {1}},
|
||||
{ "lowlevel.melbands_crest.var", {1}},
|
||||
{ "lowlevel.melbands_flatness_db.dmean", {1}},
|
||||
{ "lowlevel.melbands_flatness_db.dmean2", {1}},
|
||||
{ "lowlevel.melbands_flatness_db.dvar", {1}},
|
||||
{ "lowlevel.melbands_flatness_db.dvar2", {1}},
|
||||
{ "lowlevel.melbands_flatness_db.max", {1}},
|
||||
{ "lowlevel.melbands_flatness_db.mean", {1}},
|
||||
{ "lowlevel.melbands_flatness_db.median", {1}},
|
||||
{ "lowlevel.melbands_flatness_db.min", {1}},
|
||||
{ "lowlevel.melbands_flatness_db.var", {1}},
|
||||
{ "lowlevel.melbands_kurtosis.dmean", {1}},
|
||||
{ "lowlevel.melbands_kurtosis.dmean2", {1}},
|
||||
{ "lowlevel.melbands_kurtosis.dvar", {1}},
|
||||
{ "lowlevel.melbands_kurtosis.dvar2", {1}},
|
||||
{ "lowlevel.melbands_kurtosis.max", {1}},
|
||||
{ "lowlevel.melbands_kurtosis.mean", {1}},
|
||||
{ "lowlevel.melbands_kurtosis.median", {1}},
|
||||
{ "lowlevel.melbands_kurtosis.min", {1}},
|
||||
{ "lowlevel.melbands_kurtosis.var", {1}},
|
||||
{ "lowlevel.melbands_skewness.dmean", {1}},
|
||||
{ "lowlevel.melbands_skewness.dmean2", {1}},
|
||||
{ "lowlevel.melbands_skewness.dvar", {1}},
|
||||
{ "lowlevel.melbands_skewness.dvar2", {1}},
|
||||
{ "lowlevel.melbands_skewness.max", {1}},
|
||||
{ "lowlevel.melbands_skewness.mean", {1}},
|
||||
{ "lowlevel.melbands_skewness.median", {1}},
|
||||
{ "lowlevel.melbands_skewness.min", {1}},
|
||||
{ "lowlevel.melbands_skewness.var", {1}},
|
||||
{ "lowlevel.melbands_spread.dmean", {1}},
|
||||
{ "lowlevel.melbands_spread.dmean2", {1}},
|
||||
{ "lowlevel.melbands_spread.dvar", {1}},
|
||||
{ "lowlevel.melbands_spread.dvar2", {1}},
|
||||
{ "lowlevel.melbands_spread.max", {1}},
|
||||
{ "lowlevel.melbands_spread.mean", {1}},
|
||||
{ "lowlevel.melbands_spread.median", {1}},
|
||||
{ "lowlevel.melbands_spread.min", {1}},
|
||||
{ "lowlevel.melbands_spread.var", {1}},
|
||||
{ "lowlevel.mfcc.mean", {13}},
|
||||
{ "lowlevel.pitch_salience.dmean", {1}},
|
||||
{ "lowlevel.pitch_salience.dmean2", {1}},
|
||||
{ "lowlevel.pitch_salience.dvar", {1}},
|
||||
{ "lowlevel.pitch_salience.dvar2", {1}},
|
||||
{ "lowlevel.pitch_salience.max", {1}},
|
||||
{ "lowlevel.pitch_salience.mean", {1}},
|
||||
{ "lowlevel.pitch_salience.median", {1}},
|
||||
{ "lowlevel.pitch_salience.min", {1}},
|
||||
{ "lowlevel.pitch_salience.var", {1}},
|
||||
{ "lowlevel.silence_rate_30dB.dmean", {1}},
|
||||
{ "lowlevel.silence_rate_30dB.dmean2", {1}},
|
||||
{ "lowlevel.silence_rate_30dB.dvar", {1}},
|
||||
{ "lowlevel.silence_rate_30dB.dvar2", {1}},
|
||||
{ "lowlevel.silence_rate_30dB.max", {1}},
|
||||
{ "lowlevel.silence_rate_30dB.mean", {1}},
|
||||
{ "lowlevel.silence_rate_30dB.median", {1}},
|
||||
{ "lowlevel.silence_rate_30dB.min", {1}},
|
||||
{ "lowlevel.silence_rate_30dB.var", {1}},
|
||||
{ "lowlevel.silence_rate_60dB.dmean", {1}},
|
||||
{ "lowlevel.silence_rate_60dB.dmean2", {1}},
|
||||
{ "lowlevel.silence_rate_60dB.dvar", {1}},
|
||||
{ "lowlevel.silence_rate_60dB.dvar2", {1}},
|
||||
{ "lowlevel.silence_rate_60dB.max", {1}},
|
||||
{ "lowlevel.silence_rate_60dB.mean", {1}},
|
||||
{ "lowlevel.silence_rate_60dB.median", {1}},
|
||||
{ "lowlevel.silence_rate_60dB.min", {1}},
|
||||
{ "lowlevel.silence_rate_60dB.var", {1}},
|
||||
{ "lowlevel.spectral_centroid.dmean", {1}},
|
||||
{ "lowlevel.spectral_centroid.dmean2", {1}},
|
||||
{ "lowlevel.spectral_centroid.dvar", {1}},
|
||||
{ "lowlevel.spectral_centroid.dvar2", {1}},
|
||||
{ "lowlevel.spectral_centroid.max", {1}},
|
||||
{ "lowlevel.spectral_centroid.mean", {1}},
|
||||
{ "lowlevel.spectral_centroid.median", {1}},
|
||||
{ "lowlevel.spectral_centroid.min", {1}},
|
||||
{ "lowlevel.spectral_centroid.var", {1}},
|
||||
{ "lowlevel.spectral_complexity.dmean", {1}},
|
||||
{ "lowlevel.spectral_complexity.dmean2", {1}},
|
||||
{ "lowlevel.spectral_complexity.dvar", {1}},
|
||||
{ "lowlevel.spectral_complexity.dvar2", {1}},
|
||||
{ "lowlevel.spectral_complexity.max", {1}},
|
||||
{ "lowlevel.spectral_complexity.mean", {1}},
|
||||
{ "lowlevel.spectral_complexity.median", {1}},
|
||||
{ "lowlevel.spectral_complexity.min", {1}},
|
||||
{ "lowlevel.spectral_complexity.var", {1}},
|
||||
{ "lowlevel.spectral_contrast_coeffs.dmean", {6}},
|
||||
{ "lowlevel.spectral_contrast_coeffs.dmean2", {6}},
|
||||
{ "lowlevel.spectral_contrast_coeffs.dvar", {6}},
|
||||
{ "lowlevel.spectral_contrast_coeffs.dvar2", {6}},
|
||||
{ "lowlevel.spectral_contrast_coeffs.max", {6}},
|
||||
{ "lowlevel.spectral_contrast_coeffs.mean", {6}},
|
||||
{ "lowlevel.spectral_contrast_coeffs.median", {6}},
|
||||
{ "lowlevel.spectral_contrast_coeffs.min", {6}},
|
||||
{ "lowlevel.spectral_contrast_coeffs.var", {6}},
|
||||
{ "lowlevel.spectral_contrast_valleys.dmean", {6}},
|
||||
{ "lowlevel.spectral_contrast_valleys.dmean2", {6}},
|
||||
{ "lowlevel.spectral_contrast_valleys.dvar", {6}},
|
||||
{ "lowlevel.spectral_contrast_valleys.dvar2", {6}},
|
||||
{ "lowlevel.spectral_contrast_valleys.max", {6}},
|
||||
{ "lowlevel.spectral_contrast_valleys.mean", {6}},
|
||||
{ "lowlevel.spectral_contrast_valleys.median", {6}},
|
||||
{ "lowlevel.spectral_contrast_valleys.min", {6}},
|
||||
{ "lowlevel.spectral_contrast_valleys.var", {6}},
|
||||
{ "lowlevel.spectral_decrease.dmean", {1}},
|
||||
{ "lowlevel.spectral_decrease.dmean2", {1}},
|
||||
{ "lowlevel.spectral_decrease.dvar", {1}},
|
||||
{ "lowlevel.spectral_decrease.dvar2", {1}},
|
||||
{ "lowlevel.spectral_decrease.max", {1}},
|
||||
{ "lowlevel.spectral_decrease.mean", {1}},
|
||||
{ "lowlevel.spectral_decrease.median", {1}},
|
||||
{ "lowlevel.spectral_decrease.min", {1}},
|
||||
{ "lowlevel.spectral_decrease.var", {1}},
|
||||
{ "lowlevel.spectral_energy.dmean", {1}},
|
||||
{ "lowlevel.spectral_energy.dmean2", {1}},
|
||||
{ "lowlevel.spectral_energy.dvar", {1}},
|
||||
{ "lowlevel.spectral_energy.dvar2", {1}},
|
||||
{ "lowlevel.spectral_energy.max", {1}},
|
||||
{ "lowlevel.spectral_energy.mean", {1}},
|
||||
{ "lowlevel.spectral_energy.median", {1}},
|
||||
{ "lowlevel.spectral_energy.min", {1}},
|
||||
{ "lowlevel.spectral_energy.var", {1}},
|
||||
{ "lowlevel.spectral_energyband_high.dmean", {1}},
|
||||
{ "lowlevel.spectral_energyband_high.dmean2", {1}},
|
||||
{ "lowlevel.spectral_energyband_high.dvar", {1}},
|
||||
{ "lowlevel.spectral_energyband_high.dvar2", {1}},
|
||||
{ "lowlevel.spectral_energyband_high.max", {1}},
|
||||
{ "lowlevel.spectral_energyband_high.mean", {1}},
|
||||
{ "lowlevel.spectral_energyband_high.median", {1}},
|
||||
{ "lowlevel.spectral_energyband_high.min", {1}},
|
||||
{ "lowlevel.spectral_energyband_high.var", {1}},
|
||||
{ "lowlevel.spectral_energyband_low.dmean", {1}},
|
||||
{ "lowlevel.spectral_energyband_low.dmean2", {1}},
|
||||
{ "lowlevel.spectral_energyband_low.dvar", {1}},
|
||||
{ "lowlevel.spectral_energyband_low.dvar2", {1}},
|
||||
{ "lowlevel.spectral_energyband_low.max", {1}},
|
||||
{ "lowlevel.spectral_energyband_low.mean", {1}},
|
||||
{ "lowlevel.spectral_energyband_low.median", {1}},
|
||||
{ "lowlevel.spectral_energyband_low.min", {1}},
|
||||
{ "lowlevel.spectral_energyband_low.var", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_high.dmean", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_high.dmean2", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_high.dvar", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_high.dvar2", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_high.max", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_high.mean", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_high.median", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_high.min", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_high.var", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_low.dmean", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_low.dmean2", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_low.dvar", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_low.dvar2", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_low.max", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_low.mean", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_low.median", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_low.min", {1}},
|
||||
{ "lowlevel.spectral_energyband_middle_low.var", {1}},
|
||||
{ "lowlevel.spectral_entropy.dmean", {1}},
|
||||
{ "lowlevel.spectral_entropy.dmean2", {1}},
|
||||
{ "lowlevel.spectral_entropy.dvar", {1}},
|
||||
{ "lowlevel.spectral_entropy.dvar2", {1}},
|
||||
{ "lowlevel.spectral_entropy.max", {1}},
|
||||
{ "lowlevel.spectral_entropy.mean", {1}},
|
||||
{ "lowlevel.spectral_entropy.median", {1}},
|
||||
{ "lowlevel.spectral_entropy.min", {1}},
|
||||
{ "lowlevel.spectral_entropy.var", {1}},
|
||||
{ "lowlevel.spectral_flux.dmean", {1}},
|
||||
{ "lowlevel.spectral_flux.dmean2", {1}},
|
||||
{ "lowlevel.spectral_flux.dvar", {1}},
|
||||
{ "lowlevel.spectral_flux.dvar2", {1}},
|
||||
{ "lowlevel.spectral_flux.max", {1}},
|
||||
{ "lowlevel.spectral_flux.mean", {1}},
|
||||
{ "lowlevel.spectral_flux.median", {1}},
|
||||
{ "lowlevel.spectral_flux.min", {1}},
|
||||
{ "lowlevel.spectral_flux.var", {1}},
|
||||
{ "lowlevel.spectral_kurtosis.dmean", {1}},
|
||||
{ "lowlevel.spectral_kurtosis.dmean2", {1}},
|
||||
{ "lowlevel.spectral_kurtosis.dvar", {1}},
|
||||
{ "lowlevel.spectral_kurtosis.dvar2", {1}},
|
||||
{ "lowlevel.spectral_kurtosis.max", {1}},
|
||||
{ "lowlevel.spectral_kurtosis.mean", {1}},
|
||||
{ "lowlevel.spectral_kurtosis.median", {1}},
|
||||
{ "lowlevel.spectral_kurtosis.min", {1}},
|
||||
{ "lowlevel.spectral_kurtosis.var", {1}},
|
||||
{ "lowlevel.spectral_rms.dmean", {1}},
|
||||
{ "lowlevel.spectral_rms.dmean2", {1}},
|
||||
{ "lowlevel.spectral_rms.dvar", {1}},
|
||||
{ "lowlevel.spectral_rms.dvar2", {1}},
|
||||
{ "lowlevel.spectral_rms.max", {1}},
|
||||
{ "lowlevel.spectral_rms.mean", {1}},
|
||||
{ "lowlevel.spectral_rms.median", {1}},
|
||||
{ "lowlevel.spectral_rms.min", {1}},
|
||||
{ "lowlevel.spectral_rms.var", {1}},
|
||||
{ "lowlevel.spectral_rolloff.dmean", {1}},
|
||||
{ "lowlevel.spectral_rolloff.dmean2", {1}},
|
||||
{ "lowlevel.spectral_rolloff.dvar", {1}},
|
||||
{ "lowlevel.spectral_rolloff.dvar2", {1}},
|
||||
{ "lowlevel.spectral_rolloff.max", {1}},
|
||||
{ "lowlevel.spectral_rolloff.mean", {1}},
|
||||
{ "lowlevel.spectral_rolloff.median", {1}},
|
||||
{ "lowlevel.spectral_rolloff.min", {1}},
|
||||
{ "lowlevel.spectral_rolloff.var", {1}},
|
||||
{ "lowlevel.spectral_skewness.dmean", {1}},
|
||||
{ "lowlevel.spectral_skewness.dmean2", {1}},
|
||||
{ "lowlevel.spectral_skewness.dvar", {1}},
|
||||
{ "lowlevel.spectral_skewness.dvar2", {1}},
|
||||
{ "lowlevel.spectral_skewness.max", {1}},
|
||||
{ "lowlevel.spectral_skewness.mean", {1}},
|
||||
{ "lowlevel.spectral_skewness.median", {1}},
|
||||
{ "lowlevel.spectral_skewness.min", {1}},
|
||||
{ "lowlevel.spectral_skewness.var", {1}},
|
||||
{ "lowlevel.spectral_spread.dmean", {1}},
|
||||
{ "lowlevel.spectral_spread.dmean2", {1}},
|
||||
{ "lowlevel.spectral_spread.dvar", {1}},
|
||||
{ "lowlevel.spectral_spread.dvar2", {1}},
|
||||
{ "lowlevel.spectral_spread.max", {1}},
|
||||
{ "lowlevel.spectral_spread.mean", {1}},
|
||||
{ "lowlevel.spectral_spread.median", {1}},
|
||||
{ "lowlevel.spectral_spread.min", {1}},
|
||||
{ "lowlevel.spectral_spread.var", {1}},
|
||||
{ "lowlevel.spectral_strongpeak.dmean", {1}},
|
||||
{ "lowlevel.spectral_strongpeak.dmean2", {1}},
|
||||
{ "lowlevel.spectral_strongpeak.dvar", {1}},
|
||||
{ "lowlevel.spectral_strongpeak.dvar2", {1}},
|
||||
{ "lowlevel.spectral_strongpeak.max", {1}},
|
||||
{ "lowlevel.spectral_strongpeak.mean", {1}},
|
||||
{ "lowlevel.spectral_strongpeak.median", {1}},
|
||||
{ "lowlevel.spectral_strongpeak.min", {1}},
|
||||
{ "lowlevel.spectral_strongpeak.var", {1}},
|
||||
{ "lowlevel.zerocrossingrate.dmean", {1}},
|
||||
{ "lowlevel.zerocrossingrate.dmean2", {1}},
|
||||
{ "lowlevel.zerocrossingrate.dvar", {1}},
|
||||
{ "lowlevel.zerocrossingrate.dvar2", {1}},
|
||||
{ "lowlevel.zerocrossingrate.max", {1}},
|
||||
{ "lowlevel.zerocrossingrate.mean", {1}},
|
||||
{ "lowlevel.zerocrossingrate.median", {1}},
|
||||
{ "lowlevel.zerocrossingrate.min", {1}},
|
||||
{ "lowlevel.zerocrossingrate.var", {1}},
|
||||
};
|
||||
|
||||
FeatureDef
|
||||
getFeatureDef(const FeatureName& featureName)
|
||||
{
|
||||
auto it {featureDefinitions.find(featureName)};
|
||||
if (it == std::cend(featureDefinitions))
|
||||
throw LmsException {"Unhandled requested feature '" + featureName + "'"};
|
||||
|
||||
return it->second;
|
||||
}
|
||||
|
||||
FeatureNames
|
||||
getFeatureNames()
|
||||
{
|
||||
FeatureNames res;
|
||||
|
||||
std::transform(std::cbegin(featureDefinitions), std::cend(featureDefinitions),
|
||||
std::inserter(res, std::begin(res)), [](auto itFeature) { return itFeature.first; });
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
} // namespace Recommendation
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright (C) 2019 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace Recommendation {
|
||||
|
||||
using FeatureName = std::string;
|
||||
using FeatureNames = std::unordered_set<FeatureName>;
|
||||
using FeatureValue = double;
|
||||
using FeatureValues = std::vector<FeatureValue>;
|
||||
using FeatureValuesMap = std::unordered_map<FeatureName, FeatureValues>;
|
||||
|
||||
struct FeatureDef
|
||||
{
|
||||
std::size_t nbDimensions {};
|
||||
};
|
||||
|
||||
FeatureDef getFeatureDef(const FeatureName& featureName);
|
||||
FeatureNames getFeatureNames();
|
||||
|
||||
struct FeatureSettings
|
||||
{
|
||||
double weight {};
|
||||
};
|
||||
using FeatureSettingsMap = std::unordered_map<FeatureName, FeatureSettings>;
|
||||
|
||||
} // namespace Recommendation
|
||||
@@ -0,0 +1,455 @@
|
||||
/*
|
||||
* Copyright (C) 2018 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "FeaturesEngine.hpp"
|
||||
|
||||
#include <numeric>
|
||||
|
||||
#include "database/Artist.hpp"
|
||||
#include "database/Db.hpp"
|
||||
#include "database/Release.hpp"
|
||||
#include "database/Session.hpp"
|
||||
#include "database/Track.hpp"
|
||||
#include "database/TrackArtistLink.hpp"
|
||||
#include "database/TrackFeatures.hpp"
|
||||
#include "database/TrackList.hpp"
|
||||
#include "som/DataNormalizer.hpp"
|
||||
#include "utils/Logger.hpp"
|
||||
#include "utils/Random.hpp"
|
||||
|
||||
|
||||
namespace Recommendation {
|
||||
|
||||
std::unique_ptr<IEngine> createFeaturesEngine(Database::Db& db)
|
||||
{
|
||||
return std::make_unique<FeaturesEngine>(db);
|
||||
}
|
||||
|
||||
const FeatureSettingsMap&
|
||||
FeaturesEngine::getDefaultTrainFeatureSettings()
|
||||
{
|
||||
static const FeatureSettingsMap defaultTrainFeatureSettings
|
||||
{
|
||||
{ "lowlevel.spectral_energyband_high.mean", {1}},
|
||||
{ "lowlevel.spectral_rolloff.median", {1}},
|
||||
{ "lowlevel.spectral_contrast_valleys.var", {1}},
|
||||
{ "lowlevel.erbbands.mean", {1}},
|
||||
{ "lowlevel.gfcc.mean", {1}},
|
||||
};
|
||||
|
||||
return defaultTrainFeatureSettings;
|
||||
}
|
||||
|
||||
static
|
||||
std::optional<FeatureValuesMap>
|
||||
getTrackFeatureValues(FeaturesEngine::FeaturesFetchFunc func, Database::TrackId trackId, const std::unordered_set<FeatureName>& featureNames)
|
||||
{
|
||||
return func(trackId, featureNames);
|
||||
}
|
||||
|
||||
static
|
||||
std::optional<FeatureValuesMap>
|
||||
getTrackFeatureValuesFromDb(Database::Session& session, Database::TrackId trackId, const std::unordered_set<FeatureName>& featureNames)
|
||||
{
|
||||
auto func = [&](Database::TrackId trackId, const std::unordered_set<FeatureName>& featureNames)
|
||||
{
|
||||
std::optional<FeatureValuesMap> res;
|
||||
|
||||
auto transaction {session.createSharedTransaction()};
|
||||
|
||||
Database::Track::pointer track {Database::Track::getById(session, trackId)};
|
||||
if (!track)
|
||||
return res;
|
||||
|
||||
res = track->getTrackFeatures()->getFeatureValuesMap(featureNames);
|
||||
if (res->empty())
|
||||
res.reset();
|
||||
|
||||
return res;
|
||||
};
|
||||
|
||||
return getTrackFeatureValues(func, trackId, featureNames);
|
||||
}
|
||||
|
||||
static
|
||||
std::optional<SOM::InputVector>
|
||||
convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions)
|
||||
{
|
||||
std::size_t i {};
|
||||
std::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
|
||||
for (const auto& [featureName, values] : featureValuesMap)
|
||||
{
|
||||
if (values.size() != getFeatureDef(featureName).nbDimensions)
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size();
|
||||
res.reset();
|
||||
break;
|
||||
}
|
||||
|
||||
for (double val : values)
|
||||
(*res)[i++] = val;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
static
|
||||
SOM::InputVector
|
||||
getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions)
|
||||
{
|
||||
SOM::InputVector weights {nbDimensions};
|
||||
std::size_t index {};
|
||||
for (const auto& [featureName, featureSettings] : featureSettingsMap)
|
||||
{
|
||||
const std::size_t featureNbDimensions {getFeatureDef(featureName).nbDimensions};
|
||||
|
||||
for (std::size_t i {}; i < featureNbDimensions; ++i)
|
||||
weights[index++] = (1. / featureNbDimensions * featureSettings.weight);
|
||||
}
|
||||
|
||||
assert(index == nbDimensions);
|
||||
|
||||
return weights;
|
||||
}
|
||||
|
||||
void
|
||||
FeaturesEngine::loadFromTraining(const TrainSettings& trainSettings, const ProgressCallback& progressCallback)
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier...";
|
||||
|
||||
std::unordered_set<FeatureName> featureNames;
|
||||
std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)),
|
||||
[](const auto& itFeatureSetting) { return itFeatureSetting.first; });
|
||||
|
||||
const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0},
|
||||
[](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; })};
|
||||
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Features dimension = " << nbDimensions;
|
||||
|
||||
Database::Session& session {_db.getTLSSession()};
|
||||
|
||||
std::vector<Database::TrackId> trackIds;
|
||||
{
|
||||
auto transaction {session.createSharedTransaction()};
|
||||
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Tracks with features...";
|
||||
trackIds = Database::Track::getAllIdsWithFeatures(session);
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Tracks with features DONE (found " << trackIds.size() << " tracks)";
|
||||
}
|
||||
|
||||
std::vector<SOM::InputVector> samples;
|
||||
std::vector<Database::TrackId> samplesTrackIds;
|
||||
|
||||
samples.reserve(trackIds.size());
|
||||
samplesTrackIds.reserve(trackIds.size());
|
||||
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features...";
|
||||
for (Database::TrackId trackId : trackIds)
|
||||
{
|
||||
if (_loadCancelled)
|
||||
return;
|
||||
|
||||
std::optional<FeatureValuesMap> featureValuesMap;
|
||||
|
||||
if (_featuresFetchFunc)
|
||||
featureValuesMap = getTrackFeatureValues(_featuresFetchFunc, trackId, featureNames);
|
||||
else
|
||||
featureValuesMap = getTrackFeatureValuesFromDb(session, trackId, featureNames);
|
||||
|
||||
if (!featureValuesMap)
|
||||
continue;
|
||||
|
||||
std::optional<SOM::InputVector> inputVector {convertFeatureValuesMapToInputVector(*featureValuesMap, nbDimensions)};
|
||||
if (!inputVector)
|
||||
continue;
|
||||
|
||||
samples.emplace_back(std::move(*inputVector));
|
||||
samplesTrackIds.emplace_back(trackId);
|
||||
}
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features DONE";
|
||||
|
||||
if (samples.empty())
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, INFO) << "Nothing to classify!";
|
||||
return;
|
||||
}
|
||||
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Normalizing data...";
|
||||
SOM::DataNormalizer dataNormalizer {nbDimensions};
|
||||
|
||||
dataNormalizer.computeNormalizationFactors(samples);
|
||||
for (auto& sample : samples)
|
||||
dataNormalizer.normalizeData(sample);
|
||||
|
||||
SOM::Coordinate size {static_cast<SOM::Coordinate>(std::sqrt(samples.size() / trainSettings.sampleCountPerNeuron))};
|
||||
if (size < 2)
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, WARNING) << "Very few tracks (" << samples.size() << ") are being used by the features engine, expect bad behaviors";
|
||||
size = 2;
|
||||
}
|
||||
LMS_LOG(RECOMMENDATION, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network";
|
||||
|
||||
SOM::Network network {size, size, nbDimensions};
|
||||
|
||||
SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)};
|
||||
network.setDataWeights(weights);
|
||||
|
||||
auto somProgressCallback{[&](const SOM::Network::CurrentIteration& iter)
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount;
|
||||
progressCallback(Progress {iter.idIteration, iter.iterationCount});
|
||||
}};
|
||||
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Training network...";
|
||||
network.train(samples, trainSettings.iterationCount,
|
||||
progressCallback ? somProgressCallback : SOM::Network::ProgressCallback {},
|
||||
[this] { return _loadCancelled; });
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Training network DONE";
|
||||
|
||||
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks...";
|
||||
TrackPositions trackPositions;
|
||||
for (std::size_t i {}; i < samples.size(); ++i)
|
||||
{
|
||||
if (_loadCancelled)
|
||||
return;
|
||||
|
||||
const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
|
||||
|
||||
trackPositions[samplesTrackIds[i]].push_back(position);
|
||||
}
|
||||
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks DONE";
|
||||
|
||||
load(std::move(network), std::move(trackPositions));
|
||||
}
|
||||
|
||||
void
|
||||
FeaturesEngine::loadFromCache(FeaturesEngineCache cache)
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier from cache...";
|
||||
|
||||
load(std::move(cache._network), cache._trackPositions);
|
||||
}
|
||||
|
||||
IEngine::TrackContainer
|
||||
FeaturesEngine::getSimilarTracksFromTrackList(Database::TrackListId trackListId, std::size_t maxCount) const
|
||||
{
|
||||
const TrackContainer trackIds {[&]
|
||||
{
|
||||
TrackContainer res;
|
||||
|
||||
Database::Session& session {_db.getTLSSession()};
|
||||
|
||||
auto transaction {session.createSharedTransaction()};
|
||||
|
||||
const Database::TrackList::pointer trackList {Database::TrackList::getById(session, trackListId)};
|
||||
if (trackList)
|
||||
res = trackList->getTrackIds();
|
||||
|
||||
return res;
|
||||
}()};
|
||||
|
||||
return getSimilarTracks(trackIds, maxCount);
|
||||
}
|
||||
|
||||
IEngine::TrackContainer
|
||||
FeaturesEngine::getSimilarTracks(const std::vector<Database::TrackId>& tracksIds, std::size_t maxCount) const
|
||||
{
|
||||
auto similarTrackIds {getSimilarObjects(tracksIds, _trackMatrix, _trackPositions, maxCount)};
|
||||
|
||||
Database::Session& session {_db.getTLSSession()};
|
||||
|
||||
{
|
||||
// Report only existing ids, as tracks may have been removed a long time ago (refreshing the SOM takes some time)
|
||||
auto transaction {session.createSharedTransaction()};
|
||||
|
||||
similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds),
|
||||
[&](Database::TrackId trackId)
|
||||
{
|
||||
return !Database::Track::exists(session, trackId);
|
||||
}), std::end(similarTrackIds));
|
||||
}
|
||||
|
||||
return similarTrackIds;
|
||||
}
|
||||
|
||||
IEngine::ReleaseContainer
|
||||
FeaturesEngine::getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const
|
||||
{
|
||||
auto similarReleaseIds {getSimilarObjects({releaseId}, _releaseMatrix, _releasePositions, maxCount)};
|
||||
|
||||
Database::Session& session {_db.getTLSSession()};
|
||||
|
||||
if (!similarReleaseIds.empty())
|
||||
{
|
||||
// Report only existing ids
|
||||
auto transaction {session.createSharedTransaction()};
|
||||
|
||||
similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds),
|
||||
[&](Database::ReleaseId releaseId)
|
||||
{
|
||||
return !Database::Release::exists(session, releaseId);
|
||||
}), std::end(similarReleaseIds));
|
||||
}
|
||||
|
||||
return similarReleaseIds;
|
||||
}
|
||||
|
||||
std::vector<Database::ArtistId>
|
||||
FeaturesEngine::getSimilarArtists(Database::ArtistId artistId, EnumSet<Database::TrackArtistLinkType> linkTypes, std::size_t maxCount) const
|
||||
{
|
||||
auto getSimilarArtistIdsForLinkType {[&] (Database::TrackArtistLinkType linkType)
|
||||
{
|
||||
std::vector<Database::ArtistId> similarArtistIds;
|
||||
|
||||
const auto itArtists {_artistMatrix.find(linkType)};
|
||||
if (itArtists == std::cend(_artistMatrix))
|
||||
{
|
||||
return similarArtistIds;
|
||||
}
|
||||
|
||||
return getSimilarObjects({artistId}, itArtists->second, _artistPositions, maxCount);
|
||||
}};
|
||||
|
||||
std::unordered_set<Database::ArtistId> similarArtistIds;
|
||||
|
||||
for (Database::TrackArtistLinkType linkType : linkTypes)
|
||||
{
|
||||
const auto similarArtistIdsForLinkType {getSimilarArtistIdsForLinkType(linkType)};
|
||||
similarArtistIds.insert(std::begin(similarArtistIdsForLinkType), std::end(similarArtistIdsForLinkType));
|
||||
}
|
||||
|
||||
std::vector<Database::ArtistId> res(std::cbegin(similarArtistIds), std::cend(similarArtistIds));
|
||||
|
||||
Database::Session& session {_db.getTLSSession()};
|
||||
{
|
||||
// Report only existing ids
|
||||
auto transaction {session.createSharedTransaction()};
|
||||
|
||||
res.erase(std::remove_if(std::begin(res), std::end(res),
|
||||
[&](Database::ArtistId artistId)
|
||||
{
|
||||
return !Database::Artist::exists(session, artistId);
|
||||
}), std::end(res));
|
||||
}
|
||||
|
||||
while (res.size() > maxCount)
|
||||
res.erase(Random::pickRandom(res));
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
FeaturesEngineCache
|
||||
FeaturesEngine::toCache() const
|
||||
{
|
||||
return FeaturesEngineCache {*_network, _trackPositions};
|
||||
}
|
||||
|
||||
void
|
||||
FeaturesEngine::load(bool forceReload, const ProgressCallback& progressCallback)
|
||||
{
|
||||
if (forceReload)
|
||||
{
|
||||
FeaturesEngineCache::invalidate();
|
||||
}
|
||||
else if (const std::optional<FeaturesEngineCache> cache {FeaturesEngineCache::read()})
|
||||
{
|
||||
loadFromCache(*cache);
|
||||
return;
|
||||
}
|
||||
|
||||
TrainSettings trainSettings;
|
||||
trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings();
|
||||
|
||||
loadFromTraining(trainSettings, progressCallback);
|
||||
if (!_loadCancelled)
|
||||
toCache().write();
|
||||
}
|
||||
|
||||
void
|
||||
FeaturesEngine::requestCancelLoad()
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Requesting init cancellation";
|
||||
_loadCancelled = true;
|
||||
}
|
||||
|
||||
void
|
||||
FeaturesEngine::load(const SOM::Network& network, const TrackPositions& trackPositions)
|
||||
{
|
||||
using namespace Database;
|
||||
|
||||
_networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian();
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian;
|
||||
|
||||
const SOM::Coordinate width {network.getWidth()};
|
||||
const SOM::Coordinate height {network.getHeight()};
|
||||
|
||||
_releaseMatrix = ReleaseMatrix {width, height};
|
||||
_trackMatrix = TrackMatrix {width, height};
|
||||
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Constructing maps...";
|
||||
|
||||
Database::Session& session {_db.getTLSSession()};
|
||||
|
||||
for (const auto& [trackId, positions] : trackPositions)
|
||||
{
|
||||
if (_loadCancelled)
|
||||
return;
|
||||
|
||||
auto transaction {session.createSharedTransaction()};
|
||||
|
||||
const Track::pointer track {Database::Track::getById(session, trackId)};
|
||||
if (!track)
|
||||
continue;
|
||||
|
||||
for (const SOM::Position& position : positions)
|
||||
{
|
||||
Utils::push_back_if_not_present(_trackPositions[trackId], position);
|
||||
Utils::push_back_if_not_present(_trackMatrix[position], trackId);
|
||||
|
||||
if (Release::pointer release {track->getRelease()})
|
||||
{
|
||||
const ReleaseId releaseId {release->getId()};
|
||||
Utils::push_back_if_not_present(_releasePositions[releaseId], position);
|
||||
Utils::push_back_if_not_present(_releaseMatrix[position], releaseId);
|
||||
}
|
||||
for (const TrackArtistLink::pointer& artistLink : track->getArtistLinks())
|
||||
{
|
||||
const ArtistId artistId {artistLink->getArtist()->getId()};
|
||||
|
||||
Utils::push_back_if_not_present(_artistPositions[artistId], position);
|
||||
auto itArtists {_artistMatrix.find(artistLink->getType())};
|
||||
if (itArtists == std::cend(_artistMatrix))
|
||||
{
|
||||
auto [it, inserted] = _artistMatrix.try_emplace(artistLink->getType(), ArtistMatrix {width, height});
|
||||
assert(inserted);
|
||||
itArtists = it;
|
||||
}
|
||||
Utils::push_back_if_not_present(itArtists->second[position], artistId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_network = std::make_unique<SOM::Network>(network);
|
||||
|
||||
LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully loaded!";
|
||||
}
|
||||
|
||||
} // ns Recommendation
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
* Copyright (C) 2018 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <unordered_map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "recommendation/IEngine.hpp"
|
||||
#include "som/DataNormalizer.hpp"
|
||||
#include "som/Network.hpp"
|
||||
#include "utils/Utils.hpp"
|
||||
#include "FeaturesEngineCache.hpp"
|
||||
#include "FeaturesDefs.hpp"
|
||||
|
||||
namespace Database
|
||||
{
|
||||
class Session;
|
||||
}
|
||||
|
||||
namespace Recommendation {
|
||||
|
||||
using FeatureWeight = double;
|
||||
|
||||
class FeaturesEngine : public IEngine
|
||||
{
|
||||
public:
|
||||
FeaturesEngine(Database::Db& db) : _db {db} {}
|
||||
|
||||
FeaturesEngine(const FeaturesEngine&) = delete;
|
||||
FeaturesEngine(FeaturesEngine&&) = delete;
|
||||
FeaturesEngine& operator=(const FeaturesEngine&) = delete;
|
||||
FeaturesEngine& operator=(FeaturesEngine&&) = delete;
|
||||
|
||||
using FeaturesFetchFunc = std::function<std::optional<std::unordered_map<std::string, std::vector<double>>>(Database::TrackId, const std::unordered_set<std::string>& /*features*/)>;
|
||||
// Default is to retrieve the features from the database (may be slow).
|
||||
// Use this only if you want to train different searchers with some cached data
|
||||
static void setFeaturesFetchFunc(FeaturesFetchFunc func) { _featuresFetchFunc = func; }
|
||||
|
||||
static const FeatureSettingsMap& getDefaultTrainFeatureSettings();
|
||||
|
||||
private:
|
||||
void load(bool forceReload, const ProgressCallback& progressCallback) override;
|
||||
void requestCancelLoad() override;
|
||||
void cancelLoad() override {}
|
||||
|
||||
TrackContainer getSimilarTracksFromTrackList(Database::TrackListId tracklistId, std::size_t maxCount) const override;
|
||||
TrackContainer getSimilarTracks(const std::vector<Database::TrackId>& tracksId, std::size_t maxCount) const override;
|
||||
ReleaseContainer getSimilarReleases(Database::ReleaseId releaseId, std::size_t maxCount) const override;
|
||||
ArtistContainer getSimilarArtists(Database::ArtistId artistId, EnumSet<Database::TrackArtistLinkType> linkTypes, std::size_t maxCount) const override;
|
||||
|
||||
void loadFromCache(FeaturesEngineCache cache);
|
||||
|
||||
// Use training (may be very slow)
|
||||
struct TrainSettings
|
||||
{
|
||||
std::size_t iterationCount {10};
|
||||
float sampleCountPerNeuron {4};
|
||||
FeatureSettingsMap featureSettingsMap;
|
||||
};
|
||||
void loadFromTraining(const TrainSettings& trainSettings, const ProgressCallback& progressCallback);
|
||||
|
||||
template <typename IdType>
|
||||
using ObjectPositions = std::unordered_map<IdType, std::vector<SOM::Position>>;
|
||||
|
||||
using ArtistPositions = ObjectPositions<Database::ArtistId>;
|
||||
using ReleasePositions = ObjectPositions<Database::ReleaseId>;
|
||||
using TrackPositions = ObjectPositions<Database::TrackId>;
|
||||
|
||||
template <typename IdType>
|
||||
using ObjectMatrix = SOM::Matrix<std::vector<IdType>>;
|
||||
using ArtistMatrix = ObjectMatrix<Database::ArtistId>;
|
||||
using ReleaseMatrix = ObjectMatrix<Database::ReleaseId>;
|
||||
using TrackMatrix = ObjectMatrix<Database::TrackId>;
|
||||
|
||||
void load(const SOM::Network& network, const TrackPositions& tracksPosition);
|
||||
|
||||
FeaturesEngineCache toCache() const;
|
||||
|
||||
template <typename IdType>
|
||||
static std::vector<SOM::Position> getMatchingRefVectorsPosition(const std::vector<IdType>& ids, const ObjectPositions<IdType>& objectPositions);
|
||||
|
||||
template <typename IdType>
|
||||
static std::vector<IdType> getObjectsIds(const std::vector<SOM::Position>& positions, const ObjectMatrix<IdType>& objectsMatrix);
|
||||
|
||||
template <typename IdType>
|
||||
std::vector<IdType> getSimilarObjects(const std::vector<IdType>& ids,
|
||||
const ObjectMatrix<IdType>& objectMatrix,
|
||||
const ObjectPositions<IdType>& objectPositions,
|
||||
std::size_t maxCount) const;
|
||||
|
||||
Database::Db& _db;
|
||||
bool _loadCancelled {};
|
||||
std::unique_ptr<SOM::Network> _network;
|
||||
double _networkRefVectorsDistanceMedian {};
|
||||
|
||||
ArtistPositions _artistPositions;
|
||||
std::unordered_map<Database::TrackArtistLinkType, ArtistMatrix> _artistMatrix;
|
||||
|
||||
ReleasePositions _releasePositions;
|
||||
ReleaseMatrix _releaseMatrix;
|
||||
|
||||
TrackPositions _trackPositions;
|
||||
TrackMatrix _trackMatrix;
|
||||
|
||||
static inline FeaturesFetchFunc _featuresFetchFunc;
|
||||
};
|
||||
|
||||
template <typename IdType>
|
||||
std::vector<SOM::Position>
|
||||
FeaturesEngine::getMatchingRefVectorsPosition(const std::vector<IdType>& ids, const ObjectPositions<IdType>& objectPositions)
|
||||
{
|
||||
std::vector<SOM::Position> res;
|
||||
|
||||
if (ids.empty())
|
||||
return res;
|
||||
|
||||
for (const IdType id : ids)
|
||||
{
|
||||
auto it = objectPositions.find(id);
|
||||
if (it == objectPositions.end())
|
||||
continue;
|
||||
|
||||
for (const SOM::Position& position : it->second)
|
||||
Utils::push_back_if_not_present(res, position);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename IdType>
|
||||
std::vector<IdType>
|
||||
FeaturesEngine::getObjectsIds(const std::vector<SOM::Position>& positions, const ObjectMatrix<IdType>& objectMatrix)
|
||||
{
|
||||
std::vector<IdType> res;
|
||||
|
||||
for (const SOM::Position& position : positions)
|
||||
{
|
||||
for (const IdType id : objectMatrix.get(position))
|
||||
Utils::push_back_if_not_present(res, id);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename IdType>
|
||||
std::vector<IdType>
|
||||
FeaturesEngine::getSimilarObjects(const std::vector<IdType>& ids,
|
||||
const ObjectMatrix<IdType>& objectMatrix,
|
||||
const ObjectPositions<IdType>& objectPositions,
|
||||
std::size_t maxCount) const
|
||||
{
|
||||
std::vector<IdType> res;
|
||||
|
||||
std::vector<SOM::Position> searchedRefVectorsPosition {getMatchingRefVectorsPosition(ids, objectPositions)};
|
||||
if (searchedRefVectorsPosition.empty())
|
||||
return res;
|
||||
|
||||
while (1)
|
||||
{
|
||||
std::vector<IdType> closestObjectIds {getObjectsIds(searchedRefVectorsPosition, objectMatrix)};
|
||||
|
||||
// Remove objects that are already in input or already reported
|
||||
closestObjectIds.erase(std::remove_if(std::begin(closestObjectIds), std::end(closestObjectIds),
|
||||
[&](IdType id)
|
||||
{
|
||||
return std::find(std::cbegin(ids), std::cend(ids), id) != std::cend(ids);
|
||||
})
|
||||
, std::end(closestObjectIds));
|
||||
|
||||
for (IdType id : closestObjectIds)
|
||||
{
|
||||
if (res.size() == maxCount)
|
||||
break;
|
||||
|
||||
Utils::push_back_if_not_present(res, id);
|
||||
}
|
||||
|
||||
if (res.size() == maxCount)
|
||||
break;
|
||||
|
||||
// If there is not enough objects, try again with closest neighbour until there is too much distance
|
||||
const std::optional<SOM::Position> closestRefVectorPosition {_network->getClosestRefVectorPosition(searchedRefVectorsPosition, _networkRefVectorsDistanceMedian * 0.75)};
|
||||
if (!closestRefVectorPosition)
|
||||
break;
|
||||
|
||||
Utils::push_back_if_not_present(searchedRefVectorsPosition, closestRefVectorPosition.value());
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
} // ns Recommendation
|
||||
@@ -0,0 +1,255 @@
|
||||
/*
|
||||
* Copyright (C) 2018 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "FeaturesEngineCache.hpp"
|
||||
|
||||
#include <boost/property_tree/ptree.hpp>
|
||||
#include <boost/property_tree/xml_parser.hpp>
|
||||
|
||||
#include "utils/IConfig.hpp"
|
||||
#include "utils/Logger.hpp"
|
||||
#include "utils/Service.hpp"
|
||||
|
||||
namespace Recommendation {
|
||||
|
||||
|
||||
static
|
||||
std::filesystem::path getCacheDirectory()
|
||||
{
|
||||
return Service<IConfig>::get()->getPath("working-dir") / "cache" / "features";
|
||||
}
|
||||
|
||||
static std::filesystem::path getCacheNetworkFilePath()
|
||||
{
|
||||
return getCacheDirectory() / "network";
|
||||
}
|
||||
|
||||
static std::filesystem::path getCacheTrackPositionsFilePath()
|
||||
{
|
||||
return getCacheDirectory() / "track_positions";
|
||||
}
|
||||
|
||||
static
|
||||
bool
|
||||
networkToCacheFile(const SOM::Network& network, std::filesystem::path path)
|
||||
{
|
||||
try
|
||||
{
|
||||
boost::property_tree::ptree root;
|
||||
|
||||
root.put("width", network.getWidth());
|
||||
root.put("height", network.getHeight());
|
||||
root.put("dim_count", network.getInputDimCount());
|
||||
|
||||
for (SOM::InputVector::value_type weight : network.getDataWeights())
|
||||
root.add("weights.weight", weight);
|
||||
|
||||
for (SOM::Coordinate x = 0; x < network.getWidth(); ++x)
|
||||
{
|
||||
for (SOM::Coordinate y = 0; y < network.getWidth(); ++y)
|
||||
{
|
||||
const auto& refVector = network.getRefVector({x, y});
|
||||
|
||||
boost::property_tree::ptree node;
|
||||
for (auto value : refVector)
|
||||
node.add("values.value", value);
|
||||
|
||||
node.put("coord_x", x);
|
||||
node.put("coord_y", y);
|
||||
|
||||
root.add_child("ref_vectors.ref_vector", node);
|
||||
}
|
||||
}
|
||||
|
||||
boost::property_tree::write_xml(path.string(), root);
|
||||
|
||||
LMS_LOG(RECOMMENDATION, DEBUG) << "Created network cache";
|
||||
return true;
|
||||
}
|
||||
catch (boost::property_tree::ptree_error& error)
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create network cache: " << error.what();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<SOM::Network>
|
||||
FeaturesEngineCache::createNetworkFromCacheFile(const std::filesystem::path& path)
|
||||
{
|
||||
if (!std::filesystem::exists(path))
|
||||
return std::nullopt;
|
||||
|
||||
try
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, INFO) << "Reading network from cache...";
|
||||
|
||||
boost::property_tree::ptree root;
|
||||
|
||||
boost::property_tree::read_xml(path.string(), root);
|
||||
|
||||
SOM::Coordinate width {root.get<SOM::Coordinate>("width")};
|
||||
SOM::Coordinate height {root.get<SOM::Coordinate>("height")};
|
||||
std::size_t dimCount {root.get<std::size_t>("dim_count")};
|
||||
|
||||
SOM::Network res {width, height, dimCount};
|
||||
|
||||
{
|
||||
SOM::InputVector weights {dimCount};
|
||||
std::size_t i {};
|
||||
for (const auto& val : root.get_child("weights"))
|
||||
weights[i++] = val.second.get_value<double>();
|
||||
|
||||
res.setDataWeights(weights);
|
||||
}
|
||||
|
||||
for (const auto& node : root.get_child("ref_vectors"))
|
||||
{
|
||||
SOM::Coordinate x {node.second.get<SOM::Coordinate>("coord_x")};
|
||||
SOM::Coordinate y {node.second.get<SOM::Coordinate>("coord_y")};
|
||||
|
||||
SOM::InputVector refVector {dimCount};
|
||||
std::size_t i {};
|
||||
for (const auto& val : node.second.get_child("values"))
|
||||
refVector[i++] = val.second.get_value<SOM::InputVector::value_type>();
|
||||
|
||||
res.setRefVector({x, y}, refVector);
|
||||
}
|
||||
|
||||
LMS_LOG(RECOMMENDATION, INFO) << "Successfully read network from cache";
|
||||
|
||||
return res;
|
||||
}
|
||||
catch (boost::property_tree::ptree_error& error)
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot read network cache: " << error.what();
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
FeaturesEngineCache::objectPositionToCacheFile(const TrackPositions& trackPositions, const std::filesystem::path& path)
|
||||
{
|
||||
try
|
||||
{
|
||||
boost::property_tree::ptree root;
|
||||
|
||||
for (const auto& [id, positions] : trackPositions)
|
||||
{
|
||||
boost::property_tree::ptree node;
|
||||
|
||||
node.put("id", id.getValue());
|
||||
|
||||
for (const SOM::Position& position : positions)
|
||||
{
|
||||
boost::property_tree::ptree positionNode;
|
||||
positionNode.put("x", position.x);
|
||||
positionNode.put("y", position.y);
|
||||
|
||||
node.add_child("position.position", positionNode);
|
||||
}
|
||||
|
||||
root.add_child("objects.object", node);
|
||||
}
|
||||
|
||||
boost::property_tree::write_xml(path.string(), root);
|
||||
return true;
|
||||
}
|
||||
catch (boost::property_tree::ptree_error& error)
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot cache object position: " << error.what();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<FeaturesEngineCache::TrackPositions>
|
||||
FeaturesEngineCache::createObjectPositionsFromCacheFile(const std::filesystem::path& path)
|
||||
{
|
||||
try
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, INFO) << "Reading object position from cache...";
|
||||
|
||||
boost::property_tree::ptree root;
|
||||
|
||||
boost::property_tree::read_xml(path.string(), root);
|
||||
|
||||
TrackPositions res;
|
||||
|
||||
for (const auto& object : root.get_child("objects"))
|
||||
{
|
||||
const Database::TrackId id {object.second.get<Database::IdType::ValueType>("id")};
|
||||
for (const auto& position : object.second.get_child("position"))
|
||||
{
|
||||
auto x = position.second.get<SOM::Coordinate>("x");
|
||||
auto y = position.second.get<SOM::Coordinate>("y");
|
||||
|
||||
res[id].push_back({x, y});
|
||||
}
|
||||
}
|
||||
|
||||
LMS_LOG(RECOMMENDATION, INFO) << "Successfully read object position from cache";
|
||||
|
||||
return res;
|
||||
}
|
||||
catch (boost::property_tree::ptree_error& error)
|
||||
{
|
||||
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create object position from cache file: " << error.what();
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
FeaturesEngineCache::invalidate()
|
||||
{
|
||||
std::filesystem::remove(getCacheNetworkFilePath());
|
||||
std::filesystem::remove(getCacheTrackPositionsFilePath());
|
||||
}
|
||||
|
||||
std::optional<FeaturesEngineCache>
|
||||
FeaturesEngineCache::read()
|
||||
{
|
||||
auto network{createNetworkFromCacheFile(getCacheNetworkFilePath())};
|
||||
if (!network)
|
||||
return std::nullopt;
|
||||
|
||||
auto trackPositions{createObjectPositionsFromCacheFile(getCacheTrackPositionsFilePath())};
|
||||
if (!trackPositions)
|
||||
return std::nullopt;
|
||||
|
||||
return FeaturesEngineCache {std::move(*network), std::move(*trackPositions)};
|
||||
}
|
||||
|
||||
void
|
||||
FeaturesEngineCache::write() const
|
||||
{
|
||||
std::filesystem::create_directories(Service<IConfig>::get()->getPath("working-dir") / "cache" / "features");
|
||||
|
||||
if (!networkToCacheFile(_network, getCacheNetworkFilePath())
|
||||
|| !objectPositionToCacheFile(_trackPositions, getCacheTrackPositionsFilePath()))
|
||||
{
|
||||
invalidate();
|
||||
}
|
||||
}
|
||||
|
||||
FeaturesEngineCache::FeaturesEngineCache(SOM::Network network, TrackPositions trackPositions)
|
||||
: _network {std::move(network)},
|
||||
_trackPositions {std::move(trackPositions)}
|
||||
{
|
||||
}
|
||||
|
||||
} // namespace Recommendation
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright (C) 2018 Emeric Poupon
|
||||
*
|
||||
* This file is part of LMS.
|
||||
*
|
||||
* LMS is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* LMS is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <filesystem>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
|
||||
#include "database/Types.hpp"
|
||||
#include "som/Network.hpp"
|
||||
|
||||
namespace Recommendation {
|
||||
|
||||
class FeaturesEngineCache
|
||||
{
|
||||
public:
|
||||
static void invalidate();
|
||||
|
||||
static std::optional<FeaturesEngineCache> read();
|
||||
void write() const;
|
||||
|
||||
private:
|
||||
using TrackPositions = std::unordered_map<Database::TrackId, std::vector<SOM::Position>>;
|
||||
|
||||
FeaturesEngineCache(SOM::Network network, TrackPositions trackPositions);
|
||||
|
||||
static std::optional<SOM::Network> createNetworkFromCacheFile(const std::filesystem::path& path);
|
||||
static std::optional<TrackPositions> createObjectPositionsFromCacheFile(const std::filesystem::path& path);
|
||||
static bool objectPositionToCacheFile(const TrackPositions& trackPositions, const std::filesystem::path& path);
|
||||
|
||||
friend class FeaturesEngine;
|
||||
|
||||
SOM::Network _network;
|
||||
TrackPositions _trackPositions;
|
||||
};
|
||||
|
||||
} // namespace Recommendation
|
||||
Reference in New Issue
Block a user