owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bug on training + removed some debug info

Browse Source
This commit is contained in:
emeric
2019-03-09 14:18:23 +01:00
parent d5ead9193f
commit 0a94aa2875
5 changed files with 79 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/database/TrackFeatures.cpp
+1 -2
View File
@@ -84,8 +84,7 @@ TrackFeatures::getFeatures(std::map<std::string /*name*/, std::vector<double> /*
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(DB, ERROR) << "ptree exception: " << error.what();
std::cout << "ptree exception: " << error.what() << std::endl;
LMS_LOG(SIMILARITY, ERROR) << "Track " << _track.id() << ": ptree exception: " << error.what();
return false;
}
}
src/similarity/features/SimilarityFeaturesScannerAddon.cpp
-5
View File
@@ -123,8 +123,6 @@ FeaturesScannerAddon::updateSearcher()
FeaturesCache cache{searcher->toCache()};
cache.write();
searcher->dump(_db.getSession(), std::cout);
LMS_LOG(DBUPDATER, INFO) << "New features similarity searcher instanciated";
}
else
@@ -139,15 +137,12 @@ FeaturesScannerAddon::fetchFeatures(Database::IdType trackId, const std::string&
LMS_LOG(DBUPDATER, DEBUG) << "Fetching low level features for track '" << MBID << "'";
std::string data {AcousticBrainz::extractLowLevelFeatures(MBID)};
if (data.empty())
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot extract features using AcousticBrainz!";
return false;
}
// TODO check if the expected features are here
Wt::Dbo::Transaction transaction{_db.getSession()};
Wt::Dbo::ptr<Database::Track> track {Database::Track::getById(_db.getSession(), trackId)};
src/similarity/features/SimilarityFeaturesSearcher.cpp
+62 -52
View File
@@ -66,9 +66,56 @@ getFeatureInfoMapNbDimensions(const FeatureInfoMap& featureInfoMap)
return std::accumulate(featureInfoMap.begin(), featureInfoMap.end(), 0, [](std::size_t sum, auto it) { return sum + it.second.nbDimensions; });
}
static
boost::optional<SOM::InputVector>
getInputVectorFromTrack(const Database::Track::pointer& track, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
{
boost::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
std::map<std::string, std::vector<double>> features;
for (auto itFeatureInfo : featuresInfo)
features[itFeatureInfo.first] = {};
if (!track->getTrackFeatures()->getFeatures(features))
return res;
std::size_t i {};
for (const auto& feature : features)
{
// Check dimensions for each feature
auto it {featuresInfo.find(feature.first)};
if (it == featuresInfo.end() || it->second.nbDimensions != feature.second.size())
{
LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << feature.first << "'. Expected " << it->second.nbDimensions << ", got " << feature.second.size();
res.reset();
break;
}
for (double val : feature.second)
(*res)[i++] = val;
}
return res;
}
static
SOM::InputVector
getInputVectorWeights(const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
{
SOM::InputVector weights {nbDimensions};
std::size_t index {};
for (const auto& featureInfo : featuresInfo)
{
for (std::size_t i {}; i < featureInfo.second.nbDimensions; ++i)
weights[index++] = (1. / featureInfo.second.nbDimensions * featureInfo.second.weight);
}
return weights;
}
FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested)
{
Wt::Dbo::Transaction transaction(session);
Wt::Dbo::Transaction transaction{session};
FeatureInfoMap featuresInfo {getFeatureInfoMap(session)};
std::size_t nbDimensions {getFeatureInfoMapNbDimensions(featuresInfo)};
@@ -82,42 +129,18 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
std::vector<SOM::InputVector> samples;
std::vector<Database::IdType> tracksIds;
samples.reserve(tracks.size());
tracksIds.reserve(tracks.size());
LMS_LOG(SIMILARITY, DEBUG) << "Extracting features...";
for (const Database::Track::pointer& track : tracks)
{
if (stopRequested)
return;
SOM::InputVector sample {nbDimensions};
boost::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(track, featuresInfo, nbDimensions)};
std::map<std::string, std::vector<double>> features;
for (auto itFeatureInfo : featuresInfo)
features[itFeatureInfo.first] = {};
if (!track->getTrackFeatures()->getFeatures(features))
continue;
bool ok {true};
std::size_t i {};
for (const auto& feature : features)
{
// Check dimensions for each feature
auto it {featuresInfo.find(feature.first)};
if (it == featuresInfo.end() || it->second.nbDimensions != feature.second.size())
{
LMS_LOG(SIMILARITY, WARNING) << "Dimension mismatch for feature '" << feature.first << "'. Expected " << it->second.nbDimensions << ", got " << feature.second.size();
ok = false;
break;
}
for (double val : feature.second)
sample[i++] = val;
}
if (!ok)
continue;
samples.emplace_back(std::move(sample));
samples.emplace_back(std::move(*inputVector));
tracksIds.emplace_back(track.id());
}
LMS_LOG(SIMILARITY, DEBUG) << "Extracting features DONE";
@@ -131,27 +154,18 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
}
LMS_LOG(SIMILARITY, DEBUG) << "Normalizing data...";
SOM::DataNormalizer dataNormalizer(nbDimensions);
SOM::DataNormalizer dataNormalizer {nbDimensions};
dataNormalizer.computeNormalizationFactors(samples);
for (auto& sample : samples)
dataNormalizer.normalizeData(sample);
SOM::InputVector weights {nbDimensions};
{
std::size_t index {};
for (const auto& featureInfo : featuresInfo)
{
for (std::size_t i {}; i < featureInfo.second.nbDimensions; ++i)
weights[index++] = (1. / featureInfo.second.nbDimensions * featureInfo.second.weight);
}
}
SOM::Coordinate size {static_cast<SOM::Coordinate>(std::sqrt(samples.size() / 4))};
LMS_LOG(SIMILARITY, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network";
SOM::Network network {size, size, nbDimensions};
std::cout << "Weights = '" << weights << "'";
SOM::InputVector weights {getInputVectorWeights(featuresInfo, nbDimensions)};
network.setDataWeights(weights);
auto progressIndicator{[](const auto& iter)
@@ -175,13 +189,9 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
if (stopRequested)
return;
Wt::Dbo::Transaction transaction {session};
const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
const auto& sample = samples[i];
auto trackId = tracksIds[i];
auto position = network.getClosestRefVectorPosition(sample);
trackPositions[trackId].insert(position);
trackPositions[tracksIds[i]].insert(position);
}
LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE";
@@ -290,9 +300,9 @@ FeaturesSearcher::init(Wt::Dbo::Session& session,
SOM::Coordinate width {_network->getWidth()};
SOM::Coordinate height {_network->getHeight()};
_artistsMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
_releasesMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
_tracksMap = SOM::Matrix<std::set<Database::IdType>>(width, height);
_artistsMap = SOM::Matrix<std::set<Database::IdType>>{width, height};
_releasesMap = SOM::Matrix<std::set<Database::IdType>>{width, height};
_tracksMap = SOM::Matrix<std::set<Database::IdType>>{width, height};
Wt::Dbo::Transaction transaction {session};
@@ -301,7 +311,7 @@ FeaturesSearcher::init(Wt::Dbo::Session& session,
Database::IdType trackId {itTrackCoord.first};
const std::set<SOM::Position>& positionSet {itTrackCoord.second};
auto track {Database::Track::getById(session, trackId)};
Database::Track::pointer track {Database::Track::getById(session, trackId)};
if (!track)
continue;
src/similarity/features/som/Network.cpp
+5 -5
View File
@@ -289,8 +289,8 @@ Network::train(const std::vector<InputVector>& inputData, std::size_t nbIteratio
{
bool stopRequested {false};
std::vector<const InputVector*> inputDataShuffled;
inputDataShuffled.reserve(inputData.size());
inputDataShuffled.reserve(inputData.size());
for (const auto& input : inputData)
inputDataShuffled.push_back(&input);
@@ -311,16 +311,16 @@ Network::train(const std::vector<InputVector>& inputData, std::size_t nbIteratio
for (const InputVector* input : inputDataShuffled)
{
if (requestStopCallback)
{
stopRequested = requestStopCallback();
break;
}
if (stopRequested)
return;
updateRefVectors(getClosestRefVectorPosition(*input), *input, learningFactor, curIter);
}
if (stopRequested)
break;
return;
}
}
tools/similarity/LmsSimilarity.cpp
+11 -6
View File
@@ -66,9 +66,9 @@ int main(int argc, char *argv[])
{
try
{
const std::size_t width = 10;
const std::size_t height = 10;
const std::size_t nbIterations = 20;
const std::size_t width = 5;
const std::size_t height = 5;
const std::size_t nbIterations = 10;
std::size_t nbTracks = 5000;
const std::map<std::string, std::size_t> featuresSettings =
@@ -122,14 +122,19 @@ int main(int argc, char *argv[])
SOM::DataNormalizer normalizer {nbDims};
SOM::InputVector weights {nbDims};
for (const auto& featureSettings : featuresSettings)
{
for (std::size_t i {}; i < featureSettings.second; ++i)
weights[i] = SOM::InputVector::value_type{1. / featureSettings.second};
std::size_t index {};
for (const auto& featureSettings : featuresSettings)
{
for (std::size_t i {}; i < featureSettings.second; ++i)
weights[index++] = SOM::InputVector::value_type{1. / featureSettings.second};
}
}
network.setDataWeights(weights);
std::cout << "Weights: " << weights << std::endl;
std::cout << "Normalizing..." << std::endl;
normalizer.computeNormalizationFactors(tracksFeatures);