owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP on the clusterer

Browse Source
This commit is contained in:
emeric
2018-12-14 13:41:22 +01:00
parent de712936eb
commit b76c60f437
12 changed files with 372 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/classifier/Clusterer.hpp → src/clusterer/Clusterer.hpp
+41 -9
View File
@@ -19,7 +19,7 @@
#pragma once
#include <cmath>
#include "SOM.hpp"
#include "DataNormalizer.hpp"
@@ -31,9 +31,16 @@ class Clusterer
{
public:
using SampleType = std::pair<SOM::InputVector /* key */, T /* value*/ >;
using Cluster = std::vector<T>;
Clusterer(const std::vector<SampleType>& samples, std::size_t inputDimCount, std::size_t iterationCount);
const std::vector<T>& getClusterValues(const SOM::InputVector& data) const;
const Cluster& getCluster(const SOM::InputVector& data) const;
// Sorted results (best first)
std::vector<Cluster> getClusters(const SOM::InputVector& data, std::size_t nbClusters) const;
const std::vector<Cluster>& getAllClusters() const;
void dump(std::ostream& os) const;
@@ -55,8 +62,8 @@ class Clusterer
template<typename T>
Clusterer<T>::Clusterer(const std::vector<SampleType>& samples, std::size_t inputDimCount, std::size_t iterationCount)
:
_width(3),
_height(3),
_width(std::sqrt(samples.size()/20)),
_height(std::sqrt(samples.size()/20)),
_dataNormalizer(inputDimCount),
_network(_width, _height, inputDimCount)
{
@@ -116,8 +123,8 @@ Clusterer<T>::train(const std::vector<std::pair<SOM::InputVector, T>>& samples,
}
template<typename T>
const std::vector<T>&
Clusterer<T>::getClusterValues(const SOM::InputVector& inputVector) const
const typename Clusterer<T>::Cluster&
Clusterer<T>::getCluster(const SOM::InputVector& inputVector) const
{
auto inputVectorNormalized = inputVector;
_dataNormalizer.normalizeData(inputVectorNormalized);
@@ -125,16 +132,42 @@ Clusterer<T>::getClusterValues(const SOM::InputVector& inputVector) const
return getValues(_network.classify(inputVectorNormalized));
}
template<typename T>
std::vector<typename Clusterer<T>::Cluster>
Clusterer<T>::getClusters(const SOM::InputVector& inputVector, std::size_t nbClusters) const
{
auto inputVectorNormalized = inputVector;
_dataNormalizer.normalizeData(inputVectorNormalized);
std::vector<typename Clusterer<T>::Cluster> res;
for (auto& cluster : _network.classify(inputVectorNormalized, nbClusters))
{
res.push_back(getValues(cluster));
}
return res;
}
template<typename T>
const std::vector<typename Clusterer<T>::Cluster>&
Clusterer<T>::getAllClusters() const
{
return _values;
}
template<typename T>
void
Clusterer<T>::dump(std::ostream& os) const
{
os << "Normalizer:" << std::endl;
_dataNormalizer.dump(os);
os << std::endl;
os << "Internal network:" << std::endl;
_network.dump(os);
os << "Values: " << std::endl;
for (std::size_t x = 0; x < _width; ++x)
for (std::size_t y = 0; y < _height; ++y)
{
for (std::size_t y = 0; y < _height; ++y)
for (std::size_t x = 0; x < _width; ++x)
{
os << "[";
for (const auto& value : getValues({x, y}))
@@ -146,4 +179,3 @@ Clusterer<T>::dump(std::ostream& os) const
}
src/classifier/DataNormalizer.cpp → src/clusterer/DataNormalizer.cpp
+43 -10
View File
@@ -19,22 +19,42 @@
#include "DataNormalizer.hpp"
#include <iostream>
#include <algorithm>
#include <numeric>
namespace SOM
{
template<typename T>
static
T
variance(const std::vector<T>& vec)
{
std::size_t size = vec.size();
if (size == 1)
return T{0.};
T mean = std::accumulate(vec.begin(), vec.end(), T{0.}) / size;
return std::accumulate(vec.begin(), vec.end(), T{0.},
[mean, size] (T accumulator, const T& val)
{
return accumulator + ((val - mean) * (val - mean) / (size - 1));
});
}
DataNormalizer::DataNormalizer(std::size_t inputDimCount)
: _inputDimCount(inputDimCount)
{
}
void
DataNormalizer::computeNormalizationFactors(const std::vector<InputVector>& inputVectors)
{
if (inputVectors.empty())
throw SOMException("Empty input vectors");
// For each dimension of the input, compute the min/max
_minmax.clear();
_minmax.resize(_inputDimCount);
@@ -54,6 +74,18 @@ DataNormalizer::computeNormalizationFactors(const std::vector<InputVector>& inpu
}
}
InputVector::value_type
DataNormalizer::normalizeValue(InputVector::value_type value, std::size_t dimId) const
{
// clamp
if (value > _minmax[dimId].max)
value = _minmax[dimId].max;
else if (value < _minmax[dimId].min)
value = _minmax[dimId].min;
return (value - _minmax[dimId].min) / (_minmax[dimId].max - _minmax[dimId].min);
}
void
DataNormalizer::normalizeData(InputVector& a) const
{
@@ -61,14 +93,15 @@ DataNormalizer::normalizeData(InputVector& a) const
for (std::size_t dimId = 0; dimId < _inputDimCount; ++dimId)
{
// clamp
if (a[dimId] > _minmax[dimId].max)
a[dimId] = _minmax[dimId].max;
else if (a[dimId] < _minmax[dimId].min)
a[dimId] = _minmax[dimId].min;
a[dimId] = (a[dimId] - _minmax[dimId].min) / (_minmax[dimId].max - _minmax[dimId].min);
a[dimId] = normalizeValue(a[dimId], dimId);
}
}
void
DataNormalizer::dump(std::ostream& os) const
{
for (std::size_t i = 0; i < _inputDimCount; ++i)
os << "(" << _minmax[i].min << ", " << _minmax[i].max << ")";
}
} // namespace SOM
src/classifier/DataNormalizer.hpp → src/clusterer/DataNormalizer.hpp
+7
View File
@@ -19,6 +19,9 @@
#pragma once
#include <vector>
#include <ostream>
#include "SOM.hpp"
namespace SOM
@@ -33,7 +36,11 @@ class DataNormalizer
void normalizeData(InputVector& data) const;
void dump(std::ostream& os) const;
private:
InputVector::value_type normalizeValue(InputVector::value_type value, std::size_t dimensionId) const;
std::size_t _inputDimCount;
struct minmax
src/classifier/SOM.cpp → src/clusterer/SOM.cpp
+38 -1
View File
@@ -204,7 +204,6 @@ Network::getRefVector(std::size_t x, std::size_t y) const
return _refVectors[x + y*_width];
}
void
Network::dump(std::ostream& os) const
{
@@ -242,6 +241,44 @@ Network::classify(const InputVector& data) const
return getClosestRefVector(data);
}
std::vector<Coords>
Network::classify(const InputVector& data, std::size_t size) const
{
struct Entry
{
Coords coords;
InputVector refVector;
};
std::vector<Entry> sortedEntries;
for (std::size_t x = 0; x < _width; ++x)
{
for (std::size_t y = 0; y < _height; ++y)
{
sortedEntries.push_back( Entry{{x, y}, getRefVector(x, y)} );
}
}
const InputVector& closestRefVector = getRefVector(getClosestRefVector(data));
std::sort(sortedEntries.begin(), sortedEntries.end(),
[&](const Entry& a, const Entry& b)
{
return _distanceFunc(a.refVector, closestRefVector, _weights) < _distanceFunc(b.refVector, closestRefVector, _weights);
});
std::vector<Coords> res;
for (const Entry& entry : sortedEntries)
{
res.push_back(entry.coords);
if (res.size() == size)
break;
}
return res;
}
static InputVector::value_type
computeCoordsNorm(Coords c1, Coords c2)
{
src/classifier/SOM.hpp → src/clusterer/SOM.hpp
+4
View File
@@ -61,6 +61,9 @@ class Network
// data must be normalized
Coords classify(const InputVector& data) const;
// ordered from closest to farthest
std::vector<Coords> classify(const InputVector& data, std::size_t size) const;
void dump(std::ostream& os) const;
// For each ref vector, update formula is:
@@ -86,6 +89,7 @@ class Network
InputVector& getRefVector(std::size_t x, std::size_t y);
const InputVector& getRefVector(std::size_t x, std::size_t y) const;
const InputVector& getRefVector(Coords coords) const { return getRefVector(coords.x, coords.y); }
Coords getClosestRefVector(const InputVector& data) const;
void updateRefVectors(Coords closestRefVectorCoords, const InputVector& input, Progress progress);
src/database/DatabaseHandler.cpp
+1 -1
View File
@@ -196,7 +196,7 @@ Handler::createConnectionPool(boost::filesystem::path p)
auto connection = std::make_unique<Wt::Dbo::backend::Sqlite3>(p.string());
connection->executeSql("pragma journal_mode=WAL");
connection->setProperty("show-queries", "true");
// connection->setProperty("show-queries", "true");
auto pool = std::make_unique<Wt::Dbo::FixedSqlConnectionPool>(std::move(connection), 1);
pool->setTimeout(std::chrono::seconds(10));