owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into multi-artists

Browse Source
This commit is contained in:
emeric
2019-03-10 14:22:01 +01:00
parent 4d01a1e996 080fa856f4
commit 9538aa9de9
6 changed files with 62 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/database/Track.cpp
+5 -5
View File
@@ -118,17 +118,17 @@ Track::getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session)
return std::vector<pointer>(res.begin(), res.end());
}
std::vector<Track::pointer>
Track::getAllWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit)
std::vector<IdType>
Track::getAllIdsWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit)
{
int size {limit ? static_cast<int>(*limit) : -1};
Wt::Dbo::collection<pointer> res = session.query<pointer>
("SELECT t FROM track t")
Wt::Dbo::collection<IdType> res = session.query<IdType>
("SELECT t.id FROM track t")
.where("EXISTS (SELECT * from track_features t_f WHERE t_f.track_id = t.id)")
.limit(size);
return std::vector<pointer>(res.begin(), res.end());
return std::vector<IdType>(res.begin(), res.end());
}
std::vector<Cluster::pointer>
src/database/Track.hpp
+6 -6
View File
@@ -64,13 +64,13 @@ class Track : public Wt::Dbo::Dbo<Track>
bool& moreExpected);
static Wt::Dbo::collection< pointer > getAll(Wt::Dbo::Session& session);
static std::vector<IdType> getAllIds(Wt::Dbo::Session& session); // nested transaction
static std::vector<IdType> getAllIds(Wt::Dbo::Session& session); // nested transaction
static std::vector<boost::filesystem::path> getAllPaths(Wt::Dbo::Session& session); // nested transaction
static std::vector<pointer> getMBIDDuplicates(Wt::Dbo::Session& session);
static std::vector<pointer> getChecksumDuplicates(Wt::Dbo::Session& session);
static std::vector<pointer> getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1);
static std::vector<pointer> getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction
static std::vector<pointer> getAllWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit = {}); // nested transaction
static std::vector<pointer> getMBIDDuplicates(Wt::Dbo::Session& session);
static std::vector<pointer> getChecksumDuplicates(Wt::Dbo::Session& session);
static std::vector<pointer> getLastAdded(Wt::Dbo::Session& session, Wt::WDateTime after, int size = 1);
static std::vector<pointer> getAllWithMBIDAndMissingFeatures(Wt::Dbo::Session& session); // nested transaction
static std::vector<IdType> getAllIdsWithFeatures(Wt::Dbo::Session& session, boost::optional<std::size_t> limit = {}); // nested transaction
// Create utility
static pointer create(Wt::Dbo::Session& session, const boost::filesystem::path& p);
src/similarity/features/AcousticBrainzUtils.cpp
+6
View File
@@ -81,6 +81,12 @@ getJsonData(const std::string& mbid)
std::string
extractLowLevelFeatures(const std::string& mbid)
{
if (boost::filesystem::exists("/storage/emeric/lms-dev/features/" + mbid))
{
std::ifstream ifs{std::string{"/storage/emeric/lms-dev/features/" + mbid}.c_str()};
return std::string {std::istreambuf_iterator<char>{ifs}, std::istreambuf_iterator<char>{}};
}
return getJsonData(mbid);
}
src/similarity/features/SimilarityFeaturesScannerAddon.cpp
+2 -2
View File
@@ -106,10 +106,10 @@ void
FeaturesScannerAddon::updateSearcher()
{
Wt::Dbo::Transaction transaction {_db.getSession()};
auto tracks {Database::Track::getAllWithFeatures(_db.getSession())};
std::vector<Database::IdType> trackIds {Database::Track::getAllIdsWithFeatures(_db.getSession())};
transaction.commit();
if (tracks.empty())
if (trackIds.empty())
{
LMS_LOG(DBUPDATER, INFO) << "No track suitable for features similarity clustering";
std::atomic_store(&_searcher, std::shared_ptr<FeaturesSearcher>{});
src/similarity/features/SimilarityFeaturesSearcher.cpp
+26 -17
View File
@@ -68,7 +68,7 @@ getFeatureInfoMapNbDimensions(const FeatureInfoMap& featureInfoMap)
static
boost::optional<SOM::InputVector>
getInputVectorFromTrack(const Database::Track::pointer& track, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
getInputVectorFromTrack(Wt::Dbo::Session& session, Database::IdType trackId, const FeatureInfoMap& featuresInfo, std::size_t nbDimensions)
{
boost::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
@@ -76,6 +76,12 @@ getInputVectorFromTrack(const Database::Track::pointer& track, const FeatureInfo
for (auto itFeatureInfo : featuresInfo)
features[itFeatureInfo.first] = {};
Wt::Dbo::Transaction transaction {session};
Database::Track::pointer track {Database::Track::getById(session, trackId)};
if (!track)
return res;
if (!track->getTrackFeatures()->getFeatures(features))
return res;
@@ -115,7 +121,7 @@ getInputVectorWeights(const FeatureInfoMap& featuresInfo, std::size_t nbDimensio
FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequested)
{
Wt::Dbo::Transaction transaction{session};
Wt::Dbo::Transaction transaction {session};
FeatureInfoMap featuresInfo {getFeatureInfoMap(session)};
std::size_t nbDimensions {getFeatureInfoMapNbDimensions(featuresInfo)};
@@ -123,31 +129,33 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
LMS_LOG(SIMILARITY, DEBUG) << "Features dimension = " << nbDimensions;
LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features...";
auto tracks {Database::Track::getAllWithFeatures(session)};
std::vector<Database::IdType> trackIds {Database::Track::getAllIdsWithFeatures(session)};
LMS_LOG(SIMILARITY, DEBUG) << "Getting Tracks with features DONE";
std::vector<SOM::InputVector> samples;
std::vector<Database::IdType> tracksIds;
transaction.commit();
samples.reserve(tracks.size());
tracksIds.reserve(tracks.size());
std::vector<SOM::InputVector> samples;
std::vector<Database::IdType> samplesTrackIds;
samples.reserve(trackIds.size());
samplesTrackIds.reserve(trackIds.size());
LMS_LOG(SIMILARITY, DEBUG) << "Extracting features...";
for (const Database::Track::pointer& track : tracks)
for (Database::IdType trackId : trackIds)
{
if (stopRequested)
return;
boost::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(track, featuresInfo, nbDimensions)};
boost::optional<SOM::InputVector> inputVector {getInputVectorFromTrack(session, trackId, featuresInfo, nbDimensions)};
if (!inputVector)
continue;
samples.emplace_back(std::move(*inputVector));
tracksIds.emplace_back(track.id());
samplesTrackIds.emplace_back(trackId);
}
LMS_LOG(SIMILARITY, DEBUG) << "Extracting features DONE";
transaction.commit();
if (tracksIds.empty())
if (samples.empty())
{
LMS_LOG(SIMILARITY, INFO) << "Nothing to classify!";
return;
@@ -191,12 +199,13 @@ FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, bool& stopRequeste
const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
trackPositions[tracksIds[i]].insert(position);
trackPositions[samplesTrackIds[i]].insert(position);
}
LMS_LOG(SIMILARITY, DEBUG) << "Classifying tracks DONE";
init(session, std::move(network), std::move(trackPositions));
}
FeaturesSearcher::FeaturesSearcher(Wt::Dbo::Session& session, FeaturesCache cache)
@@ -269,12 +278,12 @@ FeaturesSearcher::dump(Wt::Dbo::Session& session, std::ostream& os) const
if (!track)
continue;
os << "\t - " << track->getName() << " - ";
os << "\t";
if (track->getArtist())
os << track->getArtist()->getName() << " - ";
if (track->getRelease())
os << track->getRelease()->getName();
os << std::endl;
os << track->getRelease()->getName() << " - ";
os << track->getName() << std::endl;
}
}