owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[DB] Reworked feature extraction

Browse Source
This commit is contained in:
emeric
2016-05-23 20:06:13 +02:00
parent d211c76cf8
commit af97d6af4d
16 changed files with 444 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/database/DatabaseFeatureExtractor.cpp
+82
View File
@@ -0,0 +1,82 @@
/*
* Copyright (C) 2016 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "config/Config.hpp"
#include "logger/Logger.hpp"
#include "feature/FeatureExtractor.hpp"
#include "feature/FeatureStore.hpp"
#include "DatabaseFeatureExtractor.hpp"
namespace Database {
static std::string getMBID(Track::id_type trackId)
{
Wt::Dbo::Transaction transaction(UpdaterDboSession());
Track::pointer track = Track::getById(UpdaterDboSession(), trackId);
return track->getMBID();
}
void
FeatureExtractor::processDatabaseUpdate(Updater::Stats stats)
{
bool fetchHighLevel = Config::instance().getBool("tag-highlevel-acousticbrainz", false);
bool fetchLowLevel = Config::instance().getBool("tag-similarity-acousticbrainz", false);
if (!fetchHighLevel && !fetchLowLevel)
{
LMS_LOG(DBUPDATER, INFO) << "No need to extract features";
return;
}
LMS_LOG(DBUPDATER, INFO) << "Processing tracks in order to extract features...";
std::vector<Track::id_type> trackIds = Track::getAllIds(UpdaterDboSession());
for (auto trackId : trackIds)
{
if (UpdaterQuitRequested())
return;
std::string mbid = getMBID(trackId);
if (mbid.empty())
{
LMS_LOG(DBUPDATER, DEBUG) << "No MBID for track " << trackId << ", skipping";
continue;
}
if (fetchLowLevel && !Feature::Store::instance().exists(UpdaterDboSession(), trackId, "low_level"))
{
boost::property_tree::ptree feature;
if (::Feature::Extractor::getLowLevel(feature, mbid))
Feature::Store::instance().set(UpdaterDboSession(), trackId, "low_level", feature);
}
if (fetchHighLevel && !Feature::Store::instance().exists(UpdaterDboSession(), trackId, "high_level"))
{
boost::property_tree::ptree feature;
if (::Feature::Extractor::getHighLevel(feature, mbid))
Feature::Store::instance().set(UpdaterDboSession(), trackId, "high_level", feature);
}
}
LMS_LOG(DBUPDATER, INFO) << "Features have been extracted";
}
} // namespace Database
src/database/DatabaseFeatureExtractor.hpp
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2016 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "database/DatabaseUpdater.hpp"
namespace Database {
class FeatureExtractor
{
public:
void processDatabaseUpdate(Updater::Stats stats);
};
} // namespace Database
src/database/DatabaseHandler.cpp
+4 -8
View File
@@ -83,13 +83,12 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool)
_session.mapClass<Database::Artist>("artist");
_session.mapClass<Database::Cluster>("cluster");
_session.mapClass<Database::Track>("track");
_session.mapClass<Database::Feature>("feature");
_session.mapClass<Database::Playlist>("playlist");
_session.mapClass<Database::PlaylistEntry>("playlist_entry");
_session.mapClass<Database::Release>("release");
_session.mapClass<Database::Video>("video");
_session.mapClass<Database::MediaDirectory>("media_directory");
_session.mapClass<Database::MediaDirectorySettings>("media_directory_settings");
_session.mapClass<Database::Setting>("setting");
_session.mapClass<Database::User>("user");
_session.mapClass<Database::AuthInfo>("auth_info");
@@ -101,13 +100,10 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool)
_session.createTables();
_session.execute("CREATE INDEX artist_name_idx ON artist(name)");
_session.execute("CREATE INDEX cluster_name_idx ON cluster(name)");
_session.execute("CREATE INDEX cluster_type_idx ON cluster(type)");
_session.execute("CREATE INDEX cluster_name_type_idx ON cluster(name, type)");
_session.execute("CREATE INDEX release_name_idx ON release(name)");
_session.execute("CREATE INDEX track_name_idx ON track(name)");
_session.execute("CREATE INDEX feature_type_idx ON feature(type)");
_session.execute("CREATE INDEX feature_track_type_idx ON feature(track_id,type)");
_session.execute("CREATE INDEX track_artist_idx ON track(artist_id)");
_session.execute("CREATE INDEX track_release_idx ON track(release_id)");
_session.execute("CREATE INDEX cluster_type_idx ON cluster(type)");
}
catch(std::exception& e) {
LMS_LOG(DB, ERROR) << "Cannot create tables: " << e.what();
src/database/DatabaseUpdater.cpp
+62 -64
View File
@@ -162,7 +162,6 @@ Updater::stop(void)
{
_running = false;
// TODO cancel all jobs (timer, ...)
_scheduleTimer.cancel();
_ioService.stop();
@@ -235,64 +234,67 @@ Updater::scheduleScan( boost::posix_time::ptime time)
void
Updater::process(boost::system::error_code err)
{
if (!err)
if (err)
return;
updateFileExtensions();
Stats stats;
checkAudioFiles(stats);
checkVideoFiles(stats);
std::vector<RootDirectory> rootDirectories;
{
updateFileExtensions();
Wt::Dbo::Transaction transaction(_db->getSession());
Stats stats;
checkAudioFiles(stats);
checkVideoFiles(stats);
std::vector<RootDirectory> rootDirectories;
{
Wt::Dbo::Transaction transaction(_db->getSession());
for (MediaDirectory::pointer directory : MediaDirectory::getAll(_db->getSession()))
rootDirectories.push_back( RootDirectory( directory->getType(), directory->getPath() ));
}
for (RootDirectory rootDirectory : rootDirectories)
{
if (!_running)
break;
LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory.path << "'...";
processRootDirectory(rootDirectory, stats);
LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory.path << "' DONE";
}
if (_running)
checkDuplicatedAudioFiles(stats);
LMS_LOG(DBUPDATER, INFO) << "Scan complete. Scanned = " << stats.nbScanned << ", Skipped = " << stats.nbSkipped << ", Changes = " << stats.nbChanges() << " (added = " << stats.nbAdded << ", nbRemoved = " << stats.nbRemoved << ", nbModified = " << stats.nbModified << "), Scan errors = " << stats.nbScanErrors << ", Not imported = " << stats.nbNotImported;
// Update database stats
boost::posix_time::ptime now = boost::posix_time::second_clock::local_time();
{
Wt::Dbo::Transaction transaction(_db->getSession());
Database::MediaDirectorySettings::pointer settings = Database::MediaDirectorySettings::get(_db->getSession());
if (stats.nbChanges() > 0)
settings.modify()->setLastUpdate(now);
// Save the last scan only if it has been completed
if (_running)
settings.modify()->setLastScan(now);
// If the manual scan was required we can now set it to done
// Update only if the scan is complete!
if (settings->getManualScanRequested() && _running)
settings.modify()->setManualScanRequested(false);
}
scanComplete().emit(stats);
if (_running)
processNextJob();
for (MediaDirectory::pointer directory : MediaDirectory::getAll(_db->getSession()))
rootDirectories.push_back( RootDirectory( directory->getType(), directory->getPath() ));
}
for (RootDirectory rootDirectory : rootDirectories)
{
if (!_running)
break;
LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory.path << "'...";
processRootDirectory(rootDirectory, stats);
LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory.path << "' DONE";
}
if (_running)
{
checkDuplicatedAudioFiles(stats);
LMS_LOG(DBUPDATER, INFO) << "Processed all files, now calling listeners...";
scanComplete().emit(stats);
}
LMS_LOG(DBUPDATER, INFO) << "Scan complete. Scanned = " << stats.nbScanned << ", Skipped = " << stats.nbSkipped << ", Changes = " << stats.nbChanges() << " (added = " << stats.nbAdded << ", nbRemoved = " << stats.nbRemoved << ", nbModified = " << stats.nbModified << "), Scan errors = " << stats.nbScanErrors << ", Not imported = " << stats.nbNotImported;
// Update database stats
boost::posix_time::ptime now = boost::posix_time::second_clock::local_time();
{
Wt::Dbo::Transaction transaction(_db->getSession());
Database::MediaDirectorySettings::pointer settings = Database::MediaDirectorySettings::get(_db->getSession());
if (stats.nbChanges() > 0)
settings.modify()->setLastUpdate(now);
// Save the last scan only if it has been completed
if (_running)
settings.modify()->setLastScan(now);
// If the manual scan was required we can now set it to done
// Update only if the scan is complete!
if (settings->getManualScanRequested() && _running)
settings.modify()->setManualScanRequested(false);
}
if (_running)
processNextJob();
}
void
@@ -538,12 +540,9 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
{
LMS_LOG(DBUPDATER, INFO) << "Updating '" << file << "'";
// TODO Remove the songs from its clusters
// TODO Remove the features of this song
track.remove();
track.flush();
track = Track::create(_db->getSession(), file);
// Remove the songs from its clusters
for (auto cluster : track->getClusters())
cluster.remove();
stats.nbModified++;
}
@@ -614,8 +613,7 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
_sigTrackChanged.emit(true, track.id(), track->getMBID(), track->getPath());
}
void
void
Updater::processRootDirectory(RootDirectory rootDirectory, Stats& stats)
{
boost::system::error_code ec;
src/database/DatabaseUpdater.hpp
+14 -1
View File
@@ -73,6 +73,9 @@ class Updater
std::mutex& getMutex(void) { return _mutex; }
Database::Handler& getDb(void) { return *_db; }
bool quitRequested(void) const { return !_running;}
private:
Updater();
@@ -133,8 +136,18 @@ class Updater
MetaData::TagLibParser _metadataParser;
}; // class Updater
// Helper to get the updater session data
static inline Wt::Dbo::Session& UpdaterDboSession()
{
return Updater::instance().getDb().getSession();
}
static inline bool UpdaterQuitRequested()
{
return Updater::instance().quitRequested();
}
} // Database
src/database/MediaDirectory.cpp
+1 -4
View File
@@ -17,8 +17,6 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include <boost/foreach.hpp>
#include "Types.hpp"
static std::string pathsToString(const std::vector<boost::filesystem::path>& paths)
@@ -111,8 +109,7 @@ MediaDirectory::create(Wt::Dbo::Session& session, boost::filesystem::path p, Typ
void
MediaDirectory::eraseAll(Wt::Dbo::Session& session)
{
std::vector<MediaDirectory::pointer> dirs = getAll(session);
BOOST_FOREACH(MediaDirectory::pointer dir, dirs)
for (auto dir : getAll(session))
dir.remove();
}