owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[DB] Reworked settings for acousticbrainz tagging

Browse Source
This commit is contained in:
emeric
2016-05-28 18:32:09 +02:00
parent b1b139ffcc
commit 2ea4ba7f22
23 changed files with 787 additions and 355 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/database/DatabaseClassifier.cpp
+15 -20
View File
@@ -42,7 +42,7 @@ Classifier::processTrackUpdate(bool added, Track::id_type trackId, std::string m
if (mbid.empty())
{
// TODO compute from file
LMS_LOG(CLASSIFICATION, INFO) << "File '" << path << "' has no MBID: skipping feature extraction";
LMS_LOG(DBUPDATER, INFO) << "File '" << path << "' has no MBID: skipping feature extraction";
return;
}
@@ -332,7 +332,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_centroid.mean", 1, 1.0 },
// { "lowlevel.spectral_centroid.median", 1, 1.0 },
// { "lowlevel.spectral_centroid.min", 1, 1.0 },
// { "lowlevel.spectral_centroid.var", 1, 1.0 },
{ "lowlevel.spectral_centroid.var", 1, 1.0 },
// { "lowlevel.spectral_complexity.dmean", 1, 1.0 },
// { "lowlevel.spectral_complexity.dmean2", 1, 1.0 },
// { "lowlevel.spectral_complexity.dvar", 1, 1.0 },
@@ -341,7 +341,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_complexity.mean", 1, 1.0 },
// { "lowlevel.spectral_complexity.median", 1, 1.0 },
// { "lowlevel.spectral_complexity.min", 1, 1.0 },
// { "lowlevel.spectral_complexity.var", 1, 1.0 },
{ "lowlevel.spectral_complexity.var", 1, 1.0 },
// { "lowlevel.spectral_decrease.dmean", 1, 1.0 },
// { "lowlevel.spectral_decrease.dmean2", 1, 1.0 },
// { "lowlevel.spectral_decrease.dvar", 1, 1.0 },
@@ -350,7 +350,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_decrease.mean", 1, 1.0 },
// { "lowlevel.spectral_decrease.median", 1, 1.0 },
// { "lowlevel.spectral_decrease.min", 1, 1.0 },
// { "lowlevel.spectral_decrease.var", 1, 1.0 },
{ "lowlevel.spectral_decrease.var", 1, 1.0 },
// { "lowlevel.spectral_energy.dmean", 1, 1.0 },
// { "lowlevel.spectral_energy.dmean2", 1, 1.0 },
// { "lowlevel.spectral_energy.dvar", 1, 1.0 },
@@ -359,7 +359,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_energy.mean", 1, 1.0 },
// { "lowlevel.spectral_energy.median", 1, 1.0 },
// { "lowlevel.spectral_energy.min", 1, 1.0 },
// { "lowlevel.spectral_energy.var", 1, 1.0 },
{ "lowlevel.spectral_energy.var", 1, 1.0 },
// { "lowlevel.spectral_energyband_low.dmean", 1, 1.0 },
// { "lowlevel.spectral_energyband_low.dmean2", 1, 1.0 },
// { "lowlevel.spectral_energyband_low.dvar", 1, 1.0 },
@@ -368,7 +368,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_energyband_low.mean", 1, 1.0 },
// { "lowlevel.spectral_energyband_low.median", 1, 1.0 },
// { "lowlevel.spectral_energyband_low.min", 1, 1.0 },
// { "lowlevel.spectral_energyband_low.var", 1, 1.0 },
{ "lowlevel.spectral_energyband_low.var", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_high.dmean", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_high.dmean2", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_high.dvar", 1, 1.0 },
@@ -377,7 +377,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_energyband_middle_high.mean", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_high.median", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_high.min", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_high.var", 1, 1.0 },
{ "lowlevel.spectral_energyband_middle_high.var", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_low.dmean", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_low.dmean2", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_low.dvar", 1, 1.0 },
@@ -386,7 +386,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_energyband_middle_low.mean", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_low.median", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_low.min", 1, 1.0 },
// { "lowlevel.spectral_energyband_middle_low.var", 1, 1.0 },
{ "lowlevel.spectral_energyband_middle_low.var", 1, 1.0 },
// { "lowlevel.spectral_entropy.dmean", 1, 1.0 },
// { "lowlevel.spectral_entropy.dmean2", 1, 1.0 },
// { "lowlevel.spectral_entropy.dvar", 1, 1.0 },
@@ -395,7 +395,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_entropy.mean", 1, 1.0 },
// { "lowlevel.spectral_entropy.median", 1, 1.0 },
// { "lowlevel.spectral_entropy.min", 1, 1.0 },
// { "lowlevel.spectral_entropy.var", 1, 1.0 },
{ "lowlevel.spectral_entropy.var", 1, 1.0 },
// { "lowlevel.spectral_flux.dmean", 1, 1.0 },
// { "lowlevel.spectral_flux.dmean2", 1, 1.0 },
// { "lowlevel.spectral_flux.dvar", 1, 1.0 },
@@ -422,7 +422,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_rms.mean", 1, 1.0 },
// { "lowlevel.spectral_rms.median", 1, 1.0 },
// { "lowlevel.spectral_rms.min", 1, 1.0 },
// { "lowlevel.spectral_rms.var", 1, 1.0 },
{ "lowlevel.spectral_rms.var", 1, 1.0 },
// { "lowlevel.spectral_rolloff.dmean", 1, 1.0 },
// { "lowlevel.spectral_rolloff.dmean2", 1, 1.0 },
// { "lowlevel.spectral_rolloff.dvar", 1, 1.0 },
@@ -449,7 +449,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_spread.mean", 1, 1.0 },
// { "lowlevel.spectral_spread.median", 1, 1.0 },
// { "lowlevel.spectral_spread.min", 1, 1.0 },
// { "lowlevel.spectral_spread.var", 1, 1.0 },
{ "lowlevel.spectral_spread.var", 1, 1.0 },
// { "lowlevel.spectral_strongpeak.dmean", 1, 1.0 },
// { "lowlevel.spectral_strongpeak.dmean2", 1, 1.0 },
// { "lowlevel.spectral_strongpeak.dvar", 1, 1.0 },
@@ -458,7 +458,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.spectral_strongpeak.mean", 1, 1.0 },
// { "lowlevel.spectral_strongpeak.median", 1, 1.0 },
// { "lowlevel.spectral_strongpeak.min", 1, 1.0 },
// { "lowlevel.spectral_strongpeak.var", 1, 1.0 },
{ "lowlevel.spectral_strongpeak.var", 1, 1.0 },
// { "lowlevel.zerocrossingrate.dmean", 1, 1.0 },
// { "lowlevel.zerocrossingrate.dmean2", 1, 1.0 },
// { "lowlevel.zerocrossingrate.dvar", 1, 1.0 },
@@ -467,7 +467,7 @@ static std::vector<FeatureDesc> features =
{ "lowlevel.zerocrossingrate.mean", 1, 1.0 },
// { "lowlevel.zerocrossingrate.median", 1, 1.0 },
// { "lowlevel.zerocrossingrate.min", 1, 1.0 },
// { "lowlevel.zerocrossingrate.var", 1, 1.0 },
{ "lowlevel.zerocrossingrate.var", 1, 1.0 },
// { "rhythm.beats_count", 1, 1.0 },
// { "rhythm.beats_loudness.dmean", 1, 1.0 },
// { "rhythm.beats_loudness.dmean2", 1, 1.0 },
@@ -477,7 +477,7 @@ static std::vector<FeatureDesc> features =
{ "rhythm.beats_loudness.mean", 1, 1.0 },
// { "rhythm.beats_loudness.median", 1, 1.0 },
// { "rhythm.beats_loudness.min", 1, 1.0 },
// { "rhythm.beats_loudness.var", 1, 1.0 },
{ "rhythm.beats_loudness.var", 1, 1.0 },
// { "rhythm.bpm", 1, 1.0 },
// { "rhythm.bpm_histogram_first_peak_bpm.dmean", 1, 1.0 },
// { "rhythm.bpm_histogram_first_peak_bpm.dmean2", 1, 1.0 },
@@ -664,12 +664,7 @@ Classifier::processDatabaseUpdate(Updater::Stats stats)
LMS_LOG(DBUPDATER, DEBUG) << "Getting track ids";
std::vector<Track::id_type> trackIdsAll;
{
Wt::Dbo::Transaction transaction(_db.getSession());
trackIdsAll = Database::Track::getAllIds(_db.getSession());
}
std::vector<Track::id_type> trackIdsAll = Database::Track::getAllIds(_db.getSession());
::std::random_shuffle ( trackIdsAll.begin(), trackIdsAll.end() );
src/database/DatabaseFeatureExtractor.cpp
+3 -3
View File
@@ -17,12 +17,12 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "config/Config.hpp"
#include "logger/Logger.hpp"
#include "feature/FeatureExtractor.hpp"
#include "feature/FeatureStore.hpp"
#include "Setting.hpp"
#include "DatabaseFeatureExtractor.hpp"
namespace Database {
@@ -37,8 +37,8 @@ static std::string getMBID(Track::id_type trackId)
void
FeatureExtractor::processDatabaseUpdate(Updater::Stats stats)
{
bool fetchHighLevel = Config::instance().getBool("tag-highlevel-acousticbrainz", false);
bool fetchLowLevel = Config::instance().getBool("tag-similarity-acousticbrainz", false);
bool fetchHighLevel = Setting::getBool(UpdaterDboSession(), "tags_highlevel_acousticbrainz");
bool fetchLowLevel = Setting::getBool(UpdaterDboSession(), "tags_similarity_acousticbrainz");
if (!fetchHighLevel && !fetchLowLevel)
{
src/database/DatabaseHandler.cpp
+21 -6
View File
@@ -28,6 +28,8 @@
#include <Wt/Auth/PasswordStrengthValidator>
#include <Wt/Auth/PasswordVerifier>
#include "Setting.hpp"
#include "logger/Logger.hpp"
#include "DatabaseHandler.hpp"
@@ -79,7 +81,6 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool)
{
_session.setConnectionPool(connectionPool);
_session.mapClass<Database::Artist>("artist");
_session.mapClass<Database::Cluster>("cluster");
_session.mapClass<Database::Track>("track");
@@ -99,11 +100,6 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool)
Wt::Dbo::Transaction transaction(_session);
_session.createTables();
_session.execute("CREATE INDEX artist_name_idx ON artist(name)");
_session.execute("CREATE INDEX release_name_idx ON release(name)");
_session.execute("CREATE INDEX track_artist_idx ON track(artist_id)");
_session.execute("CREATE INDEX track_release_idx ON track(release_id)");
_session.execute("CREATE INDEX cluster_type_idx ON cluster(type)");
}
catch(std::exception& e) {
LMS_LOG(DB, ERROR) << "Cannot create tables: " << e.what();
@@ -112,7 +108,26 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool)
{
Wt::Dbo::Transaction transaction(_session);
// Indexes
_session.execute("PRAGMA journal_mode=WAL");
_session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
_session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
_session.execute("CREATE INDEX IF NOT EXISTS track_artist_idx ON track(artist_id)");
_session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)");
_session.execute("CREATE INDEX IF NOT EXISTS cluster_type_idx ON cluster(type)");
// Default values
if (!Setting::exists(_session, "audio_file_extensions"))
Setting::setString(_session, "audio_file_extensions", ".mp3 .ogg .oga .aac .m4a .flac .wav .wma .aif .aiff .ape .mpc .shn" );
if (!Setting::exists(_session, "video_file_extensions"))
Setting::setString(_session, "video_file_extensions", ".flv .avi .mpg .mpeg .mp4 .m4v .mkv .mov .wmv .ogv .divx .m2ts");
if (!Setting::exists(_session, "tags_highlevel_acousticbrainz"))
Setting::setBool(_session, "tags_highlevel_acousticbrainz", true);
if (!Setting::exists(_session, "tags_similarity_acousticbrainz"))
Setting::setBool(_session, "tags_similarity_acousticbrain", false);
}
_users = new UserDatabase(_session);
src/database/DatabaseUpdater.cpp
+41 -51
View File
@@ -28,6 +28,7 @@
#include "utils/Utils.hpp"
#include "utils/Path.hpp"
#include "Setting.hpp"
#include "Types.hpp"
#include "DatabaseUpdater.hpp"
@@ -70,7 +71,6 @@ getNextFirstOfMonth(const boost::gregorian::date& current)
bool
isFileSupported(const boost::filesystem::path& file, const std::vector<boost::filesystem::path> extensions)
{
boost::filesystem::path fileExtension = file.extension();
for (auto& extension : extensions)
@@ -87,9 +87,9 @@ getRootDirectoriesByType(Wt::Dbo::Session& session, Database::MediaDirectory::Ty
{
Wt::Dbo::Transaction transaction(session);
std::vector<boost::filesystem::path> res;
std::vector<Database::MediaDirectory::pointer> rootDirs = Database::MediaDirectory::getByType(session, type);
std::vector<boost::filesystem::path> res;
for (auto rootDir : rootDirs)
res.push_back(rootDir->getPath());
@@ -170,48 +170,43 @@ Updater::stop(void)
void
Updater::processNextJob(void)
{
Wt::Dbo::Transaction transaction(_db->getSession());
MediaDirectorySettings::pointer settings = MediaDirectorySettings::get(_db->getSession());
if (settings->getManualScanRequested()) {
if (Setting::getBool(_db->getSession(), "manual_scan_requested", false))
{
LMS_LOG(DBUPDATER, INFO) << "Manual scan requested!";
scheduleScan( boost::posix_time::seconds(0) );
}
else
{
boost::posix_time::ptime now = boost::posix_time::second_clock::local_time();
boost::posix_time::time_duration startTime = settings->getUpdateStartTime();
boost::posix_time::time_duration startTime = Setting::getDuration(_db->getSession(), "update_start_time");
boost::gregorian::date nextScanDate;
switch( settings->getUpdatePeriod() )
std::string updatePeriod = Setting::getString(_db->getSession(), "update_period", "never");
if (updatePeriod == "daily")
{
case Database::MediaDirectorySettings::Never:
// Nothing to do
break;
case Database::MediaDirectorySettings::Daily:
if (now.time_of_day() < startTime)
nextScanDate = now.date();
else
nextScanDate = getNextDay(now.date());
break;
case Database::MediaDirectorySettings::Weekly:
if (now.time_of_day() < startTime && now.date().day_of_week() == 1)
nextScanDate = now.date();
else
nextScanDate = getNextMonday(now.date());
break;
case Database::MediaDirectorySettings::Monthly:
if (now.time_of_day() < startTime && now.date().day() == 1)
nextScanDate = now.date();
else
nextScanDate = getNextFirstOfMonth(now.date());
break;
if (now.time_of_day() < startTime)
nextScanDate = now.date();
else
nextScanDate = getNextDay(now.date());
}
else if (updatePeriod == "weekly")
{
if (now.time_of_day() < startTime && now.date().day_of_week() == 1)
nextScanDate = now.date();
else
nextScanDate = getNextMonday(now.date());
}
else if (updatePeriod == "monthly")
{
if (now.time_of_day() < startTime && now.date().day() == 1)
nextScanDate = now.date();
else
nextScanDate = getNextFirstOfMonth(now.date());
}
if (!nextScanDate.is_special())
scheduleScan( boost::posix_time::ptime (nextScanDate, settings->getUpdateStartTime() ) );
scheduleScan( boost::posix_time::ptime (nextScanDate, startTime) );
}
}
@@ -274,27 +269,17 @@ Updater::process(boost::system::error_code err)
// Update database stats
boost::posix_time::ptime now = boost::posix_time::second_clock::local_time();
{
Wt::Dbo::Transaction transaction(_db->getSession());
Database::MediaDirectorySettings::pointer settings = Database::MediaDirectorySettings::get(_db->getSession());
if (stats.nbChanges() > 0)
settings.modify()->setLastUpdate(now);
// Save the last scan only if it has been completed
if (_running)
settings.modify()->setLastScan(now);
// If the manual scan was required we can now set it to done
// Update only if the scan is complete!
if (settings->getManualScanRequested() && _running)
settings.modify()->setManualScanRequested(false);
}
if (stats.nbChanges() > 0)
Setting::setTime(_db->getSession(), "last_update", now);
// Save the last scan only if it has been completed
if (_running)
{
Setting::setTime(_db->getSession(), "last_scan", now);
Setting::setBool(_db->getSession(), "manual_scan_requested", false);
processNextJob();
}
}
void
@@ -302,8 +287,13 @@ Updater::updateFileExtensions()
{
Wt::Dbo::Transaction transaction(_db->getSession());
_audioFileExtensions = MediaDirectorySettings::get(_db->getSession())->getAudioFileExtensions();
_videoFileExtensions = MediaDirectorySettings::get(_db->getSession())->getVideoFileExtensions();
_audioFileExtensions.clear();
for (auto extension : splitString(Setting::getString(_db->getSession(), "audio_file_extensions"), " "))
_audioFileExtensions.push_back( extension );
_videoFileExtensions.clear();
for (auto extension : splitString(Setting::getString(_db->getSession(), "video_file_extensions"), " "))
_videoFileExtensions.push_back( extension );
}
Artist::pointer
src/database/MediaDirectory.cpp
+8 -74
View File
@@ -17,89 +17,18 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "utils/Utils.hpp"
#include "Types.hpp"
static std::string pathsToString(const std::vector<boost::filesystem::path>& paths)
{
std::ostringstream oss;
bool first = true;
for (auto& path : paths)
{
if (!first)
oss << " ";
oss << path.string();
first = false;
}
return oss.str();
}
static std::vector<boost::filesystem::path> stringToPaths(const std::string value)
{
std::vector<std::string> res;
std::istringstream iss(value);
std::copy(std::istream_iterator<std::string>(iss), std::istream_iterator<std::string>(), std::back_inserter(res));
return std::vector<boost::filesystem::path>(res.begin(), res.end());
}
namespace Database {
MediaDirectorySettings::MediaDirectorySettings()
: _manualScanRequested(false),
_updatePeriod(Never),
_audioFileExtensions(".mp3 .ogg .oga .aac .m4a .flac .wav .wma .aif .aiff .ape .mpc .shn"),
_videoFileExtensions(".flv .avi .mpg .mpeg .mp4 .m4v .mkv .mov .wmv .ogv .divx .m2ts")
{
}
MediaDirectory::MediaDirectory(boost::filesystem::path p, Type type)
: _type(type),
_path(p.string())
_path(stringTrimEnd(p.string(), "/\\"))
{
}
MediaDirectorySettings::pointer
MediaDirectorySettings::get(Wt::Dbo::Session& session)
{
MediaDirectorySettings::pointer res;
res = session.find<MediaDirectorySettings>().where("id = ?").bind(1);
// TODO bind necessary?
if (!res)
res = session.add( new MediaDirectorySettings());
return res;
}
std::vector<boost::filesystem::path>
MediaDirectorySettings::getAudioFileExtensions(void) const
{
return stringToPaths(_audioFileExtensions);
}
std::vector<boost::filesystem::path>
MediaDirectorySettings::getVideoFileExtensions(void) const
{
return stringToPaths(_videoFileExtensions);
}
void
MediaDirectorySettings::setAudioFileExtensions(std::vector<boost::filesystem::path> extensions)
{
_audioFileExtensions = pathsToString(extensions);
}
void
MediaDirectorySettings::setVideoFileExtensions(std::vector<boost::filesystem::path> extensions)
{
_videoFileExtensions = pathsToString(extensions);
}
MediaDirectory::pointer
MediaDirectory::create(Wt::Dbo::Session& session, boost::filesystem::path p, Type type)
{
@@ -135,5 +64,10 @@ MediaDirectory::get(Wt::Dbo::Session& session, boost::filesystem::path p, Type t
return session.find<MediaDirectory>().where("path = ?").where("type = ?").bind( p.string()).bind(type);
}
boost::filesystem::path
MediaDirectory::getPath(void) const
{
return boost::filesystem::path(stringTrimEnd(_path, "/\\"));
}
} // namespace Database
src/database/MediaDirectory.hpp
+1 -69
View File
@@ -26,73 +26,8 @@
#include <Wt/Dbo/Dbo>
#include <Wt/Dbo/WtSqlTraits>
namespace Database {
class MediaDirectory;
class MediaDirectorySettings
{
public:
enum UpdatePeriod {
Never,
Daily,
Weekly,
Monthly
};
typedef Wt::Dbo::ptr<MediaDirectorySettings> pointer;
MediaDirectorySettings();
// accessors
static pointer get(Wt::Dbo::Session& session);
// write accessors
void setManualScanRequested(bool value) { _manualScanRequested = value;}
void setUpdatePeriod(UpdatePeriod period) { _updatePeriod = period;}
void setUpdateStartTime(boost::posix_time::time_duration dur) { _updateStartTime = dur;}
void setLastUpdate(boost::posix_time::ptime time) { _lastUpdate = time; }
void setLastScan(boost::posix_time::ptime time) { _lastScan = time; }
void setAudioFileExtensions(std::vector<boost::filesystem::path> extensions);
void setVideoFileExtensions(std::vector<boost::filesystem::path> extensions);
// Read accessors
bool getManualScanRequested(void) const { return _manualScanRequested; }
UpdatePeriod getUpdatePeriod(void) const { return _updatePeriod; }
boost::posix_time::time_duration getUpdateStartTime(void) const { return _updateStartTime; }
boost::posix_time::ptime getLastUpdated(void) const { return _lastUpdate; }
boost::posix_time::ptime getLastScan(void) const { return _lastScan; }
std::vector<boost::filesystem::path> getAudioFileExtensions(void) const;
std::vector<boost::filesystem::path> getVideoFileExtensions(void) const;
template<class Action>
void persist(Action& a)
{
Wt::Dbo::field(a, _manualScanRequested, "manual_scan_requested");
Wt::Dbo::field(a, _updatePeriod, "update_period");
Wt::Dbo::field(a, _updateStartTime, "update_start_time");
Wt::Dbo::field(a, _audioFileExtensions, "audio_file_extensions");
Wt::Dbo::field(a, _videoFileExtensions, "video_file_extensions");
Wt::Dbo::field(a, _lastUpdate, "last_update");
Wt::Dbo::field(a, _lastScan, "last_scan");
Wt::Dbo::hasMany(a, _mediaDirectories, Wt::Dbo::ManyToOne, "media_directory_settings");
}
private:
bool _manualScanRequested; // Immadiate scan has been requested by user
UpdatePeriod _updatePeriod; // How long between updates
boost::posix_time::time_duration _updateStartTime; // Time of day to begin the update
std::string _audioFileExtensions; // Extension of the audio files to be scanned
std::string _videoFileExtensions; // Extension of the video files to be scanned
boost::posix_time::ptime _lastUpdate; // last time the database has changed
boost::posix_time::ptime _lastScan; // last time the database has been scanned
Wt::Dbo::collection< Wt::Dbo::ptr<MediaDirectory> > _mediaDirectories; // list of media directories
};
class MediaDirectory
{
public:
@@ -116,14 +51,13 @@ class MediaDirectory
static void eraseAll(Wt::Dbo::Session& session);
Type getType(void) const { return _type; }
boost::filesystem::path getPath(void) const { return boost::filesystem::path(_path); }
boost::filesystem::path getPath(void) const;
template<class Action>
void persist(Action& a)
{
Wt::Dbo::field(a, _type, "type");
Wt::Dbo::field(a, _path, "path");
Wt::Dbo::belongsTo(a, _settings, "media_directory_settings", Wt::Dbo::OnDeleteCascade);
}
private:
@@ -131,8 +65,6 @@ class MediaDirectory
Type _type;
std::string _path;
MediaDirectorySettings::pointer _settings; // back pointer
};
} // namespace Database
src/database/Setting.cpp
+152
View File
@@ -0,0 +1,152 @@
/*
* Copyright (C) 2016 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Setting.hpp"
namespace Database {
bool
Setting::exists(Wt::Dbo::Session& session, std::string setting)
{
Wt::Dbo::Transaction transaction(session);
return (getByName(session, setting) != Setting::pointer());
}
std::string
Setting::getString(Wt::Dbo::Session& session, std::string setting, std::string defaultValue)
{
Wt::Dbo::Transaction transaction(session);
pointer res = getByName(session, setting);
if (!res)
return defaultValue;
return res->_value;
}
bool
Setting::getBool(Wt::Dbo::Session& session, std::string setting, bool defaultValue)
{
Wt::Dbo::Transaction transaction(session);
pointer res = getByName(session, setting);
if (!res)
return defaultValue;
return (res->_value == "true");
}
boost::posix_time::time_duration
Setting::getDuration(Wt::Dbo::Session& session, std::string setting, boost::posix_time::time_duration defaultValue)
{
Wt::Dbo::Transaction transaction(session);
pointer res = getByName(session, setting);
if (!res)
return defaultValue;
return boost::posix_time::duration_from_string(res->_value);
}
boost::posix_time::ptime
Setting::getTime(Wt::Dbo::Session& session, std::string setting, boost::posix_time::ptime defaultValue)
{
Wt::Dbo::Transaction transaction(session);
pointer res = getByName(session, setting);
if (!res)
return defaultValue;
return boost::posix_time::time_from_string(res->_value);
}
int
Setting::getInt(Wt::Dbo::Session& session, std::string setting, int defaultValue)
{
Wt::Dbo::Transaction transaction(session);
pointer res = getByName(session, setting);
if (!res)
return defaultValue;
return std::stoi(res->_value);
}
Setting::pointer
Setting::create(Wt::Dbo::Session& session, std::string name)
{
return session.add<Setting>(new Setting(name));
}
Setting::pointer
Setting::getByName(Wt::Dbo::Session& session, std::string name)
{
return session.find<Setting>().where("name = ?").bind(name);
}
Setting::pointer
Setting::getOrCreateByName(Wt::Dbo::Session& session, std::string name)
{
pointer res = getByName(session, name);
if (!res)
res = create(session, name);
return res;
}
void
Setting::setString(Wt::Dbo::Session& session, std::string setting, std::string value)
{
Wt::Dbo::Transaction transaction(session);
getOrCreateByName(session, setting).modify()->_value = value;
}
void
Setting::setBool(Wt::Dbo::Session& session, std::string setting, bool value)
{
Wt::Dbo::Transaction transaction(session);
getOrCreateByName(session, setting).modify()->_value = (value ? "true" : "false");
}
void
Setting::setDuration(Wt::Dbo::Session& session, std::string setting, boost::posix_time::time_duration value)
{
Wt::Dbo::Transaction transaction(session);
getOrCreateByName(session, setting).modify()->_value = boost::posix_time::to_simple_string(value);
}
void
Setting::setTime(Wt::Dbo::Session& session, std::string setting, boost::posix_time::ptime value)
{
Wt::Dbo::Transaction transaction(session);
getOrCreateByName(session, setting).modify()->_value = boost::posix_time::to_simple_string(value);
}
void
Setting::setInt(Wt::Dbo::Session& session, std::string setting, int value)
{
Wt::Dbo::Transaction transaction(session);
getOrCreateByName(session, setting).modify()->_value = std::to_string(value);
}
} // namespace Database
src/database/Setting.hpp
+75
View File
@@ -0,0 +1,75 @@
/*
* Copyright (C) 2016 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <Wt/Dbo/Dbo>
#include <boost/date_time/posix_time/posix_time_types.hpp>
namespace Database {
// class meant to store general settings
class Setting
{
public:
Setting() {}
// check if a setting exists or not
static bool exists(Wt::Dbo::Session& session, std::string setting);
// Getters
// Nested transactions
static std::string getString(Wt::Dbo::Session& session, std::string setting, std::string defaultValue = "");
static bool getBool(Wt::Dbo::Session& session, std::string setting, bool defaultValue = false);
static boost::posix_time::time_duration getDuration(Wt::Dbo::Session& session, std::string setting, boost::posix_time::time_duration defaultDuration = boost::posix_time::seconds(0) );
static boost::posix_time::ptime getTime(Wt::Dbo::Session& session, std::string setting, boost::posix_time::ptime defaultTime = boost::posix_time::ptime());
static int getInt(Wt::Dbo::Session& session, std::string setting, int defaultValue = 0);
// Setters
// Nested transactions
static void setString(Wt::Dbo::Session& session, std::string setting, std::string value);
static void setBool(Wt::Dbo::Session& session, std::string setting, bool value);
static void setDuration(Wt::Dbo::Session& session, std::string setting, boost::posix_time::time_duration value);
static void setTime(Wt::Dbo::Session& session, std::string setting, boost::posix_time::ptime time);
static void setInt(Wt::Dbo::Session& session, std::string setting, int value);
template<class Action>
void persist(Action& a)
{
Wt::Dbo::field(a, _name, "name");
Wt::Dbo::field(a, _value, "value");
}
private:
Setting(std::string name) : _name(name) {}
typedef Wt::Dbo::ptr<Setting> pointer;
static pointer getByName(Wt::Dbo::Session& session, std::string name);
static pointer create(Wt::Dbo::Session& session, std::string name);
static pointer getOrCreateByName(Wt::Dbo::Session& session, std::string name);
std::string _name;
std::string _value;
};
} // namespace Database
src/database/cluster/DatabaseHighLevelCluster.cpp
+226
View File
@@ -0,0 +1,226 @@
/*
* Copyright (C) 2016 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "DatabaseHighLevelCluster.hpp"
#include "config/Config.hpp"
#include "logger/Logger.hpp"
#include "feature/FeatureStore.hpp"
namespace Database {
static Cluster::pointer getCluster(std::string type, std::string value)
{
Cluster::pointer cluster = ( Cluster::get(UpdaterDboSession(), type, value) );
if (!cluster)
cluster = Cluster::create(UpdaterDboSession(), type, value);
return cluster;
}
static std::list<std::string> getClustersFromFeature(Feature::Type& feature)
{
struct HighLevelNodeDesc
{
std::string node;
std::map<std::string, std::string> valueMapping;
};
static const std::vector<HighLevelNodeDesc> nodes =
{
{
"highlevel.danceability",
{
{"not_danceable", "Not danceable"},
{"danceable", "Danceable"},
},
},
{
"highlevel.gender",
{
{"male", "Male"},
{"female", "Female"},
},
},
{
"highlevel.mood_acoustic",
{
{"not_acoustic", "Not acoustic"},
{"acoustic", "Acoustic"},
},
},
{
"highlevel.mood_happy",
{
{"not_happy", "Not happy"},
{"happy", "Happy"},
},
},
{
"highlevel.mood_aggressive",
{
{"not_aggressive", "Not aggressive"},
{"aggressive", "Aggressive"},
},
},
{
"highlevel.mood_electronic",
{
{"not_electronic", "Not electronic"},
{"electronic", "Electronic"},
},
},
{
"highlevel.mood_party",
{
{"not_party", "Not party"},
{"party", "Party"},
},
},
{
"highlevel.mood_relaxed",
{
{"not_relaxed", "Not relaxed"},
{"relaxed", "Relaxed"},
},
},
{
"highlevel.mood_sad",
{
{"not_sad", "Not sad"},
{"sad", "Sad"},
},
},
{
"highlevel.timbre",
{
{"bright", "Bright"},
{"dark", "Dark"},
},
},
{
"highlevel.tonal_atonal",
{
{"atonal", "Atonal"},
{"tonal", "Tonal"},
},
},
{
"highlevel.voice_instrumental",
{
{"instrumental", "Instrumental"},
{"voice", "Voice"},
},
},
};
// Extract info and build clusters
std::list<std::string> newClusterNames;
for (auto node : nodes)
{
auto value = feature.get_child_optional(node.node + ".value");
auto probability = feature.get_child_optional(node.node + ".probability");
if (!probability || !value)
{
LMS_LOG(DBUPDATER, DEBUG) << "Missing " << node.node;
continue;
}
if (std::stod(probability->data()) < 0.90)
{
LMS_LOG(DBUPDATER, DEBUG) << "Probability too low for " << node.node << "(" << std::stod(probability->data()) << ")";
continue;
}
if (node.valueMapping[value->data()] == "")
{
LMS_LOG(DBUPDATER, DEBUG) << "Unknown value '" << value->data() << "'";
continue;
}
newClusterNames.push_back(node.valueMapping[value->data()]);
}
return newClusterNames;
}
void
HighLevelCluster::processDatabaseUpdate(Updater::Stats stats)
{
bool createTags = Config::instance().getBool("tag-highlevel-acousticbrainz", false);
LMS_LOG(DBUPDATER, INFO) << "Creating high level based clusters...";
std::vector<Track::id_type> trackIds = Track::getAllIds(UpdaterDboSession());
LMS_LOG(DBUPDATER, DEBUG) << "Got " << trackIds.size() << " tracks";
for (auto trackId : trackIds)
{
if (UpdaterQuitRequested())
return;
LMS_LOG(DBUPDATER, DEBUG) << "Processing track " << trackId;
// Get current cluster names
std::list<std::string> newClusterNames;
if (createTags)
{
Feature::Type feature;
if (!Feature::Store::instance().get(UpdaterDboSession(), trackId, "high_level", feature))
continue;
newClusterNames = getClustersFromFeature(feature);
}
Wt::Dbo::Transaction transaction(UpdaterDboSession());
auto track = Track::getById(UpdaterDboSession(), trackId);
auto clusters = track->getClusters();
for (auto cluster : clusters)
{
// Check if removed
if (cluster->getType() != "high_level")
continue;
auto it = std::find(newClusterNames.begin(), newClusterNames.end(), cluster->getName());
if (it == newClusterNames.end())
cluster.remove();
else
newClusterNames.erase(it);
}
// Add previsouly missing clusters
for (auto newName : newClusterNames)
{
auto cluster = getCluster("high_level", newName);
cluster.modify()->addTrack(track);
}
}
LMS_LOG(DBUPDATER, INFO) << "High level based clusters processed";
}
} // namespace Database
src/database/cluster/DatabaseHighLevelCluster.hpp
+33
View File
@@ -0,0 +1,33 @@
/*
* Copyright (C) 2016 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "database/DatabaseUpdater.hpp"
namespace Database {
class HighLevelCluster
{
public:
void processDatabaseUpdate(Updater::Stats stats);
};
} // namespace Database
src/database/cluster/DatabaseSimilarityCluster.hpp
View File
src/database/tagger/GenreTagger.cpp
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright (C) 2016 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "GenreTagger.hpp"
namespace Database {
void
GenreTagger::processTrackUpdate(bool added, Track::id_type trackId, std::string mbid, boost::filesystem::path p)
{
}
void
GenreTagger::processDatabaseUpdate(Updater::Stats stats)
{
// TODO: if disabled, delete all the genre tags
// TODO: if enabled, reconstruct all the genre tags from tracks
}
} // namespace Database
src/database/tagger/GenreTagger.hpp
+37
View File
@@ -0,0 +1,37 @@
/*
* Copyright (C) 2016 Emeric Poupon
*
* This file is part of LMS.
*
* LMS is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* LMS is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "database/DatabaseUpdater.hpp"
namespace Database {
class GenreTagger
{
public:
void processTrackUpdate(bool added, Track::id_type trackId, std::string mbid, boost::filesystem::path p);
void processDatabaseUpdate(Updater::Stats stats);
private:
};
} // namespace Database