diff --git a/TODO b/TODO index 15b32001..d55ee917 100644 --- a/TODO +++ b/TODO @@ -12,6 +12,8 @@ - handle access rights problems (instead of aborting) - add a global play counter for tracks. This will help people to spot most popular files - Use the WServer::post method to notify the end of the database scan? (with results?) +- Make clusters based on different metadata (albumgrouping, genre, mood, etc.). Make it configurable somewhere as it is very difficult to handle them all? +- Use albumartist when available (useful for compilations) [Metadata] - WMA covers: add support @@ -31,6 +33,8 @@ - Add a hint for the user to get the nature of tag (created from genre ? created from audio features?) - Implement a play queue - Implement a decent player + - Tags cloud for artist/release + - Dedicate a color for each tag type [REST API] - Make a dedicated REST API. Maybe use the SubSonic API or Ampache API? diff --git a/src/Makefile.am b/src/Makefile.am index 54f36b85..7842e2dd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -15,14 +15,12 @@ lms_SOURCES = \ $(srcdir)/database/SqlQuery.cpp \ $(srcdir)/database/Track.cpp \ $(srcdir)/database/User.cpp \ - $(srcdir)/database/updater/DatabaseUpdater.cpp \ - $(srcdir)/database/updater/DatabaseFeatureExtractor.cpp \ - $(srcdir)/database/updater/DatabaseHighLevelCluster.cpp \ $(srcdir)/feature/FeatureExtractor.cpp \ $(srcdir)/feature/FeatureStore.cpp \ $(srcdir)/image/Image.cpp \ $(srcdir)/metadata/AvFormat.cpp \ $(srcdir)/metadata/TagLibParser.cpp \ + $(srcdir)/scanner/MediaScanner.cpp \ $(srcdir)/ui/ArtistView.cpp \ $(srcdir)/ui/ArtistsView.cpp \ $(srcdir)/ui/Explore.cpp \ diff --git a/src/database/DatabaseHandler.cpp b/src/database/DatabaseHandler.cpp index 85164449..c063bbf0 100644 --- a/src/database/DatabaseHandler.cpp +++ b/src/database/DatabaseHandler.cpp @@ -17,6 +17,8 @@ * along with LMS. If not, see . */ +#include + #include #include @@ -83,22 +85,25 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool) _session.mapClass("artist"); _session.mapClass("cluster"); - _session.mapClass("track"); + _session.mapClass("cluster_type"); + _session.mapClass("media_directory"); _session.mapClass("playlist"); _session.mapClass("playlist_entry"); _session.mapClass("release"); - _session.mapClass("media_directory"); _session.mapClass("setting"); + _session.mapClass("track"); - _session.mapClass("user"); _session.mapClass("auth_info"); _session.mapClass("auth_identity"); _session.mapClass("auth_token"); + _session.mapClass("user"); try { Wt::Dbo::Transaction transaction(_session); _session.createTables(); + + LMS_LOG(DB, INFO) << "Tables created"; } catch(std::exception& e) { LMS_LOG(DB, ERROR) << "Cannot create tables: " << e.what(); @@ -108,21 +113,17 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool) Wt::Dbo::Transaction transaction(_session); // Indexes - _session.execute("PRAGMA journal_mode=WAL"); + // _session.execute("PRAGMA journal_mode=WAL"); _session.execute("CREATE INDEX IF NOT EXISTS track_path_idx ON track(file_path)"); _session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)"); _session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)"); _session.execute("CREATE INDEX IF NOT EXISTS track_artist_idx ON track(artist_id)"); _session.execute("CREATE INDEX IF NOT EXISTS track_release_idx ON track(release_id)"); - _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_idx ON cluster(type)"); + _session.execute("CREATE INDEX IF NOT EXISTS cluster_name_idx ON cluster(name)"); + _session.execute("CREATE INDEX IF NOT EXISTS cluster_type_name_idx ON cluster_type(name)"); + // TODO move this // Default values - if (!Setting::exists(_session, "audio_file_extensions")) - Setting::setString(_session, "audio_file_extensions", ".mp3 .ogg .oga .aac .m4a .flac .wav .wma .aif .aiff .ape .mpc .shn" ); - - if (!Setting::exists(_session, "video_file_extensions")) - Setting::setString(_session, "video_file_extensions", ".flv .avi .mpg .mpeg .mp4 .m4v .mkv .mov .wmv .ogv .divx .m2ts"); - if (!Setting::exists(_session, "tags_highlevel_acousticbrainz")) Setting::setBool(_session, "tags_highlevel_acousticbrainz", true); @@ -184,7 +185,7 @@ Handler::createConnectionPool(boost::filesystem::path p) Wt::Dbo::backend::Sqlite3 *connection = new Wt::Dbo::backend::Sqlite3(p.string()); - connection->executeSql("pragma journal_mode=WAL"); +// connection->executeSql("pragma journal_mode=WAL"); connection->setProperty("show-queries", "true"); return new Wt::Dbo::FixedSqlConnectionPool(connection, 1); diff --git a/src/database/DatabaseHandler.hpp b/src/database/DatabaseHandler.hpp index 6b07984e..28f5600a 100644 --- a/src/database/DatabaseHandler.hpp +++ b/src/database/DatabaseHandler.hpp @@ -21,6 +21,7 @@ #define DATABASE_HANDLER_HPP #include +#include #include #include diff --git a/src/database/MediaDirectory.cpp b/src/database/MediaDirectory.cpp index cd2a0cc2..88b30e88 100644 --- a/src/database/MediaDirectory.cpp +++ b/src/database/MediaDirectory.cpp @@ -23,16 +23,15 @@ namespace Database { -MediaDirectory::MediaDirectory(boost::filesystem::path p, Type type) -: _type(type), - _path(stringTrimEnd(p.string(), "/\\")) +MediaDirectory::MediaDirectory(boost::filesystem::path p) +: _path(stringTrimEnd(p.string(), "/\\")) { } MediaDirectory::pointer -MediaDirectory::create(Wt::Dbo::Session& session, boost::filesystem::path p, Type type) +MediaDirectory::create(Wt::Dbo::Session& session, boost::filesystem::path p) { - return session.add( new MediaDirectory( p, type ) ); + return session.add( new MediaDirectory(p) ); } void @@ -50,20 +49,6 @@ MediaDirectory::getAll(Wt::Dbo::Session& session) return std::vector(res.begin(), res.end()); } -std::vector -MediaDirectory::getByType(Wt::Dbo::Session& session, Type type) -{ - Wt::Dbo::collection< MediaDirectory::pointer > res = session.find().where("type = ?").bind (type); - - return std::vector(res.begin(), res.end()); -} - -MediaDirectory::pointer -MediaDirectory::get(Wt::Dbo::Session& session, boost::filesystem::path p, Type type) -{ - return session.find().where("path = ?").where("type = ?").bind( p.string()).bind(type); -} - boost::filesystem::path MediaDirectory::getPath(void) const { diff --git a/src/database/MediaDirectory.hpp b/src/database/MediaDirectory.hpp index e00892dd..3bc2e497 100644 --- a/src/database/MediaDirectory.hpp +++ b/src/database/MediaDirectory.hpp @@ -31,39 +31,29 @@ namespace Database { class MediaDirectory { public: - typedef Wt::Dbo::ptr pointer; - enum Type { - Audio = 1, - }; - MediaDirectory() {} - MediaDirectory(boost::filesystem::path p, Type type); + MediaDirectory(boost::filesystem::path p); // Accessors - static pointer create(Wt::Dbo::Session& session, boost::filesystem::path p, Type type); + static pointer create(Wt::Dbo::Session& session, boost::filesystem::path p); static std::vector getAll(Wt::Dbo::Session& session); - static std::vector getByType(Wt::Dbo::Session& session, Type type); - static pointer get(Wt::Dbo::Session& session, boost::filesystem::path p, Type type); static void eraseAll(Wt::Dbo::Session& session); + static void eraseByPath(Wt::Dbo::Session& session, boost::filesystem::path p); - Type getType(void) const { return _type; } boost::filesystem::path getPath(void) const; template void persist(Action& a) { - Wt::Dbo::field(a, _type, "type"); Wt::Dbo::field(a, _path, "path"); } private: - Type _type; std::string _path; - }; } // namespace Database diff --git a/src/database/Track.cpp b/src/database/Track.cpp index 5853a293..d913980f 100644 --- a/src/database/Track.cpp +++ b/src/database/Track.cpp @@ -224,44 +224,24 @@ Cluster::Cluster() { } -Cluster::Cluster(std::string type, std::string name) -: -_type( std::string(type, 0, _maxTypeLength)), -_name( std::string(name, 0, _maxNameLength)) +Cluster::Cluster(Wt::Dbo::ptr type, std::string name) + : _name(std::string(name, 0, _maxNameLength)), + _clusterType(type) { } -Wt::Dbo::collection -Cluster::getAll(Wt::Dbo::Session& session) -{ - return session.find(); -} - Cluster::pointer -Cluster::get(Wt::Dbo::Session& session, std::string type, std::string name) -{ - // TODO use like search - return session.find().where("type = ?").where("name = ?").bind( std::string(type, 0, _maxTypeLength)).bind( std::string(name, 0, _maxNameLength)); -} - -std::vector -Cluster::getByType(Wt::Dbo::Session& session, std::string type) -{ - Wt::Dbo::collection res = session.find().where("type = ?").bind( std::string(type, 0, _maxTypeLength)).orderBy("name"); - return std::vector(res.begin(), res.end()); -} - -Cluster::pointer -Cluster::create(Wt::Dbo::Session& session, std::string type, std::string name) +Cluster::create(Wt::Dbo::Session& session, Wt::Dbo::ptr type, std::string name) { return session.add(new Cluster(type, name)); } -void -Cluster::remove(Wt::Dbo::Session& session, std::string type) +std::vector +Cluster::getAll(Wt::Dbo::Session& session) { - Wt::Dbo::Transaction transaction(session); - session.execute("DELETE FROM cluster WHERE type = ?").bind(type); + Wt::Dbo::collection res = session.find(); + + return std::vector(res.begin(), res.end()); } Wt::Dbo::Query @@ -278,15 +258,6 @@ Cluster::getQuery(Wt::Dbo::Session& session, SearchFilter filter) return query; } -std::vector -Cluster::getAllTypes(Wt::Dbo::Session& session) -{ - Wt::Dbo::collection res - = session.query("SELECT type from cluster").groupBy("type").orderBy("type"); - - return std::vector(res.begin(), res.end()); -} - std::vector Cluster::getByFilter(Wt::Dbo::Session& session, SearchFilter filter, int offset, int size) { @@ -295,6 +266,56 @@ Cluster::getByFilter(Wt::Dbo::Session& session, SearchFilter filter, int offset, return std::vector(res.begin(), res.end()); } +ClusterType::ClusterType(std::string name) + : _name(name) +{ +} + +ClusterType::pointer +ClusterType::getByName(Wt::Dbo::Session& session, std::string name) +{ + return session.find().where("name = ?").bind(name); +} + +std::vector +ClusterType::getAll(Wt::Dbo::Session& session) +{ + Wt::Dbo::collection res = session.find(); + + return std::vector(res.begin(), res.end()); +} + +ClusterType::pointer +ClusterType::create(Wt::Dbo::Session& session, std::string name) +{ + return session.add(new ClusterType(name)); +} + +Cluster::pointer +ClusterType::getCluster(std::string name) const +{ + assert(self()); + assert(self()->id() != Wt::Dbo::dbo_traits::invalidId() ); + assert(session()); + + return session()->find() + .where("name = ?").bind(name) + .where("cluster_type_id = ").bind(self()->id()); +} + +std::vector +ClusterType::getClusters() const +{ + assert(self()); + assert(self()->id() != Wt::Dbo::dbo_traits::invalidId() ); + assert(session()); + + Wt::Dbo::collection res = session()->find() + .where("cluster_type_id = ").bind(self()->id()) + .orderBy("name"); + + return std::vector(res.begin(), res.end()); +} } // namespace Database diff --git a/src/database/Track.hpp b/src/database/Track.hpp index 7c0f1bc2..878ef796 100644 --- a/src/database/Track.hpp +++ b/src/database/Track.hpp @@ -40,64 +40,89 @@ class Artist; class Release; class Track; class PlaylistEntry; +class ClusterType; -class Cluster +class Cluster : public Wt::Dbo::Dbo { public: - - enum class Type - { - Genre = 1, - Mood = 2, - }; - typedef Wt::Dbo::ptr pointer; typedef Wt::Dbo::dbo_traits::IdType id_type; Cluster(); - Cluster(std::string type, std::string name); + Cluster(Wt::Dbo::ptr type, std::string name); // Find utility - static pointer get(Wt::Dbo::Session& session, std::string type, std::string name); static std::vector getByFilter(Wt::Dbo::Session& session, SearchFilter filter, int offset = -1, int size = -1); - static Wt::Dbo::collection getAll(Wt::Dbo::Session& session); - static std::vector getAllTypes(Wt::Dbo::Session& session); - static std::vector getByType(Wt::Dbo::Session& session, std::string type); + static std::vector getAll(Wt::Dbo::Session& session); // Create utility - static pointer create(Wt::Dbo::Session& session, std::string type, std::string name); - - // Remove utility - static void remove(Wt::Dbo::Session& session, std::string type); // nested transaction + static pointer create(Wt::Dbo::Session& session, Wt::Dbo::ptr type, std::string name); // Accessors const std::string& getName(void) const { return _name; } - const std::string& getType(void) const { return _type; } - const Wt::Dbo::collection< Wt::Dbo::ptr >& getTracks() const { return _tracks;} + Wt::Dbo::ptr getType() const { return _clusterType; } + const Wt::Dbo::collection>& getTracks() const { return _tracks; } - void addTrack(Wt::Dbo::Session& session, Wt::Dbo::dbo_traits::IdType trackId); void addTrack(Wt::Dbo::ptr track) { _tracks.insert(track); } template - void persist(Action& a) - { - Wt::Dbo::field(a, _name, "name"); - Wt::Dbo::field(a, _type, "type"); - Wt::Dbo::hasMany(a, _tracks, Wt::Dbo::ManyToMany, "track_cluster", "", Wt::Dbo::OnDeleteCascade); - } + void persist(Action& a) + { + Wt::Dbo::field(a, _name, "name"); + + Wt::Dbo::belongsTo(a, _clusterType, "cluster_type", Wt::Dbo::OnDeleteCascade); + Wt::Dbo::hasMany(a, _tracks, Wt::Dbo::ManyToMany, "track_cluster", "", Wt::Dbo::OnDeleteCascade); + } private: static Wt::Dbo::Query getQuery(Wt::Dbo::Session& session, SearchFilter filter); static const std::size_t _maxNameLength = 128; - static const std::size_t _maxTypeLength = 128; - std::string _type; std::string _name; + Wt::Dbo::ptr _clusterType; Wt::Dbo::collection< Wt::Dbo::ptr > _tracks; }; + +class ClusterType : public Wt::Dbo::Dbo +{ + public: + + using pointer = Wt::Dbo::ptr; + using id_type = Wt::Dbo::dbo_traits::IdType; + + ClusterType() {} + ClusterType(std::string name); + + static pointer getByName(Wt::Dbo::Session& session, std::string name); + static std::vector getAll(Wt::Dbo::Session& session); + + static pointer create(Wt::Dbo::Session& session, std::string name); + static void remove(Wt::Dbo::Session& session, std::string name); + + // Accessors + const std::string& getName(void) const { return _name; } + std::vector getClusters() const; + Cluster::pointer getCluster(std::string name) const; + + template + void persist(Action& a) + { + Wt::Dbo::field(a, _name, "name"); + Wt::Dbo::hasMany(a, _clusters, Wt::Dbo::ManyToOne, "cluster_type"); + } + + private: + + static const std::size_t _maxNameLength = 128; + + std::string _name; + Wt::Dbo::collection< Wt::Dbo::ptr > _clusters; +}; + + class Track { public: diff --git a/src/database/updater/DatabaseHighLevelCluster.cpp b/src/database/updater/DatabaseHighLevelCluster.cpp index 60d54918..5cc37607 100644 --- a/src/database/updater/DatabaseHighLevelCluster.cpp +++ b/src/database/updater/DatabaseHighLevelCluster.cpp @@ -26,11 +26,15 @@ namespace Database { -static Cluster::pointer getCluster(std::string type, std::string value) +static Cluster::pointer getCluster(std::string type, std::string name) { - Cluster::pointer cluster = ( Cluster::get(UpdaterDboSession(), type, value) ); + ClusterType::pointer clusterType = ClusterType::getByName(UpdaterDboSession(), type); + if (!clusterType) + clusterType = ClusterType::create(UpdaterDboSession(), type); + + auto cluster = clusterType->getCluster(name); if (!cluster) - cluster = Cluster::create(UpdaterDboSession(), type, value); + cluster = Cluster::create(UpdaterDboSession(), clusterType, name); return cluster; } @@ -196,7 +200,7 @@ HighLevelCluster::handleFilesUpdated(void) for (auto cluster : clusters) { // Check if removed - if (cluster->getType() != "high_level") + if (cluster->getType()->getName() != "high_level") continue; auto it = std::find(newClusterNames.begin(), newClusterNames.end(), cluster->getName()); diff --git a/src/main/main.cpp b/src/main/main.cpp index bcd80006..c2b24416 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -29,9 +29,9 @@ #include "image/Image.hpp" #include "feature/FeatureExtractor.hpp" -#include "database/updater/DatabaseUpdater.hpp" -#include "database/updater/DatabaseFeatureExtractor.hpp" -#include "database/updater/DatabaseHighLevelCluster.hpp" +#include "scanner/MediaScanner.hpp" +//#include "database/updater/DatabaseFeatureExtractor.hpp" +//#include "database/updater/DatabaseHighLevelCluster.hpp" #include "ui/LmsApplication.hpp" @@ -125,20 +125,19 @@ int main(int argc, char* argv[]) std::unique_ptr connectionPool( Database::Handler::createConnectionPool(Config::instance().getPath("working-dir") / "lms.db")); - Database::Updater& dbUpdater = Database::Updater::instance(); - dbUpdater.setConnectionPool(*connectionPool); + Scanner::MediaScanner scanner(*connectionPool); // Instanciate the updater's event handler. Order is important - dbUpdater.registerEventHandler(std::make_shared()); - dbUpdater.registerEventHandler(std::make_shared()); +// dbUpdater.registerEventHandler(std::make_shared()); +// dbUpdater.registerEventHandler(std::make_shared()); // bind entry point server.addEntryPoint(Wt::Application, boost::bind(UserInterface::LmsApplication::create, _1, boost::ref(*connectionPool))); // Start - LMS_LOG(MAIN, INFO) << "Starting database updater..."; - dbUpdater.start(); + LMS_LOG(MAIN, INFO) << "Starting Media scanner..."; + scanner.start(); LMS_LOG(MAIN, INFO) << "Starting server..."; server.start(); @@ -152,7 +151,7 @@ int main(int argc, char* argv[]) server.stop(); LMS_LOG(MAIN, INFO) << "Stopping database updater..."; - dbUpdater.stop(); + scanner.stop(); res = EXIT_SUCCESS; } diff --git a/src/metadata/AvFormat.cpp b/src/metadata/AvFormat.cpp index da5fc3bb..9b13a731 100644 --- a/src/metadata/AvFormat.cpp +++ b/src/metadata/AvFormat.cpp @@ -30,19 +30,23 @@ namespace MetaData { -bool -AvFormat::parse(const boost::filesystem::path& p, Items& items) +AvFormat::AvFormat(const std::map& clusterMap) +: _clusterMap(clusterMap) { +} + +boost::optional +AvFormat::parse(const boost::filesystem::path& p) +{ + Items items; Av::MediaFile mediaFile(p); if (!mediaFile.open()) - return false; + return boost::none; if (!mediaFile.scan()) - return false; - - std::map metadata = mediaFile.getMetaData(); + return boost::none; // Stream info { @@ -63,41 +67,6 @@ AvFormat::parse(const boost::filesystem::path& p, Items& items) items.insert( std::make_pair(MetaData::Type::AudioStreams, audioStreams)); } - { - std::vector videoStreams; - - std::vector streams = mediaFile.getStreams(Av::Stream::Type::Video); - - for (Av::Stream& stream : streams) - { - VideoStream videoStream; - videoStream.desc = stream.desc; - videoStream.bitRate = stream.bitrate; - - videoStreams.push_back(videoStream); - } - - if (!videoStreams.empty()) - items.insert( std::make_pair(MetaData::Type::VideoStreams, videoStreams)); - } - - { - std::vector subtitleStreams; - - std::vector streams = mediaFile.getStreams(Av::Stream::Type::Subtitle); - - for (Av::Stream& stream : streams) - { - SubtitleStream subtitleStream; - subtitleStream.desc = stream.desc; - - subtitleStreams.push_back(subtitleStream); - } - - if (!subtitleStreams.empty()) - items.insert( std::make_pair(MetaData::Type::SubtitleStreams, subtitleStreams)); - } - // Duration items.insert( std::make_pair(MetaData::Type::Duration, mediaFile.getDuration() )); @@ -106,19 +75,27 @@ AvFormat::parse(const boost::filesystem::path& p, Items& items) // Embedded MetaData // Make sure to convert strings into UTF-8 - std::map::const_iterator it; - for (it = metadata.begin(); it != metadata.end(); ++it) + + MetaData::Clusters clusters; + + std::map metadataMap = mediaFile.getMetaData(); + for (auto metadata : metadataMap) { - if (boost::iequals(it->first, "artist")) - items.insert( std::make_pair(MetaData::Type::Artist, stringTrim( stringToUTF8(it->second)) )); - else if (boost::iequals(it->first, "album")) - items.insert( std::make_pair(MetaData::Type::Album, stringTrim( stringToUTF8(it->second)) )); - else if (boost::iequals(it->first, "title")) - items.insert( std::make_pair(MetaData::Type::Title, stringTrim( stringToUTF8(it->second)) )); - else if (boost::iequals(it->first, "track")) + const std::string tag = boost::to_upper_copy(metadata.first); + const std::string value = metadata.second; +#if 0 + std::cout << "TAG = " << tag << ", VAL = " << value << std::endl; +#endif + if (tag == "ARTIST") + items.insert( std::make_pair(MetaData::Type::Artist, stringTrim( stringToUTF8(value)) )); + else if (tag == "ALBUM") + items.insert( std::make_pair(MetaData::Type::Album, stringTrim( stringToUTF8(value)) )); + else if (tag == "TITLE") + items.insert( std::make_pair(MetaData::Type::Title, stringTrim( stringToUTF8(value)) )); + else if (tag == "TRACK") { // Expecting 'Number/Total' - auto strings = splitString(it->second, "/"); + auto strings = splitString(value, "/"); if (strings.size() > 0) { @@ -134,10 +111,10 @@ AvFormat::parse(const boost::filesystem::path& p, Items& items) } } } - else if (boost::iequals(it->first, "disc")) + else if (tag == "DISC") { // Expecting 'Number/Total' - auto strings = splitString(it->second, "/"); + auto strings = splitString(value, "/"); if (strings.size() > 0) { @@ -153,49 +130,57 @@ AvFormat::parse(const boost::filesystem::path& p, Items& items) } } } - else if (boost::iequals(it->first, "date") - || boost::iequals(it->first, "year") - || boost::iequals(it->first, "WM/Year")) + else if (tag == "DATE" + || tag == "YEAR" + || tag == "WM/Year") { boost::posix_time::ptime p; - if (readAsPosixTime(it->second, p)) + if (readAsPosixTime(value, p)) items.insert( std::make_pair(MetaData::Type::Date, p)); } - else if (boost::iequals(it->first, "TDOR") // Original release time (ID3v2 2.4) - || boost::iequals(it->first, "TORY")) // Original release year + else if (tag == "TDOR" // Original release time (ID3v2 2.4) + || tag == "TORY") // Original release year { boost::posix_time::ptime p; - if (readAsPosixTime(it->second, p)) + if (readAsPosixTime(value, p)) items.insert( std::make_pair(MetaData::Type::OriginalDate, p)); } - else if (boost::iequals(it->first, "genre")) + else if (tag == "MUSICBRAINZ ARTIST ID" + || tag == "MUSICBRAINZ_ARTISTID") { - // TODO use splitStrings - std::list genres; - if (readList(it->second, ";,\\", genres)) - items.insert( std::make_pair(MetaData::Type::Genres, genres)); + items.insert( std::make_pair(MetaData::Type::MusicBrainzArtistID, stringTrim( stringToUTF8(value)) )); + } + else if (tag == "MUSICBRAINZ ALBUM ID" + || tag == "MUSICBRAINZ_ALBUMID") + { + items.insert( std::make_pair(MetaData::Type::MusicBrainzAlbumID, stringTrim( stringToUTF8(value)) )); + } + else if (tag == "MUSICBRAINZ RELEASE TRACK ID" + || tag == "MUSICBRAINZ_RELEASETRACKID" + || tag == "MUSICBRAINZ_TRACKID") + { + items.insert( std::make_pair(MetaData::Type::MusicBrainzTrackID, stringTrim( stringToUTF8(value)) )); + } + else if (tag == "ACOUSTID ID") + { + items.insert( std::make_pair(MetaData::Type::AcoustID, stringTrim( stringToUTF8(value)) )); + } + else if (_clusterMap.find(tag) != _clusterMap.end()) + { + std::vector clusterNames = splitString(value, ";,\\"); - } - else if (boost::iequals(it->first, "MusicBrainz Artist Id") - || boost::iequals(it->first, "MUSICBRAINZ_ARTISTID")) - { - items.insert( std::make_pair(MetaData::Type::MusicBrainzArtistID, stringTrim( stringToUTF8(it->second)) )); - } - else if (boost::iequals(it->first, "MusicBrainz Album Id") - || boost::iequals(it->first, "MUSICBRAINZ_ALBUMID")) - { - items.insert( std::make_pair(MetaData::Type::MusicBrainzAlbumID, stringTrim( stringToUTF8(it->second)) )); - } - else if (boost::iequals(it->first, "MusicBrainz Release Track Id") - || boost::iequals(it->first, "MUSICBRAINZ_RELEASETRACKID") - || boost::iequals(it->first, "MUSICBRAINZ_TRACKID")) - { - items.insert( std::make_pair(MetaData::Type::MusicBrainzTrackID, stringTrim( stringToUTF8(it->second)) )); + if (!clusterNames.empty()) + { + clusters[_clusterMap[tag]] = std::set(clusterNames.begin(), clusterNames.end()); + } } } - return true; + if (!clusters.empty()) + items.insert( std::make_pair(MetaData::Type::Clusters, clusters) ); + + return items; } } // namespace MetaData diff --git a/src/metadata/AvFormat.hpp b/src/metadata/AvFormat.hpp index 4c08d943..8c3c9504 100644 --- a/src/metadata/AvFormat.hpp +++ b/src/metadata/AvFormat.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Emeric Poupon + * Copyright (C) 2018 Emeric Poupon * * This file is part of LMS. * @@ -17,8 +17,10 @@ * along with LMS. If not, see . */ -#ifndef METADATA_AVFORMAT_HPP -#define METADATA_AVFORMAT_HPP +#pragma once + +#include +#include #include "MetaData.hpp" @@ -30,13 +32,19 @@ class AvFormat : public Parser { public: - bool parse(const boost::filesystem::path& p, Items& items); + AvFormat(const std::map& clusterMap + = { + {"GENRE", "Genre" }, + {"ALBUMGROUPING", "Group" } + }); + + + boost::optional parse(const boost::filesystem::path& p); private: + std::map _clusterMap; }; - } // namespace MetaData -#endif diff --git a/src/metadata/MetaData.hpp b/src/metadata/MetaData.hpp index 019184a3..6bbea293 100644 --- a/src/metadata/MetaData.hpp +++ b/src/metadata/MetaData.hpp @@ -21,8 +21,10 @@ #define METADATA_HPP #include +#include #include +#include #include namespace MetaData @@ -30,10 +32,11 @@ namespace MetaData enum class Type { + // Name Type of the value Artist, // string Title, // string Album, // string - Genres, // list + Clusters, // Clusters, ex: { "genre", {"death metal", "brutal death"} }, { "albumgrouping", {"metal"} } Duration, // boost::posix_time::time_duration TrackNumber, // size_t DiscNumber, // size_t @@ -43,12 +46,11 @@ namespace MetaData OriginalDate, // boost::posix_time::ptime HasCover, // bool AudioStreams, // vector - VideoStreams, // vector - SubtitleStreams, // vector MusicBrainzArtistID, // string MusicBrainzAlbumID, // string MusicBrainzTrackID, // string MusicBrainzRecordingID, // string + AcoustID, // string }; // Used by Streams @@ -58,28 +60,16 @@ namespace MetaData std::size_t bitRate; }; - struct VideoStream - { - std::string desc; - std::size_t bitRate; - }; - - struct SubtitleStream - { - std::string desc; - }; - // Type and associated data // See enum Type's comments - typedef std::map Items; + using Items = std::map; + using Clusters = std::map>; class Parser { public: - typedef std::shared_ptr pointer; - - virtual bool parse(const boost::filesystem::path& p, Items& items) = 0; + virtual boost::optional parse(const boost::filesystem::path& p) = 0; }; diff --git a/src/metadata/TagLibParser.cpp b/src/metadata/TagLibParser.cpp index 7afa6140..5508ef46 100644 --- a/src/metadata/TagLibParser.cpp +++ b/src/metadata/TagLibParser.cpp @@ -32,18 +32,25 @@ namespace MetaData { -bool -TagLibParser::parse(const boost::filesystem::path& p, Items& items) +TagLibParser::TagLibParser(const std::map& clusterMap) +: _clusterMap(clusterMap) +{ +} + +boost::optional +TagLibParser::parse(const boost::filesystem::path& p) { TagLib::FileRef f(p.string().c_str(), true, // read audio properties TagLib::AudioProperties::Average); if (f.isNull()) - return false; + return boost::none; if (!f.audioProperties()) - return false; + return boost::none; + + Items items; { TagLib::AudioProperties *properties = f.audioProperties(); @@ -70,32 +77,46 @@ TagLibParser::parse(const boost::filesystem::path& p, Items& items) if (f.tag()) { - TagLib::PropertyMap tags = f.file()->properties(); + MetaData::Clusters clusters; + TagLib::PropertyMap properties = f.file()->properties(); - for(TagLib::PropertyMap::ConstIterator itElem = tags.begin(); itElem != tags.end(); ++itElem) + for(auto property : properties) { - const std::string tag = itElem->first.to8Bit(true); - const TagLib::StringList &values = itElem->second; + const std::string tag = property.first.upper().to8Bit(true); + const TagLib::StringList& values = property.second; if (tag.empty() || values.isEmpty() || values.front().isEmpty()) continue; // TODO validate MBID format +#if 0 + std::cout << "TAG = '" << tag << "'" << std::endl; + for (auto value : values) + { + std::cout << "\t'" << value.to8Bit(true) << "'" << std::endl; + } +#endif + if (tag == "ARTIST") items.insert( std::make_pair(MetaData::Type::Artist, stringTrim( values.front().to8Bit(true)))); else if (tag == "ALBUM") items.insert( std::make_pair(MetaData::Type::Album, stringTrim( values.front().to8Bit(true)))); else if (tag == "TITLE") items.insert( std::make_pair(MetaData::Type::Title, stringTrim( values.front().to8Bit(true)))); - else if (tag == "MUSICBRAINZ_RELEASETRACKID") + else if (tag == "MUSICBRAINZ_RELEASETRACKID" + || tag == "MUSICBRAINZ RELEASE TRACK ID") + { items.insert( std::make_pair(MetaData::Type::MusicBrainzTrackID, stringTrim( values.front().to8Bit(true)))); + } else if (tag == "MUSICBRAINZ_ARTISTID") items.insert( std::make_pair(MetaData::Type::MusicBrainzArtistID, stringTrim( values.front().to8Bit(true)))); else if (tag == "MUSICBRAINZ_ALBUMID") items.insert( std::make_pair(MetaData::Type::MusicBrainzAlbumID, stringTrim( values.front().to8Bit(true)))); else if (tag == "MUSICBRAINZ_TRACKID") - items.insert( std::make_pair(MetaData::Type::MusicBrainzRecordingID, stringTrim( tags["MUSICBRAINZ_TRACKID"].front().to8Bit(true)))); + items.insert( std::make_pair(MetaData::Type::MusicBrainzRecordingID, stringTrim( values.front().to8Bit(true)))); + else if (tag == "ACOUSTID_ID") + items.insert( std::make_pair(MetaData::Type::AcoustID, stringTrim( values.front().to8Bit(true)))); else if (tag == "TRACKNUMBER") { // Expecting 'Number/Total' @@ -160,24 +181,29 @@ TagLibParser::parse(const boost::filesystem::path& p, Items& items) items.insert( std::make_pair(MetaData::Type::OriginalDate, p)); } } - else if (tag == "GENRE") - { - std::list genreList; - for (TagLib::StringList::ConstIterator itGenre = values.begin(); itGenre != values.end(); ++itGenre) - genreList.push_back(itGenre->to8Bit(true)); - - items.insert( std::make_pair(MetaData::Type::Genres, genreList) ); - } else if (tag == "METADATA_BLOCK_PICTURE") { // Only add once if (items.find(MetaData::Type::HasCover) == items.end()) items.insert( std::make_pair(MetaData::Type::HasCover, true)); } + // Check if a hit a cluster tag + else if (_clusterMap.find(tag) != _clusterMap.end()) + { + std::set clusterNames; + for (const auto& value : values) + clusterNames.insert(value.to8Bit(true)); + + if (!clusterNames.empty()) + clusters[_clusterMap[tag]] = clusterNames; + } } + + if (!clusters.empty()) + items.insert( std::make_pair(MetaData::Type::Clusters, clusters) ); } - return true; + return items; } } // namespace MetaData diff --git a/src/metadata/TagLibParser.hpp b/src/metadata/TagLibParser.hpp index a05364aa..baefc112 100644 --- a/src/metadata/TagLibParser.hpp +++ b/src/metadata/TagLibParser.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Emeric Poupon + * Copyright (C) 2018 Emeric Poupon * * This file is part of LMS. * @@ -19,6 +19,9 @@ #pragma once +#include +#include + #include "MetaData.hpp" namespace MetaData @@ -28,10 +31,19 @@ namespace MetaData class TagLibParser : public Parser { public: - bool parse(const boost::filesystem::path& p, Items& items); + + // Provide a map for TagLib name -> Cluster name + TagLibParser(const std::map& clusterMap + = { + {"GENRE", "Genre" }, + {"ALBUMGROUPING", "Group" } + }); + + boost::optional parse(const boost::filesystem::path& p); private: + std::map _clusterMap; }; } // namespace MetaData diff --git a/src/database/updater/DatabaseUpdater.cpp b/src/scanner/MediaScanner.cpp similarity index 59% rename from src/database/updater/DatabaseUpdater.cpp rename to src/scanner/MediaScanner.cpp index b0484f40..60f19f55 100644 --- a/src/database/updater/DatabaseUpdater.cpp +++ b/src/scanner/MediaScanner.cpp @@ -32,8 +32,7 @@ #include "utils/Path.hpp" #include "utils/Utils.hpp" - -#include "DatabaseUpdater.hpp" +#include "MediaScanner.hpp" namespace { @@ -85,20 +84,6 @@ isFileSupported(const boost::filesystem::path& file, const std::vector -getRootDirectoriesByType(Wt::Dbo::Session& session, Database::MediaDirectory::Type type) -{ - Wt::Dbo::Transaction transaction(session); - - std::vector rootDirs = Database::MediaDirectory::getByType(session, type); - - std::vector res; - for (auto rootDir : rootDirs) - res.push_back(rootDir->getPath()); - - return res; -} - bool isPathInParentPath(const boost::filesystem::path& path, const boost::filesystem::path& parentPath) { @@ -118,40 +103,33 @@ isPathInParentPath(const boost::filesystem::path& path, const boost::filesystem: } // namespace -namespace Database { +namespace Scanner { -Updater& Updater::instance(void) -{ - static Updater updater; - return updater; -} +using namespace Database; -Updater::Updater() - : _running(true), -_scheduleTimer(_ioService) +MediaScanner::MediaScanner(Wt::Dbo::SqlConnectionPool& connectionPool) + : _running(false), +_scheduleTimer(_ioService), +_db(connectionPool) { _ioService.setThreadCount(1); + + Wt::Dbo::Transaction transaction(_db.getSession()); + + if (!Setting::exists(_db.getSession(), "file_extensions")) + Setting::setString(_db.getSession(), "file_extensions", ".mp3 .ogg .oga .aac .m4a .flac .wav .wma .aif .aiff .ape .mpc .shn" ); } void -Updater::setConnectionPool(Wt::Dbo::SqlConnectionPool& connectionPool) -{ - _db = new Database::Handler(connectionPool); -} - -void -Updater::restart(void) +MediaScanner::restart(void) { stop(); start(); } void -Updater::start(void) +MediaScanner::start(void) { - if (_db == nullptr) - throw std::logic_error("uninitialized db!"); - _running = true; // post some jobs in the io_service @@ -161,7 +139,7 @@ Updater::start(void) } void -Updater::stop(void) +MediaScanner::stop(void) { _running = false; @@ -171,9 +149,9 @@ Updater::stop(void) } void -Updater::processNextJob(void) +MediaScanner::processNextJob(void) { - if (Setting::getBool(_db->getSession(), "manual_scan_requested", false)) + if (Setting::getBool(_db.getSession(), "manual_scan_requested", false)) { LMS_LOG(DBUPDATER, INFO) << "Manual scan requested!"; scheduleScan( boost::posix_time::seconds(0) ); @@ -181,11 +159,11 @@ Updater::processNextJob(void) else { boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); - boost::posix_time::time_duration startTime = Setting::getDuration(_db->getSession(), "update_start_time"); + boost::posix_time::time_duration startTime = Setting::getDuration(_db.getSession(), "update_start_time"); boost::gregorian::date nextScanDate; - std::string updatePeriod = Setting::getString(_db->getSession(), "update_period", "never"); + std::string updatePeriod = Setting::getString(_db.getSession(), "update_period", "never"); if (updatePeriod == "daily") { if (now.time_of_day() < startTime) @@ -214,72 +192,58 @@ Updater::processNextJob(void) } void -Updater::scheduleScan( boost::posix_time::time_duration duration) +MediaScanner::scheduleScan( boost::posix_time::time_duration duration) { LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan in " << duration; _scheduleTimer.expires_from_now(duration); - _scheduleTimer.async_wait( boost::bind( &Updater::process, this, boost::asio::placeholders::error) ); + _scheduleTimer.async_wait( boost::bind( &MediaScanner::process, this, boost::asio::placeholders::error) ); } void -Updater::scheduleScan( boost::posix_time::ptime time) +MediaScanner::scheduleScan( boost::posix_time::ptime time) { LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan at " << time; _scheduleTimer.expires_at(time); - _scheduleTimer.async_wait( boost::bind( &Updater::process, this, boost::asio::placeholders::error) ); + _scheduleTimer.async_wait( boost::bind( &MediaScanner::process, this, boost::asio::placeholders::error) ); } void -Updater::process(boost::system::error_code err) +MediaScanner::process(boost::system::error_code err) { if (err) return; - updateFileExtensions(); + refreshScanSettings(); Stats stats; checkAudioFiles(stats); - std::vector rootDirectories; - { - Wt::Dbo::Transaction transaction(_db->getSession()); - - for (MediaDirectory::pointer directory : MediaDirectory::getAll(_db->getSession())) - rootDirectories.push_back( RootDirectory( directory->getType(), directory->getPath() )); - } - - for (RootDirectory rootDirectory : rootDirectories) + for (auto rootDirectory : _rootDirectories) { if (!_running) break; - LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory.path << "'..."; + LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory << "'..."; processRootDirectory(rootDirectory, stats); - LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory.path << "' DONE"; + LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory << "' DONE"; } if (_running) - { checkDuplicatedAudioFiles(stats); - LMS_LOG(DBUPDATER, INFO) << "Processed all files, now calling listeners..."; - for (auto eventHandler : _eventHandlers) - eventHandler->handleFilesUpdated(); - } - LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.nbAdded << ", removed = " << stats.nbRemoved << ", updated = " << stats.nbUpdated << "), Not changed = " << stats.nbNoChange << ", Scanned = " << stats.nbScanned << " (errors = " << stats.nbScanErrors << ", not imported = " << stats.nbNotImported << ")"; // Update database stats boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); if (stats.nbChanges() > 0) - Setting::setTime(_db->getSession(), "last_update", now); + Setting::setTime(_db.getSession(), "last_update", now); // Save the last scan only if it has been completed if (_running) { - Setting::setTime(_db->getSession(), "last_scan", now); - Setting::setBool(_db->getSession(), "manual_scan_requested", false); + Setting::setTime(_db.getSession(), "last_scan", now); + Setting::setBool(_db.getSession(), "manual_scan_requested", false); processNextJob(); @@ -288,26 +252,30 @@ Updater::process(boost::system::error_code err) } void -Updater::updateFileExtensions() +MediaScanner::refreshScanSettings() { - Wt::Dbo::Transaction transaction(_db->getSession()); + Wt::Dbo::Transaction transaction(_db.getSession()); - _audioFileExtensions.clear(); - for (auto extension : splitString(Setting::getString(_db->getSession(), "audio_file_extensions"), " ")) - _audioFileExtensions.push_back( extension ); + _fileExtensions.clear(); + for (auto extension : splitString(Setting::getString(_db.getSession(), "file_extensions"), " ")) + _fileExtensions.push_back( extension ); + + _rootDirectories.clear(); + for (auto rootDir : Database::MediaDirectory::getAll(_db.getSession())) + _rootDirectories.push_back(rootDir->getPath()); } Artist::pointer -Updater::getArtist( const boost::filesystem::path& file, const std::string& name, const std::string& mbid) +MediaScanner::getArtist( const boost::filesystem::path& file, const std::string& name, const std::string& mbid) { Artist::pointer artist; // First try to get by MBID if (!mbid.empty()) { - artist = Artist::getByMBID( _db->getSession(), mbid ); + artist = Artist::getByMBID( _db.getSession(), mbid ); if (!artist) - artist = Artist::create( _db->getSession(), name, mbid); + artist = Artist::create( _db.getSession(), name, mbid); return artist; } @@ -315,7 +283,7 @@ Updater::getArtist( const boost::filesystem::path& file, const std::string& name // Fall back on artist name (collisions may occur) if (!name.empty()) { - for (Artist::pointer sameNamedArtist : Artist::getByName( _db->getSession(), name )) + for (Artist::pointer sameNamedArtist : Artist::getByName( _db.getSession(), name )) { if (sameNamedArtist->getMBID().empty()) { @@ -326,7 +294,7 @@ Updater::getArtist( const boost::filesystem::path& file, const std::string& name // No Artist found with the same name and without MBID -> creating if (!artist) - artist = Artist::create( _db->getSession(), name); + artist = Artist::create( _db.getSession(), name); return artist; } @@ -335,16 +303,16 @@ Updater::getArtist( const boost::filesystem::path& file, const std::string& name } Release::pointer -Updater::getRelease( const boost::filesystem::path& file, const std::string& name, const std::string& mbid) +MediaScanner::getRelease( const boost::filesystem::path& file, const std::string& name, const std::string& mbid) { Release::pointer release; // First try to get by MBID if (!mbid.empty()) { - release = Release::getByMBID( _db->getSession(), mbid ); + release = Release::getByMBID( _db.getSession(), mbid ); if (!release) - release = Release::create( _db->getSession(), name, mbid); + release = Release::create( _db.getSession(), name, mbid); return release; } @@ -352,7 +320,7 @@ Updater::getRelease( const boost::filesystem::path& file, const std::string& nam // Fall back on release name (collisions may occur) if (!name.empty()) { - for (Release::pointer sameNamedRelease : Release::getByName( _db->getSession(), name )) + for (Release::pointer sameNamedRelease : Release::getByName( _db.getSession(), name )) { if (sameNamedRelease->getMBID().empty()) { @@ -363,7 +331,7 @@ Updater::getRelease( const boost::filesystem::path& file, const std::string& nam // No release found with the same name and without MBID -> creating if (!release) - release = Release::create( _db->getSession(), name); + release = Release::create( _db.getSession(), name); return release; } @@ -372,32 +340,39 @@ Updater::getRelease( const boost::filesystem::path& file, const std::string& nam } std::vector -Updater::getGenreClusters( const std::list& names) +MediaScanner::getClusters( const MetaData::Clusters& clustersNames) { - std::vector< Cluster::pointer > genres; + std::vector< Cluster::pointer > clusters; - for (const std::string& name : names) + for (auto clusterNames : clustersNames) { - Cluster::pointer genre ( Cluster::get(_db->getSession(), "Genre", name) ); - if (!genre) - genre = Cluster::create(_db->getSession(), "Genre", name); + ClusterType::pointer clusterType = ClusterType::getByName(_db.getSession(), clusterNames.first); + if (!clusterType) + clusterType = ClusterType::create(_db.getSession(), clusterNames.first); - genres.push_back( genre ); + for (auto clusterName : clusterNames.second) + { + Cluster::pointer cluster = clusterType->getCluster(clusterName); + if (!cluster) + cluster = Cluster::create(_db.getSession(), clusterType, clusterName); + + clusters.push_back(cluster); + } } - return genres; + return clusters; } void -Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) +MediaScanner::processAudioFile( const boost::filesystem::path& file, Stats& stats) { boost::posix_time::ptime lastWriteTime (boost::posix_time::from_time_t( boost::filesystem::last_write_time( file ) ) ); // Skip file if last write is the same { - Wt::Dbo::Transaction transaction(_db->getSession()); + Wt::Dbo::Transaction transaction(_db.getSession()); - Wt::Dbo::ptr track = Track::getByPath(_db->getSession(), file); + Wt::Dbo::ptr track = Track::getByPath(_db.getSession(), file); if (track && track->getLastWriteTime() == lastWriteTime) { @@ -406,8 +381,8 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) } } - MetaData::Items items; - if (!_metadataParser.parse(file, items)) + boost::optional items = _metadataParser.parse(file); + if (!items) { stats.nbScanErrors++; return; @@ -418,15 +393,15 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) std::vector checksum ; computeCrc(file, checksum); - Wt::Dbo::Transaction transaction(_db->getSession()); + Wt::Dbo::Transaction transaction(_db.getSession()); - Wt::Dbo::ptr track = Track::getByPath(_db->getSession(), file); + Wt::Dbo::ptr track = Track::getByPath(_db.getSession(), file); // We estimate this is a audio file if: // - we found a least one audio stream // - the duration is not null - if (items.find(MetaData::Type::AudioStreams) == items.end() - || boost::any_cast >(items[MetaData::Type::AudioStreams]).empty()) + if ((*items).find(MetaData::Type::AudioStreams) == (*items).end() + || boost::any_cast> ((*items)[MetaData::Type::AudioStreams]).empty()) { LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file << "' (no audio stream found)"; @@ -439,8 +414,8 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) stats.nbNotImported++; return; } - if (items.find(MetaData::Type::Duration) == items.end() - || boost::any_cast(items[MetaData::Type::Duration]).total_seconds() <= 0) + if ((*items).find(MetaData::Type::Duration) == (*items).end() + || boost::any_cast((*items)[MetaData::Type::Duration]).total_seconds() <= 0) { LMS_LOG(DBUPDATER, INFO) << "Skipped '" << file << "' (no duration or duration <= 0)"; @@ -456,9 +431,9 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) // ***** Title std::string title; - if (items.find(MetaData::Type::Title) != items.end()) + if ((*items).find(MetaData::Type::Title) != (*items).end()) { - title = boost::any_cast(items[MetaData::Type::Title]); + title = boost::any_cast((*items)[MetaData::Type::Title]); } else { @@ -470,13 +445,15 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) // ***** Clusters std::vector< Cluster::pointer > genres; { - std::list genreList; + MetaData::Clusters clusterNames; - if (items.find(MetaData::Type::Genres) != items.end()) - genreList = boost::any_cast< std::list > (items[MetaData::Type::Genres]); + if ((*items).find(MetaData::Type::Clusters) != (*items).end()) + { + clusterNames = boost::any_cast ((*items)[MetaData::Type::Clusters]); + } // TODO rename - genres = getGenreClusters( genreList ); + genres = getClusters( clusterNames ); } // ***** Artist @@ -485,11 +462,11 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) std::string artistName; std::string artistMusicBrainzID; - if (items.find(MetaData::Type::MusicBrainzArtistID) != items.end()) - artistMusicBrainzID = boost::any_cast(items[MetaData::Type::MusicBrainzArtistID] ); + if ((*items).find(MetaData::Type::MusicBrainzArtistID) != (*items).end()) + artistMusicBrainzID = boost::any_cast((*items)[MetaData::Type::MusicBrainzArtistID] ); - if (items.find(MetaData::Type::Artist) != items.end()) - artistName = boost::any_cast(items[MetaData::Type::Artist]); + if ((*items).find(MetaData::Type::Artist) != (*items).end()) + artistName = boost::any_cast((*items)[MetaData::Type::Artist]); artist = getArtist(file, artistName, artistMusicBrainzID); } @@ -500,11 +477,11 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) std::string releaseName; std::string releaseMusicBrainzID; - if (items.find(MetaData::Type::MusicBrainzAlbumID) != items.end()) - releaseMusicBrainzID = boost::any_cast(items[MetaData::Type::MusicBrainzAlbumID] ); + if ((*items).find(MetaData::Type::MusicBrainzAlbumID) != (*items).end()) + releaseMusicBrainzID = boost::any_cast((*items)[MetaData::Type::MusicBrainzAlbumID] ); - if (items.find(MetaData::Type::Album) != items.end()) - releaseName = boost::any_cast(items[MetaData::Type::Album]); + if ((*items).find(MetaData::Type::Album) != (*items).end()) + releaseName = boost::any_cast((*items)[MetaData::Type::Album]); release = getRelease(file, releaseName, releaseMusicBrainzID); } @@ -515,7 +492,7 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) if (!track) { // Create a new song - track = Track::create(_db->getSession(), file); + track = Track::create(_db.getSession(), file); LMS_LOG(DBUPDATER, INFO) << "Adding '" << file << "'"; stats.nbAdded++; } @@ -537,7 +514,7 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) track.modify()->setRelease(release); track.modify()->setLastWriteTime(lastWriteTime); track.modify()->setName(title); - track.modify()->setDuration( boost::any_cast(items[MetaData::Type::Duration]) ); + track.modify()->setDuration( boost::any_cast((*items)[MetaData::Type::Duration]) ); track.modify()->setAddedTime( boost::posix_time::second_clock::local_time() ); { @@ -555,53 +532,54 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) track.modify()->setGenres( trackClusterList ); } - if (items.find(MetaData::Type::TrackNumber) != items.end()) - track.modify()->setTrackNumber( boost::any_cast(items[MetaData::Type::TrackNumber]) ); + if ((*items).find(MetaData::Type::TrackNumber) != (*items).end()) + track.modify()->setTrackNumber( boost::any_cast((*items)[MetaData::Type::TrackNumber]) ); - if (items.find(MetaData::Type::TotalTrack) != items.end()) - track.modify()->setTotalTrackNumber( boost::any_cast(items[MetaData::Type::TotalTrack]) ); + if ((*items).find(MetaData::Type::TotalTrack) != (*items).end()) + track.modify()->setTotalTrackNumber( boost::any_cast((*items)[MetaData::Type::TotalTrack]) ); - if (items.find(MetaData::Type::DiscNumber) != items.end()) - track.modify()->setDiscNumber( boost::any_cast(items[MetaData::Type::DiscNumber]) ); + if ((*items).find(MetaData::Type::DiscNumber) != (*items).end()) + track.modify()->setDiscNumber( boost::any_cast((*items)[MetaData::Type::DiscNumber]) ); - if (items.find(MetaData::Type::TotalDisc) != items.end()) - track.modify()->setTotalDiscNumber( boost::any_cast(items[MetaData::Type::TotalDisc]) ); + if ((*items).find(MetaData::Type::TotalDisc) != (*items).end()) + track.modify()->setTotalDiscNumber( boost::any_cast((*items)[MetaData::Type::TotalDisc]) ); - if (items.find(MetaData::Type::Date) != items.end()) - track.modify()->setDate( boost::any_cast(items[MetaData::Type::Date]) ); + if ((*items).find(MetaData::Type::Date) != (*items).end()) + track.modify()->setDate( boost::any_cast((*items)[MetaData::Type::Date]) ); - if (items.find(MetaData::Type::OriginalDate) != items.end()) + if ((*items).find(MetaData::Type::OriginalDate) != (*items).end()) { - track.modify()->setOriginalDate( boost::any_cast(items[MetaData::Type::OriginalDate]) ); + track.modify()->setOriginalDate( boost::any_cast((*items)[MetaData::Type::OriginalDate]) ); // If a file has an OriginalDate but no date, set the date to ease filtering - if (items.find(MetaData::Type::Date) == items.end()) - track.modify()->setDate( boost::any_cast(items[MetaData::Type::OriginalDate]) ); + if ((*items).find(MetaData::Type::Date) == (*items).end()) + track.modify()->setDate( boost::any_cast((*items)[MetaData::Type::OriginalDate]) ); } - if (items.find(MetaData::Type::MusicBrainzRecordingID) != items.end()) + if ((*items).find(MetaData::Type::MusicBrainzRecordingID) != (*items).end()) { - track.modify()->setMBID( boost::any_cast(items[MetaData::Type::MusicBrainzRecordingID]) ); + track.modify()->setMBID( boost::any_cast((*items)[MetaData::Type::MusicBrainzRecordingID]) ); } - if (items.find(MetaData::Type::HasCover) != items.end()) + if ((*items).find(MetaData::Type::HasCover) != (*items).end()) { - bool hasCover = boost::any_cast(items[MetaData::Type::HasCover]); + bool hasCover = boost::any_cast((*items)[MetaData::Type::HasCover]); track.modify()->setCoverType( hasCover ? Track::CoverType::Embedded : Track::CoverType::None ); } - transaction.commit(); + // TODO check added/modified + _sigAddedTrack.emit(track); - _sigTrackChanged.emit(true, track.id(), track->getMBID(), track->getPath()); + transaction.commit(); } void -Updater::processRootDirectory(RootDirectory rootDirectory, Stats& stats) +MediaScanner::processRootDirectory(boost::filesystem::path rootDirectory, Stats& stats) { boost::system::error_code ec; - boost::filesystem::recursive_directory_iterator itPath(rootDirectory.path, ec); + boost::filesystem::recursive_directory_iterator itPath(rootDirectory, ec); boost::filesystem::recursive_directory_iterator itEnd; while (!ec && itPath != itEnd) @@ -614,20 +592,15 @@ Updater::processRootDirectory(RootDirectory rootDirectory, Stats& stats) if (boost::filesystem::is_regular(path)) { - switch( rootDirectory.type ) - { - case Database::MediaDirectory::Audio: - if (isFileSupported(path, _audioFileExtensions)) - processAudioFile(path, stats ); - - break; - } + if (isFileSupported(path, _fileExtensions)) + processAudioFile(path, stats ); } } } -bool -Updater::checkFile(const boost::filesystem::path& p, const std::vector& rootDirs, const std::vector& extensions) +// Check if a file exists and is still in a root directory +static bool +checkFile(const boost::filesystem::path& p, const std::vector& rootDirs, const std::vector& extensions) { try { @@ -678,12 +651,11 @@ Updater::checkFile(const boost::filesystem::path& p, const std::vector trackPaths = Track::getAllPaths(_db->getSession());; - std::vector rootDirs = getRootDirectoriesByType(_db->getSession(), Database::MediaDirectory::Audio); + std::vector trackPaths = Track::getAllPaths(_db.getSession());; LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks..."; for (auto& trackPath : trackPaths) @@ -691,11 +663,11 @@ Updater::checkAudioFiles( Stats& stats ) if (!_running) return; - if (!checkFile(trackPath, rootDirs, _audioFileExtensions)) + if (!checkFile(trackPath, _rootDirectories, _fileExtensions)) { - Wt::Dbo::Transaction transaction(_db->getSession()); + Wt::Dbo::Transaction transaction(_db.getSession()); - Track::pointer track = Track::getByPath(_db->getSession(), trackPath); + Track::pointer track = Track::getByPath(_db.getSession(), trackPath); if (track) { track.remove(); @@ -706,25 +678,25 @@ Updater::checkAudioFiles( Stats& stats ) LMS_LOG(DBUPDATER, DEBUG) << "Checking Clusters..."; { - Wt::Dbo::Transaction transaction(_db->getSession()); + Wt::Dbo::Transaction transaction(_db.getSession()); // Now process orphan Cluster (no track) - auto genres = Cluster::getAll(_db->getSession()); - for (auto genre : genres) + auto clusters = Cluster::getAll(_db.getSession()); + for (auto cluster : clusters) { - if (genre->getTracks().size() == 0) + if (cluster->getTracks().size() == 0) { - LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan genre '" << genre->getName() << "'"; - genre.remove(); + LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan cluster '" << cluster->getName() << "'"; + cluster.remove(); } } } LMS_LOG(DBUPDATER, DEBUG) << "Checking artists..."; { - Wt::Dbo::Transaction transaction(_db->getSession()); + Wt::Dbo::Transaction transaction(_db.getSession()); - auto artists = Artist::getAllOrphans(_db->getSession()); + auto artists = Artist::getAllOrphans(_db.getSession()); for (auto artist : artists) { LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan artist '" << artist->getName() << "'"; @@ -734,9 +706,9 @@ Updater::checkAudioFiles( Stats& stats ) LMS_LOG(DBUPDATER, DEBUG) << "Checking releases..."; { - Wt::Dbo::Transaction transaction(_db->getSession()); + Wt::Dbo::Transaction transaction(_db.getSession()); - auto releases = Release::getAllOrphans(_db->getSession()); + auto releases = Release::getAllOrphans(_db.getSession()); for (auto release : releases) { LMS_LOG(DBUPDATER, DEBUG) << "Removing orphan release '" << release->getName() << "'"; @@ -748,19 +720,19 @@ Updater::checkAudioFiles( Stats& stats ) } void -Updater::checkDuplicatedAudioFiles(Stats& stats) +MediaScanner::checkDuplicatedAudioFiles(Stats& stats) { LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files"; - Wt::Dbo::Transaction transaction(_db->getSession()); + Wt::Dbo::Transaction transaction(_db.getSession()); - std::vector tracks = Database::Track::getMBIDDuplicates(_db->getSession()); + std::vector tracks = Database::Track::getMBIDDuplicates(_db.getSession()); for (Track::pointer track : tracks) { LMS_LOG(DBUPDATER, INFO) << "Found duplicated MBID [" << track->getMBID() << "], file: " << track->getPath() << " - " << track->getArtist()->getName() << " - " << track->getName(); } - tracks = Database::Track::getChecksumDuplicates(_db->getSession()); + tracks = Database::Track::getChecksumDuplicates(_db.getSession()); for (Track::pointer track : tracks) { LMS_LOG(DBUPDATER, INFO) << "Found duplicated checksum [" << bufferToString(track->getChecksum()) << "], file: " << track->getPath() << " - " << track->getArtist()->getName() << " - " << track->getName(); @@ -771,4 +743,4 @@ Updater::checkDuplicatedAudioFiles(Stats& stats) } -} // namespace Database +} // namespace Scanner diff --git a/src/database/updater/DatabaseUpdater.hpp b/src/scanner/MediaScanner.hpp similarity index 50% rename from src/database/updater/DatabaseUpdater.hpp rename to src/scanner/MediaScanner.hpp index f9945b70..fd8d1e5f 100644 --- a/src/database/updater/DatabaseUpdater.hpp +++ b/src/scanner/MediaScanner.hpp @@ -19,9 +19,6 @@ #pragma once -#include -#include - #include #include @@ -31,21 +28,14 @@ #include "database/DatabaseHandler.hpp" -namespace Database { +namespace Scanner { -class UpdaterEventHandler; - -class Updater +class MediaScanner { public: - static Updater& instance(); - - void setConnectionPool(Wt::Dbo::SqlConnectionPool& connectionPool); - - void setAudioExtensions(const std::vector& extensions); - void setVideoExtensions(const std::vector& extensions); + MediaScanner(Wt::Dbo::SqlConnectionPool& connectionPool); void start(); void stop(); @@ -64,37 +54,21 @@ class Updater std::size_t nbChanges() const { return nbAdded + nbRemoved + nbUpdated;} }; - // Emitted when the whole database has been scanned (and all the event handlers have been called) + + // Called just after track addition + Wt::Signal& addedTrack() { return _sigAddedTrack; } + + // Called just before track removal + Wt::Signal& removedTrack() { return _sigRemovedTrack; } + + // Called just after track modification + Wt::Signal& modifiedTrack() { return _sigModifiedTrack; } + + // Called just after scan complete Wt::Signal& scanComplete() { return _sigScanComplete; } - // Emitted when a track changed - // true -> added or modified, false -> to be deleted - // id of the track - // musicbrainz recordid - // path of the track - typedef Wt::Signal SigTrackChanged; - SigTrackChanged& trackChanged() { return _sigTrackChanged; } - - std::mutex& getMutex(void) { return _mutex; } - - Database::Handler& getDb(void) { return *_db; } - - bool quitRequested(void) const { return !_running;} - - void registerEventHandler(std::shared_ptr handler) { _eventHandlers.push_back(handler); } - private: - Updater(); - - struct RootDirectory - { - Database::MediaDirectory::Type type; - boost::filesystem::path path; - - RootDirectory(Database::MediaDirectory::Type t, boost::filesystem::path p) : type(t), path(p) {} - }; - // Job handling void processNextJob(); void scheduleScan(boost::posix_time::time_duration duration); @@ -103,19 +77,13 @@ class Updater // Update database (scheduled callback) void process(boost::system::error_code ec); - // Check if a file exists and is still in a root directory - static bool checkFile(const boost::filesystem::path& p, - const std::vector& rootDirectories, - const std::vector& extensions); - - - void processRootDirectory( RootDirectory rootDirectory, Stats& stats); + void processRootDirectory( boost::filesystem::path rootDirectory, Stats& stats); // Helpers Database::Artist::pointer getArtist( const boost::filesystem::path& file, const std::string& name, const std::string& MBID); Database::Release::pointer getRelease( const boost::filesystem::path& file, const std::string& name, const std::string& MBID); - std::vector getGenreClusters( const std::list& names); - void updateFileExtensions(); + std::vector getClusters( const MetaData::Clusters& names); + void refreshScanSettings(); // Audio void checkAudioFiles( Stats& stats ); @@ -128,47 +96,23 @@ class Updater bool _running; Wt::WIOService _ioService; + Wt::Signal _sigScanComplete; - Wt::Signal _sigArtistChanged; - Wt::Signal _sigReleaseChanged; - SigTrackChanged _sigTrackChanged; - std::mutex _mutex; + Wt::Signal _sigModifiedTrack; + Wt::Signal _sigAddedTrack; + Wt::Signal _sigRemovedTrack; boost::asio::deadline_timer _scheduleTimer; - Database::Handler* _db = nullptr; + Database::Handler _db; - std::vector _audioFileExtensions; - std::vector _videoFileExtensions; + // Scan settings + std::vector _fileExtensions; + std::vector _rootDirectories; MetaData::TagLibParser _metadataParser; - std::list > _eventHandlers; +}; // class MediaScanner -}; // class Updater - -// Helper to get the updater session data -static inline Wt::Dbo::Session& UpdaterDboSession() -{ - return Updater::instance().getDb().getSession(); -} - -static inline bool UpdaterQuitRequested() -{ - return Updater::instance().quitRequested(); -} - - -class UpdaterEventHandler -{ - public: - - // called when all the files have been scanned by the updater - virtual void handleFilesUpdated(void) = 0; - - private: - -}; - -} // Database +} // Scanner diff --git a/src/ui/Filters.cpp b/src/ui/Filters.cpp index 9ecdc704..efc2ae07 100644 --- a/src/ui/Filters.cpp +++ b/src/ui/Filters.cpp @@ -56,14 +56,15 @@ Filters::showDialog() { Wt::Dbo::Transaction transaction(DboSession()); - auto types = Database::Cluster::getAllTypes(DboSession()); + auto types = Database::ClusterType::getAll(DboSession()); for (auto type : types) - typeCombo->addItem(Wt::WString::fromUTF8(type)); + typeCombo->addItem(Wt::WString::fromUTF8(type->getName())); + if (!types.empty()) { - auto values = Database::Cluster::getByType(DboSession(), types.front()); + auto values = types.front()->getClusters(); for (auto value : values) { @@ -76,13 +77,15 @@ Filters::showDialog() typeCombo->changed().connect(std::bind([=] { - auto type = typeCombo->valueText().toUTF8(); + auto name = typeCombo->valueText().toUTF8(); valueCombo->clear(); Wt::Dbo::Transaction transaction(DboSession()); - auto values = Database::Cluster::getByType(DboSession(), type); + auto clusterType = Database::ClusterType::getByName(DboSession(), name); + + auto values = clusterType->getClusters(); for (auto value : values) { if (_filterIds.find(value.id()) == _filterIds.end()) @@ -108,7 +111,11 @@ Filters::showDialog() Wt::Dbo::Transaction transaction(DboSession()); - auto cluster = Database::Cluster::get(DboSession(), type, value); + auto clusterType = Database::ClusterType::getByName(DboSession(), type); + if (!clusterType) + return; + + auto cluster = clusterType->getCluster(value); if (!cluster) return; diff --git a/test/Makefile.am b/test/Makefile.am index 0df0354b..411aef1a 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -1,7 +1,7 @@ TESTS = sql-query database-user -check_PROGRAMS = sql-query database-user test-wt test-avmetadata +check_PROGRAMS = test-avmetadata database_basics_SOURCES = \ $(srcdir)/CheckDbBasics.cpp \ @@ -88,7 +88,7 @@ test_wt_audio_LDADD=$(MAGICKXX_LIBS) test_avmetadata_SOURCES = TestAvMetadata.cpp \ - $(top_srcdir)/src/logger/Logger.cpp \ + $(top_srcdir)/src/utils/Logger.cpp \ $(top_srcdir)/src/utils/Utils.cpp \ $(top_srcdir)/src/metadata/AvFormat.cpp \ $(top_srcdir)/src/metadata/TagLibParser.cpp \ diff --git a/test/TestAvMetadata.cpp b/test/TestAvMetadata.cpp index b42d53f8..c91d0ea3 100644 --- a/test/TestAvMetadata.cpp +++ b/test/TestAvMetadata.cpp @@ -26,20 +26,20 @@ int main(int argc, char *argv[]) MetaData::AvFormat avFormatParser; MetaData::TagLibParser tagLibParser; - std::vector parsers = { &avFormatParser, &tagLibParser }; + std::vector parsers = {&avFormatParser, &tagLibParser }; for (auto& parser : parsers) { - MetaData::Items items; + boost::optional items = parser->parse(argv[1]); - if (!parser->parse(argv[1], items)) + if (!items) { std::cout << "Parsing failed" << std::endl; continue; } std::cout << "Items:" << std::endl; - for (auto item : items) + for (auto item : (*items)) { switch (item.first) { @@ -55,9 +55,15 @@ int main(int argc, char *argv[]) std::cout << "Album: " << boost::any_cast(item.second) << std::endl; break; - case MetaData::Type::Genres: - for (auto& genre : boost::any_cast >(item.second)) - std::cout << "Genre: " << genre << std::endl; + case MetaData::Type::Clusters: + for (const auto& cluster : boost::any_cast(item.second)) + { + std::cout << "Cluster: " << cluster.first << std::endl; + for (const auto name : cluster.second) + { + std::cout << "\t" << name << std::endl; + } + } break; case MetaData::Type::Duration: @@ -113,6 +119,10 @@ int main(int argc, char *argv[]) std::cout << "MusicBrainzRecordingID: " << boost::any_cast(item.second) << std::endl; break; + case MetaData::Type::AcoustID: + std::cout << "AcoustID: " << boost::any_cast(item.second) << std::endl; + break; + default: break; }