From b9a770784904a0edcd7fbe8d7e9bd83144660833 Mon Sep 17 00:00:00 2001 From: emeric Date: Mon, 1 Feb 2016 20:48:58 +0100 Subject: [PATCH] [DB] Added detection for duplicated audio files (mbid or checksum) --- src/database-updater/DatabaseUpdater.cpp | 32 +++++++++++++++++++++++- src/database-updater/DatabaseUpdater.hpp | 1 + src/database/Track.cpp | 15 +++++++++++ src/database/Track.hpp | 3 +++ src/utils/Utils.cpp | 12 +++++++++ src/utils/Utils.hpp | 3 +++ 6 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/database-updater/DatabaseUpdater.cpp b/src/database-updater/DatabaseUpdater.cpp index ca0caf8f..7857d1f3 100644 --- a/src/database-updater/DatabaseUpdater.cpp +++ b/src/database-updater/DatabaseUpdater.cpp @@ -23,6 +23,7 @@ #include #include "logger/Logger.hpp" +#include "utils/Utils.hpp" #include "database/Types.hpp" @@ -244,7 +245,9 @@ Updater::process(boost::system::error_code err) LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory.path << "' DONE"; } - LMS_LOG(DBUPDATER, INFO) << "Scan complete. Changes = " << stats.nbChanges() << ", Errors = " << stats.nbScanErrors; + checkDuplicatedAudioFiles(stats); + + LMS_LOG(DBUPDATER, INFO) << "Scan complete. Changes = " << stats.nbChanges() << "(added = " << stats.nbAdded << ", nbRemoved = " << stats.nbRemoved << ", nbModified = " << stats.nbModified << "), Errors = " << stats.nbScanErrors; // Update database stats boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); @@ -394,6 +397,9 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) if (!_metadataParser.parse(file, items)) return; + std::vector checksum ; + computeCrc(file, checksum); + Wt::Dbo::Transaction transaction(_db.getSession()); Wt::Dbo::ptr track = Track::getByPath(_db.getSession(), file); @@ -502,6 +508,7 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats) assert(track); + track.modify()->setChecksum(checksum); track.modify()->setArtist(artist); track.modify()->setRelease(release); track.modify()->setLastWriteTime(lastWriteTime); @@ -719,6 +726,29 @@ Updater::checkAudioFiles( Stats& stats ) LMS_LOG(DBUPDATER, INFO) << "Check audio files done!"; } +void +Updater::checkDuplicatedAudioFiles(Stats& stats) +{ + LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files"; + + Wt::Dbo::Transaction transaction(_db.getSession()); + + std::vector tracks = Database::Track::getMBIDDuplicates(_db.getSession()); + for (Track::pointer track : tracks) + { + LMS_LOG(DBUPDATER, INFO) << "Found duplicated MBID [" << track->getMBID() << "], file: " << track->getPath() << " - " << track->getArtist()->getName() << " - " << track->getName(); + } + + tracks = Database::Track::getChecksumDuplicates(_db.getSession()); + for (Track::pointer track : tracks) + { + LMS_LOG(DBUPDATER, INFO) << "Found duplicated checksum [" << bufferToString(track->getChecksum()) << "], file: " << track->getPath() << " - " << track->getArtist()->getName() << " - " << track->getName(); + } + + + LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!"; +} + void Updater::checkVideoFiles( Stats& stats ) { diff --git a/src/database-updater/DatabaseUpdater.hpp b/src/database-updater/DatabaseUpdater.hpp index ed90ebdf..39924667 100644 --- a/src/database-updater/DatabaseUpdater.hpp +++ b/src/database-updater/DatabaseUpdater.hpp @@ -84,6 +84,7 @@ class Updater // Audio void checkAudioFiles( Stats& stats ); + void checkDuplicatedAudioFiles( Stats& stats ); void processAudioFile( const boost::filesystem::path& file, Stats& stats); // Video diff --git a/src/database/Track.cpp b/src/database/Track.cpp index 8ca5f038..f8f8a811 100644 --- a/src/database/Track.cpp +++ b/src/database/Track.cpp @@ -87,6 +87,20 @@ Track::getAllPaths(Wt::Dbo::Session& session) return std::vector(res.begin(), res.end()); } +std::vector +Track::getMBIDDuplicates(Wt::Dbo::Session& session) +{ + Wt::Dbo::collection res = session.query( "SELECT track FROM track WHERE mbid in (SELECT mbid FROM track WHERE mbid <> '' GROUP BY mbid HAVING COUNT (*) > 1)").orderBy("track.mbid"); + return std::vector(res.begin(), res.end()); +} + +std::vector +Track::getChecksumDuplicates(Wt::Dbo::Session& session) +{ + Wt::Dbo::collection res = session.query( "SELECT track FROM track WHERE checksum in (SELECT checksum FROM track WHERE Length(checksum) > 0 GROUP BY checksum HAVING COUNT(*) > 1)").orderBy("track.checksum"); + return std::vector(res.begin(), res.end()); +} + std::vector< Genre::pointer > Track::getGenres(void) const { @@ -167,6 +181,7 @@ Track::updateUIQueryModel(Wt::Dbo::Session& session, Wt::Dbo::QueryModel< UIQuer } + Genre::Genre() { } diff --git a/src/database/Track.hpp b/src/database/Track.hpp index cd96cf17..29c8dc6f 100644 --- a/src/database/Track.hpp +++ b/src/database/Track.hpp @@ -53,6 +53,7 @@ class Genre static pointer getNone(Wt::Dbo::Session& session); static std::vector getByFilter(Wt::Dbo::Session& session, SearchFilter filter, int offset = -1, int size = -1); static Wt::Dbo::collection getAll(Wt::Dbo::Session& session); + // MVC models for the user interface // Genre ID, name, track count typedef boost::tuple UIQueryResult; @@ -106,6 +107,8 @@ class Track static std::vector getByFilter(Wt::Dbo::Session& session, SearchFilter filter, int offset = -1, int size = -1); static Wt::Dbo::collection< pointer > getAll(Wt::Dbo::Session& session); static std::vector getAllPaths(Wt::Dbo::Session& session); + static std::vector getMBIDDuplicates(Wt::Dbo::Session& session); + static std::vector getChecksumDuplicates(Wt::Dbo::Session& session); // Utility fonctions // MVC models for the user interface diff --git a/src/utils/Utils.cpp b/src/utils/Utils.cpp index 517a9e4e..3f968730 100644 --- a/src/utils/Utils.cpp +++ b/src/utils/Utils.cpp @@ -118,4 +118,16 @@ stringToUTF8(const std::string& str) return boost::locale::conv::to_utf(str, "UTF-8"); } +std::string +bufferToString(const std::vector& data) +{ + std::ostringstream oss; + + for (unsigned char c : data) + { + oss << std::setw(2) << std::setfill('0') << std::hex << (int)c; + } + + return oss.str(); +} diff --git a/src/utils/Utils.hpp b/src/utils/Utils.hpp index 5c9013a7..e8c71ea6 100644 --- a/src/utils/Utils.hpp +++ b/src/utils/Utils.hpp @@ -44,6 +44,9 @@ stringTrim(const std::string& str, const std::string& whitespace = " \t"); std::string stringToUTF8(const std::string& str); +std::string +bufferToString(const std::vector& data); + template static inline bool readAs(const std::string& str, T& data) {