[DB] Added detection for duplicated audio files (mbid or checksum)

src/database-updater/DatabaseUpdater.cpp

@@ -23,6 +23,7 @@
 #include <boost/asio/placeholders.hpp>
 
 #include "logger/Logger.hpp"
+#include "utils/Utils.hpp"
 
 #include "database/Types.hpp"
 
@@ -244,7 +245,9 @@ Updater::process(boost::system::error_code err)
 			LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory.path << "' DONE";
 		}
 
-		LMS_LOG(DBUPDATER, INFO) << "Scan complete. Changes = " << stats.nbChanges() << ", Errors = " << stats.nbScanErrors;
+		checkDuplicatedAudioFiles(stats);
+
+		LMS_LOG(DBUPDATER, INFO) << "Scan complete. Changes = " << stats.nbChanges() << "(added = " << stats.nbAdded << ", nbRemoved = " << stats.nbRemoved << ", nbModified = " << stats.nbModified << "), Errors = " << stats.nbScanErrors;
 
 		// Update database stats
 		boost::posix_time::ptime now = boost::posix_time::second_clock::local_time();
@@ -394,6 +397,9 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
 	if (!_metadataParser.parse(file, items))
 		return;
 
+	std::vector<unsigned char> checksum ;
+	computeCrc(file, checksum);
+
 	Wt::Dbo::Transaction transaction(_db.getSession());
 
 	Wt::Dbo::ptr<Track> track = Track::getByPath(_db.getSession(), file);
@@ -502,6 +508,7 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
 
 		assert(track);
 
+		track.modify()->setChecksum(checksum);
 		track.modify()->setArtist(artist);
 		track.modify()->setRelease(release);
 		track.modify()->setLastWriteTime(lastWriteTime);
@@ -719,6 +726,29 @@ Updater::checkAudioFiles( Stats& stats )
 	LMS_LOG(DBUPDATER, INFO) << "Check audio files done!";
 }
 
+void
+Updater::checkDuplicatedAudioFiles(Stats& stats)
+{
+	LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files";
+
+	Wt::Dbo::Transaction transaction(_db.getSession());
+
+	std::vector<Track::pointer> tracks = Database::Track::getMBIDDuplicates(_db.getSession());
+	for (Track::pointer track : tracks)
+	{
+		LMS_LOG(DBUPDATER, INFO) << "Found duplicated MBID [" << track->getMBID() << "], file: " << track->getPath() << " - " << track->getArtist()->getName() << " - " << track->getName();
+	}
+
+	tracks = Database::Track::getChecksumDuplicates(_db.getSession());
+	for (Track::pointer track : tracks)
+	{
+		LMS_LOG(DBUPDATER, INFO) << "Found duplicated checksum [" << bufferToString(track->getChecksum()) << "], file: " << track->getPath() << " - " << track->getArtist()->getName() << " - " << track->getName();
+	}
+
+
+	LMS_LOG(DBUPDATER, INFO) << "Checking duplicated audio files done!";
+}
+
 void
 Updater::checkVideoFiles( Stats& stats )
 {

src/database-updater/DatabaseUpdater.hpp

@@ -84,6 +84,7 @@ class Updater
 
 		// Audio
 		void checkAudioFiles( Stats& stats );
+		void checkDuplicatedAudioFiles( Stats& stats );
 		void processAudioFile( const boost::filesystem::path& file, Stats& stats);
 
 		// Video

src/database/Track.cpp

@@ -87,6 +87,20 @@ Track::getAllPaths(Wt::Dbo::Session& session)
 	return std::vector<boost::filesystem::path>(res.begin(), res.end());
 }
 
+std::vector<Track::pointer>
+Track::getMBIDDuplicates(Wt::Dbo::Session& session)
+{
+	Wt::Dbo::collection<pointer> res = session.query<pointer>( "SELECT track FROM track WHERE mbid in (SELECT mbid FROM track WHERE mbid <> '' GROUP BY mbid HAVING COUNT (*) > 1)").orderBy("track.mbid");
+	return std::vector<pointer>(res.begin(), res.end());
+}
+
+std::vector<Track::pointer>
+Track::getChecksumDuplicates(Wt::Dbo::Session& session)
+{
+	Wt::Dbo::collection<pointer> res = session.query<pointer>( "SELECT track FROM track WHERE checksum in (SELECT checksum FROM track WHERE Length(checksum) > 0 GROUP BY checksum HAVING COUNT(*) > 1)").orderBy("track.checksum");
+	return std::vector<pointer>(res.begin(), res.end());
+}
+
 std::vector< Genre::pointer >
 Track::getGenres(void) const
 {
@@ -167,6 +181,7 @@ Track::updateUIQueryModel(Wt::Dbo::Session& session, Wt::Dbo::QueryModel< UIQuer
 
 }
 
+
 Genre::Genre()
 {
 }

src/database/Track.hpp

@@ -53,6 +53,7 @@ class Genre
 		static pointer getNone(Wt::Dbo::Session& session);
 		static std::vector<pointer> getByFilter(Wt::Dbo::Session& session, SearchFilter filter, int offset = -1, int size = -1);
 		static Wt::Dbo::collection<Genre::pointer> getAll(Wt::Dbo::Session& session);
+
 		// MVC models for the user interface
 		// Genre ID, name, track count
 		typedef boost::tuple<id_type, std::string, int> UIQueryResult;
@@ -106,6 +107,8 @@ class Track
 		static std::vector<pointer> 	getByFilter(Wt::Dbo::Session& session, SearchFilter filter, int offset = -1, int size = -1);
 		static Wt::Dbo::collection< pointer > getAll(Wt::Dbo::Session& session);
 		static std::vector<boost::filesystem::path> getAllPaths(Wt::Dbo::Session& session);
+		static std::vector<pointer> getMBIDDuplicates(Wt::Dbo::Session& session);
+		static std::vector<pointer> getChecksumDuplicates(Wt::Dbo::Session& session);
 
 		// Utility fonctions
 		// MVC models for the user interface

src/utils/Utils.cpp

@@ -118,4 +118,16 @@ stringToUTF8(const std::string& str)
 	return boost::locale::conv::to_utf<char>(str, "UTF-8");
 }
 
+std::string
+bufferToString(const std::vector<unsigned char>& data)
+{
+	std::ostringstream oss;
+
+	for (unsigned char c : data)
+	{
+		oss << std::setw(2) << std::setfill('0') << std::hex << (int)c;
+	}
+
+	return oss.str();
+}
 

src/utils/Utils.hpp

@@ -44,6 +44,9 @@ stringTrim(const std::string& str, const std::string& whitespace = " \t");
 std::string
 stringToUTF8(const std::string& str);
 
+std::string
+bufferToString(const std::vector<unsigned char>& data);
+
 template<typename T>
 static inline bool readAs(const std::string& str, T& data)
 {