[DB] Better reporting database scan results (skipped, import errors)

src/database-updater/DatabaseUpdater.cpp

@@ -245,9 +245,10 @@ Updater::process(boost::system::error_code err)
 			LMS_LOG(DBUPDATER, INFO) << "Processing root directory '" << rootDirectory.path << "' DONE";
 		}
 
-		checkDuplicatedAudioFiles(stats);
+		if (_running)
+			checkDuplicatedAudioFiles(stats);
 
-		LMS_LOG(DBUPDATER, INFO) << "Scan complete. Changes = " << stats.nbChanges() << " (added = " << stats.nbAdded << ", nbRemoved = " << stats.nbRemoved << ", nbModified = " << stats.nbModified << "), Errors = " << stats.nbScanErrors;
+		LMS_LOG(DBUPDATER, INFO) << "Scan complete. Scanned = " << stats.nbScanned << ", Skipped = " << stats.nbSkipped << ", Changes = " << stats.nbChanges() << " (added = " << stats.nbAdded << ", nbRemoved = " << stats.nbRemoved << ", nbModified = " << stats.nbModified << "), Scan errors = " << stats.nbScanErrors << ", Not imported = " << stats.nbNotImported;
 
 		// Update database stats
 		boost::posix_time::ptime now = boost::posix_time::second_clock::local_time();
@@ -384,13 +385,17 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
 	boost::posix_time::ptime lastWriteTime (boost::posix_time::from_time_t( boost::filesystem::last_write_time( file ) ) );
 
 	// Skip file if last write is the same
+	// We chose not to compare
 	{
 		Wt::Dbo::Transaction transaction(_db.getSession());
 
 		Wt::Dbo::ptr<Track> track = Track::getByPath(_db.getSession(), file);
 
 		if (track && track->getLastWriteTime() == lastWriteTime)
+		{
+			stats.nbSkipped++;
 			return;
+		}
 	}
 
 	MetaData::Items items;
@@ -400,6 +405,8 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
 		return;
 	}
 
+	stats.nbScanned++;
+
 	std::vector<unsigned char> checksum ;
 	computeCrc(file, checksum);
 
@@ -421,7 +428,7 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
 			track.remove();
 			stats.nbRemoved++;
 		}
-		stats.nbScanErrors++;
+		stats.nbNotImported++;
 		return;
 	}
 	if (items.find(MetaData::Type::Duration) == items.end()
@@ -435,147 +442,147 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
 			track.remove();
 			stats.nbRemoved++;
 		}
-		stats.nbScanErrors++;
+		stats.nbNotImported++;
 		return;
 	}
 
 	// ***** Title
 	std::string title;
-	if (items.find(MetaData::Type::Title) != items.end()) {
+	if (items.find(MetaData::Type::Title) != items.end())
-			title = boost::any_cast<std::string>(items[MetaData::Type::Title]);
+	{
-		}
+		title = boost::any_cast<std::string>(items[MetaData::Type::Title]);
-		else
+	}
+	else
+	{
+		// TODO parse file name guess track etc.
+		// For now juste use file name as title
+		title = file.filename().string();
+	}
+
+	// ***** Genres
+	std::vector< Genre::pointer > genres;
+	{
+		std::list<std::string> genreList;
+
+		if (items.find(MetaData::Type::Genres) != items.end())
+			genreList = boost::any_cast< std::list<std::string> > (items[MetaData::Type::Genres]);
+
+		genres = getGenres( genreList );
+	}
+	assert( !genres.empty() );
+
+	//  ***** Artist
+	Artist::pointer artist;
+	{
+		std::string artistName;
+		std::string artistMusicBrainzID;
+
+		if (items.find(MetaData::Type::MusicBrainzArtistID) != items.end())
+			artistMusicBrainzID = boost::any_cast<std::string>(items[MetaData::Type::MusicBrainzArtistID] );
+
+		if (items.find(MetaData::Type::Artist) != items.end())
+			artistName = boost::any_cast<std::string>(items[MetaData::Type::Artist]);
+
+		artist = getArtist(file, artistName, artistMusicBrainzID);
+	}
+	assert(artist);
+
+	//  ***** Release
+	Release::pointer release;
+	{
+		std::string releaseName;
+		std::string releaseMusicBrainzID;
+
+		if (items.find(MetaData::Type::MusicBrainzAlbumID) != items.end())
+			releaseMusicBrainzID = boost::any_cast<std::string>(items[MetaData::Type::MusicBrainzAlbumID] );
+
+		if (items.find(MetaData::Type::Album) != items.end())
+			releaseName = boost::any_cast<std::string>(items[MetaData::Type::Album]);
+
+		release = getRelease(file, releaseName, releaseMusicBrainzID);
+	}
+	assert(release);
+
+	// If file already exist, update data
+	// Otherwise, create it
+	if (!track)
+	{
+		// Create a new song
+		track = Track::create(_db.getSession(), file);
+		LMS_LOG(DBUPDATER, INFO) << "Adding '" << file << "'";
+		stats.nbAdded++;
+	}
+	else
+	{
+		LMS_LOG(DBUPDATER, INFO) << "Updating '" << file << "'";
+		stats.nbModified++;
+	}
+
+	assert(track);
+
+	track.modify()->setChecksum(checksum);
+	track.modify()->setArtist(artist);
+	track.modify()->setRelease(release);
+	track.modify()->setLastWriteTime(lastWriteTime);
+	track.modify()->setName(title);
+	track.modify()->setDuration( boost::any_cast<boost::posix_time::time_duration>(items[MetaData::Type::Duration]) );
+	track.modify()->setAddedTime( boost::posix_time::second_clock::local_time() );
+
+	{
+		std::string trackGenreList;
+		// Product genre list
+		for (Genre::pointer genre : genres)
 		{
-			// TODO parse file name guess track etc.
+			if (!trackGenreList.empty())
-			// For now juste use file name as title
+				trackGenreList += ", ";
-			title = file.filename().string();
+			trackGenreList += genre->getName();
 		}
 
+		track.modify()->setGenres( trackGenreList );
+	}
+	track.modify()->setGenres( genres );
 
-		// ***** Genres
+	if (items.find(MetaData::Type::TrackNumber) != items.end())
-		std::vector< Genre::pointer > genres;
+		track.modify()->setTrackNumber( boost::any_cast<std::size_t>(items[MetaData::Type::TrackNumber]) );
-		{
-			std::list<std::string> genreList;
 
-			if (items.find(MetaData::Type::Genres) != items.end())
+	if (items.find(MetaData::Type::TotalTrack) != items.end())
-				genreList = boost::any_cast< std::list<std::string> > (items[MetaData::Type::Genres]);
+		track.modify()->setTotalTrackNumber( boost::any_cast<std::size_t>(items[MetaData::Type::TotalTrack]) );
 
-			genres = getGenres( genreList );
+	if (items.find(MetaData::Type::DiscNumber) != items.end())
-		}
+		track.modify()->setDiscNumber( boost::any_cast<std::size_t>(items[MetaData::Type::DiscNumber]) );
-		assert( !genres.empty() );
 
-		//  ***** Artist
+	if (items.find(MetaData::Type::TotalDisc) != items.end())
-		Artist::pointer artist;
+		track.modify()->setTotalDiscNumber( boost::any_cast<std::size_t>(items[MetaData::Type::TotalDisc]) );
-		{
-			std::string artistName;
-			std::string artistMusicBrainzID;
 
-			if (items.find(MetaData::Type::MusicBrainzArtistID) != items.end())
+	if (items.find(MetaData::Type::Date) != items.end())
-				artistMusicBrainzID = boost::any_cast<std::string>(items[MetaData::Type::MusicBrainzArtistID] );
+		track.modify()->setDate( boost::any_cast<boost::posix_time::ptime>(items[MetaData::Type::Date]) );
 
-			if (items.find(MetaData::Type::Artist) != items.end())
+	if (items.find(MetaData::Type::OriginalDate) != items.end())
-				artistName = boost::any_cast<std::string>(items[MetaData::Type::Artist]);
+	{
+		track.modify()->setOriginalDate( boost::any_cast<boost::posix_time::ptime>(items[MetaData::Type::OriginalDate]) );
 
-			artist = getArtist(file, artistName, artistMusicBrainzID);
+		// If a file has an OriginalDate but no date, set the date to ease filtering
-		}
+		if (items.find(MetaData::Type::Date) == items.end())
-		assert(artist);
+			track.modify()->setDate( boost::any_cast<boost::posix_time::ptime>(items[MetaData::Type::OriginalDate]) );
+	}
 
-		//  ***** Release
+	if (items.find(MetaData::Type::MusicBrainzTrackID) != items.end())
-		Release::pointer release;
+	{
-		{
+		track.modify()->setMBID( boost::any_cast<std::string>(items[MetaData::Type::MusicBrainzTrackID]) );
-			std::string releaseName;
+	}
-			std::string releaseMusicBrainzID;
 
-			if (items.find(MetaData::Type::MusicBrainzAlbumID) != items.end())
+	if (items.find(MetaData::Type::HasCover) != items.end())
-				releaseMusicBrainzID = boost::any_cast<std::string>(items[MetaData::Type::MusicBrainzAlbumID] );
+	{
+		bool hasCover = boost::any_cast<bool>(items[MetaData::Type::HasCover]);
 
-			if (items.find(MetaData::Type::Album) != items.end())
+		track.modify()->setCoverType( hasCover ? Track::CoverType::Embedded : Track::CoverType::None );
-				releaseName = boost::any_cast<std::string>(items[MetaData::Type::Album]);
+	}
 
-			release = getRelease(file, releaseName, releaseMusicBrainzID);
+	transaction.commit();
-		}
-		assert(release);
-
-		// If file already exist, update data
-		// Otherwise, create it
-		if (!track)
-		{
-			// Create a new song
-			track = Track::create(_db.getSession(), file);
-			LMS_LOG(DBUPDATER, INFO) << "Adding '" << file << "'";
-			stats.nbAdded++;
-		}
-		else
-		{
-			LMS_LOG(DBUPDATER, INFO) << "Updating '" << file << "'";
-			stats.nbModified++;
-		}
-
-		assert(track);
-
-		track.modify()->setChecksum(checksum);
-		track.modify()->setArtist(artist);
-		track.modify()->setRelease(release);
-		track.modify()->setLastWriteTime(lastWriteTime);
-		track.modify()->setName(title);
-		track.modify()->setDuration( boost::any_cast<boost::posix_time::time_duration>(items[MetaData::Type::Duration]) );
-		track.modify()->setAddedTime( boost::posix_time::second_clock::local_time() );
-
-		{
-			std::string trackGenreList;
-			// Product genre list
-			for (Genre::pointer genre : genres)
-			{
-				if (!trackGenreList.empty())
-					trackGenreList += ", ";
-				trackGenreList += genre->getName();
-			}
-
-			track.modify()->setGenres( trackGenreList );
-		}
-		track.modify()->setGenres( genres );
-
-		if (items.find(MetaData::Type::TrackNumber) != items.end())
-			track.modify()->setTrackNumber( boost::any_cast<std::size_t>(items[MetaData::Type::TrackNumber]) );
-
-		if (items.find(MetaData::Type::TotalTrack) != items.end())
-			track.modify()->setTotalTrackNumber( boost::any_cast<std::size_t>(items[MetaData::Type::TotalTrack]) );
-
-		if (items.find(MetaData::Type::DiscNumber) != items.end())
-			track.modify()->setDiscNumber( boost::any_cast<std::size_t>(items[MetaData::Type::DiscNumber]) );
-
-		if (items.find(MetaData::Type::TotalDisc) != items.end())
-			track.modify()->setTotalDiscNumber( boost::any_cast<std::size_t>(items[MetaData::Type::TotalDisc]) );
-
-		if (items.find(MetaData::Type::Date) != items.end())
-			track.modify()->setDate( boost::any_cast<boost::posix_time::ptime>(items[MetaData::Type::Date]) );
-
-		if (items.find(MetaData::Type::OriginalDate) != items.end())
-		{
-			track.modify()->setOriginalDate( boost::any_cast<boost::posix_time::ptime>(items[MetaData::Type::OriginalDate]) );
-
-			// If a file has an OriginalDate but no date, set the date to ease filtering
-			if (items.find(MetaData::Type::Date) == items.end())
-				track.modify()->setDate( boost::any_cast<boost::posix_time::ptime>(items[MetaData::Type::OriginalDate]) );
-		}
-
-		if (items.find(MetaData::Type::MusicBrainzTrackID) != items.end())
-		{
-			track.modify()->setMBID( boost::any_cast<std::string>(items[MetaData::Type::MusicBrainzTrackID]) );
-		}
-
-		if (items.find(MetaData::Type::HasCover) != items.end())
-		{
-			bool hasCover = boost::any_cast<bool>(items[MetaData::Type::HasCover]);
-
-			track.modify()->setCoverType( hasCover ? Track::CoverType::Embedded : Track::CoverType::None );
-		}
-
-		transaction.commit();
 }
 
 
-void
+	void
 Updater::processRootDirectory(RootDirectory rootDirectory, Stats& stats)
 {
 
@@ -781,80 +788,74 @@ Updater::checkVideoFiles( Stats& stats )
 
 	LMS_LOG(DBUPDATER, DEBUG) << "Check video files done!";
 }
+
 void
 Updater::processVideoFile( const boost::filesystem::path& file, Stats& stats)
 {
-	try {
+	// Check last update time
+	boost::posix_time::ptime lastWriteTime (boost::posix_time::from_time_t( boost::filesystem::last_write_time( file ) ) );
 
-		// Check last update time
+	Wt::Dbo::Transaction transaction(_db.getSession());
-	        boost::posix_time::ptime lastWriteTime (boost::posix_time::from_time_t( boost::filesystem::last_write_time( file ) ) );
 
-		Wt::Dbo::Transaction transaction(_db.getSession());
+	// Skip file if last write is the same
+	Wt::Dbo::ptr<Video> video = Video::getByPath(_db.getSession(), file);
+	if (video && video->getLastWriteTime() == lastWriteTime)
+		return;
 
-		// Skip file if last write is the same
+	MetaData::Items items;
-		Wt::Dbo::ptr<Video> video = Video::getByPath(_db.getSession(), file);
+	if (!_metadataParser.parse(file, items))
-		if (video && video->getLastWriteTime() == lastWriteTime)
+		return;
-			return;
 
-		MetaData::Items items;
+	// We estimate this is a video if:
-		_metadataParser.parse(file, items);
+	// - we found a least one video stream
-
+	// - the duration is not null
-		// We estimate this is a video if:
+	if (items.find(MetaData::Type::VideoStreams) == items.end()
-		// - we found a least one video stream
+			|| boost::any_cast<std::vector<MetaData::VideoStream> >(items[MetaData::Type::VideoStreams]).empty())
-		// - the duration is not null
-		if (items.find(MetaData::Type::VideoStreams) == items.end()
-		|| boost::any_cast<std::vector<MetaData::VideoStream> >(items[MetaData::Type::VideoStreams]).empty())
-		{
-			LMS_LOG(DBUPDATER, ERROR) << "Skipped '" << file << "' (no video stream found)";
-
-			// If the video exists here, delete it!
-			if (video) {
-				video.remove();
-				stats.nbRemoved++;
-			}
-			return;
-		}
-		if (items.find(MetaData::Type::Duration) == items.end()
-		|| boost::any_cast<boost::posix_time::time_duration>(items[MetaData::Type::Duration]).total_seconds() == 0)
-		{
-			LMS_LOG(DBUPDATER, ERROR) << "Skipped '" << file << "' (no duration or duration 0)";
-
-			// If Track exists here, delete it!
-			if (video) {
-				video.remove();
-				stats.nbRemoved++;
-			}
-			return;
-		}
-
-		// If video already exist, update data
-		// Otherwise, create it
-		// Today we are very aggressive, but we could also guess names from path, etc.
-		if (!video)
-		{
-			video = Video::create(_db.getSession(), file);
-			LMS_LOG(DBUPDATER, DEBUG) << "Adding '" << file << "'";
-			stats.nbAdded++;
-		}
-		else
-		{
-			LMS_LOG(DBUPDATER, DEBUG) << "Updating '" << file << "'";
-			stats.nbModified++;
-		}
-
-		assert(video);
-
-		video.modify()->setName( file.filename().string() );
-		video.modify()->setDuration( boost::any_cast<boost::posix_time::time_duration>(items[MetaData::Type::Duration]) );
-		video.modify()->setLastWriteTime(lastWriteTime);
-
-		transaction.commit();
-	}
-	catch( std::exception& e )
 	{
-		LMS_LOG(DBUPDATER, ERROR) << "Exception while parsing video file : '" << file << "': '" << e.what() << "' => skipping!";
+		LMS_LOG(DBUPDATER, ERROR) << "Skipped '" << file << "' (no video stream found)";
-		stats.nbScanErrors++;
+
+		// If the video exists here, delete it!
+		if (video) {
+			video.remove();
+			stats.nbRemoved++;
+		}
+		return;
 	}
+	if (items.find(MetaData::Type::Duration) == items.end()
+			|| boost::any_cast<boost::posix_time::time_duration>(items[MetaData::Type::Duration]).total_seconds() == 0)
+	{
+		LMS_LOG(DBUPDATER, ERROR) << "Skipped '" << file << "' (no duration or duration 0)";
+
+		// If Track exists here, delete it!
+		if (video) {
+			video.remove();
+			stats.nbRemoved++;
+		}
+		return;
+	}
+
+	// If video already exist, update data
+	// Otherwise, create it
+	// Today we are very aggressive, but we could also guess names from path, etc.
+	if (!video)
+	{
+		video = Video::create(_db.getSession(), file);
+		LMS_LOG(DBUPDATER, DEBUG) << "Adding '" << file << "'";
+		stats.nbAdded++;
+	}
+	else
+	{
+		LMS_LOG(DBUPDATER, DEBUG) << "Updating '" << file << "'";
+		stats.nbModified++;
+	}
+
+	assert(video);
+
+	video.modify()->setName( file.filename().string() );
+	video.modify()->setDuration( boost::any_cast<boost::posix_time::time_duration>(items[MetaData::Type::Duration]) );
+	video.modify()->setLastWriteTime(lastWriteTime);
+
+	transaction.commit();
 }
 
 } // namespace DatabaseUpdater

src/database-updater/DatabaseUpdater.hpp

@@ -44,10 +44,13 @@ class Updater
 
 		struct Stats
 		{
+			std::size_t	nbSkipped = 0;		// no change since last scan
+			std::size_t	nbScanned = 0;
+			std::size_t	nbScanErrors = 0;	// cannot scan file
+			std::size_t	nbNotImported = 0;	// Not imported (criteria not filled)
 			std::size_t	nbAdded = 0;
 			std::size_t	nbRemoved = 0;
 			std::size_t	nbModified = 0;
-			std::size_t	nbScanErrors = 0;
 
 			std::size_t nbChanges() const { return nbAdded + nbRemoved + nbModified;}
 		};