[DB] Faster scan thanks to index on file path + stats improvements + bugfix

src/database/DatabaseHandler.cpp

@@ -110,6 +110,7 @@ Handler::Handler(Wt::Dbo::SqlConnectionPool& connectionPool)
 
 		// Indexes
 		_session.execute("PRAGMA journal_mode=WAL");
+		_session.execute("CREATE INDEX IF NOT EXISTS track_path_idx ON track(file_path)");
 		_session.execute("CREATE INDEX IF NOT EXISTS artist_name_idx ON artist(name)");
 		_session.execute("CREATE INDEX IF NOT EXISTS release_name_idx ON release(name)");
 		_session.execute("CREATE INDEX IF NOT EXISTS track_artist_idx ON track(artist_id)");

src/database/updater/DatabaseHighLevelCluster.cpp

@@ -142,19 +142,16 @@ static std::list<std::string> getClustersFromFeature(Feature::Type& feature, dou
 
 		if (!probability || !value)
 		{
-			LMS_LOG(DBUPDATER, DEBUG) << "Missing " << node.node;
+			LMS_LOG(DBUPDATER, ERROR) << "Missing " << node.node;
 			continue;
 		}
 
-		if (std::stod(probability->data()) < 0.90)
+		if (std::stod(probability->data()) < minProb)
-		{
-			LMS_LOG(DBUPDATER, DEBUG) << "Probability too low for " << node.node << "(" << std::stod(probability->data()) << ")";
 			continue;
-		}
 
 		if (node.valueMapping[value->data()] == "")
 		{
-			LMS_LOG(DBUPDATER, DEBUG) << "Unknown value '" << value->data() << "'";
+			LMS_LOG(DBUPDATER, ERROR) << "Unknown value '" << value->data() << "'";
 			continue;
 		}
 

src/database/updater/DatabaseUpdater.cpp

@@ -263,10 +263,11 @@ Updater::process(boost::system::error_code err)
 		checkDuplicatedAudioFiles(stats);
 
 		LMS_LOG(DBUPDATER, INFO) << "Processed all files, now calling listeners...";
-		scanComplete().emit(stats);
+		for (auto eventHandler : _eventHandlers)
+			eventHandler->handleFilesUpdated();
 	}
 
-	LMS_LOG(DBUPDATER, INFO) << "Scan complete. Scanned = " << stats.nbScanned << ", Skipped = " << stats.nbSkipped << ", Changes = " << stats.nbChanges() << " (added = " << stats.nbAdded << ", nbRemoved = " << stats.nbRemoved << ", nbModified = " << stats.nbModified << "), Scan errors = " << stats.nbScanErrors << ", Not imported = " << stats.nbNotImported;
+	LMS_LOG(DBUPDATER, INFO) << "Scan " << (_running ? "complete" : "aborted") << ". Changes = " << stats.nbChanges() << " (added = " << stats.nbAdded << ", removed = " << stats.nbRemoved << ", updated = " << stats.nbUpdated << "), Not changed = " << stats.nbNoChange << ", Scanned = " << stats.nbScanned << " (errors = " << stats.nbScanErrors << ", not imported = " << stats.nbNotImported << ")";
 
 	// Update database stats
 	boost::posix_time::ptime now = boost::posix_time::second_clock::local_time();
@@ -280,6 +281,8 @@ Updater::process(boost::system::error_code err)
 		Setting::setBool(_db->getSession(), "manual_scan_requested", false);
 
 		processNextJob();
+
+		scanComplete().emit(stats);
 	}
 }
 
@@ -404,12 +407,9 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
 
 		if (track && track->getLastWriteTime() == lastWriteTime)
 		{
-			stats.nbSkipped++;
+			stats.nbNoChange++;
-			transaction.rollback();
 			return;
 		}
-
-		transaction.rollback();
 	}
 
 	MetaData::Items items;
@@ -535,7 +535,7 @@ Updater::processAudioFile( const boost::filesystem::path& file, Stats& stats)
 		for (auto cluster : track->getClusters())
 			cluster.remove();
 
-		stats.nbModified++;
+		stats.nbUpdated++;
 	}
 
 	assert(track);
@@ -868,7 +868,7 @@ Updater::processVideoFile( const boost::filesystem::path& file, Stats& stats)
 	else
 	{
 		LMS_LOG(DBUPDATER, DEBUG) << "Updating '" << file << "'";
-		stats.nbModified++;
+		stats.nbUpdated++;
 	}
 
 	assert(video);

src/database/updater/DatabaseUpdater.hpp

@@ -53,15 +53,15 @@ class Updater
 
 		struct Stats
 		{
-			std::size_t	nbSkipped = 0;		// no change since last scan
+			std::size_t	nbNoChange = 0;		// no change since last scan
-			std::size_t	nbScanned = 0;
+			std::size_t	nbScanned = 0;		// total scanned filed
 			std::size_t	nbScanErrors = 0;	// cannot scan file
-			std::size_t	nbNotImported = 0;	// Not imported (criteria not filled)
+			std::size_t	nbNotImported = 0;	// Scanned, but not imported (criteria not filled)
-			std::size_t	nbAdded = 0;
+			std::size_t	nbAdded = 0;		// Added in DB
-			std::size_t	nbRemoved = 0;
+			std::size_t	nbRemoved = 0;		// removed from DB
-			std::size_t	nbModified = 0;
+			std::size_t	nbUpdated = 0;		// updated file in DB
 
-			std::size_t nbChanges() const { return nbAdded + nbRemoved + nbModified;}
+			std::size_t nbChanges() const { return nbAdded + nbRemoved + nbUpdated;}
 		};
 
 		// Emitted when the whole database has been scanned (and all the event handlers have been called)