Migrated scrobbling stuff

src/libs/services/scanner/CMakeLists.txt

@@ -0,0 +1,29 @@
+
+add_library(lmsscanner SHARED
+	impl/AcousticBrainzUtils.cpp
+	impl/Scanner.cpp
+	impl/ScannerStats.cpp
+	)
+
+target_include_directories(lmsscanner INTERFACE
+	include
+	)
+
+target_include_directories(lmsscanner PRIVATE
+	include
+	)
+
+target_link_libraries(lmsscanner PRIVATE
+	lmsdatabase
+	lmsmetadata
+	lmsrecommendation
+	lmsutils
+	)
+
+target_link_libraries(lmsscanner PUBLIC
+	std::filesystem
+	Wt::Wt
+	)
+
+install(TARGETS lmsscanner DESTINATION lib)
+

src/libs/services/scanner/impl/AcousticBrainzUtils.cpp

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "AcousticBrainzUtils.hpp"
+
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/json_parser.hpp>
+
+#include <Wt/WIOService.h>
+#include <Wt/Http/Client.h>
+
+#include "utils/IConfig.hpp"
+#include "utils/Logger.hpp"
+#include "utils/Service.hpp"
+#include "utils/UUID.hpp"
+
+
+namespace AcousticBrainz
+{
+
+static
+std::string
+getJsonData(const UUID& mbid)
+{
+	static constexpr std::string_view defaultAPIURL {"https://acousticbrainz.org"};
+
+	const std::string url {std::string {Service<IConfig>::get()->getString("acousticbrainz-api-base-url", defaultAPIURL)} + "/api/v1/" + std::string {mbid.getAsString()} + "/low-level"};
+
+	boost::asio::io_service ioService;
+
+	Wt::Http::Client client {ioService};
+	client.setFollowRedirect(true);
+	client.setSslCertificateVerificationEnabled(true);
+	client.setMaximumResponseSize(256*1024);
+
+	if (!client.get(url))
+	{
+		LMS_LOG(DBUPDATER, ERROR) << "Cannot perform a GET request to url '" << url << "'";
+		return {};
+	}
+
+	std::string response;
+	client.done().connect([&](Wt::AsioWrapper::error_code ec, const Wt::Http::Message &msg)
+	{
+		if (ec)
+		{
+			LMS_LOG(DBUPDATER, ERROR) << "GET request to url '" << url << "' failed: " << ec.message();
+			return;
+		}
+
+		if (msg.status() != 200)
+		{
+			LMS_LOG(DBUPDATER, ERROR) << "GET request to url '" << url << "' failed: status = " << msg.status() << ", body = " << msg.body();
+			return;
+		}
+
+		response = msg.body();
+	});
+
+	ioService.run();
+
+	return response;
+}
+
+std::string
+extractLowLevelFeatures(const UUID& recordingMBID)
+{
+	return getJsonData(recordingMBID);
+}
+
+} // namespace Scanner::AcousticBrainz

src/libs/services/scanner/impl/AcousticBrainzUtils.hpp

@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2018 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <string>
+
+class UUID;
+
+namespace AcousticBrainz
+{
+	std::string extractLowLevelFeatures(const UUID& recordingMBID);
+}
+

src/libs/services/scanner/impl/Scanner.hpp

@@ -0,0 +1,122 @@
+/*
+ * Copyright (C) 2013 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <chrono>
+#include <shared_mutex>
+#include <optional>
+#include <unordered_set>
+
+#include <Wt/WDateTime.h>
+#include <Wt/WIOService.h>
+#include <Wt/WSignal.h>
+
+#include <boost/asio/system_timer.hpp>
+
+#include "database/Types.hpp"
+#include "database/ScanSettings.hpp"
+#include "database/Session.hpp"
+#include "metadata/IParser.hpp"
+#include "scanner/IScanner.hpp"
+#include "utils/Path.hpp"
+
+class UUID;
+
+namespace Recommendation
+{
+	class IEngine;
+}
+
+namespace Scanner {
+
+class Scanner : public IScanner
+{
+	public:
+		Scanner(Database::Db& db, Recommendation::IEngine& recommendationEngine);
+		~Scanner();
+
+		Scanner(const Scanner&) = delete;
+		Scanner(Scanner&&) = delete;
+		Scanner& operator=(const Scanner&) = delete;
+		Scanner& operator=(Scanner&&) = delete;
+
+		void requestReload() override;
+		void requestImmediateScan(bool force) override;
+
+		Status	getStatus() const override;
+		Events&	getEvents() override { return _events; }
+
+	private:
+		void start();
+		void stop();
+
+		// Job handling
+		void scheduleNextScan();
+		void scheduleScan(bool force, const Wt::WDateTime& dateTime = {});
+
+		void abortScan();
+
+		// Update database (scheduled callback)
+		void scan(bool force);
+
+		void scanMediaDirectory( const std::filesystem::path& mediaDirectory, bool forceScan, ScanStats& stats);
+		bool fetchTrackFeatures(Database::TrackId trackId, const UUID& MBID);
+		void fetchTrackFeatures(ScanStats& stats);
+
+		// Helpers
+		void refreshScanSettings();
+
+		void countAllFiles(ScanStats& stats);
+		void removeMissingTracks(ScanStats& stats);
+		void removeOrphanEntries();
+		void checkDuplicatedAudioFiles(ScanStats& stats);
+		void scanAudioFile(const std::filesystem::path& file, bool forceScan, ScanStats& stats);
+		void notifyInProgressIfNeeded(const ScanStepStats& stats);
+		void notifyInProgress(const ScanStepStats& stats);
+		void reloadSimilarityEngine(ScanStats& stats);
+
+		Recommendation::IEngine&				_recommendationEngine;
+
+		std::mutex								_controlMutex;
+		std::atomic<bool>						_abortScan {};
+		Wt::WIOService							_ioService;
+		boost::asio::system_timer				_scheduleTimer {_ioService};
+		Events									_events;
+		std::chrono::system_clock::time_point	_lastScanInProgressEmit {};
+		Database::Session						_dbSession;
+		std::unique_ptr<MetaData::IParser>		_metadataParser;
+
+		mutable std::shared_mutex			_statusMutex;
+		State								_curState {State::NotScheduled};
+		std::optional<ScanStats> 			_lastCompleteScanStats;
+		std::optional<ScanStepStats> 		_currentScanStepStats;
+		Wt::WDateTime						_nextScheduledScan;
+
+		// Current scan settings
+		std::size_t				_scanVersion {};
+		Wt::WTime				_startTime;
+		Database::ScanSettings::UpdatePeriod 	_updatePeriod {Database::ScanSettings::UpdatePeriod::Never};
+		std::unordered_set<std::filesystem::path>		_fileExtensions;
+		std::filesystem::path					_mediaDirectory;
+		Database::ScanSettings::RecommendationEngineType _recommendationEngineType;
+};
+
+} // Scanner
+

src/libs/services/scanner/impl/ScannerStats.cpp

@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "scanner/ScannerStats.hpp"
+
+namespace Scanner {
+
+ScanError::ScanError(const std::filesystem::path& _file, ScanErrorType _error, const std::string& _systemError)
+: file {_file},
+error {_error},
+systemError {_systemError}
+{
+}
+
+std::size_t
+ScanStats::nbFiles() const
+{
+	return skips + additions + updates;
+}
+
+std::size_t
+ScanStats::nbChanges() const
+{
+	return additions + deletions + updates;
+}
+
+unsigned
+ScanStepStats::progress() const
+{
+	return (processedElems / static_cast<float>(totalElems ? totalElems : 1)) * 100;
+}
+
+} // namespace Scanner
+

src/libs/services/scanner/include/scanner/IScanner.hpp

@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2013 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <optional>
+
+#include "ScannerEvents.hpp"
+#include "ScannerStats.hpp"
+
+namespace Database
+{
+	class Db;
+}
+
+namespace Recommendation
+{
+	class IEngine;
+}
+
+namespace Scanner
+{
+
+	class IScanner
+	{
+		public:
+			virtual ~IScanner() = default;
+
+			// Async requests
+			virtual void requestReload() = 0;
+			virtual void requestImmediateScan(bool force) = 0;
+
+			enum class State
+			{
+				NotScheduled,
+				Scheduled,
+				InProgress,
+			};
+
+			struct Status
+			{
+				State								currentState {State::NotScheduled};
+				Wt::WDateTime						nextScheduledScan;
+				std::optional<ScanStats>			lastCompleteScanStats;
+				std::optional<ScanStepStats> 		currentScanStepStats;
+			};
+
+			virtual Status getStatus() const = 0;
+
+			virtual Events& getEvents() = 0;
+	};
+
+	std::unique_ptr<IScanner> createScanner(Database::Db& db, Recommendation::IEngine& recommendationEngine);
+
+} // Scanner
+

src/libs/services/scanner/include/scanner/ScannerEvents.hpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2020 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/WDateTime.h>
+#include <Wt/WSignal.h>
+
+#include "ScannerStats.hpp"
+
+namespace Scanner
+{
+
+	struct Events
+	{
+		// Called just after scan start
+		Wt::Signal<> 				scanStarted;
+
+		// Called just after scan complete (true if changes have been made)
+		Wt::Signal<ScanStats>		scanComplete;
+
+		// Called during scan in progress
+		Wt::Signal<ScanStepStats>	scanInProgress;
+
+		// Called after a schedule
+		Wt::Signal<Wt::WDateTime>	scanScheduled;
+	};
+
+} // ns Scanner
+

src/libs/services/scanner/include/scanner/ScannerStats.hpp

@@ -0,0 +1,107 @@
+/*
+ * Copyright (C) 2019 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <Wt/WDateTime.h>
+
+#include <filesystem>
+#include <vector>
+
+#include "database/Types.hpp"
+
+namespace Scanner {
+
+	enum class ScanErrorType
+	{
+		CannotReadFile,		// cannot read file
+		CannotParseFile,	// cannot parse file
+		NoAudioTrack,		// no audio track found
+		BadDuration,		// bad duration
+	};
+
+	enum class DuplicateReason
+	{
+		SameHash,
+		SameMBID,
+	};
+
+	struct ScanError
+	{
+		std::filesystem::path	file;
+		ScanErrorType		error;
+		std::string		systemError;
+
+		ScanError(const std::filesystem::path& file, ScanErrorType error, const std::string& systemError = "");
+	};
+
+	struct ScanDuplicate
+	{
+		Database::TrackId	trackId;
+		DuplicateReason		reason;
+	};
+
+	enum class ScanProgressStep : unsigned
+	{
+		ChekingForMissingFiles = 0,
+		DiscoveringFiles,
+		ScanningFiles,
+		FetchingTrackFeatures,
+		ReloadingSimilarityEngine,
+	};
+	static inline constexpr unsigned ScanProgressStepCount {5};
+
+	// reduced scan stats
+	struct ScanStepStats
+	{
+		Wt::WDateTime   startTime;
+
+		ScanProgressStep currentStep;
+
+		std::size_t	totalElems {};
+		std::size_t	processedElems {};
+
+		unsigned		progress() const;
+	};
+
+	struct ScanStats
+	{
+		Wt::WDateTime	startTime;
+		Wt::WDateTime	stopTime;
+
+		std::size_t	filesScanned {};	// Total number of files scanned (estimated)
+
+		std::size_t	skips {};			// no change since last scan
+		std::size_t	scans {};			// actually scanned filed
+
+		std::size_t	additions {};		// added in DB
+		std::size_t	deletions {};		// removed from DB
+		std::size_t	updates {};			// updated file in DB
+
+		std::size_t	featuresFetched {};	// features fetched in DB
+
+		std::vector<ScanError>		errors;
+		std::vector<ScanDuplicate>	duplicates;
+
+		std::size_t	nbFiles() const;
+		std::size_t	nbChanges() const;
+	};
+
+}
+