Send pending listens that have not been successfully sent

src/libs/services/database/impl/Listen.cpp

@@ -155,17 +155,24 @@ namespace Database
 		return session.getDboSession().find<Listen>().where("id = ?").bind(id).resultValue();
 	}
 
-	RangeResults<Listen::pointer>
+	RangeResults<ListenId>
-	Listen::find(Session& session, UserId userId, Scrobbler scrobbler, Range range)
+	Listen::find(Session& session, const FindParameters& parameters)
 	{
 		session.checkSharedLocked();
 
-		auto query {session.getDboSession().find<Listen>()
+		auto query {session.getDboSession().query<ListenId>("SELECT id FROM listen")
-						.where("user_id = ?").bind(userId)
+								.orderBy("date_time")};
-						.where("scrobbler = ?").bind(scrobbler)
-						.orderBy("date_time")};
 
-		return execQuery(query, range);
+		if (parameters.user.isValid())
+			query.where("user_id = ?").bind(parameters.user);
+
+		if (parameters.scrobbler)
+			query.where("scrobbler = ?").bind(*parameters.scrobbler);
+
+		if (parameters.scrobblingState)
+			query.where("scrobbling_state = ?").bind(*parameters.scrobblingState);
+
+		return execQuery(query, parameters.range);
 	}
 
 	Listen::pointer

src/libs/services/database/impl/Migration.cpp

@@ -75,6 +75,13 @@ namespace Database::Migration
 			Db& _db;
 	};
 
+	static
+	std::string
+	dateTimeToDbFormat(const Wt::WDateTime& dateTime)
+	{
+		return dateTime.toString("yyyy'-'MM'-'dd'T'hh':'mm':'ss'.000'", false).toUTF8();
+	}
+
 	static
 	void
 	migrateFromV5(Session& session)
@@ -450,7 +457,7 @@ CREATE TABLE "starred_track" (
 		// Can't migrate using class mapping as mapping may evolve in the future
 
 		// use time_t to avoid rounding issues later
-		const Wt::WDateTime now {Wt::WDateTime::fromTime_t(Wt::WDateTime::currentDateTime().toTime_t())};
+		const std::string now {dateTimeToDbFormat(Wt::WDateTime::fromTime_t(Wt::WDateTime::currentDateTime().toTime_t()))};
 
 		std::map<IdType::ValueType, Scrobbler> userScrobblers;
 		auto getScrobbler {[&](IdType::ValueType userId)
@@ -480,7 +487,7 @@ CREATE TABLE "starred_track" (
 				session.getDboSession().execute("INSERT INTO " + newTableName + " ('version', 'scrobbler', 'date_time', '" + colName + "', 'user_id') VALUES (?, ?, ?, ?, ?)")
 					.bind(0)
 					.bind(getScrobbler(userId))
-					.bind(now.toString().toUTF8())
+					.bind(now)
 					.bind(entryId)
 					.bind(userId);
 			}
@@ -523,7 +530,7 @@ CREATE TABLE "listen" (
 			{
 				session.getDboSession().execute("INSERT INTO listen ('version', 'date_time', 'scrobbler', 'scrobbling_state', 'track_id', 'user_id') VALUES (?, ?, ?, ?, ?, ?)")
 					.bind(0)
-					.bind(dateTime.toString().toUTF8())
+					.bind(dateTimeToDbFormat(dateTime))
 					.bind(scrobbler)
 					.bind(ScrobblingState::Synchronized) // consider sync is done to avoid duplicate submissions
 					.bind(trackId)

src/libs/services/database/impl/User.cpp

@@ -44,13 +44,16 @@ User::getCount(Session& session)
 }
 
 RangeResults<UserId>
-User::find(Session& session, Range range)
+User::find(Session& session, const FindParameters& params)
 {
 	session.checkSharedLocked();
 
 	auto query {session.getDboSession().query<UserId>("SELECT id FROM user")};
 
-	return execQuery(query, range);
+	if (params.scrobbler)
+		query.where("scrobbler = ?").bind(*params.scrobbler);
+
+	return execQuery(query, params.range);
 }
 
 User::pointer

src/libs/services/database/include/services/database/Listen.hpp

@@ -44,11 +44,24 @@ class Listen : public Object<Listen, ListenId>
 		Listen() = default;
 		Listen(ObjectPtr<User> user, ObjectPtr<Track> track, Scrobbler scrobbler, const Wt::WDateTime& dateTime);
 
+		struct FindParameters
+		{
+			UserId							user;
+			std::optional<Scrobbler>		scrobbler;
+			std::optional<ScrobblingState>	scrobblingState;
+			Range							range;
+
+			FindParameters& setUser(UserId _user) { user = _user; return *this; }
+			FindParameters& setScrobbler(Scrobbler _scrobbler) { scrobbler = _scrobbler; return *this; }
+			FindParameters& setScrobblingState(ScrobblingState _scrobblingState) { scrobblingState = _scrobblingState; return *this; }
+			FindParameters& setRange(Range _range) {range = _range; return *this; }
+		};
+
 		// Accessors
 		static std::size_t				getCount(Session& session);
 		static pointer 					find(Session& session, ListenId id);
-		static RangeResults<pointer>	find(Session& session, UserId userId, Scrobbler scrobbler, Range = {});
 		static pointer					find(Session& session, UserId userId, TrackId trackId, Scrobbler scrobbler, const Wt::WDateTime& dateTime);
+		static RangeResults<ListenId>	find(Session& session, const FindParameters& parameters);
 
 		// Create
 		static pointer create(Session& session, ObjectPtr<User> user, ObjectPtr<Track> track, Scrobbler scrobbler, const Wt::WDateTime& dateTime);
@@ -88,8 +101,10 @@ class Listen : public Object<Listen, ListenId>
 														const std::vector<ClusterId>& clusterIds,
 														Range range = {});
 
-		ScrobblingState getScrobblingState() const { return _scrobblingState; }
+		ScrobblingState			getScrobblingState() const { return _scrobblingState; }
-		ObjectPtr<User> getUser() const { return _user; }
+		ObjectPtr<User>			getUser() const { return _user; }
+		ObjectPtr<Track>		getTrack() const { return _track; }
+		const Wt::WDateTime&	getDateTime() const { return _dateTime; }
 
 		void			setScrobblingState(ScrobblingState state) { _scrobblingState = state; }
 

src/libs/services/database/include/services/database/User.hpp

@@ -45,6 +45,15 @@ class User : public Object<User, UserId>
 			std::string hash;
 		};
 
+		struct FindParameters
+		{
+			std::optional<Scrobbler>		scrobbler;
+			Range							range;
+
+			FindParameters& setScrobbler(Scrobbler _scrobbler) { scrobbler = _scrobbler; return *this; }
+			FindParameters& setRange(Range _range) {range = _range; return *this; }
+		};
+
 		static inline const std::size_t 	MinNameLength {3};
 		static inline const std::size_t 	MaxNameLength {15};
 		static inline const bool		defaultSubsonicTranscodeEnable {true};
@@ -63,7 +72,7 @@ class User : public Object<User, UserId>
 		static std::size_t			getCount(Session& session);
 		static pointer				find(Session& session, UserId id);
 		static pointer				find(Session& session, std::string_view loginName);
-		static RangeResults<UserId>	find(Session& session, Range range);
+		static RangeResults<UserId>	find(Session& session, const FindParameters& params);
 		static pointer				findDemoUser(Session& session);
 
 		// accessors

src/libs/services/database/test/DatabaseTest.cpp

@@ -89,7 +89,7 @@ TEST_F(DatabaseFixture, SingleUser)
 {
 	{
 		auto transaction {session.createSharedTransaction()};
-		EXPECT_TRUE(User::find(session, Range {}).results.empty());
+		EXPECT_TRUE(User::find(session, User::FindParameters {}).results.empty());
 		EXPECT_EQ(User::getCount(session), 0);
 	}
 
@@ -98,7 +98,7 @@ TEST_F(DatabaseFixture, SingleUser)
 	{
 		auto transaction {session.createSharedTransaction()};
 
-		EXPECT_EQ(User::find(session, Range {}).results.size(), 1);
+		EXPECT_EQ(User::find(session, User::FindParameters {}).results.size(), 1);
 		EXPECT_EQ(User::getCount(session), 1);
 	}
 }

src/libs/services/database/test/Listen.cpp

@@ -66,17 +66,28 @@ TEST_F(DatabaseFixture, Listen_get)
 	{
 		auto transaction {session.createSharedTransaction()};
 
-		auto listens {Listen::find(session, user->getId(), Scrobbler::ListenBrainz)};
+		auto listens {Listen::find(session, Listen::FindParameters{}.setUser(user->getId()).setScrobbler(Scrobbler::ListenBrainz))};
 		EXPECT_EQ(listens.results.size(), 0);
 	}
 
 	{
 		auto transaction {session.createSharedTransaction()};
 
-		auto listens {Listen::find(session, user->getId(), Scrobbler::Internal)};
+		{
-		EXPECT_EQ(listens.moreResults, false);
+			auto listens {Listen::find(session, Listen::FindParameters{}.setUser(user->getId()).setScrobbler(Scrobbler::Internal))};
-		ASSERT_EQ(listens.results.size(), 1);
+			EXPECT_EQ(listens.moreResults, false);
-		EXPECT_EQ(listens.results.front()->getId(), listen->getId());
+			ASSERT_EQ(listens.results.size(), 1);
+			EXPECT_EQ(listens.results.front(), listen->getId());
+		}
+
+		{
+			auto listens {Listen::find(session, Listen::FindParameters{}.setUser(user->getId()).setScrobbler(Scrobbler::Internal).setScrobblingState(ScrobblingState::PendingAdd))};
+			EXPECT_EQ(listens.results.size(), 1);
+		}
+		{
+			auto listens {Listen::find(session, Listen::FindParameters{}.setUser(user->getId()).setScrobbler(Scrobbler::Internal).setScrobblingState(ScrobblingState::Synchronized))};
+			EXPECT_EQ(listens.results.size(), 0);
+		}
 	}
 }
 
@@ -91,11 +102,11 @@ TEST_F(DatabaseFixture, Listen_get_multi)
 	{
 		auto transaction {session.createSharedTransaction()};
 
-		auto listens {Listen::find(session, user->getId(), Scrobbler::Internal)};
+		auto listens {Listen::find(session, Listen::FindParameters{}.setUser(user->getId()).setScrobbler(Scrobbler::Internal))};
 		ASSERT_EQ(listens.results.size(), 3);
-		EXPECT_EQ(listens.results[0]->getId(), listen1.getId());
+		EXPECT_EQ(listens.results[0], listen1.getId());
-		EXPECT_EQ(listens.results[1]->getId(), listen2.getId());
+		EXPECT_EQ(listens.results[1], listen2.getId());
-		EXPECT_EQ(listens.results[2]->getId(), listen3.getId());
+		EXPECT_EQ(listens.results[2], listen3.getId());
 	}
 }
 

src/libs/services/scrobbling/impl/ScrobblingService.cpp

@@ -105,7 +105,8 @@ namespace Scrobbling
 		Session& session {_db.getTLSSession()};
 		auto transaction {session.createSharedTransaction()};
 
-		return Database::Listen::getRecentArtists(session, userId, *scrobbler, clusterIds, linkType, range);
+		res = Database::Listen::getRecentArtists(session, userId, *scrobbler, clusterIds, linkType, range);
+		return res;
 	}
 
 	ScrobblingService::ReleaseContainer
@@ -120,7 +121,8 @@ namespace Scrobbling
 		Session& session {_db.getTLSSession()};
 		auto transaction {session.createSharedTransaction()};
 
-		return Database::Listen::getRecentReleases(session, userId, *scrobbler, clusterIds, range);
+		res = Database::Listen::getRecentReleases(session, userId, *scrobbler, clusterIds, range);
+		return res;
 	}
 
 	ScrobblingService::TrackContainer
@@ -135,7 +137,8 @@ namespace Scrobbling
 		Session& session {_db.getTLSSession()};
 		auto transaction {session.createSharedTransaction()};
 
-		return Database::Listen::getRecentTracks(session, userId, *scrobbler, clusterIds, range);
+		res = Database::Listen::getRecentTracks(session, userId, *scrobbler, clusterIds, range);
+		return res;
 	}
 
 	// Top
@@ -151,7 +154,8 @@ namespace Scrobbling
 		Session& session {_db.getTLSSession()};
 		auto transaction {session.createSharedTransaction()};
 
-		return Database::Listen::getTopArtists(session, userId, *scrobbler, clusterIds, linkType, range);
+		res = Database::Listen::getTopArtists(session, userId, *scrobbler, clusterIds, linkType, range);
+		return res;
 	}
 
 	ScrobblingService::ReleaseContainer
@@ -166,7 +170,8 @@ namespace Scrobbling
 		Session& session {_db.getTLSSession()};
 		auto transaction {session.createSharedTransaction()};
 
-		return Database::Listen::getTopReleases(session, userId, *scrobbler, clusterIds, range);
+		res = Database::Listen::getTopReleases(session, userId, *scrobbler, clusterIds, range);
+		return res;
 	}
 
 	ScrobblingService::TrackContainer
@@ -181,7 +186,8 @@ namespace Scrobbling
 		Session& session {_db.getTLSSession()};
 		auto transaction {session.createSharedTransaction()};
 
-		return Database::Listen::getTopTracks(session, userId, *scrobbler, clusterIds, range);
+		res = Database::Listen::getTopTracks(session, userId, *scrobbler, clusterIds, range);
+		return res;
 	}
 
 	void

src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.cpp

@@ -69,7 +69,7 @@ namespace Scrobbling::ListenBrainz
 	void
 	Scrobbler::listenStarted(const Listen& listen)
 	{
-		enqueListen(listen, Wt::WDateTime {});
+		_listensSynchronizer.enqueListenNow(listen);
 	}
 
 	void
@@ -78,23 +78,14 @@ namespace Scrobbling::ListenBrainz
 		if (duration && !canBeScrobbled(_db.getTLSSession(), listen.trackId, *duration))
 			return;
 
-		const Listen timedListen {listen};
+		const TimedListen timedListen {listen, Wt::WDateTime::currentDateTime()};
-		const Wt::WDateTime now {Wt::WDateTime::currentDateTime()};
+		_listensSynchronizer.enqueListen(timedListen);
-
-		enqueListen(timedListen, now);
 	}
 
 	void
-	Scrobbler::addTimedListen(const TimedListen& listen)
+	Scrobbler::addTimedListen(const TimedListen& timedListen)
 	{
-		assert(listen.listenedAt.isValid());
+		_listensSynchronizer.enqueListen(timedListen);
-		enqueListen(listen, listen.listenedAt);
-	}
-
-	void
-	Scrobbler::enqueListen(const Listen& listen, const Wt::WDateTime& timePoint)
-	{
-		_listensSynchronizer.enqueListen(listen, timePoint);
 	}
 } // namespace Scrobbling::ListenBrainz
 

src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.hpp

@@ -50,9 +50,6 @@ namespace Scrobbling::ListenBrainz
 			void listenFinished(const Listen& listen, std::optional<std::chrono::seconds> duration) override;
 			void addTimedListen(const TimedListen& listen) override;
 
-			// Submit listens
-			void enqueListen(const Listen& listen, const Wt::WDateTime& timePoint);
-
 			boost::asio::io_context&		_ioContext;
 			Database::Db&					_db;
 			std::string						_baseAPIUrl;

src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp

@@ -42,6 +42,7 @@
 #include "Utils.hpp"
 
 #define LOG(sev)	LMS_LOG(SCROBBLING, sev) << "[listenbrainz Synchronizer] - "
+#define LOG_EX(sev)	LMS_LOG_EX(Module::SCROBBLING, sev) << "[listenbrainz Synchronizer] - "
 
 namespace
 {
@@ -173,6 +174,8 @@ namespace
 	{
 		using namespace Database;
 
+		//LOG(DEBUG) << "Trying to match track' " << Wt::Json::serialize(metadata) << "'";
+
 		// first try to get the associated track using MBIDs, and then fallback on names
 		if (metadata.type("additional_info") == Wt::Json::Type::Object)
 		{
@@ -302,7 +305,20 @@ namespace Scrobbling::ListenBrainz
 	{
 		LOG(INFO) << "Starting Listens synchronizer, maxSyncListenCount = " << _maxSyncListenCount << ", _syncListensPeriod = " << _syncListensPeriod.count() << " hours";
 
-		scheduleGetListens(std::chrono::seconds {30});
+		scheduleSync(std::chrono::seconds {30});
+	}
+
+	void
+	ListensSynchronizer::enqueListen(const TimedListen& listen)
+	{
+		assert(listen.listenedAt.isValid());
+		enqueListen(listen, listen.listenedAt);
+	}
+
+	void
+	ListensSynchronizer::enqueListenNow(const Listen& listen)
+	{
+		enqueListen(listen, {});
 	}
 
 	void
@@ -313,15 +329,22 @@ namespace Scrobbling::ListenBrainz
 
 		if (timePoint.isValid())
 		{
+			const TimedListen timedListen {listen, timePoint};
 			// We want the listen to be sent again later in case of failure, so we just save it as pending send
-			const Database::ListenId listenId {saveListen(TimedListen {listen, timePoint})};
+			saveListen(timedListen, Database::ScrobblingState::PendingAdd);
-			if (!listenId.isValid())
-				return;
 
 			request.priority = Http::ClientRequestParameters::Priority::Normal;
 			request.onSuccessFunc = [=](std::string_view)
 			{
-				onListenSent(listenId);
+				_strand.dispatch([=]
+				{
+					if (saveListen(timedListen, Database::ScrobblingState::Synchronized))
+					{
+						UserContext& context {getUserContext(listen.userId)};
+						if (context.listenCount)
+							(*context.listenCount)++;
+					}
+				});
 			};
 			// on failure, this listen will be sent during the next sync
 		}
@@ -341,62 +364,93 @@ namespace Scrobbling::ListenBrainz
 
 		const std::optional<UUID> listenBrainzToken {Utils::getListenBrainzToken(_db.getTLSSession(), listen.userId)};
 		if (!listenBrainzToken)
+		{
+			LOG(DEBUG) << "No listenbrainz token found: skipping";
 			return;
+		}
 
 		request.message.addBodyText(bodyText);
 		request.message.addHeader("Authorization", "Token " + std::string {listenBrainzToken->getAsString()});
 		request.message.addHeader("Content-Type", "application/json");
 		_client.sendPOSTRequest(std::move(request));
-
 	}
 
-	Database::ListenId
+	bool
-	ListensSynchronizer::saveListen(const TimedListen& listen)
+	ListensSynchronizer::saveListen(const TimedListen& listen, Database::ScrobblingState scrobblingState)
 	{
 		using namespace Database;
 
 		Session& session {_db.getTLSSession()};
 		auto transaction {session.createUniqueTransaction()};
 
-		if (Database::Listen::find(session, listen.userId, listen.trackId, Database::Scrobbler::ListenBrainz, listen.listenedAt))
+		Database::Listen::pointer dbListen {Database::Listen::find(session, listen.userId, listen.trackId, Database::Scrobbler::ListenBrainz, listen.listenedAt)};
-			return {};
+		if (!dbListen)
+		{
+			const User::pointer user {User::find(session, listen.userId)};
+			if (!user)
+				return false;
 
-		const User::pointer user {User::find(session, listen.userId)};
+			const Track::pointer track {Track::find(session, listen.trackId)};
-		if (!user)
+			if (!track)
-			return {};
+				return false;
 
-		const Track::pointer track {Track::find(session, listen.trackId)};
+			dbListen = Database::Listen::create(session, user, track, Database::Scrobbler::ListenBrainz, listen.listenedAt);
-		if (!track)
+			dbListen.modify()->setScrobblingState(scrobblingState);
-			return {};
 
-		const auto dbListen {Database::Listen::create(session, user, track, Database::Scrobbler::ListenBrainz, listen.listenedAt)};
+			LOG(DEBUG) << "LISTEN CREATED for user " << user->getLoginName() << ", track '" << track->getName() << "' AT " << listen.listenedAt.toString();
-		assert(dbListen->getScrobblingState() == Database::ScrobblingState::PendingAdd);
 
-		return dbListen->getId();
+			return true;
+		}
+
+		if (dbListen->getScrobblingState() == scrobblingState)
+			return false;
+
+		dbListen.modify()->setScrobblingState(scrobblingState);
+		return true;
 	}
 
 	void
-	ListensSynchronizer::onListenSent(Database::ListenId listenId)
+	ListensSynchronizer::enquePendingListens()
 	{
-		_strand.dispatch([=]
+		std::vector<TimedListen> pendingListens;
+
 		{
 			Database::Session& session {_db.getTLSSession()};
+
 			auto transaction {session.createUniqueTransaction()};
 
-			if (Database::Listen::pointer listen {Database::Listen::find(session, listenId)})
+			Database::Listen::FindParameters params;
-			{
+			params.setScrobbler(Database::Scrobbler::ListenBrainz)
-				listen.modify()->setScrobblingState(Database::ScrobblingState::Synchronized);
+					.setScrobblingState(Database::ScrobblingState::PendingAdd)
+					.setRange(Database::Range {0, 100}); // don't flood too much?
 
-				UserContext& context {getUserContext(listen->getUser()->getId())};
+			const Database::RangeResults results {Database::Listen::find(session, params)};
-				if (context.listenCount)
+			pendingListens.reserve(results.results.size());
-					(*context.listenCount)++;
+
+			for (Database::ListenId listenId : results.results)
+			{
+				const Database::Listen::pointer listen {Database::Listen::find(session, listenId)};
+
+				TimedListen timedListen;
+				timedListen.listenedAt = listen->getDateTime();
+				timedListen.userId = listen->getUser()->getId();
+				timedListen.trackId = listen->getTrack()->getId();
+
+				pendingListens.push_back(std::move(timedListen));
 			}
-		});
+		}
+
+		LOG(DEBUG) << "Queing " << pendingListens.size() << " pending listen";
+
+		for (const TimedListen& pendingListen : pendingListens)
+			enqueListen(pendingListen);
 	}
 
 	ListensSynchronizer::UserContext&
 	ListensSynchronizer::getUserContext(Database::UserId userId)
 	{
+		assert(_strand.running_in_this_thread());
+
 		auto itContext {_userContexts.find(userId)};
 		if (itContext == std::cend(_userContexts))
 		{
@@ -408,24 +462,24 @@ namespace Scrobbling::ListenBrainz
 	}
 
 	bool
-	ListensSynchronizer::isFetching() const
+	ListensSynchronizer::isSyncing() const
 	{
 		return std::any_of(std::cbegin(_userContexts), std::cend(_userContexts), [](const auto& contextEntry)
 				{
 					const auto& [userId, context] {contextEntry};
-					return context.fetching;
+					return context.syncing;
 				});
 	}
 
 	void
-	ListensSynchronizer::scheduleGetListens(std::chrono::seconds fromNow)
+	ListensSynchronizer::scheduleSync(std::chrono::seconds fromNow)
 	{
 		if (_syncListensPeriod.count() == 0 || _maxSyncListenCount == 0)
 			return;
 
 		LOG(DEBUG) << "Scheduled sync in " << fromNow.count() << " seconds...";
-		_getListensTimer.expires_after(fromNow);
+		_syncTimer.expires_after(fromNow);
-		_getListensTimer.async_wait(boost::asio::bind_executor(_strand, [this] (const boost::system::error_code& ec)
+		_syncTimer.async_wait(boost::asio::bind_executor(_strand, [this] (const boost::system::error_code& ec)
 		{
 			if (ec == boost::asio::error::operation_aborted)
 			{
@@ -437,38 +491,37 @@ namespace Scrobbling::ListenBrainz
 				throw Exception {"GetListens timer failure: " + std::string {ec.message()} };
 			}
 
-			startGetListens();
+			startSync();
 		}));
 	}
 
 	void
-	ListensSynchronizer::startGetListens()
+	ListensSynchronizer::startSync()
 	{
-		LOG(DEBUG) << "GetListens started!";
+		LOG(DEBUG) << "Starting sync!";
 
-		assert(!isFetching());
+		assert(!isSyncing());
+
+		enquePendingListens();
 
 		Database::RangeResults<Database::UserId> userIds;
 		{
 			Database::Session& session {_db.getTLSSession()};
 			auto transaction {session.createSharedTransaction()};
-			userIds = Database::User::find(_db.getTLSSession(), Database::Range {});
+			userIds = Database::User::find(_db.getTLSSession(), Database::User::FindParameters{}.setScrobbler(Database::Scrobbler::ListenBrainz));
 		}
 
 		for (const Database::UserId userId : userIds.results)
-		{
+			startSync(getUserContext(userId));
-			if (Utils::getListenBrainzToken(_db.getTLSSession(), userId))
-				startGetListens(getUserContext(userId));
-		}
 
-		if (!isFetching())
+		if (!isSyncing())
-			scheduleGetListens(_syncListensPeriod);
+			scheduleSync(_syncListensPeriod);
 	}
 
 	void
-	ListensSynchronizer::startGetListens(UserContext& context)
+	ListensSynchronizer::startSync(UserContext& context)
 	{
-		context.fetching = true;
+		context.syncing = true;
 		context.listenBrainzUserName = "";
 		context.maxDateTime = {};
 		context.fetchedListenCount = 0;
@@ -479,15 +532,15 @@ namespace Scrobbling::ListenBrainz
 	}
 
 	void
-	ListensSynchronizer::onGetListensEnded(UserContext& context)
+	ListensSynchronizer::onSyncEnded(UserContext& context)
 	{
 		_strand.dispatch([this, &context]
 		{
-			LOG(DEBUG) << "Fetch done for user " << context.userId.getValue() << ", fetched: " << context.fetchedListenCount << ", matched: " << context.matchedListenCount << ", imported: " << context.importedListenCount;
+			LOG_EX(context.importedListenCount > 0 ? Severity::INFO : Severity::DEBUG) << "Sync done for user '" << context.listenBrainzUserName << "', fetched: " << context.fetchedListenCount << ", matched: " << context.matchedListenCount << ", imported: " << context.importedListenCount;
-			context.fetching = false;
+			context.syncing = false;
 
-			if (!isFetching())
+			if (!isSyncing())
-				scheduleGetListens(_syncListensPeriod);
+				scheduleSync(_syncListensPeriod);
 		});
 	}
 
@@ -499,7 +552,7 @@ namespace Scrobbling::ListenBrainz
 		const std::optional<UUID> listenBrainzToken {Utils::getListenBrainzToken(_db.getTLSSession(), context.userId)};
 		if (!listenBrainzToken)
 		{
-			onGetListensEnded(context);
+			onSyncEnded(context);
 			return;
 		}
 
@@ -512,14 +565,14 @@ namespace Scrobbling::ListenBrainz
 				context.listenBrainzUserName = parseValidateToken(msgBody);
 				if (context.listenBrainzUserName.empty())
 				{
-					onGetListensEnded(context);
+					onSyncEnded(context);
 					return;
 				}
 				enqueGetListenCount(context);
 			};
 		request.onFailureFunc = [this, &context]
 			{
-				onGetListensEnded(context);
+				onSyncEnded(context);
 			};
 
 		_client.sendGETRequest(std::move(request));
@@ -544,7 +597,7 @@ namespace Scrobbling::ListenBrainz
 
 				if (!needSync)
 				{
-					onGetListensEnded(context);
+					onSyncEnded(context);
 					return;
 				}
 
@@ -553,7 +606,7 @@ namespace Scrobbling::ListenBrainz
 			};
 		request.onFailureFunc = [this, &context]
 			{
-				onGetListensEnded(context);
+				onSyncEnded(context);
 			};
 
 		_client.sendGETRequest(std::move(request));
@@ -572,7 +625,7 @@ namespace Scrobbling::ListenBrainz
 				processGetListensResponse(msgBody, context);
 				if (context.fetchedListenCount >= _maxSyncListenCount || !context.maxDateTime.isValid())
 				{
-					onGetListensEnded(context);
+					onSyncEnded(context);
 					return;
 				}
 
@@ -580,7 +633,7 @@ namespace Scrobbling::ListenBrainz
 			};
 		request.onFailureFunc = [=, &context]
 			{
-				onGetListensEnded(context);
+				onSyncEnded(context);
 			};
 
 		_client.sendGETRequest(std::move(request));
@@ -597,27 +650,10 @@ namespace Scrobbling::ListenBrainz
 		context.matchedListenCount += parseResult.matchedListens.size();
 		context.maxDateTime = parseResult.oldestEntry;
 
-		if (parseResult.matchedListens.empty())
-			return;
-
-		auto transaction {session.createUniqueTransaction()};
-
-		Database::User::pointer user {Database::User::find(session, context.userId)};
-		if (!user)
-			return;
-
 		for (const TimedListen& listen : parseResult.matchedListens)
 		{
-			const Database::Track::pointer track {Database::Track::find(session, listen.trackId)};
+			if (saveListen(listen, Database::ScrobblingState::Synchronized))
-			if (!track)
+				context.importedListenCount++;
-				continue;
-
-			if (Database::Listen::find(session, listen.userId, listen.trackId, Database::Scrobbler::ListenBrainz, listen.listenedAt))
-				continue;
-
-			Database::Listen::create(session, user, track, Database::Scrobbler::ListenBrainz, listen.listenedAt);
-			context.importedListenCount++;
 		}
 	}
 } // namespace Scrobbling::ListenBrainz
-

src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.hpp

@@ -50,11 +50,14 @@ namespace Scrobbling::ListenBrainz
 		public:
 			ListensSynchronizer(boost::asio::io_context& ioContext, Database::Db& db, Http::IClient& client);
 
-			void enqueListen(const Listen& listen, const Wt::WDateTime& timePoint);
+			void enqueListen(const TimedListen& listen);
+			void enqueListenNow(const Listen& listen);
 
 		private:
-			Database::ListenId	saveListen(const TimedListen& listen);
+			void enqueListen(const Listen& listen, const Wt::WDateTime& timePoint);
-			void				onListenSent(Database::ListenId listenId);
+			bool saveListen(const TimedListen& listen, Database::ScrobblingState scrobblinState);
+
+			void enquePendingListens();
 
 			struct UserContext
 			{
@@ -66,10 +69,10 @@ namespace Scrobbling::ListenBrainz
 				UserContext& operator=(UserContext&&) = delete;
 
 				const Database::UserId	userId;
-				bool					fetching {};
+				bool					syncing {};
 				std::optional<std::size_t> listenCount {};
 
-				// resetted at each fetch
+				// resetted at each sync
 				std::string		listenBrainzUserName; // need to be resolved first
 				Wt::WDateTime	maxDateTime;
 				std::size_t		fetchedListenCount{};
@@ -78,20 +81,20 @@ namespace Scrobbling::ListenBrainz
 			};
 
 			UserContext& getUserContext(Database::UserId userId);
-			bool isFetching() const;
+			bool isSyncing() const;
-			void scheduleGetListens(std::chrono::seconds fromNow);
+			void scheduleSync(std::chrono::seconds fromNow);
-			void startGetListens();
+			void startSync();
-			void startGetListens(UserContext& context);
+			void startSync(UserContext& context);
-			void onGetListensEnded(UserContext& context);
+			void onSyncEnded(UserContext& context);
 			void enqueValidateToken(UserContext& context);
 			void enqueGetListenCount(UserContext& context);
 			void enqueGetListens(UserContext& context);
-			void									processGetListensResponse(std::string_view body, UserContext& context);
+			void processGetListensResponse(std::string_view body, UserContext& context);
 
 			boost::asio::io_context&		_ioContext;
 			boost::asio::io_context::strand	_strand {_ioContext};
 			Database::Db&					_db;
-			boost::asio::steady_timer		_getListensTimer {_ioContext};
+			boost::asio::steady_timer		_syncTimer {_ioContext};
 			Http::IClient&					_client;
 
 			std::unordered_map<Database::UserId, UserContext> _userContexts;

src/libs/services/scrobbling/impl/listenbrainz/Utils.cpp

@@ -19,14 +19,9 @@
 
 #include "Utils.hpp"
 
-#include <string_view>
-
 #include "services/database/Session.hpp"
-#include "services/database/TrackList.hpp"
 #include "services/database/User.hpp"
 
-static constexpr std::string_view historyTracklistName {"__scrobbler_listenbrainz_history__"};
-
 namespace Scrobbling::ListenBrainz::Utils
 {
 	std::optional<UUID>
@@ -38,9 +33,6 @@ namespace Scrobbling::ListenBrainz::Utils
 		if (!user)
 			return std::nullopt;
 
-		if (user->getScrobbler() != Database::Scrobbler::ListenBrainz)
-			return std::nullopt;
-
 		return user->getListenBrainzToken();
 	}
 }

src/libs/services/scrobbling/impl/listenbrainz/Utils.hpp

@@ -32,5 +32,5 @@ namespace Database
 
 namespace Scrobbling::ListenBrainz::Utils
 {
-	std::optional<UUID>							getListenBrainzToken(Database::Session& session, Database::UserId userId);
+	std::optional<UUID>	getListenBrainzToken(Database::Session& session, Database::UserId userId);
 }

src/libs/subsonic/impl/SubsonicResource.cpp

@@ -1394,7 +1394,7 @@ handleGetUsersRequest(RequestContext& context)
 	Response response {Response::createOkResponse(context.serverProtocolVersion)};
 	Response::Node& usersNode {response.createNode("users")};
 
-	const auto userIds {User::find(context.dbSession, Range {})};
+	const auto userIds {User::find(context.dbSession, User::FindParameters {})};
 	for (const UserId userId : userIds.results)
 	{
 		const User::pointer user {User::find(context.dbSession, userId)};

src/libs/utils/impl/http/SendQueue.cpp

@@ -26,7 +26,7 @@
 #include "utils/Logger.hpp"
 #include "utils/String.hpp"
 
-#define LOG(sev)	LMS_LOG(SCROBBLING, sev) << "[listenbrainz SendQueue] - "
+#define LOG(sev)	LMS_LOG(SCROBBLING, sev) << "[Http SendQueue] - "
 
 namespace StringUtils
 {
@@ -143,7 +143,7 @@ namespace Http
 	{
 		if (ec == boost::asio::error::operation_aborted)
 		{
-			LOG(DEBUG) << "SendQueue: client aborted";
+			LOG(DEBUG) << "Client aborted";
 			return;
 		}
 
@@ -228,7 +228,7 @@ namespace Http
 		{
 			if (ec == boost::asio::error::operation_aborted)
 			{
-				LOG(DEBUG) << "SendQueue: throttle aborted";
+				LOG(DEBUG) << "Throttle aborted";
 				return;
 			}
 			else if (ec)

src/libs/utils/include/utils/Logger.hpp

@@ -85,5 +85,6 @@ class Logger
 		virtual void processLog(const Log& log) = 0;
 };
 
-#define LMS_LOG(module, severity)	Log(Service<Logger>::get(), Module::module, Severity::severity).getOstream()
+#define LMS_LOG(module, severity)		Log(Service<Logger>::get(), Module::module, Severity::severity).getOstream()
+#define LMS_LOG_EX(module, severity)	Log(Service<Logger>::get(), module, severity).getOstream()
 

src/lms/ui/admin/UsersView.cpp

@@ -71,7 +71,7 @@ UsersView::refreshView()
 
 	auto transaction {LmsApp->getDbSession().createSharedTransaction()};
 
-	for (const UserId userId : User::find(LmsApp->getDbSession(), Range {}).results)
+	for (const UserId userId : User::find(LmsApp->getDbSession(), User::FindParameters {}).results)
 	{
 		const User::pointer user {User::find(LmsApp->getDbSession(), userId)};