diff --git a/INSTALL.md b/INSTALL.md index b93a9e1c..4be9decd 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -106,7 +106,7 @@ You can define which authentication backend to be used thanks to the `authentica * `PAM`: the user/password authentication request is forwarded to PAM (see the default [PAM configuration file](conf/pam/lms) provided). * `http-headers`: _LMS_ uses a configurable HTTP header field, typically set by a reverse proxy to handle [SSO](https://en.wikipedia.org/wiki/Single_sign-on), to extract the login name. You can customize the field to be used using the `http-headers-login-field` option. __Note__: the first created user is the admin user -### `internal` backend: reset admin password +#### `internal` backend: reset admin password Open the the database file located in `/var/lms/lms.db` using `sqlite3`: ```sh sqlite3 /var/lms/lms.db diff --git a/README.md b/README.md index 2c1302a5..c90c7199 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ A [demo instance](http://lms-demo.poupon.dev) is available. Note the administrat * Audio transcode for maximum interoperability and low bandwith requirements * Multi-value tags: artists, genres, composers, lyricists, moods, ... * [MusicBrainz Identifier](https://musicbrainz.org/doc/MusicBrainz_Identifier) support to handle duplicated artist and release names -* [ListenBrainz](https://listenbrainz.org) support for scrobbling and synchronizing listens +* [ListenBrainz](https://listenbrainz.org) support for: + * Scrobbling and synchronizing listens + * Synchronizing 'love' feedbacks * Compilation support * Disc subtitles support * ReplayGain support @@ -50,7 +52,7 @@ __Notes on the self-organizing map__: ## Subsonic API The API version implemented is 1.16.0 and has been tested on _Android_ using _Subsonic Player_, _Ultrasonic_ and _DSub_. -Since _LMS_ uses metadata tags to organize music, a compatibility mode is used to navigate through the collection when using the directory browsing commands. +Since _LMS_ uses metadata tags to organize music, a compatibility mode is used to browse the collection when using the directory browsing commands. The Subsonic API is enabled by default. @@ -79,9 +81,9 @@ $setmulti(albumartistssort,%_albumartists_sort%) ## Security considerations _Wt_ (the web framework used) has some [built-in security measures](https://www.webtoolkit.eu/wt/features#security), but _LMS_ also has some too: -* to mitigate brute force login attempts, _LMS_ uses an internal login throttler based on the client IP address. The `Client-IP` or `X-Forwarded-For` headers are used to determined the real IP adress, so make sure to properly configure your reverse proxy to filter or even erase the values (see example in [INSTALL.md](INSTALL.md)). +* to mitigate brute force login attempts, _LMS_ uses an internal login throttler based on the client IP address. The `Client-IP` or `X-Forwarded-For` headers are used to determine the real IP adress, so make sure to properly configure your reverse proxy to filter or even erase the values (see example in [INSTALL.md](INSTALL.md)). * all passwords are stored hashed and salted using [bcrypt](https://fr.wikipedia.org/wiki/Bcrypt) -* all the resources relative to the music collection (tracks, covers, etc.) are private to a session +* all the resources relative to the music collection (tracks, covers, etc.) are private to an anthenticated session ## Installation diff --git a/approot/messages_fr.xml b/approot/messages_fr.xml index 7a9c708a..e1fff9b1 100644 --- a/approot/messages_fr.xml +++ b/approot/messages_fr.xml @@ -137,7 +137,7 @@ Artistes divers -Apparitions +Apparaît dans Artistes similaires diff --git a/conf/lms.conf b/conf/lms.conf index 7817c217..ca7dae59 100644 --- a/conf/lms.conf +++ b/conf/lms.conf @@ -36,10 +36,14 @@ http-server-thread-count = 0; # ListenBrainz root API listenbrainz-api-base-url = "https://api.listenbrainz.org"; -# How many listens to retrieve when syncing (0 disables sync) +# How many listens to retrieve when syncing (0 to disable sync) listenbrainz-max-sync-listen-count = 1000; -# How often to resync listens (0 disables sync) +# How often to resync listens (0 to disable sync) listenbrainz-sync-listens-period-hours = 1; +# How many feedbacks to retrieve when syncing (0 to disables sync) +listenbrainz-max-sync-feedback-count = 1000; +# How often to resync feedbacks (0 to disable sync) +listenbrainz-sync-feedbacks-period-hours = 1; # Acousticbrainz root API acousticbrainz-api-base-url = "https://acousticbrainz.org"; diff --git a/src/libs/services/database/impl/Artist.cpp b/src/libs/services/database/impl/Artist.cpp index 1243d043..7d904bd8 100644 --- a/src/libs/services/database/impl/Artist.cpp +++ b/src/libs/services/database/impl/Artist.cpp @@ -136,7 +136,8 @@ createQuery(Session& session, const Artist::FindParameters& params) assert(params.scrobbler); query.join("starred_artist s_a ON s_a.artist_id = a.id") .where("s_a.user_id = ?").bind(params.starringUser) - .where("s_a.scrobbler = ?").bind(*params.scrobbler); + .where("s_a.scrobbler = ?").bind(*params.scrobbler) + .where("s_a.scrobbling_state <> ?").bind(ScrobblingState::PendingRemove); } if (!params.clusters.empty()) diff --git a/src/libs/services/database/impl/Migration.cpp b/src/libs/services/database/impl/Migration.cpp index fed5d7fc..a2c9616c 100644 --- a/src/libs/services/database/impl/Migration.cpp +++ b/src/libs/services/database/impl/Migration.cpp @@ -571,6 +571,17 @@ CREATE TABLE IF NOT EXISTS "track_artist_link_backup" ( session.getDboSession().execute("ALTER TABLE track_artist_link_backup RENAME TO track_artist_link"); } + static + void + migrateFromV34(Session& session) + { + // Add scrobbling state + // By default, everythin needs to be sent + session.getDboSession().execute("ALTER TABLE starred_artist ADD scrobbling_state INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*ScrobblingState::PendingAdd*/0)) + ")"); + session.getDboSession().execute("ALTER TABLE starred_release ADD scrobbling_state INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*ScrobblingState::PendingAdd*/0)) + ")"); + session.getDboSession().execute("ALTER TABLE starred_track ADD scrobbling_state INTEGER NOT NULL DEFAULT(" + std::to_string(static_cast(/*ScrobblingState::PendingAdd*/0)) + ")"); + } + void doDbMigration(Session& session) { @@ -611,6 +622,7 @@ CREATE TABLE IF NOT EXISTS "track_artist_link_backup" ( {31, migrateFromV31}, {32, migrateFromV32}, {33, migrateFromV33}, + {34, migrateFromV34}, }; while (1) diff --git a/src/libs/services/database/impl/Migration.hpp b/src/libs/services/database/impl/Migration.hpp index 108c8c6c..65b8b284 100644 --- a/src/libs/services/database/impl/Migration.hpp +++ b/src/libs/services/database/impl/Migration.hpp @@ -26,7 +26,7 @@ namespace Database class Session; using Version = std::size_t; - static constexpr Version LMS_DATABASE_VERSION {34}; + static constexpr Version LMS_DATABASE_VERSION {35}; class VersionInfo { public: diff --git a/src/libs/services/database/impl/Release.cpp b/src/libs/services/database/impl/Release.cpp index 0d18bf03..3ca55456 100644 --- a/src/libs/services/database/impl/Release.cpp +++ b/src/libs/services/database/impl/Release.cpp @@ -64,7 +64,8 @@ createQuery(Session& session, const Release::FindParameters& params) assert(params.scrobbler); query.join("starred_release s_r ON s_r.release_id = r.id") .where("s_r.user_id = ?").bind(params.starringUser) - .where("s_r.scrobbler = ?").bind(*params.scrobbler); + .where("s_r.scrobbler = ?").bind(*params.scrobbler) + .where("s_r.scrobbling_state <> ?").bind(ScrobblingState::PendingRemove); } if (params.artist.isValid()) diff --git a/src/libs/services/database/impl/StarredArtist.cpp b/src/libs/services/database/impl/StarredArtist.cpp index ed617665..1960a66e 100644 --- a/src/libs/services/database/impl/StarredArtist.cpp +++ b/src/libs/services/database/impl/StarredArtist.cpp @@ -22,6 +22,7 @@ #include #include "services/database/Artist.hpp" +#include "services/database/Session.hpp" #include "services/database/User.hpp" #include "IdTypeTraits.hpp" #include "Utils.hpp" diff --git a/src/libs/services/database/impl/StarredRelease.cpp b/src/libs/services/database/impl/StarredRelease.cpp index c72331d5..22b556bf 100644 --- a/src/libs/services/database/impl/StarredRelease.cpp +++ b/src/libs/services/database/impl/StarredRelease.cpp @@ -22,6 +22,7 @@ #include #include "services/database/Release.hpp" +#include "services/database/Session.hpp" #include "services/database/User.hpp" #include "IdTypeTraits.hpp" #include "Utils.hpp" diff --git a/src/libs/services/database/impl/StarredTrack.cpp b/src/libs/services/database/impl/StarredTrack.cpp index 6543654f..b2ddb486 100644 --- a/src/libs/services/database/impl/StarredTrack.cpp +++ b/src/libs/services/database/impl/StarredTrack.cpp @@ -22,6 +22,7 @@ #include #include "services/database/Track.hpp" +#include "services/database/Session.hpp" #include "services/database/User.hpp" #include "IdTypeTraits.hpp" #include "Utils.hpp" @@ -60,6 +61,23 @@ namespace Database .resultValue(); } + RangeResults + StarredTrack::find(Session& session, const FindParameters& params) + { + session.checkSharedLocked(); + + auto query {session.getDboSession().query("SELECT DISTINCT s_t.id FROM starred_track s_t")}; + + if (params.scrobbler) + query.where("s_t.scrobbler = ?").bind(*params.scrobbler); + if (params.scrobblingState) + query.where("s_t.scrobbling_state = ?").bind(*params.scrobblingState); + if (params.user.isValid()) + query.where("s_t.user_id = ?").bind(params.user); + + return execQuery(query, params.range); + } + StarredTrack::pointer StarredTrack::create(Session& session, ObjectPtr track, ObjectPtr user, Scrobbler scrobbler) { diff --git a/src/libs/services/database/impl/Track.cpp b/src/libs/services/database/impl/Track.cpp index 04c6e943..20700bb0 100644 --- a/src/libs/services/database/impl/Track.cpp +++ b/src/libs/services/database/impl/Track.cpp @@ -56,7 +56,8 @@ createQuery(Session& session, const Track::FindParameters& params) assert(params.scrobbler); query.join("starred_track s_t ON s_t.track_id = t.id") .where("s_t.user_id = ?").bind(params.starringUser) - .where("s_t.scrobbler = ?").bind(*params.scrobbler); + .where("s_t.scrobbler = ?").bind(*params.scrobbler) + .where("s_t.scrobbling_state <> ?").bind(ScrobblingState::PendingRemove); } if (!params.clusters.empty()) diff --git a/src/libs/services/database/include/services/database/StarredArtist.hpp b/src/libs/services/database/include/services/database/StarredArtist.hpp index 0abb51e0..7c731578 100644 --- a/src/libs/services/database/include/services/database/StarredArtist.hpp +++ b/src/libs/services/database/include/services/database/StarredArtist.hpp @@ -23,14 +23,11 @@ #include #include "services/database/ArtistId.hpp" -#include "services/database/IdType.hpp" #include "services/database/Object.hpp" -#include "services/database/Session.hpp" +#include "services/database/StarredArtistId.hpp" #include "services/database/Types.hpp" #include "services/database/UserId.hpp" -LMS_DECLARE_IDTYPE(StarredArtistId) - namespace Database { class Artist; @@ -56,15 +53,18 @@ namespace Database ObjectPtr getUser() const { return _user; } Scrobbler getScrobbler() const { return _scrobbler; } const Wt::WDateTime& getDateTime() const { return _dateTime; } + ScrobblingState getScrobblingState() const { return _scrobblingState; } // Setters void setDateTime(const Wt::WDateTime& dateTime); + void setScrobblingState(ScrobblingState state) { _scrobblingState = state; } template void persist(Action& a) { - Wt::Dbo::field(a, _scrobbler, "scrobbler"); - Wt::Dbo::field(a, _dateTime, "date_time"); + Wt::Dbo::field(a, _scrobbler, "scrobbler"); + Wt::Dbo::field(a, _scrobblingState, "scrobbling_state"); + Wt::Dbo::field(a, _dateTime, "date_time"); Wt::Dbo::belongsTo(a, _artist, "artist", Wt::Dbo::OnDeleteCascade); Wt::Dbo::belongsTo(a, _user, "user", Wt::Dbo::OnDeleteCascade); @@ -72,6 +72,7 @@ namespace Database private: Scrobbler _scrobbler; // for which scrobbler + ScrobblingState _scrobblingState {ScrobblingState::PendingAdd}; Wt::WDateTime _dateTime; // when it was starred Wt::Dbo::ptr _artist; diff --git a/src/libs/services/database/include/services/database/StarredArtistId.hpp b/src/libs/services/database/include/services/database/StarredArtistId.hpp new file mode 100644 index 00000000..3a5ebddf --- /dev/null +++ b/src/libs/services/database/include/services/database/StarredArtistId.hpp @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2022 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "services/database/IdType.hpp" + +LMS_DECLARE_IDTYPE(StarredArtistId) + diff --git a/src/libs/services/database/include/services/database/StarredRelease.hpp b/src/libs/services/database/include/services/database/StarredRelease.hpp index e7a1ad1e..c79ce4b2 100644 --- a/src/libs/services/database/include/services/database/StarredRelease.hpp +++ b/src/libs/services/database/include/services/database/StarredRelease.hpp @@ -22,15 +22,12 @@ #include #include -#include "services/database/ReleaseId.hpp" -#include "services/database/IdType.hpp" #include "services/database/Object.hpp" -#include "services/database/Session.hpp" +#include "services/database/ReleaseId.hpp" +#include "services/database/StarredReleaseId.hpp" #include "services/database/Types.hpp" #include "services/database/UserId.hpp" -LMS_DECLARE_IDTYPE(StarredReleaseId) - namespace Database { class Release; @@ -56,15 +53,18 @@ namespace Database ObjectPtr getUser() const { return _user; } Scrobbler getScrobbler() const { return _scrobbler; } const Wt::WDateTime& getDateTime() const { return _dateTime; } + ScrobblingState getScrobblingState() const { return _scrobblingState; } // Setters void setDateTime(const Wt::WDateTime& dateTime); + void setScrobblingState(ScrobblingState state) { _scrobblingState = state; } template void persist(Action& a) { - Wt::Dbo::field(a, _scrobbler, "scrobbler"); - Wt::Dbo::field(a, _dateTime, "date_time"); + Wt::Dbo::field(a, _scrobbler, "scrobbler"); + Wt::Dbo::field(a, _scrobblingState, "scrobbling_state"); + Wt::Dbo::field(a, _dateTime, "date_time"); Wt::Dbo::belongsTo(a, _release, "release", Wt::Dbo::OnDeleteCascade); Wt::Dbo::belongsTo(a, _user, "user", Wt::Dbo::OnDeleteCascade); @@ -72,6 +72,7 @@ namespace Database private: Scrobbler _scrobbler; // for which scrobbler + ScrobblingState _scrobblingState {ScrobblingState::PendingAdd}; Wt::WDateTime _dateTime; // when it was starred Wt::Dbo::ptr _release; diff --git a/src/libs/services/database/include/services/database/StarredReleaseId.hpp b/src/libs/services/database/include/services/database/StarredReleaseId.hpp new file mode 100644 index 00000000..380baf43 --- /dev/null +++ b/src/libs/services/database/include/services/database/StarredReleaseId.hpp @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2022 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "services/database/IdType.hpp" + +LMS_DECLARE_IDTYPE(StarredReleaseId) + diff --git a/src/libs/services/database/include/services/database/StarredTrack.hpp b/src/libs/services/database/include/services/database/StarredTrack.hpp index ab4cf1d1..1332e1ae 100644 --- a/src/libs/services/database/include/services/database/StarredTrack.hpp +++ b/src/libs/services/database/include/services/database/StarredTrack.hpp @@ -23,13 +23,11 @@ #include #include "services/database/TrackId.hpp" -#include "services/database/IdType.hpp" #include "services/database/Object.hpp" -#include "services/database/Session.hpp" +#include "services/database/StarredTrackId.hpp" #include "services/database/Types.hpp" #include "services/database/UserId.hpp" - -LMS_DECLARE_IDTYPE(StarredTrackId) +#include "utils/EnumSet.hpp" namespace Database { @@ -43,10 +41,23 @@ namespace Database StarredTrack() = default; StarredTrack(ObjectPtr track, ObjectPtr user, Scrobbler scrobbler); + struct FindParameters + { + std::optional scrobbler; // for this scrobbler + std::optional scrobblingState; // and these states + UserId user; // and this user + Range range; + + FindParameters& setScrobbler(Scrobbler _scrobbler, ScrobblingState _scrobblingState) { scrobbler = _scrobbler; scrobblingState = _scrobblingState; return *this; } + FindParameters& setUser(UserId _user) {user = _user; return *this; } + FindParameters& setRange(Range _range) {range = _range; return *this; } + }; + // Search utility static std::size_t getCount(Session& session); static pointer find(Session& session, StarredTrackId id); static pointer find(Session& session, TrackId trackId, UserId userId, Scrobbler scrobbler); + static RangeResults find(Session& session, const FindParameters& findParams); // Create utility static pointer create(Session& session, ObjectPtr track, ObjectPtr user, Scrobbler scrobbler); @@ -56,15 +67,18 @@ namespace Database ObjectPtr getUser() const { return _user; } Scrobbler getScrobbler() const { return _scrobbler; } const Wt::WDateTime& getDateTime() const { return _dateTime; } + ScrobblingState getScrobblingState() const { return _scrobblingState; } // Setters void setDateTime(const Wt::WDateTime& dateTime); + void setScrobblingState(ScrobblingState state) { _scrobblingState = state; } template void persist(Action& a) { - Wt::Dbo::field(a, _scrobbler, "scrobbler"); - Wt::Dbo::field(a, _dateTime, "date_time"); + Wt::Dbo::field(a, _scrobbler, "scrobbler"); + Wt::Dbo::field(a, _scrobblingState, "scrobbling_state"); + Wt::Dbo::field(a, _dateTime, "date_time"); Wt::Dbo::belongsTo(a, _track, "track", Wt::Dbo::OnDeleteCascade); Wt::Dbo::belongsTo(a, _user, "user", Wt::Dbo::OnDeleteCascade); @@ -72,6 +86,7 @@ namespace Database private: Scrobbler _scrobbler; // for which scrobbler + ScrobblingState _scrobblingState {ScrobblingState::PendingAdd}; Wt::WDateTime _dateTime; // when it was starred Wt::Dbo::ptr _track; diff --git a/src/libs/services/database/include/services/database/StarredTrackId.hpp b/src/libs/services/database/include/services/database/StarredTrackId.hpp new file mode 100644 index 00000000..510ef2c7 --- /dev/null +++ b/src/libs/services/database/include/services/database/StarredTrackId.hpp @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2022 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "services/database/IdType.hpp" + +LMS_DECLARE_IDTYPE(StarredTrackId) + diff --git a/src/libs/services/database/test/StarredArtist.cpp b/src/libs/services/database/test/StarredArtist.cpp index 22f11f8e..de2f5511 100644 --- a/src/libs/services/database/test/StarredArtist.cpp +++ b/src/libs/services/database/test/StarredArtist.cpp @@ -64,6 +64,24 @@ TEST_F(DatabaseFixture, StarredArtist) } } +TEST_F(DatabaseFixture, StarredArtist_PendingDestroy) +{ + ScopedArtist artist {session, "MyArtist"}; + ScopedUser user {session, "MyUser"}; + ScopedStarredArtist starredArtist {session, artist.lockAndGet(), user.lockAndGet(), Scrobbler::Internal}; + + { + auto transaction {session.createUniqueTransaction()}; + + auto artists {Artist::find(session, Artist::FindParameters {}.setStarringUser(user.getId(), Scrobbler::Internal))}; + EXPECT_EQ(artists.results.size(), 1); + + starredArtist.get().modify()->setScrobblingState(ScrobblingState::PendingRemove); + artists = Artist::find(session, Artist::FindParameters {}.setStarringUser(user.getId(), Scrobbler::Internal)); + EXPECT_EQ(artists.results.size(), 0); + } +} + TEST_F(DatabaseFixture, StarredArtist_dateTime) { ScopedArtist artist1 {session, "MyArtist1"}; diff --git a/src/libs/services/database/test/StarredRelease.cpp b/src/libs/services/database/test/StarredRelease.cpp index 2059a785..b60b149f 100644 --- a/src/libs/services/database/test/StarredRelease.cpp +++ b/src/libs/services/database/test/StarredRelease.cpp @@ -64,6 +64,24 @@ TEST_F(DatabaseFixture, StarredRelease) } } +TEST_F(DatabaseFixture, Starredrelease_PendingDestroy) +{ + ScopedRelease release {session, "MyRelease"}; + ScopedUser user {session, "MyUser"}; + ScopedStarredRelease starredRelease {session, release.lockAndGet(), user.lockAndGet(), Scrobbler::Internal}; + + { + auto transaction {session.createUniqueTransaction()}; + + auto releases {Release::find(session, Release::FindParameters {}.setStarringUser(user.getId(), Scrobbler::Internal))}; + EXPECT_EQ(releases.results.size(), 1); + + starredRelease.get().modify()->setScrobblingState(ScrobblingState::PendingRemove); + releases = Release::find(session, Release::FindParameters {}.setStarringUser(user.getId(), Scrobbler::Internal)); + EXPECT_EQ(releases.results.size(), 0); + } +} + TEST_F(DatabaseFixture, StarredRelease_dateTime) { ScopedRelease release1 {session, "MyRelease1"}; diff --git a/src/libs/services/database/test/StarredTrack.cpp b/src/libs/services/database/test/StarredTrack.cpp index 6bd00cb0..36c3faac 100644 --- a/src/libs/services/database/test/StarredTrack.cpp +++ b/src/libs/services/database/test/StarredTrack.cpp @@ -64,6 +64,24 @@ TEST_F(DatabaseFixture, StarredTrack) } } +TEST_F(DatabaseFixture, Starredtrack_PendingDestroy) +{ + ScopedTrack track {session, "MyTrack"}; + ScopedUser user {session, "MyUser"}; + ScopedStarredTrack starredTrack {session, track.lockAndGet(), user.lockAndGet(), Scrobbler::Internal}; + + { + auto transaction {session.createUniqueTransaction()}; + + auto tracks {Track::find(session, Track::FindParameters {}.setStarringUser(user.getId(), Scrobbler::Internal))}; + EXPECT_EQ(tracks.results.size(), 1); + + starredTrack.get().modify()->setScrobblingState(ScrobblingState::PendingRemove); + tracks = Track::find(session, Track::FindParameters {}.setStarringUser(user.getId(), Scrobbler::Internal)); + EXPECT_EQ(tracks.results.size(), 0); + } +} + TEST_F(DatabaseFixture, StarredTrack_dateTime) { ScopedTrack track1 {session, "MyTrack1"}; diff --git a/src/libs/services/scrobbling/CMakeLists.txt b/src/libs/services/scrobbling/CMakeLists.txt index 2ce1bb21..d39d001e 100644 --- a/src/libs/services/scrobbling/CMakeLists.txt +++ b/src/libs/services/scrobbling/CMakeLists.txt @@ -1,6 +1,7 @@ add_library(lmsscrobbling SHARED impl/internal/InternalScrobbler.cpp + impl/listenbrainz/FeedbacksSynchronizer.cpp impl/listenbrainz/ListenBrainzScrobbler.cpp impl/listenbrainz/ListensSynchronizer.cpp impl/listenbrainz/Utils.cpp diff --git a/src/libs/services/scrobbling/impl/IScrobbler.hpp b/src/libs/services/scrobbling/impl/IScrobbler.hpp index 3a41242d..7fd1f71f 100644 --- a/src/libs/services/scrobbling/impl/IScrobbler.hpp +++ b/src/libs/services/scrobbling/impl/IScrobbler.hpp @@ -23,9 +23,9 @@ #include #include -#include "services/database/ArtistId.hpp" -#include "services/database/ReleaseId.hpp" -#include "services/database/TrackListId.hpp" +#include "services/database/StarredArtistId.hpp" +#include "services/database/StarredReleaseId.hpp" +#include "services/database/StarredTrackId.hpp" #include "services/scrobbling/Listen.hpp" namespace Database @@ -47,16 +47,13 @@ namespace Scrobbling virtual void listenFinished(const Listen& listen, std::optional duration) = 0; virtual void addTimedListen(const TimedListen& listen) = 0; - // Feedback - virtual void onStarred(Database::UserId, Database::ArtistId) {}; - virtual void onUnstarred(Database::UserId, Database::ArtistId) {}; - virtual void onStarred(Database::UserId, Database::ReleaseId) {}; - virtual void onUnstarred(Database::UserId, Database::ReleaseId) {}; - virtual void onStarred(Database::UserId, Database::TrackId) {}; - virtual void onUnstarred(Database::UserId, Database::TrackId) {}; - -// virtual void star(Database::TrackId trackId) = 0; -// virtual void unstar(Database::TrackId trackId) = 0; + // Feedbacks + virtual void onStarred(Database::StarredArtistId) = 0; + virtual void onUnstarred(Database::StarredArtistId) = 0; + virtual void onStarred(Database::StarredReleaseId) = 0; + virtual void onUnstarred(Database::StarredReleaseId) = 0; + virtual void onStarred(Database::StarredTrackId) = 0; + virtual void onUnstarred(Database::StarredTrackId) = 0; }; std::unique_ptr createScrobbler(std::string_view backendName); diff --git a/src/libs/services/scrobbling/impl/ScrobblingService.impl.hpp b/src/libs/services/scrobbling/impl/ScrobblingService.impl.hpp index 1b7ef03d..be0a06ba 100644 --- a/src/libs/services/scrobbling/impl/ScrobblingService.impl.hpp +++ b/src/libs/services/scrobbling/impl/ScrobblingService.impl.hpp @@ -35,6 +35,7 @@ namespace Scrobbling if (!scrobbler) return; + typename StarredObjType::IdType starredObjId; { Session& session {_db.getTLSSession()}; auto transaction {session.createUniqueTransaction()}; @@ -53,8 +54,9 @@ namespace Scrobbling starredObj = StarredObjType::create(session, obj, user, *scrobbler); } starredObj.modify()->setDateTime(Wt::WDateTime::currentDateTime()); + starredObjId = starredObj->getId(); } - _scrobblers[*scrobbler]->onStarred(userId, objId); + _scrobblers[*scrobbler]->onStarred(starredObjId); } template @@ -65,14 +67,18 @@ namespace Scrobbling if (!scrobbler) return; + typename StarredObjType::IdType starredObjId; { Session& session {_db.getTLSSession()}; - auto transaction {session.createUniqueTransaction()}; + auto transaction {session.createSharedTransaction()}; - if (typename StarredObjType::pointer starredObj {StarredObjType::find(session, objId, userId, *scrobbler)}) - starredObj.remove(); + typename StarredObjType::pointer starredObj {StarredObjType::find(session, objId, userId, *scrobbler)}; + if (!starredObj) + return; + + starredObjId = starredObj->getId(); } - _scrobblers[*scrobbler]->onUnstarred(userId, objId); + _scrobblers[*scrobbler]->onUnstarred(starredObjId); } template @@ -86,7 +92,8 @@ namespace Scrobbling Session& session {_db.getTLSSession()}; auto transaction {session.createSharedTransaction()}; - return StarredObjType::find(session, objId, userId, *scrobbler); + typename StarredObjType::pointer starredObj {StarredObjType::find(session, objId, userId, *scrobbler)}; + return starredObj && (starredObj->getScrobblingState() != ScrobblingState::PendingRemove); } } // ns Scrobbling diff --git a/src/libs/services/scrobbling/impl/internal/InternalScrobbler.cpp b/src/libs/services/scrobbling/impl/internal/InternalScrobbler.cpp index 2a7572ad..a4d8cedc 100644 --- a/src/libs/services/scrobbling/impl/internal/InternalScrobbler.cpp +++ b/src/libs/services/scrobbling/impl/internal/InternalScrobbler.cpp @@ -22,8 +22,32 @@ #include "services/database/Db.hpp" #include "services/database/Listen.hpp" #include "services/database/Session.hpp" -#include "services/database/User.hpp" +#include "services/database/StarredArtist.hpp" +#include "services/database/StarredRelease.hpp" +#include "services/database/StarredTrack.hpp" #include "services/database/Track.hpp" +#include "services/database/User.hpp" + +namespace +{ + template + void onStarred(Database::Session& session, typename StarredObjType::IdType id) + { + auto transaction {session.createUniqueTransaction()}; + + if (auto starredObj {StarredObjType::find(session, id)}) + starredObj.modify()->setScrobblingState(Database::ScrobblingState::Synchronized); + } + + template + void onUnstarred(Database::Session& session, typename StarredObjType::IdType id) + { + auto transaction {session.createUniqueTransaction()}; + + if (auto starredObj {StarredObjType::find(session, id)}) + starredObj.remove(); + } +} namespace Scrobbling { @@ -67,5 +91,40 @@ namespace Scrobbling auto dbListen {Database::Listen::create(session, user, track, Database::Scrobbler::Internal, listen.listenedAt)}; dbListen.modify()->setScrobblingState(Database::ScrobblingState::Synchronized); } + + void + InternalScrobbler::onStarred(Database::StarredArtistId starredArtistId) + { + ::onStarred(_db.getTLSSession(), starredArtistId); + } + + void + InternalScrobbler::onUnstarred(Database::StarredArtistId starredArtistId) + { + ::onUnstarred(_db.getTLSSession(), starredArtistId); + } + + void + InternalScrobbler::onStarred(Database::StarredReleaseId starredReleaseId) + { + ::onStarred(_db.getTLSSession(), starredReleaseId); + } + + void InternalScrobbler::onUnstarred(Database::StarredReleaseId starredReleaseId) + { + ::onUnstarred(_db.getTLSSession(), starredReleaseId); + } + + void + InternalScrobbler::onStarred(Database::StarredTrackId starredTrackId) + { + ::onStarred(_db.getTLSSession(), starredTrackId); + } + + void + InternalScrobbler::onUnstarred(Database::StarredTrackId starredTrackId) + { + ::onUnstarred(_db.getTLSSession(), starredTrackId); + } } // Scrobbling diff --git a/src/libs/services/scrobbling/impl/internal/InternalScrobbler.hpp b/src/libs/services/scrobbling/impl/internal/InternalScrobbler.hpp index 1c4d7df4..4b6dd39c 100644 --- a/src/libs/services/scrobbling/impl/internal/InternalScrobbler.hpp +++ b/src/libs/services/scrobbling/impl/internal/InternalScrobbler.hpp @@ -39,6 +39,13 @@ namespace Scrobbling void listenFinished(const Listen& listen, std::optional duration) override; void addTimedListen(const TimedListen& listen) override; + void onStarred(Database::StarredArtistId) override; + void onUnstarred(Database::StarredArtistId) override; + void onStarred(Database::StarredReleaseId) override; + void onUnstarred(Database::StarredReleaseId) override; + void onStarred(Database::StarredTrackId) override; + void onUnstarred(Database::StarredTrackId) override; + Database::Db& _db; }; } // Scrobbling diff --git a/src/libs/services/scrobbling/impl/listenbrainz/FeedbackTypes.hpp b/src/libs/services/scrobbling/impl/listenbrainz/FeedbackTypes.hpp new file mode 100644 index 00000000..782e523d --- /dev/null +++ b/src/libs/services/scrobbling/impl/listenbrainz/FeedbackTypes.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2022 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include "utils/UUID.hpp" + +namespace Scrobbling::ListenBrainz +{ + // See https://listenbrainz.readthedocs.io/en/production/dev/feedback-json/#feedback-json-doc + enum class FeedbackType + { + Love = 1, + Hate = -1, + Erase = 0, + }; + + struct Feedback + { + Wt::WDateTime created; + UUID recordingMBID; + FeedbackType score; + }; +} // Scrobbling::ListenBrainz diff --git a/src/libs/services/scrobbling/impl/listenbrainz/FeedbacksSynchronizer.cpp b/src/libs/services/scrobbling/impl/listenbrainz/FeedbacksSynchronizer.cpp new file mode 100644 index 00000000..36cf922c --- /dev/null +++ b/src/libs/services/scrobbling/impl/listenbrainz/FeedbacksSynchronizer.cpp @@ -0,0 +1,588 @@ +/* + * Copyright (C) 2021 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "ListenBrainzScrobbler.hpp" + +#include +#include +#include +#include +#include + +#include "services/database/Db.hpp" +#include "services/database/Session.hpp" +#include "services/database/StarredTrack.hpp" +#include "services/database/Track.hpp" +#include "services/database/User.hpp" +#include "services/scrobbling/Exception.hpp" +#include "utils/IConfig.hpp" +#include "utils/http/IClient.hpp" +#include "utils/Service.hpp" + +#include "Utils.hpp" + +using namespace Scrobbling::ListenBrainz; +using namespace Database; + +namespace +{ + + class Exception : public Scrobbling::Exception + { + public: + using Scrobbling::Exception::Exception; + }; + + class ParseErrorException : public Exception + { + public: + using Exception::Exception; + }; + + class MBIDNotFoundException : public Exception + { + public: + MBIDNotFoundException() : Exception {"MBID not found"} {} + }; + + std::optional + parseTotalFeedbackCount(std::string_view msgBody) + { + try + { + Wt::Json::Object root; + Wt::Json::parse(std::string {msgBody}, root); + + return static_cast(root.get("total_count")); + } + catch (const Wt::WException& e) + { + LOG(ERROR) << "Cannot parse listen count response: " << e.what(); + return std::nullopt; + } + } + + Feedback + parseFeedback(const Wt::Json::Object& feedbackObj) + { + try + { + const std::optional recordingMBID {UUID::fromString(static_cast(feedbackObj.get("recording_mbid")))}; + if (!recordingMBID) + throw MBIDNotFoundException {}; + + return Feedback + { + Wt::WDateTime::fromTime_t(static_cast(feedbackObj.get("created"))), + *recordingMBID, + static_cast(static_cast(feedbackObj.get("score"))) + }; + } + catch (const Wt::WException& e) + { + LOG(DEBUG) << "Cannot parse feedback: " << e.what(); + throw Exception {}; + } + } + + struct GetFeedbacksResult + { + std::size_t totalFeedbackCount{}; + std::vector feedbacks; + }; + + GetFeedbacksResult + parseGetFeedbacks(std::string_view msgBody) + { + GetFeedbacksResult res; + + try + { + Wt::Json::Object root; + Wt::Json::parse(std::string {msgBody}, root); + + const Wt::Json::Array& feedbacks = root.get("feedback"); + + LOG(DEBUG) << "Got " << feedbacks.size() << " feedbacks"; + + if (feedbacks.empty()) + return res; + + res.totalFeedbackCount += feedbacks.size(); + + for (const Wt::Json::Value& value : feedbacks) + { + try + { + res.feedbacks.push_back(parseFeedback(value)); + } + catch (const Exception &e) + { + LOG(DEBUG) << "Cannot parse feedback: " << e.what() << ", skipping"; + } + } + } + catch (const Wt::WException& error) + { + LOG(ERROR) << "Cannot parse 'get-feedback' result: " << error.what(); + throw ParseErrorException {error.what()}; + } + + return res; + } +} + +namespace Scrobbling::ListenBrainz +{ + FeedbacksSynchronizer::FeedbacksSynchronizer(boost::asio::io_context& ioContext, Database::Db& db, Http::IClient& client) + : _ioContext {ioContext} + , _db {db} + , _client {client} + , _maxSyncFeedbackCount {Service::get()->getULong("listenbrainz-max-sync-feedback-count", 1000)} + , _syncFeedbacksPeriod {Service::get()->getULong("listenbrainz-sync-feedbacks-period-hours", 1)} + { + LOG(INFO) << "Starting Feedbacks synchronizer, maxSyncFeedbackCount = " << _maxSyncFeedbackCount << ", _syncFeedbacksPeriod = " << _syncFeedbacksPeriod.count() << " hours"; + + scheduleSync(std::chrono::seconds {30}); + } + + void + FeedbacksSynchronizer::enqueFeedback(FeedbackType type, Database::StarredTrackId starredTrackId) + { + try + { + Database::Session& session {_db.getTLSSession()}; + + auto transaction {session.createUniqueTransaction()}; + + StarredTrack::pointer starredTrack {StarredTrack::find(session, starredTrackId)}; + if (!starredTrack) + return; + + std::optional recordingMBID {starredTrack->getTrack()->getRecordingMBID()}; + + switch (type) + { + case FeedbackType::Love: + starredTrack.modify()->setScrobblingState(ScrobblingState::PendingAdd); + break; + + case FeedbackType::Erase: + if (!recordingMBID) + { + LOG(DEBUG) << "Track has no recording MBID: erasing star"; + starredTrack.remove(); + } + else + { + // Send the erase order even if it is not on the remote LB server (it may be + // queued for add, or not) + starredTrack.modify()->setScrobblingState(ScrobblingState::PendingRemove); + } + break; + + default: + throw Exception {"Unhandled feedback type"}; + } + + const std::optional listenBrainzToken {starredTrack->getUser()->getListenBrainzToken()}; + if (!listenBrainzToken) + return; + + Http::ClientPOSTRequestParameters request; + request.relativeUrl = "/1/feedback/recording-feedback"; + request.message.addHeader("Authorization", "Token " + std::string {listenBrainzToken->getAsString()}); + + Wt::Json::Object root; + root["recording_mbid"] = Wt::Json::Value {std::string {recordingMBID->getAsString()}}; + root["score"] = Wt::Json::Value {static_cast(type)}; + + request.message.addBodyText(Wt::Json::serialize(root)); + request.message.addHeader("Content-Type", "application/json"); + + request.onSuccessFunc = [=](std::string_view /*msgBody*/) + { + _strand.dispatch([=] + { + onFeedbackSent(type, starredTrackId); + }); + }; + _client.sendPOSTRequest(std::move(request)); + } + catch (Exception& e) + { + LOG(DEBUG) << "Cannot send feedback: " << e.what(); + } + } + + void + FeedbacksSynchronizer::onFeedbackSent(FeedbackType type, Database::StarredTrackId starredTrackId) + { + assert(_strand.running_in_this_thread()); + + Session& session {_db.getTLSSession()}; + auto transaction {session.createUniqueTransaction()}; + + StarredTrack::pointer starredTrack {StarredTrack::find(session, starredTrackId)}; + if (!starredTrack) + { + LOG(DEBUG) << "Starred track not found. deleted?"; + return; + } + + UserContext& userContext {getUserContext(starredTrack->getUser()->getId())}; + + switch (type) + { + case FeedbackType::Love: + starredTrack.modify()->setScrobblingState(ScrobblingState::Synchronized); + LOG(DEBUG) << "State set to synchronized"; + + if (userContext.feedbackCount) + { + (*userContext.feedbackCount)++; + LOG(DEBUG) << "Feedback count set to " << *userContext.feedbackCount << " for user '" << userContext.listenBrainzUserName <<"'"; + } + break; + + case FeedbackType::Erase: + starredTrack.remove(); + LOG(DEBUG) << "Removed starred track"; + + if (userContext.feedbackCount && *userContext.feedbackCount > 0) + { + (*userContext.feedbackCount)--; + LOG(DEBUG) << "Feedback count set to " << *userContext.feedbackCount << " for user '" << userContext.listenBrainzUserName <<"'"; + } + break; + + default: + throw Exception {"Unhandled feedback type"}; + } + } + + void + FeedbacksSynchronizer::enquePendingFeedbacks() + { + auto processPendingFeedbacks { [this] (ScrobblingState scrobblingState, FeedbackType feedbackType) + { + RangeResults pendingFeedbacks; + + { + Database::Session& session {_db.getTLSSession()}; + + auto transaction {session.createSharedTransaction()}; + + StarredTrack::FindParameters params; + params.setScrobbler(Database::Scrobbler::ListenBrainz, scrobblingState) + .setRange(Database::Range {0, 100}); // don't flood too much? + + pendingFeedbacks = StarredTrack::find(session, params); + } + + LOG(DEBUG) << "Queing " << pendingFeedbacks.results.size() << " pending '" << (feedbackType == FeedbackType::Love ? "love" : "erase") << "'feedbacks"; + + for (const StarredTrackId starredTrackId : pendingFeedbacks.results) + enqueFeedback(feedbackType, starredTrackId); + }}; + + processPendingFeedbacks(ScrobblingState::PendingAdd, FeedbackType::Love); + processPendingFeedbacks(ScrobblingState::PendingRemove, FeedbackType::Erase); + } + + FeedbacksSynchronizer::UserContext& + FeedbacksSynchronizer::getUserContext(Database::UserId userId) + { + assert(_strand.running_in_this_thread()); + + auto itContext {_userContexts.find(userId)}; + if (itContext == std::cend(_userContexts)) + { + auto [itNewContext, inserted] {_userContexts.emplace(userId, userId)}; + itContext = itNewContext; + } + + return itContext->second; + } + + bool + FeedbacksSynchronizer::isSyncing() const + { + return std::any_of(std::cbegin(_userContexts), std::cend(_userContexts), [](const auto& contextEntry) + { + const auto& [userId, context] {contextEntry}; + return context.syncing; + }); + } + + void + FeedbacksSynchronizer::scheduleSync(std::chrono::seconds fromNow) + { + if (_syncFeedbacksPeriod.count() == 0 || _maxSyncFeedbackCount == 0) + return; + + LOG(DEBUG) << "Scheduled sync in " << fromNow.count() << " seconds..."; + _syncTimer.expires_after(fromNow); + _syncTimer.async_wait(boost::asio::bind_executor(_strand, [this] (const boost::system::error_code& ec) + { + if (ec == boost::asio::error::operation_aborted) + { + LOG(DEBUG) << "getFeedbacks aborted"; + return; + } + else if (ec) + { + throw Exception {"GetFeedbacks timer failure: " + std::string {ec.message()} }; + } + + startSync(); + })); + } + + void + FeedbacksSynchronizer::startSync() + { + LOG(DEBUG) << "Starting sync!"; + + assert(!isSyncing()); + assert(_strand.running_in_this_thread()); + + enquePendingFeedbacks(); + + Database::RangeResults userIds; + { + Database::Session& session {_db.getTLSSession()}; + auto transaction {session.createSharedTransaction()}; + userIds = Database::User::find(_db.getTLSSession(), Database::User::FindParameters{}.setScrobbler(Database::Scrobbler::ListenBrainz)); + } + + for (const Database::UserId userId : userIds.results) + startSync(getUserContext(userId)); + + if (!isSyncing()) + scheduleSync(_syncFeedbacksPeriod); + } + + void + FeedbacksSynchronizer::startSync(UserContext& context) + { + context.syncing = true; + context.listenBrainzUserName = ""; + context.fetchedFeedbackCount = 0; + context.matchedFeedbackCount = 0; + context.importedFeedbackCount = 0; + + enqueValidateToken(context); + } + + void + FeedbacksSynchronizer::onSyncEnded(UserContext& context) + { + _strand.dispatch([this, &context] + { + LOG(INFO) << "Feedback sync done for user '" << context.listenBrainzUserName << "', fetched: " << context.fetchedFeedbackCount << ", matched: " << context.matchedFeedbackCount << ", imported: " << context.importedFeedbackCount; + context.syncing = false; + + if (!isSyncing()) + scheduleSync(_syncFeedbacksPeriod); + }); + } + + void + FeedbacksSynchronizer::enqueValidateToken(UserContext& context) + { + assert(context.listenBrainzUserName.empty()); + + const std::optional listenBrainzToken {Utils::getListenBrainzToken(_db.getTLSSession(), context.userId)}; + if (!listenBrainzToken) + { + onSyncEnded(context); + return; + } + + Http::ClientGETRequestParameters request; + request.priority = Http::ClientRequestParameters::Priority::Low; + request.relativeUrl = "/1/validate-token"; + request.headers = { {"Authorization", "Token " + std::string {listenBrainzToken->getAsString()}} }; + request.onSuccessFunc = [this, &context] (std::string_view msgBody) + { + context.listenBrainzUserName = Utils::parseValidateToken(msgBody); + if (context.listenBrainzUserName.empty()) + { + onSyncEnded(context); + return; + } + enqueGetFeedbackCount(context); + }; + request.onFailureFunc = [this, &context] + { + onSyncEnded(context); + }; + + _client.sendGETRequest(std::move(request)); + } + + void + FeedbacksSynchronizer::enqueGetFeedbackCount(UserContext& context) + { + assert(!context.listenBrainzUserName.empty()); + + Http::ClientGETRequestParameters request; + request.relativeUrl = "/1/feedback/user/" + std::string {context.listenBrainzUserName} + "/get-feedback?score=1&count=0"; + request.priority = Http::ClientRequestParameters::Priority::Low; + request.onSuccessFunc = [this, &context] (std::string_view msgBody) + { + std::string msgBodyCopy {msgBody}; + _strand.dispatch([this, msgBodyCopy, &context] + { + LOG(DEBUG) << "Current feedback count = " << (context.feedbackCount ? *context.feedbackCount : 0) << " for user '" << context.listenBrainzUserName << "'"; + + const auto totalFeedbackCount = parseTotalFeedbackCount(msgBodyCopy); + if (totalFeedbackCount) + LOG(DEBUG) << "Feedback count for listenbrainz user '" << context.listenBrainzUserName << "' = " << *totalFeedbackCount; + + bool needSync {totalFeedbackCount && (!context.feedbackCount || *context.feedbackCount != *totalFeedbackCount)}; + context.feedbackCount = totalFeedbackCount; + + if (needSync) + enqueGetFeedbacks(context); + else + onSyncEnded(context); + }); + }; + request.onFailureFunc = [this, &context] + { + onSyncEnded(context); + }; + + _client.sendGETRequest(std::move(request)); + } + + void + FeedbacksSynchronizer::enqueGetFeedbacks(UserContext& context) + { + assert(!context.listenBrainzUserName.empty()); + + Http::ClientGETRequestParameters request; + request.relativeUrl = "/1/feedback/user/" + context.listenBrainzUserName + "/get-feedback?offset=" + std::to_string(context.fetchedFeedbackCount); + request.priority = Http::ClientRequestParameters::Priority::Low; + request.onSuccessFunc = [this, &context] (std::string_view msgBody) + { + std::string msgBodyCopy {msgBody}; + _strand.dispatch([this, msgBodyCopy, &context] + { + try + { + const std::size_t fetchedFeedbackCount {processGetFeedbacks(msgBodyCopy, context)}; + if (fetchedFeedbackCount == 0 // no more thing available on server + || context.fetchedFeedbackCount >= context.feedbackCount // we may miss something, but we will get it next time + || context.fetchedFeedbackCount >= _maxSyncFeedbackCount) + { + onSyncEnded(context); + } + else + { + enqueGetFeedbacks(context); + } + } + catch (const Exception& e) + { + onSyncEnded(context); + } + }); + }; + request.onFailureFunc = [=, &context] + { + onSyncEnded(context); + }; + + _client.sendGETRequest(std::move(request)); + } + + std::size_t + FeedbacksSynchronizer::processGetFeedbacks(std::string_view msgBody, UserContext& context) + { + const GetFeedbacksResult parseResult {parseGetFeedbacks(msgBody)}; + + LOG(DEBUG) << "Parsed " << parseResult.totalFeedbackCount << " feedbacks, found " << parseResult.feedbacks.size() << " usable entries"; + context.fetchedFeedbackCount += parseResult.totalFeedbackCount; + + for (const Feedback& feedback : parseResult.feedbacks) + { + tryImportFeedback(feedback, context); + } + + return parseResult.totalFeedbackCount; + } + + void + FeedbacksSynchronizer::tryImportFeedback(const Feedback& feedback, UserContext& context) + { + Database::Session& session {_db.getTLSSession()}; + + bool needImport{}; + TrackId trackId; + + { + auto transaction {session.createSharedTransaction()}; + const std::vector tracks {Track::findByRecordingMBID(session, feedback.recordingMBID)}; + if (tracks.size() > 1) + { + LOG(DEBUG) << "Duplicate recording MBIDs found for '" << feedback.recordingMBID.getAsString() << "', using first entry found"; + } + else if (tracks.empty()) + { + LOG(DEBUG) << "No track found for recording MBID '" << feedback.recordingMBID.getAsString() << "'"; + return; + } + + trackId = tracks.front()->getId(); + + const StarredTrack::pointer starredTrack {StarredTrack::find(session, trackId, context.userId, Database::Scrobbler::ListenBrainz)}; + needImport = !starredTrack; + + // don't update starred date time + // no need to update state if it was found as not synchronized + // pending remove => will be removed later + // pending add => will be resent later + } + + if (needImport) + { + auto transaction {session.createUniqueTransaction()}; + + const Track::pointer track {Track::find(session, trackId)}; + if (!track) + return; + + const User::pointer user {User::find(session, context.userId)}; + if (!user) + return; + + StarredTrack::pointer starredTrack {StarredTrack::create(session, track, user, Database::Scrobbler::ListenBrainz)}; + starredTrack.modify()->setScrobblingState(ScrobblingState::Synchronized); + + context.importedFeedbackCount++; + } + else + { + context.matchedFeedbackCount++; + } + } +} // namespace Scrobbling::ListenBrainz diff --git a/src/libs/services/scrobbling/impl/listenbrainz/FeedbacksSynchronizer.hpp b/src/libs/services/scrobbling/impl/listenbrainz/FeedbacksSynchronizer.hpp new file mode 100644 index 00000000..b0115d7d --- /dev/null +++ b/src/libs/services/scrobbling/impl/listenbrainz/FeedbacksSynchronizer.hpp @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2021 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "services/database/Types.hpp" +#include "services/database/UserId.hpp" + +#include "services/scrobbling/Listen.hpp" + +#include "FeedbackTypes.hpp" + + +namespace Database +{ + class Db; +} + +namespace Http +{ + class IClient; +} + +namespace Scrobbling::ListenBrainz +{ + class FeedbacksSynchronizer + { + public: + FeedbacksSynchronizer(boost::asio::io_context& ioContext, Database::Db& db, Http::IClient& client); + + void enqueFeedback(FeedbackType type, Database::StarredTrackId starredTrackId); + + private: + void onFeedbackSent(FeedbackType type, Database::StarredTrackId starredTrackId); + + void enquePendingFeedbacks(); + + struct UserContext + { + UserContext(Database::UserId id) : userId {id} {} + + UserContext(const UserContext&) = delete; + UserContext(UserContext&&) = delete; + UserContext& operator=(const UserContext&) = delete; + UserContext& operator=(UserContext&&) = delete; + + const Database::UserId userId; + bool syncing {}; + std::optional feedbackCount {}; + + // resetted at each sync + std::string listenBrainzUserName; // need to be resolved first + + std::size_t currentOffset{}; + std::size_t fetchedFeedbackCount{}; + std::size_t matchedFeedbackCount{}; + std::size_t importedFeedbackCount{}; + }; + + UserContext& getUserContext(Database::UserId userId); + bool isSyncing() const; + void scheduleSync(std::chrono::seconds fromNow); + void startSync(); + void startSync(UserContext& context); + void onSyncEnded(UserContext& context); + void enqueValidateToken(UserContext& context); + void enqueGetFeedbackCount(UserContext& context); + void enqueGetFeedbacks(UserContext& context); + std::size_t processGetFeedbacks(std::string_view body, UserContext& context); + void tryImportFeedback(const Feedback& feedback, UserContext& context); + + boost::asio::io_context& _ioContext; + boost::asio::io_context::strand _strand {_ioContext}; + Database::Db& _db; + boost::asio::steady_timer _syncTimer {_ioContext}; + Http::IClient& _client; + + std::unordered_map _userContexts; + + const std::size_t _maxSyncFeedbackCount; + const std::chrono::hours _syncFeedbacksPeriod; + }; +} // Scrobbling::ListenBrainz + diff --git a/src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.cpp b/src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.cpp index 62a57ce0..063a9987 100644 --- a/src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.cpp +++ b/src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.cpp @@ -21,6 +21,8 @@ #include "services/database/Db.hpp" #include "services/database/Session.hpp" +#include "services/database/StarredArtist.hpp" +#include "services/database/StarredRelease.hpp" #include "services/database/Track.hpp" #include "utils/IConfig.hpp" #include "utils/http/IClient.hpp" @@ -28,16 +30,16 @@ #include "utils/Service.hpp" #include "Utils.hpp" -#define LOG(sev) LMS_LOG(SCROBBLING, sev) << "[listenbrainz] - " +using namespace Database; namespace { bool - canBeScrobbled(Database::Session& session, Database::TrackId trackId, std::chrono::seconds duration) + canBeScrobbled(Session& session, TrackId trackId, std::chrono::seconds duration) { auto transaction {session.createSharedTransaction()}; - const Database::Track::pointer track {Database::Track::find(session, trackId)}; + const Track::pointer track {Track::find(session, trackId)}; if (!track) return false; @@ -47,18 +49,40 @@ namespace return res; } + + template + void onStarred(Database::Session& session, typename StarredObjType::IdType id) + { + auto transaction {session.createUniqueTransaction()}; + + if (auto starredObj {StarredObjType::find(session, id)}) + { + // maybe in the future this will be supported by ListenBrainz so set it to PendingAdd + starredObj.modify()->setScrobblingState(Database::ScrobblingState::PendingAdd); + } + } + + template + void onUnstarred(Database::Session& session, typename StarredObjType::IdType id) + { + auto transaction {session.createUniqueTransaction()}; + + if (auto starredObj {StarredObjType::find(session, id)}) + starredObj.remove(); + } } namespace Scrobbling::ListenBrainz { - Scrobbler::Scrobbler(boost::asio::io_context& ioContext, Database::Db& db) + Scrobbler::Scrobbler(boost::asio::io_context& ioContext, Db& db) : _ioContext {ioContext} , _db {db} , _baseAPIUrl {Service::get()->getString("listenbrainz-api-base-url", "https://api.listenbrainz.org")} , _client {Http::createClient(_ioContext, _baseAPIUrl)} , _listensSynchronizer {_ioContext, db, *_client} + , _feedbacksSynchronizer {_ioContext, db, *_client} { - LOG(INFO) << "Starting ListenBrainz scrobbler... API endpoint = '" << _baseAPIUrl; + LOG(INFO) << "Starting ListenBrainz scrobbler... API endpoint = '" << _baseAPIUrl << "'"; } Scrobbler::~Scrobbler() @@ -87,5 +111,41 @@ namespace Scrobbling::ListenBrainz { _listensSynchronizer.enqueListen(timedListen); } + + void + Scrobbler::onStarred(StarredArtistId starredArtistId) + { + ::onStarred(_db.getTLSSession(), starredArtistId); + } + + void + Scrobbler::onUnstarred(StarredArtistId starredArtistId) + { + ::onUnstarred(_db.getTLSSession(), starredArtistId); + } + + void + Scrobbler::onStarred(StarredReleaseId starredReleaseId) + { + ::onStarred(_db.getTLSSession(), starredReleaseId); + } + + void + Scrobbler::onUnstarred(StarredReleaseId starredReleaseId) + { + ::onUnstarred(_db.getTLSSession(), starredReleaseId); + } + + void + Scrobbler::onStarred(StarredTrackId starredTrackId) + { + _feedbacksSynchronizer.enqueFeedback(FeedbackType::Love, starredTrackId); + } + + void + Scrobbler::onUnstarred(StarredTrackId starredtrackId) + { + _feedbacksSynchronizer.enqueFeedback(FeedbackType::Erase, starredtrackId); + } } // namespace Scrobbling::ListenBrainz diff --git a/src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.hpp b/src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.hpp index 4a7dc77c..e91159f6 100644 --- a/src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.hpp +++ b/src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.hpp @@ -19,17 +19,17 @@ #pragma once +#include #include #include #include "IScrobbler.hpp" +#include "FeedbacksSynchronizer.hpp" #include "ListensSynchronizer.hpp" namespace Database { class Db; - class Session; - class TrackList; } namespace Scrobbling::ListenBrainz @@ -50,11 +50,23 @@ namespace Scrobbling::ListenBrainz void listenFinished(const Listen& listen, std::optional duration) override; void addTimedListen(const TimedListen& listen) override; + // Submit listens + void enqueListen(const Listen& listen, const Wt::WDateTime& timePoint); + + // Star + void onStarred(Database::StarredArtistId starredArtistId) override; + void onUnstarred(Database::StarredArtistId starredArtistId) override; + void onStarred(Database::StarredReleaseId starredReleaseId) override; + void onUnstarred(Database::StarredReleaseId starredReleaseId) override; + void onStarred(Database::StarredTrackId starredTrackId) override; + void onUnstarred(Database::StarredTrackId starredTrackId) override; + boost::asio::io_context& _ioContext; Database::Db& _db; std::string _baseAPIUrl; std::unique_ptr _client; ListensSynchronizer _listensSynchronizer; + FeedbacksSynchronizer _feedbacksSynchronizer; }; } // Scrobbling::ListenBrainz diff --git a/src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp b/src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp index 329a19bb..11316b6b 100644 --- a/src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp +++ b/src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp @@ -31,19 +31,14 @@ #include "services/database/Release.hpp" #include "services/database/Session.hpp" #include "services/database/Track.hpp" -#include "services/database/TrackList.hpp" #include "services/database/User.hpp" #include "services/scrobbling/Exception.hpp" #include "utils/IConfig.hpp" #include "utils/http/IClient.hpp" -#include "utils/Logger.hpp" #include "utils/Service.hpp" #include "Utils.hpp" -#define LOG(sev) LMS_LOG(SCROBBLING, sev) << "[listenbrainz Synchronizer] - " -#define LOG_EX(sev) LMS_LOG_EX(Module::SCROBBLING, sev) << "[listenbrainz Synchronizer] - " - namespace { using namespace Scrobbling::ListenBrainz; @@ -128,29 +123,6 @@ namespace return res; } - std::string - parseValidateToken(std::string_view msgBody) - { - std::string listenBrainzUserName; - - Wt::Json::ParseError error; - Wt::Json::Object root; - if (!Wt::Json::parse(std::string {msgBody}, root, error)) - { - LOG(ERROR) << "Cannot parse 'validate-token' result: " << error.what(); - return listenBrainzUserName; - } - - if (!root.get("valid").orIfNull(false)) - { - LOG(INFO) << "Invalid listenbrainz user"; - return listenBrainzUserName; - } - - listenBrainzUserName = root.get("user_name").orIfNull(""); - return listenBrainzUserName; - } - std::optional parseListenCount(std::string_view msgBody) { @@ -536,7 +508,7 @@ namespace Scrobbling::ListenBrainz { _strand.dispatch([this, &context] { - LOG_EX(context.importedListenCount > 0 ? Severity::INFO : Severity::DEBUG) << "Sync done for user '" << context.listenBrainzUserName << "', fetched: " << context.fetchedListenCount << ", matched: " << context.matchedListenCount << ", imported: " << context.importedListenCount; + LOG(INFO) << "Sync done for user '" << context.listenBrainzUserName << "', fetched: " << context.fetchedListenCount << ", matched: " << context.matchedListenCount << ", imported: " << context.importedListenCount; context.syncing = false; if (!isSyncing()) @@ -562,7 +534,7 @@ namespace Scrobbling::ListenBrainz request.headers = { {"Authorization", "Token " + std::string {listenBrainzToken->getAsString()}} }; request.onSuccessFunc = [this, &context] (std::string_view msgBody) { - context.listenBrainzUserName = parseValidateToken(msgBody); + context.listenBrainzUserName = Utils::parseValidateToken(msgBody); if (context.listenBrainzUserName.empty()) { onSyncEnded(context); @@ -588,21 +560,24 @@ namespace Scrobbling::ListenBrainz request.priority = Http::ClientRequestParameters::Priority::Low; request.onSuccessFunc = [=, &context] (std::string_view msgBody) { - const auto listenCount = parseListenCount(msgBody); - if (listenCount) - LOG(DEBUG) << "Listen count for listenbrainz user '" << context.listenBrainzUserName << "' = " << *listenCount; - - bool needSync {listenCount && (!context.listenCount || *context.listenCount != *listenCount)}; - context.listenCount = listenCount; - - if (!needSync) + _strand.dispatch([=, &context] { - onSyncEnded(context); - return; - } + const auto listenCount = parseListenCount(msgBody); + if (listenCount) + LOG(DEBUG) << "Listen count for listenbrainz user '" << context.listenBrainzUserName << "' = " << *listenCount; - context.maxDateTime = Wt::WDateTime::currentDateTime(); - enqueGetListens(context); + bool needSync {listenCount && (!context.listenCount || *context.listenCount != *listenCount)}; + context.listenCount = listenCount; + + if (!needSync) + { + onSyncEnded(context); + return; + } + + context.maxDateTime = Wt::WDateTime::currentDateTime(); + enqueGetListens(context); + }); }; request.onFailureFunc = [this, &context] { diff --git a/src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.hpp b/src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.hpp index 45fef878..0102409e 100644 --- a/src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.hpp +++ b/src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.hpp @@ -26,16 +26,13 @@ #include #include "services/database/Types.hpp" -#include "services/database/ListenId.hpp" #include "services/database/UserId.hpp" + #include "services/scrobbling/Listen.hpp" namespace Database { class Db; - class Session; - class TrackList; - class User; } namespace Http @@ -68,9 +65,9 @@ namespace Scrobbling::ListenBrainz UserContext& operator=(const UserContext&) = delete; UserContext& operator=(UserContext&&) = delete; - const Database::UserId userId; - bool syncing {}; - std::optional listenCount {}; + const Database::UserId userId; + bool syncing {}; + std::optional listenCount {}; // resetted at each sync std::string listenBrainzUserName; // need to be resolved first diff --git a/src/libs/services/scrobbling/impl/listenbrainz/Utils.cpp b/src/libs/services/scrobbling/impl/listenbrainz/Utils.cpp index 4e9acdf6..e6ad55b7 100644 --- a/src/libs/services/scrobbling/impl/listenbrainz/Utils.cpp +++ b/src/libs/services/scrobbling/impl/listenbrainz/Utils.cpp @@ -19,6 +19,9 @@ #include "Utils.hpp" +#include +#include + #include "services/database/Session.hpp" #include "services/database/User.hpp" @@ -35,4 +38,27 @@ namespace Scrobbling::ListenBrainz::Utils return user->getListenBrainzToken(); } + + std::string + parseValidateToken(std::string_view msgBody) + { + std::string listenBrainzUserName; + + Wt::Json::ParseError error; + Wt::Json::Object root; + if (!Wt::Json::parse(std::string {msgBody}, root, error)) + { + LOG(ERROR) << "Cannot parse 'validate-token' result: " << error.what(); + return listenBrainzUserName; + } + + if (!root.get("valid").orIfNull(false)) + { + LOG(INFO) << "Invalid listenbrainz user"; + return listenBrainzUserName; + } + + listenBrainzUserName = root.get("user_name").orIfNull(""); + return listenBrainzUserName; + } } diff --git a/src/libs/services/scrobbling/impl/listenbrainz/Utils.hpp b/src/libs/services/scrobbling/impl/listenbrainz/Utils.hpp index 47bd2a0e..f3454f40 100644 --- a/src/libs/services/scrobbling/impl/listenbrainz/Utils.hpp +++ b/src/libs/services/scrobbling/impl/listenbrainz/Utils.hpp @@ -21,16 +21,19 @@ #include "utils/UUID.hpp" -#include "services/database/Types.hpp" #include "services/database/UserId.hpp" +#include "utils/Logger.hpp" + +#define LOG(sev) LMS_LOG(SCROBBLING, sev) << "[listenbrainz] - " namespace Database { class Session; - class User; } namespace Scrobbling::ListenBrainz::Utils { std::optional getListenBrainzToken(Database::Session& session, Database::UserId userId); + + std::string parseValidateToken(std::string_view msgBody); } diff --git a/src/libs/services/scrobbling/include/services/scrobbling/Exception.hpp b/src/libs/services/scrobbling/include/services/scrobbling/Exception.hpp index b60f7189..e6009e68 100644 --- a/src/libs/services/scrobbling/include/services/scrobbling/Exception.hpp +++ b/src/libs/services/scrobbling/include/services/scrobbling/Exception.hpp @@ -23,7 +23,6 @@ namespace Scrobbling { - class Exception : public LmsException { public: