From 11f8a9e38b089238fb70f1f952586ca7921b716f Mon Sep 17 00:00:00 2001 From: emeric Date: Tue, 5 Oct 2021 18:41:54 +0200 Subject: [PATCH] Reworked http client to ease reuse --- src/libs/scrobbling/CMakeLists.txt | 1 - .../listenbrainz/ListenBrainzScrobbler.cpp | 52 ++---- .../listenbrainz/ListenBrainzScrobbler.hpp | 5 +- .../impl/listenbrainz/ListensSynchronizer.cpp | 164 +++++++----------- .../impl/listenbrainz/ListensSynchronizer.hpp | 7 +- src/libs/utils/CMakeLists.txt | 2 + src/libs/utils/impl/Logger.cpp | 1 + src/libs/utils/impl/http/Client.cpp | 73 ++++++++ src/libs/utils/impl/http/Client.hpp | 47 +++++ src/libs/utils/impl/http/ClientRequest.hpp | 75 ++++++++ .../impl/http}/SendQueue.cpp | 70 ++++---- .../impl/http}/SendQueue.hpp | 76 ++------ src/libs/utils/include/utils/Logger.hpp | 1 + .../utils/http/ClientRequestParameters.hpp | 60 +++++++ src/libs/utils/include/utils/http/IClient.hpp | 40 +++++ src/lms/main.cpp | 2 + 16 files changed, 438 insertions(+), 238 deletions(-) create mode 100644 src/libs/utils/impl/http/Client.cpp create mode 100644 src/libs/utils/impl/http/Client.hpp create mode 100644 src/libs/utils/impl/http/ClientRequest.hpp rename src/libs/{scrobbling/impl/listenbrainz => utils/impl/http}/SendQueue.cpp (69%) rename src/libs/{scrobbling/impl/listenbrainz => utils/impl/http}/SendQueue.hpp (50%) create mode 100644 src/libs/utils/include/utils/http/ClientRequestParameters.hpp create mode 100644 src/libs/utils/include/utils/http/IClient.hpp diff --git a/src/libs/scrobbling/CMakeLists.txt b/src/libs/scrobbling/CMakeLists.txt index bd8a0960..d69a6f5a 100644 --- a/src/libs/scrobbling/CMakeLists.txt +++ b/src/libs/scrobbling/CMakeLists.txt @@ -3,7 +3,6 @@ add_library(lmsscrobbling SHARED impl/internal/InternalScrobbler.cpp impl/listenbrainz/ListenBrainzScrobbler.cpp impl/listenbrainz/ListensSynchronizer.cpp - impl/listenbrainz/SendQueue.cpp impl/listenbrainz/Utils.cpp impl/Scrobbling.cpp ) diff --git a/src/libs/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.cpp b/src/libs/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.cpp index c17254da..08202d9e 100644 --- a/src/libs/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.cpp +++ b/src/libs/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.cpp @@ -32,6 +32,7 @@ #include "database/TrackList.hpp" #include "database/User.hpp" #include "utils/IConfig.hpp" +#include "utils/http/IClient.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" #include "Utils.hpp" @@ -142,10 +143,10 @@ namespace Scrobbling::ListenBrainz Scrobbler::Scrobbler(boost::asio::io_context& ioContext, Database::Db& db) : _ioContext {ioContext} , _db {db} - , _sendQueue {_ioContext, Service::get()->getString("listenbrainz-api-base-url", "https://api.listenbrainz.org")} - , _listensSynchronizer {_ioContext, db, _sendQueue} + , _baseAPIUrl {Service::get()->getString("listenbrainz-api-base-url", "https://api.listenbrainz.org")} + , _listensSynchronizer {_ioContext, db, _baseAPIUrl} { - LOG(INFO) << "Starting ListenBrainz scrobbler... API endpoint = '" << _sendQueue.getAPIBaseURL(); + LOG(INFO) << "Starting ListenBrainz scrobbler... API endpoint = '" << _baseAPIUrl; } Scrobbler::~Scrobbler() @@ -187,53 +188,38 @@ namespace Scrobbling::ListenBrainz void Scrobbler::enqueListen(const Listen& listen, const Wt::WDateTime& timePoint) { - std::optional requestData {createSubmitListenRequestData(listen, timePoint)}; - if (!requestData) - return; + Http::ClientPOSTRequestParameters request; + request.url = _baseAPIUrl + "/1/submit-listens"; - SendQueue::Request submitListen {std::move(*requestData)}; if (timePoint.isValid()) { - submitListen.setPriority(SendQueue::Request::Priority::Normal); - submitListen.setOnSuccessFunc([=](std::string_view) + request.priority = Http::ClientRequestParameters::Priority::Normal; + request.onSuccessFunc = [=](std::string_view) { _listensSynchronizer.saveListen(TimedListen {listen, timePoint}); - }); + }; } else { // We want "listen now" to appear as soon as possible - submitListen.setPriority(SendQueue::Request::Priority::High); + request.priority = Http::ClientRequestParameters::Priority::High; } - _sendQueue.enqueueRequest(std::move(submitListen)); - } - - std::optional - Scrobbler::createSubmitListenRequestData(const Listen& listen, const Wt::WDateTime& timePoint) - { - Database::Session& session {_db.getTLSSession()}; - - const std::optional listenBrainzToken {Utils::getListenBrainzToken(session, listen.userId)}; - if (!listenBrainzToken) - return std::nullopt; - - SendQueue::RequestData requestData; - requestData.endpoint = "/1/submit-listens"; - requestData.type = SendQueue::RequestData::Type::POST; - - std::string bodyText {listenToJsonString(session, listen, timePoint, timePoint.isValid() ? "single" : "playing_now")}; + std::string bodyText {listenToJsonString(_db.getTLSSession(), listen, timePoint, timePoint.isValid() ? "single" : "playing_now")}; if (bodyText.empty()) { LOG(DEBUG) << "Cannot convert listen to json: skipping"; - return std::nullopt; + return; } - requestData.message.addBodyText(bodyText); - requestData.message.addHeader("Authorization", "Token " + std::string {listenBrainzToken->getAsString()}); - requestData.message.addHeader("Content-Type", "application/json"); + const std::optional listenBrainzToken {Utils::getListenBrainzToken(_db.getTLSSession(), listen.userId)}; + if (!listenBrainzToken) + return; - return requestData; + request.message.addBodyText(bodyText); + request.message.addHeader("Authorization", "Token " + std::string {listenBrainzToken->getAsString()}); + request.message.addHeader("Content-Type", "application/json"); + Service::get()->sendPOSTRequest(std::move(request)); } } // namespace Scrobbling::ListenBrainz diff --git a/src/libs/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.hpp b/src/libs/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.hpp index 30e85bad..c1b2406e 100644 --- a/src/libs/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.hpp +++ b/src/libs/scrobbling/impl/listenbrainz/ListenBrainzScrobbler.hpp @@ -24,7 +24,6 @@ #include "IScrobbler.hpp" #include "ListensSynchronizer.hpp" -#include "SendQueue.hpp" namespace Database { @@ -54,11 +53,11 @@ namespace Scrobbling::ListenBrainz // Submit listens void enqueListen(const Listen& listen, const Wt::WDateTime& timePoint); - std::optional createSubmitListenRequestData(const Listen& listen, const Wt::WDateTime& timePoint); + //std::optional createSubmitListenRequestData(const Listen& listen, const Wt::WDateTime& timePoint); boost::asio::io_context& _ioContext; Database::Db& _db; - SendQueue _sendQueue; + std::string _baseAPIUrl; ListensSynchronizer _listensSynchronizer; }; } // Scrobbling::ListenBrainz diff --git a/src/libs/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp b/src/libs/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp index 8a1809cc..8196da8b 100644 --- a/src/libs/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp +++ b/src/libs/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp @@ -34,6 +34,7 @@ #include "database/User.hpp" #include "scrobbling/Exception.hpp" #include "utils/IConfig.hpp" +#include "utils/http/IClient.hpp" #include "utils/Logger.hpp" #include "utils/Service.hpp" @@ -45,17 +46,6 @@ namespace { using namespace Scrobbling::ListenBrainz; - SendQueue::RequestData - createValidateTokenRequestData(std::string_view authToken) - { - SendQueue::RequestData requestData; - requestData.type = SendQueue::RequestData::Type::GET; - requestData.endpoint = "/1/validate-token"; - requestData.headers = { {"Authorization", "Token " + std::string {authToken}} }; - - return requestData; - } - std::string parseValidateToken(std::string_view msgBody) { @@ -79,18 +69,6 @@ namespace return listenBrainzUserName; } - SendQueue::RequestData - createListenCountRequestData(std::string_view listenBrainzUserName) - { - LOG(DEBUG) << "Getting listen count for listenbrainz user '" << listenBrainzUserName << "'"; - - SendQueue::RequestData requestData; - requestData.type = SendQueue::RequestData::Type::GET; - requestData.endpoint = "/1/user/" + std::string {listenBrainzUserName} + "/listen-count"; - - return requestData; - } - std::optional parseListenCount(std::string_view msgBody) { @@ -109,18 +87,6 @@ namespace } } - SendQueue::RequestData - createGetListensRequestData(std::string_view listenBrainzUserName, const Wt::WDateTime& maxDateTime) - { - LOG(DEBUG) << "Getting listens for listenbrainz user '" << listenBrainzUserName << "' with max_ts = " << maxDateTime.toString(); - - SendQueue::RequestData requestData; - requestData.type = SendQueue::RequestData::Type::GET; - requestData.endpoint = "/1/user/" + std::string {listenBrainzUserName} + "/listens?max_ts=" + std::to_string(maxDateTime.toTime_t()); - - return requestData; - } - Database::Track::pointer tryMatchListen(Database::Session& session, const Wt::Json::Object& metadata) { @@ -247,10 +213,10 @@ namespace namespace Scrobbling::ListenBrainz { - ListensSynchronizer::ListensSynchronizer(boost::asio::io_context& ioContext, Database::Db& db, SendQueue& sendQueue) + ListensSynchronizer::ListensSynchronizer(boost::asio::io_context& ioContext, Database::Db& db, std::string_view baseAPIUrl) : _ioContext {ioContext} , _db {db} - , _sendQueue {sendQueue} + , _baseAPIUrl {baseAPIUrl} , _maxSyncListenCount {Service::get()->getULong("listenbrainz-max-sync-listen-count", 1000)} , _syncListensPeriod {Service::get()->getULong("listenbrainz-sync-listens-period-hours", 1)} { @@ -386,31 +352,33 @@ namespace Scrobbling::ListenBrainz { assert(context.listenBrainzUserName.empty()); - std::optional requestData {createValidateTokenRequestData(context.userId)}; - if (!requestData) + const std::optional listenBrainzToken {Utils::getListenBrainzToken(_db.getTLSSession(), context.userId)}; + if (!listenBrainzToken) { onGetListensEnded(context); return; } - SendQueue::Request validateTokenRequest {std::move(*requestData)}; - validateTokenRequest.setOnSuccessFunc([this, &context] (std::string_view msgBody) - { - context.listenBrainzUserName = parseValidateToken(msgBody); - if (context.listenBrainzUserName.empty()) + Http::ClientGETRequestParameters request; + request.priority = Http::ClientRequestParameters::Priority::Low; + request.url = _baseAPIUrl + "/1/validate-token"; + request.headers = { {"Authorization", "Token " + std::string {listenBrainzToken->getAsString()}} }; + request.onSuccessFunc = [this, &context] (std::string_view msgBody) + { + context.listenBrainzUserName = parseValidateToken(msgBody); + if (context.listenBrainzUserName.empty()) + { + onGetListensEnded(context); + return; + } + enqueGetListenCount(context); + }; + request.onFailureFunc = [this, &context] { onGetListensEnded(context); - return; - } - enqueGetListenCount(context); - }); - validateTokenRequest.setOnFailureFunc([this, &context] - { - onGetListensEnded(context); - }); + }; - validateTokenRequest.setPriority(SendQueue::Request::Priority::Low); - _sendQueue.enqueueRequest(std::move(validateTokenRequest)); + Service::get()->sendGETRequest(std::move(request)); } void @@ -418,32 +386,33 @@ namespace Scrobbling::ListenBrainz { assert(!context.listenBrainzUserName.empty()); - SendQueue::Request getListenCountRequest {createListenCountRequestData(context.listenBrainzUserName)}; - getListenCountRequest.setOnSuccessFunc([=, &context] (std::string_view msgBody) - { - const auto listenCount = parseListenCount(msgBody); - if (listenCount) - LOG(DEBUG) << "Listen count for listenbrainz user '" << context.listenBrainzUserName << "' = " << *listenCount; + Http::ClientGETRequestParameters request; + request.url = _baseAPIUrl + "/1/user/" + std::string {context.listenBrainzUserName} + "/listen-count"; + request.priority = Http::ClientRequestParameters::Priority::Low; + request.onSuccessFunc = [=, &context] (std::string_view msgBody) + { + const auto listenCount = parseListenCount(msgBody); + if (listenCount) + LOG(DEBUG) << "Listen count for listenbrainz user '" << context.listenBrainzUserName << "' = " << *listenCount; - bool needSync {listenCount && (!context.listenCount || *context.listenCount != *listenCount)}; - context.listenCount = listenCount; + bool needSync {listenCount && (!context.listenCount || *context.listenCount != *listenCount)}; + context.listenCount = listenCount; - if (!needSync) + if (!needSync) + { + onGetListensEnded(context); + return; + } + + context.maxDateTime = Wt::WDateTime::currentDateTime(); + enqueGetListens(context); + }; + request.onFailureFunc = [this, &context] { onGetListensEnded(context); - return; - } + }; - context.maxDateTime = Wt::WDateTime::currentDateTime(); - enqueGetListens(context); - }); - getListenCountRequest.setOnFailureFunc([this, &context] - { - onGetListensEnded(context); - }); - - getListenCountRequest.setPriority(SendQueue::Request::Priority::Low); - _sendQueue.enqueueRequest(std::move(getListenCountRequest)); + Service::get()->sendGETRequest(std::move(request)); } void @@ -451,37 +420,26 @@ namespace Scrobbling::ListenBrainz { assert(!context.listenBrainzUserName.empty()); - SendQueue::Request getListensRequest {::createGetListensRequestData(context.listenBrainzUserName, context.maxDateTime)}; - getListensRequest.setOnSuccessFunc([=, &context] (std::string_view msgBody) - { - processGetListensResponse(msgBody, context); - if (context.fetchedListenCount >= _maxSyncListenCount || !context.maxDateTime.isValid()) + Http::ClientGETRequestParameters request; + request.url = _baseAPIUrl + "/1/user/" + context.listenBrainzUserName + "/listens?max_ts=" + std::to_string(context.maxDateTime.toTime_t()); + request.priority = Http::ClientRequestParameters::Priority::Low; + request.onSuccessFunc = [=, &context] (std::string_view msgBody) + { + processGetListensResponse(msgBody, context); + if (context.fetchedListenCount >= _maxSyncListenCount || !context.maxDateTime.isValid()) + { + onGetListensEnded(context); + return; + } + + enqueGetListens(context); + }; + request.onFailureFunc = [=, &context] { onGetListensEnded(context); - return; - } + }; - enqueGetListens(context); - }); - getListensRequest.setOnFailureFunc([=, &context] - { - onGetListensEnded(context); - }); - - getListensRequest.setPriority(SendQueue::Request::Priority::Low); - _sendQueue.enqueueRequest(std::move(getListensRequest)); - } - - std::optional - ListensSynchronizer::createValidateTokenRequestData(Database::UserId userId) - { - Database::Session& session {_db.getTLSSession()}; - - const std::optional listenBrainzToken {Utils::getListenBrainzToken(session, userId)}; - if (!listenBrainzToken) - return std::nullopt; - - return ::createValidateTokenRequestData(listenBrainzToken->getAsString()); + Service::get()->sendGETRequest(std::move(request)); } void diff --git a/src/libs/scrobbling/impl/listenbrainz/ListensSynchronizer.hpp b/src/libs/scrobbling/impl/listenbrainz/ListensSynchronizer.hpp index e2914dee..e71a8ae4 100644 --- a/src/libs/scrobbling/impl/listenbrainz/ListensSynchronizer.hpp +++ b/src/libs/scrobbling/impl/listenbrainz/ListensSynchronizer.hpp @@ -28,7 +28,6 @@ #include "database/Types.hpp" #include "scrobbling/Listen.hpp" -#include "SendQueue.hpp" namespace Database { @@ -43,7 +42,7 @@ namespace Scrobbling::ListenBrainz class ListensSynchronizer { public: - ListensSynchronizer(boost::asio::io_context& ioContext, Database::Db& db, SendQueue& sendQueue); + ListensSynchronizer(boost::asio::io_context& ioContext, Database::Db& db, std::string_view baseAPIUrl); void saveListen(const TimedListen& listen); @@ -78,14 +77,12 @@ namespace Scrobbling::ListenBrainz void enqueValidateToken(UserContext& context); void enqueGetListenCount(UserContext& context); void enqueGetListens(UserContext& context); - std::optional createValidateTokenRequestData(Database::UserId userId); - std::optional createGetListensRequestData(std::string_view listenBrainzUserName, const Wt::WDateTime& maxDateTime); void processGetListensResponse(std::string_view body, UserContext& context); boost::asio::io_context& _ioContext; boost::asio::io_context::strand _strand {_ioContext}; Database::Db& _db; - SendQueue& _sendQueue; + std::string _baseAPIUrl; boost::asio::steady_timer _getListensTimer {_ioContext}; std::unordered_map _userContexts; diff --git a/src/libs/utils/CMakeLists.txt b/src/libs/utils/CMakeLists.txt index 34744ea7..261ec14c 100644 --- a/src/libs/utils/CMakeLists.txt +++ b/src/libs/utils/CMakeLists.txt @@ -1,4 +1,6 @@ add_library(lmsutils SHARED + impl/http/Client.cpp + impl/http/SendQueue.cpp impl/ChildProcess.cpp impl/ChildProcessManager.cpp impl/Config.cpp diff --git a/src/libs/utils/impl/Logger.cpp b/src/libs/utils/impl/Logger.cpp index 3c0d6668..57d7a35d 100644 --- a/src/libs/utils/impl/Logger.cpp +++ b/src/libs/utils/impl/Logger.cpp @@ -31,6 +31,7 @@ const char* getModuleName(Module mod) case Module::DB: return "DB"; case Module::DBUPDATER: return "DB UPDATER"; case Module::FEATURE: return "FEATURE"; + case Module::HTTP: return "HTTP"; case Module::MAIN: return "MAIN"; case Module::METADATA: return "METADATA"; case Module::REMOTE: return "REMOTE"; diff --git a/src/libs/utils/impl/http/Client.cpp b/src/libs/utils/impl/http/Client.cpp new file mode 100644 index 00000000..e4a12a49 --- /dev/null +++ b/src/libs/utils/impl/http/Client.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2021 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#include "Client.hpp" +#include "utils/Exception.hpp" + +namespace Http +{ + std::unique_ptr + createClient(boost::asio::io_context& ioContext) + { + return std::make_unique(ioContext); + } + + void + Client::sendGETRequest(ClientGETRequestParameters&& GETParams) + { + SendQueue& sendQueue {getOrCreateSendQueue(GETParams.url)}; + sendQueue.sendRequest(std::make_unique(std::move(GETParams))); + } + + void + Client::sendPOSTRequest(ClientPOSTRequestParameters&& POSTParams) + { + SendQueue& sendQueue {getOrCreateSendQueue(POSTParams.url)}; + sendQueue.sendRequest(std::make_unique(std::move(POSTParams))); + } + + SendQueue& + Client::getOrCreateSendQueue(const std::string& url) + { + Wt::Http::Client::URL parsedURL; + if (!Wt::Http::Client::parseUrl(url, parsedURL)) + throw LmsException {"Cannot parse URL '" + url + "'"}; + + { + std::shared_lock lock {_sendQueuesMutex}; + + if (auto it = _sendQueues.find(parsedURL.host); it != std::cend(_sendQueues)) + return it->second; + } + + { + std::unique_lock lock {_sendQueuesMutex}; + + if (auto it = _sendQueues.find(parsedURL.host); it != std::cend(_sendQueues)) + return it->second; + + auto [it, inserted] {_sendQueues.emplace(parsedURL.host, _ioContext)}; + assert(inserted); + + return it->second; + } + } + +} // namespace Http + diff --git a/src/libs/utils/impl/http/Client.hpp b/src/libs/utils/impl/http/Client.hpp new file mode 100644 index 00000000..d6bff7f4 --- /dev/null +++ b/src/libs/utils/impl/http/Client.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2021 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include "utils/http/IClient.hpp" +#include "SendQueue.hpp" + +namespace Http +{ + class Client final : public IClient + { + public: + Client(boost::asio::io_context& ioContext) : _ioContext {ioContext} {} + + private: + void sendGETRequest(ClientGETRequestParameters&& request) override; + void sendPOSTRequest(ClientPOSTRequestParameters&& request) override; + + SendQueue& getOrCreateSendQueue(const std::string& host); + + boost::asio::io_context& _ioContext; + std::shared_mutex _sendQueuesMutex; + std::unordered_map _sendQueues; + }; +} // namespace Http + diff --git a/src/libs/utils/impl/http/ClientRequest.hpp b/src/libs/utils/impl/http/ClientRequest.hpp new file mode 100644 index 00000000..0b9f028d --- /dev/null +++ b/src/libs/utils/impl/http/ClientRequest.hpp @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2021 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include "utils/http/ClientRequestParameters.hpp" + +namespace Http +{ + class ClientRequest + { + public: + ClientRequest(ClientGETRequestParameters&& GETParams) : _parameters {std::move(GETParams)} {} + ClientRequest(ClientPOSTRequestParameters&& POSTParams) : _parameters {std::move(POSTParams)} {} + + std::size_t retryCount {}; + + const ClientRequestParameters& getParameters() const + { + const ClientRequestParameters* res; + + std::visit([&](const auto& parameters) + { + res = &static_cast(parameters); + }, _parameters); + + return *res; + } + + + enum class Type + { + GET, + POST + }; + Type getType() const + { + if (std::holds_alternative(_parameters)) + return Type::GET; + else + return Type::POST; + } + + const ClientGETRequestParameters& getGETParameters() const + { + return std::get(_parameters); + } + + const ClientPOSTRequestParameters& getPOSTParameters() const + { + return std::get(_parameters); + } + + private: + std::variant _parameters; + }; +} diff --git a/src/libs/scrobbling/impl/listenbrainz/SendQueue.cpp b/src/libs/utils/impl/http/SendQueue.cpp similarity index 69% rename from src/libs/scrobbling/impl/listenbrainz/SendQueue.cpp rename to src/libs/utils/impl/http/SendQueue.cpp index aa1cfde7..de5a0722 100644 --- a/src/libs/scrobbling/impl/listenbrainz/SendQueue.cpp +++ b/src/libs/utils/impl/http/SendQueue.cpp @@ -19,9 +19,10 @@ #include "SendQueue.hpp" +#include #include -#include "scrobbling/Exception.hpp" +#include "utils/Exception.hpp" #include "utils/Logger.hpp" #include "utils/String.hpp" @@ -57,11 +58,10 @@ namespace } } -namespace Scrobbling::ListenBrainz +namespace Http { - SendQueue::SendQueue(boost::asio::io_context& ioContext, std::string_view apiBaseURL) + SendQueue::SendQueue(boost::asio::io_context& ioContext) : _ioContext {ioContext} - , _apiBaseURL {apiBaseURL} { _client.done().connect([this](Wt::AsioWrapper::error_code ec, const Wt::Http::Message& msg) { @@ -78,11 +78,11 @@ namespace Scrobbling::ListenBrainz } void - SendQueue::enqueueRequest(Request request) + SendQueue::sendRequest(std::unique_ptr request) { - _strand.dispatch([this, request = std::move(request)]() + boost::asio::dispatch(_strand, [this, request = std::move(request)]() mutable { - _sendQueue[request._priority].emplace_back(std::move(request)); + _sendQueue[request->getParameters().priority].emplace_back(std::move(request)); if (_state == State::Idle) sendNextQueuedRequest(); @@ -93,16 +93,17 @@ namespace Scrobbling::ListenBrainz SendQueue::sendNextQueuedRequest() { assert(_state == State::Idle); + assert(!_currentRequest); for (auto& [prio, requests] : _sendQueue) { LOG(DEBUG) << "Processing prio " << static_cast(prio) << ", request count = " << requests.size(); while (!requests.empty()) { - Request request {std::move(requests.front())}; + std::unique_ptr request {std::move(requests.front())}; requests.pop_front(); - if (!sendRequest(request._requestData)) + if (!sendRequest(*request)) continue; _state = State::Sending; @@ -113,20 +114,19 @@ namespace Scrobbling::ListenBrainz } bool - SendQueue::sendRequest(const RequestData& requestData) + SendQueue::sendRequest(const ClientRequest& request) { - const std::string url {_apiBaseURL + requestData.endpoint}; + LOG(DEBUG) << "Sending request to url '" << request.getParameters().url << "'"; - LOG(DEBUG) << "Sending request type " << (requestData.type == RequestData::Type::GET ? "GET" : "POST") << " to url '" << url << "'"; - - bool res{}; - switch (requestData.type) + bool res {}; + switch (request.getType()) { - case RequestData::Type::GET: - res = _client.get(url, requestData.headers); + case ClientRequest::Type::GET: + res = _client.get(request.getParameters().url, request.getGETParameters().headers); break; - case RequestData::Type::POST: - res = _client.post(url, requestData.message); + + case ClientRequest::Type::POST: + res = _client.post(request.getParameters().url, request.getPOSTParameters().message); break; } @@ -146,43 +146,43 @@ namespace Scrobbling::ListenBrainz } assert(_currentRequest); - Request request {std::move(*_currentRequest)}; _state = State::Idle; LOG(DEBUG) << "Client done. status = " << msg.status(); if (ec) - onClientDoneError(std::move(request), ec); + onClientDoneError(std::move(_currentRequest), ec); else - onClientDoneSuccess(std::move(request), msg); + onClientDoneSuccess(std::move(_currentRequest), msg); } void - SendQueue::onClientDoneError(Request request, Wt::AsioWrapper::error_code ec) + SendQueue::onClientDoneError(std::unique_ptr request, Wt::AsioWrapper::error_code ec) { - LOG(ERROR) << "Retry " << request._retryCount << ", client error: '" << ec.message() << "'"; + LOG(ERROR) << "Retry " << request->retryCount << ", client error: '" << ec.message() << "'"; // may be a network error, try again later throttle(_defaultRetryWaitDuration); - if (request._retryCount++ < _maxRetryCount) + if (request->retryCount++ < _maxRetryCount) { - _sendQueue[request._priority].emplace_front(std::move(request)); + _sendQueue[request->getParameters().priority].emplace_front(std::move(request)); } else { LOG(ERROR) << "Too many retries, giving up operation and throttle"; - if (request._onFailureFunc) - request._onFailureFunc(); + if (request->getParameters().onFailureFunc) + request->getParameters().onFailureFunc(); } } void - SendQueue::onClientDoneSuccess(Request request, const Wt::Http::Message& msg) + SendQueue::onClientDoneSuccess(std::unique_ptr request, const Wt::Http::Message& msg) { + const ClientRequestParameters& requestParameters {request->getParameters()}; bool mustThrottle{}; if (msg.status() == 429) { - _sendQueue[request._priority].emplace_front(std::move(request)); + _sendQueue[requestParameters.priority].emplace_front(std::move(request)); mustThrottle = true; } @@ -198,14 +198,14 @@ namespace Scrobbling::ListenBrainz { if (msg.status() == 200) { - if (request._onSuccessFunc) - request._onSuccessFunc(msg.body()); + if (requestParameters.onSuccessFunc) + requestParameters.onSuccessFunc(msg.body()); } else { LOG(ERROR) << "Send error: '" << msg.body() << "'"; - if (request._onFailureFunc) - request._onFailureFunc(); + if (requestParameters.onFailureFunc) + requestParameters.onFailureFunc(); } } @@ -231,7 +231,7 @@ namespace Scrobbling::ListenBrainz } else if (ec) { - throw Exception {"Throttle timer failure: " + std::string {ec.message()} }; + throw LmsException {"Throttle timer failure: " + std::string {ec.message()} }; } _state = State::Idle; diff --git a/src/libs/scrobbling/impl/listenbrainz/SendQueue.hpp b/src/libs/utils/impl/http/SendQueue.hpp similarity index 50% rename from src/libs/scrobbling/impl/listenbrainz/SendQueue.hpp rename to src/libs/utils/impl/http/SendQueue.hpp index 9878d5d8..0e3bba24 100644 --- a/src/libs/scrobbling/impl/listenbrainz/SendQueue.hpp +++ b/src/libs/utils/impl/http/SendQueue.hpp @@ -20,19 +20,22 @@ #pragma once #include +#include +#include #include #include #include #include +#include "ClientRequest.hpp" -namespace Scrobbling::ListenBrainz +namespace Http { class SendQueue { public: - SendQueue(boost::asio::io_context& ioContext, std::string_view apiBaseURL); + SendQueue(boost::asio::io_context& ioContext); ~SendQueue(); SendQueue(const SendQueue&) = delete; @@ -40,58 +43,15 @@ namespace Scrobbling::ListenBrainz SendQueue& operator=(const SendQueue&) = delete; SendQueue& operator=(const SendQueue&&) = delete; - // generic queue operations - struct RequestData - { - enum class Type - { - GET, - POST, - }; - - Type type; - std::string endpoint; // relative URL to the base API - std::vector headers; // used by GET - Wt::Http::Message message; // used by POST - }; - - class Request - { - public: - enum class Priority - { - High, - Normal, - Low, - }; - - Request(RequestData requestData) : _requestData {std::move(requestData)} {} - - using OnSuccessFunc = std::function; - using OnFailureFunc = std::function; - - void setOnSuccessFunc(OnSuccessFunc onSuccessFunc) { _onSuccessFunc = onSuccessFunc; } - void setOnFailureFunc(OnFailureFunc onFailureFunc) { _onFailureFunc = onFailureFunc; } - void setPriority(Priority priority) { _priority = priority; } - - private: - friend class SendQueue; - RequestData _requestData; - Priority _priority {Priority::Normal}; - std::size_t _retryCount {}; - OnSuccessFunc _onSuccessFunc; - OnFailureFunc _onFailureFunc; - }; - - std::string_view getAPIBaseURL() const { return _apiBaseURL; } - void enqueueRequest(Request request); + void sendRequest(std::unique_ptr request); private: + void sendNextQueuedRequest(); - bool sendRequest(const RequestData& request); + bool sendRequest(const ClientRequest& request); void onClientDone(Wt::AsioWrapper::error_code ec, const Wt::Http::Message& msg); - void onClientDoneError(Request request, Wt::AsioWrapper::error_code ec); - void onClientDoneSuccess(Request request, const Wt::Http::Message& msg); + void onClientDoneError(std::unique_ptr request, Wt::AsioWrapper::error_code ec); + void onClientDoneSuccess(std::unique_ptr request, const Wt::Http::Message& msg); void throttle(std::chrono::seconds duration); const std::size_t _maxRetryCount {2}; @@ -99,20 +59,20 @@ namespace Scrobbling::ListenBrainz const std::chrono::seconds _minRetryWaitDuration {1}; const std::chrono::seconds _maxRetryWaitDuration {300}; + boost::asio::io_context& _ioContext; + boost::asio::io_context::strand _strand {_ioContext}; + boost::asio::steady_timer _throttleTimer {_ioContext}; + enum class State { Idle, Throttled, Sending, }; - boost::asio::io_context& _ioContext; - boost::asio::io_context::strand _strand {_ioContext}; - boost::asio::steady_timer _throttleTimer {_ioContext}; - std::string _apiBaseURL; - State _state {State::Idle}; - Wt::Http::Client _client {_ioContext}; - std::map> _sendQueue; - std::optional _currentRequest; + State _state {State::Idle}; + Wt::Http::Client _client {_ioContext}; + std::map>> _sendQueue; + std::unique_ptr _currentRequest; }; } // namespace Scrobbling::ListenBrainz diff --git a/src/libs/utils/include/utils/Logger.hpp b/src/libs/utils/include/utils/Logger.hpp index 4da7f4ba..43b2f893 100644 --- a/src/libs/utils/include/utils/Logger.hpp +++ b/src/libs/utils/include/utils/Logger.hpp @@ -43,6 +43,7 @@ enum class Module DB, DBUPDATER, FEATURE, + HTTP, MAIN, METADATA, REMOTE, diff --git a/src/libs/utils/include/utils/http/ClientRequestParameters.hpp b/src/libs/utils/include/utils/http/ClientRequestParameters.hpp new file mode 100644 index 00000000..2e11f183 --- /dev/null +++ b/src/libs/utils/include/utils/http/ClientRequestParameters.hpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2021 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include +#include + +#include + +namespace Http +{ + struct ClientRequestParameters + { + enum class Priority + { + High, + Normal, + Low, + }; + + Priority priority {Priority::Normal}; + std::string url; + + using OnSuccessFunc = std::function; + OnSuccessFunc onSuccessFunc; + + using OnFailureFunc = std::function; + OnFailureFunc onFailureFunc; + }; + + struct ClientGETRequestParameters final : public ClientRequestParameters + { + std::vector headers; + }; + + struct ClientPOSTRequestParameters final : public ClientRequestParameters + { + Wt::Http::Message message; + }; + +} // namespace Http + diff --git a/src/libs/utils/include/utils/http/IClient.hpp b/src/libs/utils/include/utils/http/IClient.hpp new file mode 100644 index 00000000..0feb33df --- /dev/null +++ b/src/libs/utils/include/utils/http/IClient.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2021 Emeric Poupon + * + * This file is part of LMS. + * + * LMS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * LMS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with LMS. If not, see . + */ + +#pragma once + +#include +#include + +#include "utils/http/ClientRequestParameters.hpp" + +namespace Http +{ + class IClient + { + public: + virtual ~IClient() = default; + + virtual void sendGETRequest(ClientGETRequestParameters&& request) = 0; + virtual void sendPOSTRequest(ClientPOSTRequestParameters&& request) = 0; + }; + + std::unique_ptr createClient(boost::asio::io_context& ioContext); +} // namespace Http + diff --git a/src/lms/main.cpp b/src/lms/main.cpp index 40a2fc03..ec7b906c 100644 --- a/src/lms/main.cpp +++ b/src/lms/main.cpp @@ -37,6 +37,7 @@ #include "scrobbling/IScrobbling.hpp" #include "ui/LmsApplication.hpp" #include "ui/LmsApplicationManager.hpp" +#include "utils/http/IClient.hpp" #include "utils/IChildProcessManager.hpp" #include "utils/IConfig.hpp" #include "utils/IOContextRunner.hpp" @@ -255,6 +256,7 @@ int main(int argc, char* argv[]) else throw LmsException {"Bad value '" + authenticationBackend + "' for 'authentication-backend'"}; + Service httpClient {Http::createClient(ioContext)}; Service coverArtService {CoverArt::createGrabber(database, argv[0], server.appRoot() + "/images/unknown-cover.jpg")}; Service recommendationEngineService {Recommendation::createEngine(database)}; Service scannerService {Scanner::createScanner(database, *recommendationEngineService)};