Reworked http client to ease reuse

src/libs/utils/CMakeLists.txt

@@ -1,4 +1,6 @@
 add_library(lmsutils SHARED
+	impl/http/Client.cpp
+	impl/http/SendQueue.cpp
 	impl/ChildProcess.cpp
 	impl/ChildProcessManager.cpp
 	impl/Config.cpp

src/libs/utils/impl/Logger.cpp

@@ -31,6 +31,7 @@ const char* getModuleName(Module mod)
 		case Module::DB:			return "DB";
 		case Module::DBUPDATER:		return "DB UPDATER";
 		case Module::FEATURE:		return "FEATURE";
+		case Module::HTTP:			return "HTTP";
 		case Module::MAIN:			return "MAIN";
 		case Module::METADATA:		return "METADATA";
 		case Module::REMOTE:		return "REMOTE";

src/libs/utils/impl/http/Client.cpp

@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2021 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "Client.hpp"
+#include "utils/Exception.hpp"
+
+namespace Http
+{
+	std::unique_ptr<IClient>
+	createClient(boost::asio::io_context& ioContext)
+	{
+		return std::make_unique<Client>(ioContext);
+	}
+
+	void
+	Client::sendGETRequest(ClientGETRequestParameters&& GETParams)
+	{
+		SendQueue& sendQueue {getOrCreateSendQueue(GETParams.url)};
+		sendQueue.sendRequest(std::make_unique<ClientRequest>(std::move(GETParams)));
+	}
+
+	void
+	Client::sendPOSTRequest(ClientPOSTRequestParameters&& POSTParams)
+	{
+		SendQueue& sendQueue {getOrCreateSendQueue(POSTParams.url)};
+		sendQueue.sendRequest(std::make_unique<ClientRequest>(std::move(POSTParams)));
+	}
+
+	SendQueue&
+	Client::getOrCreateSendQueue(const std::string& url)
+	{
+		Wt::Http::Client::URL parsedURL;
+		if (!Wt::Http::Client::parseUrl(url, parsedURL))
+			throw LmsException {"Cannot parse URL '" + url + "'"};
+
+		{
+			std::shared_lock lock {_sendQueuesMutex};
+
+			if (auto it = _sendQueues.find(parsedURL.host); it != std::cend(_sendQueues))
+				return it->second;
+		}
+
+		{
+			std::unique_lock lock {_sendQueuesMutex};
+
+			if (auto it = _sendQueues.find(parsedURL.host); it != std::cend(_sendQueues))
+				return it->second;
+
+			auto [it, inserted] {_sendQueues.emplace(parsedURL.host, _ioContext)};
+			assert(inserted);
+
+			return it->second;
+		}
+	}
+
+} // namespace Http
+

src/libs/utils/impl/http/Client.hpp

@@ -0,0 +1,47 @@
+/*
+ * Copyright (C) 2021 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <unordered_map>
+#include <string>
+#include <shared_mutex>
+
+#include "utils/http/IClient.hpp"
+#include "SendQueue.hpp"
+
+namespace Http
+{
+	class Client final : public IClient
+	{
+		public:
+			Client(boost::asio::io_context& ioContext) : _ioContext {ioContext} {}
+
+		private:
+			void sendGETRequest(ClientGETRequestParameters&& request) override;
+			void sendPOSTRequest(ClientPOSTRequestParameters&& request) override;
+
+			SendQueue&	getOrCreateSendQueue(const std::string& host);
+
+			boost::asio::io_context&					_ioContext;
+			std::shared_mutex							_sendQueuesMutex;
+			std::unordered_map<std::string, SendQueue>	_sendQueues;
+	};
+} // namespace Http
+

src/libs/utils/impl/http/ClientRequest.hpp

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2021 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <memory>
+#include <variant>
+#include "utils/http/ClientRequestParameters.hpp"
+
+namespace Http
+{
+	class ClientRequest
+	{
+		public:
+			ClientRequest(ClientGETRequestParameters&& GETParams) : _parameters {std::move(GETParams)} {}
+			ClientRequest(ClientPOSTRequestParameters&& POSTParams) : _parameters {std::move(POSTParams)} {}
+
+			std::size_t retryCount {};
+
+			const ClientRequestParameters& getParameters() const
+			{
+				const ClientRequestParameters* res;
+
+				std::visit([&](const auto& parameters)
+				{
+					res = &static_cast<const ClientRequestParameters&>(parameters);
+				}, _parameters);
+
+				return *res;
+			}
+
+
+			enum class Type
+			{
+				GET,
+				POST
+			};
+			Type getType() const
+			{
+				if (std::holds_alternative<ClientGETRequestParameters>(_parameters))
+					return Type::GET;
+				else
+					return Type::POST;
+			}
+
+			const ClientGETRequestParameters& getGETParameters() const
+			{
+				return std::get<ClientGETRequestParameters>(_parameters);
+			}
+
+			const ClientPOSTRequestParameters& getPOSTParameters() const
+			{
+				return std::get<ClientPOSTRequestParameters>(_parameters);
+			}
+
+		private:
+			std::variant<ClientGETRequestParameters, ClientPOSTRequestParameters> _parameters;
+	};
+}

src/libs/utils/impl/http/SendQueue.cpp

@@ -0,0 +1,242 @@
+/*
+ * Copyright (C) 2021 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "SendQueue.hpp"
+
+#include <boost/asio/dispatch.hpp>
+#include <boost/asio/bind_executor.hpp>
+
+#include "utils/Exception.hpp"
+#include "utils/Logger.hpp"
+#include "utils/String.hpp"
+
+#define LOG(sev)	LMS_LOG(SCROBBLING, sev) << "[listenbrainz SendQueue] - "
+
+namespace StringUtils
+{
+	template<>
+	std::optional<std::chrono::seconds>
+	readAs(std::string_view str)
+	{
+		std::optional<std::chrono::seconds> res;
+
+		if (const std::optional<std::size_t> value {StringUtils::readAs<std::size_t>(str)})
+			res = std::chrono::seconds {*value};
+
+		return res;
+	}
+}
+
+namespace
+{
+	template <typename T>
+	std::optional<T>
+	headerReadAs(const Wt::Http::Message& msg, std::string_view headerName)
+	{
+		std::optional<T> res;
+
+		if (const std::string* headerValue {msg.getHeader(std::string {headerName})})
+			res = StringUtils::readAs<T>(*headerValue);
+
+		return res;
+	}
+}
+
+namespace Http
+{
+	SendQueue::SendQueue(boost::asio::io_context& ioContext)
+	: _ioContext {ioContext}
+	{
+		_client.done().connect([this](Wt::AsioWrapper::error_code ec, const Wt::Http::Message& msg)
+		{
+			_strand.dispatch([=, msg = std::move(msg)]
+			{
+				onClientDone(ec, msg);
+			});
+		});
+	}
+
+	SendQueue::~SendQueue()
+	{
+		_client.abort();
+	}
+
+	void
+	SendQueue::sendRequest(std::unique_ptr<ClientRequest> request)
+	{
+		boost::asio::dispatch(_strand, [this, request = std::move(request)]() mutable
+		{
+			_sendQueue[request->getParameters().priority].emplace_back(std::move(request));
+
+			if (_state == State::Idle)
+				sendNextQueuedRequest();
+		});
+	}
+
+	void
+	SendQueue::sendNextQueuedRequest()
+	{
+		assert(_state == State::Idle);
+		assert(!_currentRequest);
+
+		for (auto& [prio, requests] : _sendQueue)
+		{
+			LOG(DEBUG) << "Processing prio " << static_cast<int>(prio) << ", request count = " << requests.size();
+			while (!requests.empty())
+			{
+				std::unique_ptr<ClientRequest> request {std::move(requests.front())};
+				requests.pop_front();
+
+				if (!sendRequest(*request))
+					continue;
+
+				_state = State::Sending;
+				_currentRequest = std::move(request);
+				return;
+			}
+		}
+	}
+
+	bool
+	SendQueue::sendRequest(const ClientRequest& request)
+	{
+		LOG(DEBUG) << "Sending request to url '" << request.getParameters().url << "'";
+
+		bool res {};
+		switch (request.getType())
+		{
+			case ClientRequest::Type::GET:
+				res = _client.get(request.getParameters().url, request.getGETParameters().headers);
+				break;
+
+			case ClientRequest::Type::POST:
+				res = _client.post(request.getParameters().url, request.getPOSTParameters().message);
+				break;
+		}
+
+		if (!res)
+			LOG(ERROR) << "Send failed, bad url or unsupported scheme?";
+
+		return res;
+	}
+
+	void
+	SendQueue::onClientDone(Wt::AsioWrapper::error_code ec, const Wt::Http::Message& msg)
+	{
+		if (ec == boost::asio::error::operation_aborted)
+		{
+			LOG(DEBUG) << "SendQueue: client aborted";
+			return;
+		}
+
+		assert(_currentRequest);
+		_state = State::Idle;
+
+		LOG(DEBUG) << "Client done. status = " << msg.status();
+		if (ec)
+			onClientDoneError(std::move(_currentRequest), ec);
+		else
+			onClientDoneSuccess(std::move(_currentRequest), msg);
+	}
+
+	void
+	SendQueue::onClientDoneError(std::unique_ptr<ClientRequest> request, Wt::AsioWrapper::error_code ec)
+	{
+		LOG(ERROR) << "Retry " << request->retryCount << ", client error: '" << ec.message() << "'";
+
+		// may be a network error, try again later
+		throttle(_defaultRetryWaitDuration);
+
+		if (request->retryCount++ < _maxRetryCount)
+		{
+			_sendQueue[request->getParameters().priority].emplace_front(std::move(request));
+		}
+		else
+		{
+			LOG(ERROR) << "Too many retries, giving up operation and throttle";
+			if (request->getParameters().onFailureFunc)
+				request->getParameters().onFailureFunc();
+		}
+	}
+
+	void
+	SendQueue::onClientDoneSuccess(std::unique_ptr<ClientRequest> request, const Wt::Http::Message& msg)
+	{
+		const ClientRequestParameters& requestParameters {request->getParameters()};
+		bool mustThrottle{};
+		if (msg.status() == 429)
+		{
+			_sendQueue[requestParameters.priority].emplace_front(std::move(request));
+			mustThrottle = true;
+		}
+
+		const auto remainingCount {headerReadAs<std::size_t>(msg, "X-RateLimit-Remaining")};
+		LOG(DEBUG) << "Remaining messages = " << (remainingCount ? *remainingCount : 0);
+		if (mustThrottle || (remainingCount && *remainingCount == 0))
+		{
+			const auto waitDuration {headerReadAs<std::chrono::seconds>(msg, "X-RateLimit-Reset-In")};
+			throttle(waitDuration.value_or(_defaultRetryWaitDuration));
+		}
+
+		if (!mustThrottle)
+		{
+			if (msg.status() == 200)
+			{
+				if (requestParameters.onSuccessFunc)
+					requestParameters.onSuccessFunc(msg.body());
+			}
+			else
+			{
+				LOG(ERROR) << "Send error: '" << msg.body() << "'";
+				if (requestParameters.onFailureFunc)
+					requestParameters.onFailureFunc();
+			}
+		}
+
+		if (_state == State::Idle)
+			sendNextQueuedRequest();
+	}
+
+	void
+	SendQueue::throttle(std::chrono::seconds requestedDuration)
+	{
+		assert(_state == State::Idle);
+
+		const std::chrono::seconds duration {clamp(requestedDuration, _minRetryWaitDuration, _maxRetryWaitDuration)};
+		LOG(DEBUG) << "Throttling for " << duration.count() << " seconds";
+
+		_throttleTimer.expires_after(duration);
+		_throttleTimer.async_wait([this](const boost::system::error_code& ec)
+		{
+			if (ec == boost::asio::error::operation_aborted)
+			{
+				LOG(DEBUG) << "SendQueue: throttle aborted";
+				return;
+			}
+			else if (ec)
+			{
+				throw LmsException {"Throttle timer failure: " + std::string {ec.message()} };
+			}
+
+			_state = State::Idle;
+			sendNextQueuedRequest();
+		});
+		_state = State::Throttled;
+	}
+} // namespace Scrobbling::ListenBrainz

src/libs/utils/impl/http/SendQueue.hpp

@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2021 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <deque>
+#include <vector>
+#include <string_view>
+
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/io_context_strand.hpp>
+#include <boost/asio/steady_timer.hpp>
+
+#include <Wt/Http/Client.h>
+#include "ClientRequest.hpp"
+
+namespace Http
+{
+	class SendQueue
+	{
+		public:
+			SendQueue(boost::asio::io_context& ioContext);
+			~SendQueue();
+
+			SendQueue(const SendQueue&) = delete;
+			SendQueue(const SendQueue&&) = delete;
+			SendQueue& operator=(const SendQueue&) = delete;
+			SendQueue& operator=(const SendQueue&&) = delete;
+
+			void sendRequest(std::unique_ptr<ClientRequest> request);
+
+		private:
+
+			void sendNextQueuedRequest();
+			bool sendRequest(const ClientRequest& request);
+			void onClientDone(Wt::AsioWrapper::error_code ec, const Wt::Http::Message& msg);
+			void onClientDoneError(std::unique_ptr<ClientRequest> request, Wt::AsioWrapper::error_code ec);
+			void onClientDoneSuccess(std::unique_ptr<ClientRequest> request, const Wt::Http::Message& msg);
+			void throttle(std::chrono::seconds duration);
+
+			const std::size_t			_maxRetryCount {2};
+			const std::chrono::seconds	_defaultRetryWaitDuration {30};
+			const std::chrono::seconds	_minRetryWaitDuration {1};
+			const std::chrono::seconds	_maxRetryWaitDuration {300};
+
+			boost::asio::io_context&		_ioContext;
+			boost::asio::io_context::strand	_strand {_ioContext};
+			boost::asio::steady_timer		_throttleTimer {_ioContext};
+
+			enum class State
+			{
+				Idle,
+				Throttled,
+				Sending,
+			};
+			State								_state {State::Idle};
+			Wt::Http::Client					_client {_ioContext};
+			std::map<ClientRequestParameters::Priority, std::deque<std::unique_ptr<ClientRequest>>> _sendQueue;
+			std::unique_ptr<ClientRequest>		_currentRequest;
+	};
+
+} // namespace Scrobbling::ListenBrainz
+

src/libs/utils/include/utils/Logger.hpp

@@ -43,6 +43,7 @@ enum class Module
 	DB,
 	DBUPDATER,
 	FEATURE,
+	HTTP,
 	MAIN,
 	METADATA,
 	REMOTE,

src/libs/utils/include/utils/http/ClientRequestParameters.hpp

@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2021 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <functional>
+#include <string_view>
+#include <vector>
+
+#include <Wt/Http/Message.h>
+
+namespace Http
+{
+	struct ClientRequestParameters
+	{
+		enum class Priority
+		{
+			High,
+			Normal,
+			Low,
+		};
+
+		Priority	priority {Priority::Normal};
+		std::string url;
+
+		using OnSuccessFunc = std::function<void(std::string_view msgBody)>;
+		OnSuccessFunc onSuccessFunc;
+
+		using OnFailureFunc = std::function<void()>;
+		OnFailureFunc onFailureFunc;
+	};
+
+	struct ClientGETRequestParameters final : public ClientRequestParameters
+	{
+		std::vector<Wt::Http::Message::Header> headers;
+	};
+
+	struct ClientPOSTRequestParameters final : public ClientRequestParameters
+	{
+		Wt::Http::Message message;
+	};
+
+} // namespace Http
+

src/libs/utils/include/utils/http/IClient.hpp

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2021 Emeric Poupon
+ *
+ * This file is part of LMS.
+ *
+ * LMS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * LMS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with LMS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <string_view>
+#include <boost/asio/io_context.hpp>
+
+#include "utils/http/ClientRequestParameters.hpp"
+
+namespace Http
+{
+	class IClient
+	{
+		public:
+			virtual ~IClient() = default;
+
+			virtual void sendGETRequest(ClientGETRequestParameters&& request) = 0;
+			virtual void sendPOSTRequest(ClientPOSTRequestParameters&& request) = 0;
+	};
+
+	std::unique_ptr<IClient> createClient(boost::asio::io_context& ioContext);
+} // namespace Http
+