Added podcast support, only from subsonic API for now, ref #726

src/libs/core/impl/Logger.cpp

@@ -59,6 +59,8 @@ namespace lms::core::logging
             return "MAIN";
         case Module::METADATA:
             return "METADATA";
+        case Module::PODCAST:
+            return "PODCAST";
         case Module::REMOTE:
             return "REMOTE";
         case Module::SCROBBLING:

src/libs/core/impl/MimeTypes.cpp

@@ -67,6 +67,7 @@ namespace lms::core
             { ".jpg", "image/jpeg" },
             { ".jpeg", "image/jpeg" },
             { ".png", "image/png" },
+            { ".svg", "image/svg+xml" },
             { ".webp", "image/webp" },
         };
 

src/libs/core/impl/Path.cpp

@@ -19,22 +19,15 @@
 
 #include "core/Path.hpp"
 
+#include <algorithm>
 #include <array>
+#include <cassert>
 #include <unistd.h>
 
-#include "core/ILogger.hpp"
 #include "core/String.hpp"
 
 namespace lms::core::pathUtils
 {
-    bool ensureDirectory(const std::filesystem::path& dir)
-    {
-        if (std::filesystem::exists(dir))
-            return std::filesystem::is_directory(dir);
-        else
-            return std::filesystem::create_directory(dir);
-    }
-
     bool hasFileAnyExtension(const std::filesystem::path& file, std::span<const std::filesystem::path> supportedExtensions)
     {
         const std::filesystem::path extension{ stringUtils::stringToLower(file.extension().c_str()) };

src/libs/core/impl/String.cpp

@@ -20,6 +20,8 @@
 #include "core/String.hpp"
 
 #include <algorithm>
+#include <charconv>
+#include <chrono>
 #include <cstring>
 #include <iomanip>
 #include <utility>
@@ -133,6 +135,64 @@ namespace lms::core::stringUtils
             res.push_back(str.substr(currentPos));
             return res;
         }
+
+        static std::optional<std::chrono::minutes> getRFC822ZoneOffset(std::string_view zoneStr)
+        {
+            if (zoneStr == "UT" || zoneStr == "GMT" || zoneStr == "Z") return std::chrono::hours{ 0 };
+            if (zoneStr == "EST") return -std::chrono::hours{ 5 };
+            if (zoneStr == "EDT") return -std::chrono::hours{ 4 };
+            if (zoneStr == "CST") return -std::chrono::hours{ 6 };
+            if (zoneStr == "CDT") return -std::chrono::hours{ 5 };
+            if (zoneStr == "MST") return -std::chrono::hours{ 7 };
+            if (zoneStr == "MDT") return -std::chrono::hours{ 6 };
+            if (zoneStr == "PST") return -std::chrono::hours{ 8 };
+            if (zoneStr == "PDT") return -std::chrono::hours{ 7 };
+            if (zoneStr.size() == 1)
+            {
+                const char c{ zoneStr[0] };
+                if (c == 'A') return -std::chrono::hours{ 1 };
+                if (c == 'M') return -std::chrono::hours{ 12 };
+                if (c == 'N') return std::chrono::hours{ 1 };
+                if (c == 'Y') return std::chrono::hours{ 12 };
+
+                return std::nullopt;
+            }
+
+            // (+/-)HHMM
+            if (zoneStr[0] != '+' && zoneStr[0] != '-')
+                return std::nullopt;
+
+            if (zoneStr.size() != 5)
+                return std::nullopt;
+
+            if (!std::all_of(std::cbegin(zoneStr) + 1, std::cend(zoneStr), [](char c) { return std::isdigit(c); }))
+                return std::nullopt;
+
+            int hours{};
+            const auto [p, ec]{ std::from_chars(zoneStr.data() + 1, zoneStr.data() + 3, hours) };
+            if (ec != std::errc())
+                return std::nullopt;
+
+            int minutes{};
+            const auto [p2, ec2] = std::from_chars(zoneStr.data() + 3, zoneStr.data() + 5, minutes);
+            if (ec2 != std::errc())
+                return std::nullopt;
+
+            std::chrono::minutes res{ std::chrono::hours{ hours } + std::chrono::minutes{ minutes } };
+            if (zoneStr[0] == '-')
+                res = -res;
+            return res;
+        }
+
+        std::optional<unsigned> getRFC822Month(std::string_view monthStr)
+        {
+            static const std::array<std::string_view, 12> months{ "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
+            const std::string_view* str{ std::find(std::cbegin(months), std::cend(months), monthStr) };
+            if (str == std::cend(months))
+                return {};
+
+            return std::distance(std::cbegin(months), str) + 1;
+        }
     } // namespace details
 
     template<>
@@ -435,9 +495,9 @@ namespace lms::core::stringUtils
         return str.substr(str.length() - ending.length()) == ending;
     }
 
-    std::optional<std::string> stringFromHex(const std::string& str)
+    std::optional<std::string> stringFromHex(std::string_view str)
     {
-        static const char lut[]{ "0123456789ABCDEF" };
+        constexpr char lut[]{ "0123456789ABCDEF" };
 
         if (str.length() % 2 != 0)
             return std::nullopt;
@@ -445,13 +505,13 @@ namespace lms::core::stringUtils
         std::string res;
         res.reserve(str.length() / 2);
 
-        auto it{ std::cbegin(str) };
+        const char* it{ std::cbegin(str) };
         while (it != std::cend(str))
         {
             unsigned val{};
 
-            auto itHigh{ std::lower_bound(std::cbegin(lut), std::cend(lut), std::toupper(*(it++))) };
-            auto itLow{ std::lower_bound(std::cbegin(lut), std::cend(lut), std::toupper(*(it++))) };
+            const char* itHigh{ std::lower_bound(std::cbegin(lut), std::cend(lut), std::toupper(*(it++))) };
+            const char* itLow{ std::lower_bound(std::cbegin(lut), std::cend(lut), std::toupper(*(it++))) };
 
             if (itHigh == std::cend(lut) || itLow == std::cend(lut))
                 return {};
@@ -465,6 +525,21 @@ namespace lms::core::stringUtils
         return res;
     }
 
+    std::string toHexString(std::string_view str)
+    {
+        constexpr char lut[]{ "0123456789ABCDEF" };
+
+        std::string res;
+
+        for (char c : str)
+        {
+            res.push_back(lut[(c >> 4) & 0xF]);
+            res.push_back(lut[c & 0xF]);
+        }
+
+        return res;
+    }
+
     std::string toISO8601String(const Wt::WDateTime& dateTime)
     {
         if (dateTime.isValid())
@@ -496,6 +571,47 @@ namespace lms::core::stringUtils
         return Wt::WDateTime::fromString(Wt::WString{ std::string{ dateTime } }, "yyyy-MM-ddThh:mm:ss.zzz");
     }
 
+    Wt::WDateTime fromRFC822String(std::string_view dateTime)
+    {
+        // Expect something like "[Sun,] 6 Nov 1994 08:49[:37] GMT"
+        if (dateTime.size() > 4 && dateTime[3] == ',')
+            dateTime.remove_prefix(5);
+
+        // Extract parts
+        const std::vector<std::string_view> subParts{ splitString(dateTime, ' ') };
+        if (subParts.size() != 5)
+            return {};
+
+        const std::string_view dayStr{ subParts[0] };
+        const std::string_view monthStr{ subParts[1] };
+        const std::string_view yearStr{ subParts[2] };
+        std::string timeStr{ subParts[3] };
+        const std::string_view zoneStr{ subParts[4] };
+        if (std::count(std::cbegin(timeStr), std::cend(timeStr), ':') == 1)
+            timeStr += ":00";
+
+        // Normalize zone
+        const std::optional<std::chrono::minutes> offset{ details::getRFC822ZoneOffset(zoneStr) };
+        if (!offset)
+            return {};
+
+        std::optional<unsigned> month{ details::getRFC822Month(monthStr) };
+        if (!month)
+            return {};
+
+        std::string datetimeStr;
+        datetimeStr = dayStr;
+        datetimeStr += " ";
+        datetimeStr += std::to_string(month.value());
+        datetimeStr += " ";
+        datetimeStr += yearStr;
+        datetimeStr += " ";
+        datetimeStr += timeStr;
+
+        const Wt::WDateTime res{ Wt::WDateTime::fromString(Wt::WString{ datetimeStr }, "d M yyyy HH:mm:ss") };
+        return res.addSecs(static_cast<int>(std::chrono::duration_cast<std::chrono::seconds>(offset.value()).count()));
+    }
+
     std::string formatTimestamp(std::chrono::milliseconds timestamp)
     {
         using namespace std::chrono;

src/libs/core/impl/http/Client.cpp

@@ -36,4 +36,9 @@ namespace lms::core::http
     {
         _sendQueue.sendRequest(std::make_unique<ClientRequest>(std::move(POSTParams)));
     }
+
+    void Client::abortAllRequests()
+    {
+        _sendQueue.abortAllRequests();
+    }
 } // namespace lms::core::http

src/libs/core/impl/http/Client.hpp

@@ -40,6 +40,7 @@ namespace lms::core::http
     private:
         void sendGETRequest(ClientGETRequestParameters&& request) override;
         void sendPOSTRequest(ClientPOSTRequestParameters&& request) override;
+        void abortAllRequests() override;
 
         SendQueue _sendQueue;
     };

src/libs/core/impl/http/SendQueue.cpp

@@ -19,16 +19,19 @@
 
 #include "SendQueue.hpp"
 
+#include <latch>
+
 #include <boost/asio/bind_executor.hpp>
 #include <boost/asio/dispatch.hpp>
 #include <boost/asio/post.hpp>
+#include <boost/asio/ssl/error.hpp>
 
 #include "core/Exception.hpp"
 #include "core/ILogger.hpp"
 #include "core/ITraceLogger.hpp"
 #include "core/String.hpp"
 
-#define LOG(sev, message) LMS_LOG(SCROBBLING, sev, "[Http SendQueue] - " << message)
+#define LOG(sev, message) LMS_LOG(HTTP, sev, "[Http SendQueue] - " << message)
 
 namespace lms::core::stringUtils
 {
@@ -63,7 +66,21 @@ namespace lms::core::http
     SendQueue::SendQueue(boost::asio::io_context& ioContext, std::string_view baseUrl)
         : _ioContext{ ioContext }
         , _baseUrl{ baseUrl }
+        , _abortAllRequests{ false }
+        , _state{ State::Idle }
+        , _client{ _ioContext }
     {
+        _client.setFollowRedirect(true);
+        _client.setTimeout(std::chrono::seconds{ 5 });
+
+        // not very efficient (response bodies are copied for each callback), but Wt's code already makes copies anyway
+
+        _client.bodyDataReceived().connect([this](const std::string& data) {
+            boost::asio::post(boost::asio::bind_executor(_strand, [this, data] {
+                onClientBodyDataReceived(data);
+            }));
+        });
+
         _client.done().connect([this](Wt::AsioWrapper::error_code ec, const Wt::Http::Message& msg) {
             boost::asio::post(boost::asio::bind_executor(_strand, [this, ec, msg = std::move(msg)] {
                 onClientDone(ec, msg);
@@ -73,12 +90,60 @@ namespace lms::core::http
 
     SendQueue::~SendQueue()
     {
-        _client.abort();
+        abortAllRequests();
+    }
+
+    void SendQueue::abortAllRequests()
+    {
+        LOG(DEBUG, "Aborting all requests...");
+
+        assert(!_abortAllRequests);
+        _abortAllRequests = true;
+
+        std::latch abortLatch{ 1 };
+
+        boost::asio::post(boost::asio::bind_executor(_strand, [this, &abortLatch] {
+            for (auto& [prio, requests] : _sendQueue)
+            {
+                while (!requests.empty())
+                {
+                    std::unique_ptr<ClientRequest> request{ std::move(requests.front()) };
+                    requests.pop_front();
+                    if (request->getParameters().onAbortFunc)
+                        request->getParameters().onAbortFunc();
+                }
+            }
+
+            if (_state == State::Throttled)
+                _throttleTimer.cancel();
+            else if (_state == State::Sending)
+                _client.abort();
+
+            abortLatch.count_down();
+        }));
+
+        abortLatch.wait();
+
+        while (_state != State::Idle)
+            std::this_thread::yield();
+
+        _abortAllRequests = false;
+
+        LOG(DEBUG, "All requests aborted!");
     }
 
     void SendQueue::sendRequest(std::unique_ptr<ClientRequest> request)
     {
-        boost::asio::dispatch(_strand, [this, request = std::move(request)]() mutable {
+        boost::asio::post(_strand, [this, request = std::move(request)]() mutable {
+            if (_abortAllRequests)
+            {
+                LOG(DEBUG, "Not posting request because abortAllRequests() in progress");
+                if (request->getParameters().onAbortFunc)
+                    request->getParameters().onAbortFunc();
+
+                return;
+            }
+
             _sendQueue[request->getParameters().priority].emplace_back(std::move(request));
 
             if (_state == State::Idle)
@@ -88,7 +153,7 @@ namespace lms::core::http
 
     void SendQueue::sendNextQueuedRequest()
     {
-        assert(_state == State::Idle);
+        assert(_strand.running_in_this_thread());
         assert(!_currentRequest);
 
         for (auto& [prio, requests] : _sendQueue)
@@ -100,21 +165,31 @@ namespace lms::core::http
                 requests.pop_front();
 
                 if (!sendRequest(*request))
+                {
+                    if (request->getParameters().onFailureFunc)
+                        request->getParameters().onFailureFunc();
                     continue;
+                }
 
-                _state = State::Sending;
+                setState(State::Sending);
                 _currentRequest = std::move(request);
                 return;
             }
         }
+
+        setState(State::Idle);
     }
 
     bool SendQueue::sendRequest(const ClientRequest& request)
     {
+        assert(_strand.running_in_this_thread());
+
         LMS_SCOPED_TRACE_DETAILED("SendQueue", "SendRequest");
 
-        std::string url{ _baseUrl + request.getParameters().relativeUrl };
-        LOG(DEBUG, "Sending request to url '" << url << "'");
+        const std::string url{ _baseUrl + request.getParameters().relativeUrl };
+        LOG(DEBUG, "Sending " << (request.getType() == ClientRequest::Type::GET ? "GET" : "POST") << " request to url '" << url << "'");
+
+        _client.setMaximumResponseSize(request.getParameters().onChunkReceived ? 0 : request.getParameters().responseBufferSize);
 
         bool res{};
         switch (request.getType())
@@ -134,29 +209,50 @@ namespace lms::core::http
         return res;
     }
 
+    void SendQueue::onClientBodyDataReceived(const std::string& data)
+    {
+        assert(_strand.running_in_this_thread());
+        assert(_currentRequest);
+
+        if (_currentRequest->getParameters().onChunkReceived)
+        {
+            const auto byteSpan{ std::as_bytes(std::span{ data.data(), data.size() }) };
+            if (_currentRequest->getParameters().onChunkReceived(byteSpan) == ClientRequestParameters::ChunckReceivedResult::Abort)
+                _client.abort();
+        }
+    }
+
     void SendQueue::onClientDone(Wt::AsioWrapper::error_code ec, const Wt::Http::Message& msg)
     {
         LMS_SCOPED_TRACE_DETAILED("SendQueue", "OnClientDone");
 
-        if (ec == boost::asio::error::operation_aborted)
-        {
-            LOG(DEBUG, "Client aborted");
-            return;
-        }
-
         assert(_currentRequest);
-        _state = State::Idle;
 
-        LOG(DEBUG, "Client done. status = " << msg.status());
-        if (ec)
+        LOG(DEBUG, "Client done. ec = " << ec.category().name() << " - " << ec.message() << " (" << ec.value() << "), status = " << msg.status());
+
+        if (_abortAllRequests || ec == boost::asio::error::operation_aborted)
+            onClientAborted(std::move(_currentRequest));
+        else if (ec && (ec != boost::asio::ssl::error::stream_truncated))
             onClientDoneError(std::move(_currentRequest), ec);
         else
             onClientDoneSuccess(std::move(_currentRequest), msg);
     }
 
+    void SendQueue::onClientAborted(std::unique_ptr<ClientRequest> request)
+    {
+        assert(_strand.running_in_this_thread());
+
+        if (request->getParameters().onAbortFunc)
+            request->getParameters().onAbortFunc();
+
+        sendNextQueuedRequest();
+    }
+
     void SendQueue::onClientDoneError(std::unique_ptr<ClientRequest> request, Wt::AsioWrapper::error_code ec)
     {
-        LOG(ERROR, "Retry " << request->retryCount << ", client error: '" << ec.message() << "'");
+        assert(_strand.running_in_this_thread());
+
+        LOG(WARNING, "Retry " << request->retryCount << ", client error: '" << ec.message() << "'");
 
         // may be a network error, try again later
         throttle(_defaultRetryWaitDuration);
@@ -196,42 +292,48 @@ namespace lms::core::http
             if (msg.status() == 200)
             {
                 if (requestParameters.onSuccessFunc)
-                    requestParameters.onSuccessFunc(msg.body());
+                    requestParameters.onSuccessFunc(msg);
             }
             else
             {
-                LOG(ERROR, "Send error: '" << msg.body() << "'");
+                LOG(ERROR, "Send error, status = " << msg.status() << ", body = '" << msg.body() << "'");
                 if (requestParameters.onFailureFunc)
                     requestParameters.onFailureFunc();
             }
         }
 
-        if (_state == State::Idle)
+        if (_state != State::Throttled)
             sendNextQueuedRequest();
     }
 
     void SendQueue::throttle(std::chrono::seconds requestedDuration)
     {
-        assert(_state == State::Idle);
-
         const std::chrono::seconds duration{ clamp(requestedDuration, _minRetryWaitDuration, _maxRetryWaitDuration) };
         LOG(DEBUG, "Throttling for " << duration.count() << " seconds");
 
         _throttleTimer.expires_after(duration);
         _throttleTimer.async_wait([this](const boost::system::error_code& ec) {
             if (ec == boost::asio::error::operation_aborted)
-            {
                 LOG(DEBUG, "Throttle aborted");
-                return;
-            }
             else if (ec)
-            {
                 throw LmsException{ "Throttle timer failure: " + std::string{ ec.message() } };
-            }
 
-            _state = State::Idle;
-            sendNextQueuedRequest();
+            setState(State::Idle);
+            if (!ec)
+                sendNextQueuedRequest();
         });
-        _state = State::Throttled;
+
+        setState(State::Throttled);
+    }
+
+    void SendQueue::setState(State state)
+    {
+        assert(_strand.running_in_this_thread());
+        if (_state != state)
+        {
+            LOG(DEBUG, "Changing state to " << (state == State::Idle ? "Idle" : state == State::Sending ? "Sending" :
+                                                                                                          "Throttled"));
+            _state = state;
+        }
     }
 } // namespace lms::core::http

src/libs/core/impl/http/SendQueue.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
+#include <atomic>
 #include <deque>
 #include <string_view>
-#include <vector>
 
 #include <Wt/Http/Client.h>
 #include <boost/asio/io_context.hpp>
@@ -44,10 +44,13 @@ namespace lms::core::http
         SendQueue& operator=(const SendQueue&&) = delete;
 
         void sendRequest(std::unique_ptr<ClientRequest> request);
+        void abortAllRequests();
 
     private:
         void sendNextQueuedRequest();
         bool sendRequest(const ClientRequest& request);
+        void onClientBodyDataReceived(const std::string& data);
+        void onClientAborted(std::unique_ptr<ClientRequest> request);
         void onClientDone(Wt::AsioWrapper::error_code ec, const Wt::Http::Message& msg);
         void onClientDoneError(std::unique_ptr<ClientRequest> request, Wt::AsioWrapper::error_code ec);
         void onClientDoneSuccess(std::unique_ptr<ClientRequest> request, const Wt::Http::Message& msg);
@@ -59,9 +62,9 @@ namespace lms::core::http
         const std::chrono::seconds _maxRetryWaitDuration{ 300 };
 
         boost::asio::io_context& _ioContext;
-        boost::asio::io_context::strand _strand{ _ioContext };
+        boost::asio::io_context::strand _strand{ _ioContext }; // protect _state, _sendQueue and _currentRequest
         boost::asio::steady_timer _throttleTimer{ _ioContext };
-        std::string _baseUrl;
+        const std::string _baseUrl;
 
         enum class State
         {
@@ -69,10 +72,11 @@ namespace lms::core::http
             Throttled,
             Sending,
         };
-        State _state{ State::Idle };
-        Wt::Http::Client _client{ _ioContext };
+        void setState(State state);
+        std::atomic<bool> _abortAllRequests;
+        State _state;
+        Wt::Http::Client _client;
         std::map<ClientRequestParameters::Priority, std::deque<std::unique_ptr<ClientRequest>>> _sendQueue;
         std::unique_ptr<ClientRequest> _currentRequest;
     };
-
 } // namespace lms::core::http