owen/lms
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplified logger configuration, it no longer depends on Wt

Browse Source
This commit is contained in:
emeric
2023-11-24 16:23:31 +01:00
parent 6f47c32ac2
commit 9060c0d925
104 changed files with 2603 additions and 2598 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/libs/av/impl/AudioFile.cpp
+6 -6
View File
@@ -31,7 +31,7 @@ extern "C"
#include <map>
#include <unordered_map>
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/String.hpp"
namespace Av
@@ -105,14 +105,14 @@ namespace Av
int error{ avformat_open_input(&_context, _p.string().c_str(), nullptr, nullptr) };
if (error < 0)
{
LMS_LOG(AV, ERROR) << "Cannot open " << _p.string() << ": " << averror_to_string(error);
LMS_LOG(AV, ERROR, "Cannot open " << _p.string() << ": " << averror_to_string(error));
throw AudioFileException{ error };
}
error = avformat_find_stream_info(_context, nullptr);
if (error < 0)
{
LMS_LOG(AV, ERROR) << "Cannot find stream information on " << _p.string() << ": " << averror_to_string(error);
LMS_LOG(AV, ERROR, "Cannot find stream information on " << _p.string() << ": " << averror_to_string(error));
avformat_close_input(&_context);
throw AudioFileException{ error };
}
@@ -233,7 +233,7 @@ namespace Av
if (avstream->codecpar == nullptr)
{
LMS_LOG(AV, ERROR) << "Skipping stream " << i << " since no codecpar is set";
LMS_LOG(AV, ERROR, "Skipping stream " << i << " since no codecpar is set");
continue;
}
@@ -247,7 +247,7 @@ namespace Av
else
{
picture.mimeType = "application/octet-stream";
LMS_LOG(AV, ERROR) << "CODEC ID " << avstream->codecpar->codec_id << " not handled in mime type conversion";
LMS_LOG(AV, ERROR, "CODEC ID " << avstream->codecpar->codec_id << " not handled in mime type conversion");
}
const AVPacket& pkt{ avstream->attached_pic };
@@ -271,7 +271,7 @@ namespace Av
if (!avstream->codecpar)
{
LMS_LOG(AV, ERROR) << "Skipping stream " << streamIndex << " since no codecpar is set";
LMS_LOG(AV, ERROR, "Skipping stream " << streamIndex << " since no codecpar is set");
return res;
}
src/libs/av/impl/Transcoder.cpp
+5 -5
View File
@@ -25,13 +25,13 @@
#include "utils/IChildProcessManager.hpp"
#include "utils/IConfig.hpp"
#include "utils/Path.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Service.hpp"
namespace Av::Transcoding
{
#define LOG(sev) LMS_LOG(TRANSCODING, sev) << "[" << _debugId << "] - "
#define LOG(severity, message) LMS_LOG(TRANSCODING, severity, "[" << _debugId << "] - " << message)
static std::atomic<size_t> globalId{};
static std::filesystem::path ffmpegPath;
@@ -84,7 +84,7 @@ namespace Av::Transcoding
throw Exception{ "File error '" + _inputParameters.trackPath.string() + "': " + e.what() };
}
LOG(INFO) << "Transcoding file '" << _inputParameters.trackPath.string() << "'";
LOG(INFO, "Transcoding file '" << _inputParameters.trackPath.string() << "'");
std::vector<std::string> args;
@@ -176,9 +176,9 @@ namespace Av::Transcoding
args.emplace_back("pipe:1");
LOG(DEBUG) << "Dumping args (" << args.size() << ")";
LOG(DEBUG, "Dumping args (" << args.size() << ")");
for (const std::string& arg : args)
LOG(DEBUG) << "Arg = '" << arg << "'";
LOG(DEBUG, "Arg = '" << arg << "'");
// Caution: stdin must have been closed before
try
src/libs/av/impl/TranscodingResourceHandler.cpp
+8 -8
View File
@@ -18,7 +18,7 @@
*/
#include "TranscodingResourceHandler.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Av::Transcoding
{
@@ -43,9 +43,9 @@ namespace Av::Transcoding
, _transcoder{ inputParameters, outputParameters }
{
if (_estimatedContentLength)
LMS_LOG(TRANSCODING, DEBUG) << "Estimated content length = " << *_estimatedContentLength;
LMS_LOG(TRANSCODING, DEBUG, "Estimated content length = " << *_estimatedContentLength);
else
LMS_LOG(TRANSCODING, DEBUG) << "Not using estimated content length";
LMS_LOG(TRANSCODING, DEBUG, "Not using estimated content length");
}
Wt::Http::ResponseContinuation* TranscodingResourceHandler::processRequest(const Wt::Http::Request& /*request*/, Wt::Http::Response& response)
@@ -53,11 +53,11 @@ namespace Av::Transcoding
if (_estimatedContentLength)
response.setContentLength(*_estimatedContentLength);
response.setMimeType(_transcoder.getOutputMimeType());
LMS_LOG(TRANSCODING, DEBUG) << "Transcoder finished = " << _transcoder.finished() << ", total served bytes = " << _totalServedByteCount << ", mime type = " << _transcoder.getOutputMimeType();
LMS_LOG(TRANSCODING, DEBUG, "Transcoder finished = " << _transcoder.finished() << ", total served bytes = " << _totalServedByteCount << ", mime type = " << _transcoder.getOutputMimeType());
if (_bytesReadyCount > 0)
{
LMS_LOG(TRANSCODING, DEBUG) << "Writing " << _bytesReadyCount << " bytes back to client";
LMS_LOG(TRANSCODING, DEBUG, "Writing " << _bytesReadyCount << " bytes back to client");
response.out().write(reinterpret_cast<const char*>(&_buffer[0]), _bytesReadyCount);
_totalServedByteCount += _bytesReadyCount;
@@ -70,7 +70,7 @@ namespace Av::Transcoding
continuation->waitForMoreData();
_transcoder.asyncRead(_buffer.data(), _buffer.size(), [=](std::size_t nbBytesRead)
{
LMS_LOG(TRANSCODING, DEBUG) << "Have " << nbBytesRead << " more bytes to send back";
LMS_LOG(TRANSCODING, DEBUG, "Have " << nbBytesRead << " more bytes to send back");
assert(_bytesReadyCount == 0);
_bytesReadyCount = nbBytesRead;
@@ -86,7 +86,7 @@ namespace Av::Transcoding
{
const std::size_t padSize{ *_estimatedContentLength - _totalServedByteCount };
LMS_LOG(TRANSCODING, DEBUG) << "Adding " << padSize << " padding bytes";
LMS_LOG(TRANSCODING, DEBUG, "Adding " << padSize << " padding bytes");
for (std::size_t i{}; i < padSize; ++i)
response.out().put(0);
@@ -94,7 +94,7 @@ namespace Av::Transcoding
_totalServedByteCount += padSize;
}
LMS_LOG(TRANSCODING, DEBUG) << "Transcoding finished. Total served byte count = " << _totalServedByteCount;
LMS_LOG(TRANSCODING, DEBUG, "Transcoding finished. Total served byte count = " << _totalServedByteCount);
}
return {};
src/libs/image/impl/graphicsmagick/JPEGImage.cpp
+2 -2
View File
@@ -21,7 +21,7 @@
#include "RawImage.hpp"
#include "image/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Image::GraphicsMagick
{
@@ -36,7 +36,7 @@ namespace Image::GraphicsMagick
}
catch (Magick::Exception& e)
{
LMS_LOG(COVER, ERROR) << "Caught Magick exception: " << e.what();
LMS_LOG(COVER, ERROR, "Caught Magick exception: " << e.what());
throw ImageException {std::string {"Magick read error: "} + e.what()};
}
}
src/libs/image/impl/graphicsmagick/RawImage.cpp
+13 -13
View File
@@ -23,7 +23,7 @@
#include "JPEGImage.hpp"
#include "image/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Image
{
@@ -43,16 +43,16 @@ namespace Image
Magick::InitializeMagick(path.string().c_str());
if (auto nbThreads {MagickLib::GetMagickResourceLimit(MagickLib::ThreadsResource)}; nbThreads != 1)
LMS_LOG(COVER, WARNING) << "Consider setting env var OMP_NUM_THREADS=1 to save resources";
LMS_LOG(COVER, WARNING, "Consider setting env var OMP_NUM_THREADS=1 to save resources");
if (!MagickLib::SetMagickResourceLimit(MagickLib::ThreadsResource, 1))
LMS_LOG(COVER, ERROR) << "Cannot set Magick thread resource limit to 1!";
LMS_LOG(COVER, ERROR, "Cannot set Magick thread resource limit to 1!");
if (!MagickLib::SetMagickResourceLimit(MagickLib::DiskResource, 0))
LMS_LOG(COVER, ERROR) << "Cannot set Magick disk resource limit to 0!";
LMS_LOG(COVER, ERROR, "Cannot set Magick disk resource limit to 0!");
LMS_LOG(COVER, INFO) << "Magick threads resource limit = " << GetMagickResourceLimit(MagickLib::ThreadsResource);
LMS_LOG(COVER, INFO) << "Magick Disk resource limit = " << GetMagickResourceLimit(MagickLib::DiskResource);
LMS_LOG(COVER, INFO, "Magick threads resource limit = " << GetMagickResourceLimit(MagickLib::ThreadsResource));
LMS_LOG(COVER, INFO, "Magick Disk resource limit = " << GetMagickResourceLimit(MagickLib::DiskResource));
}
}
@@ -68,16 +68,16 @@ RawImage::RawImage(const std::byte* encodedData, std::size_t encodedDataSize)
}
catch (Magick::WarningCoder& e)
{
LMS_LOG(COVER, WARNING) << "Caught Magick WarningCoder: " << e.what();
LMS_LOG(COVER, WARNING, "Caught Magick WarningCoder: " << e.what());
}
catch (Magick::Warning& e)
{
LMS_LOG(COVER, WARNING) << "Caught Magick warning: " << e.what();
LMS_LOG(COVER, WARNING, "Caught Magick warning: " << e.what());
throw ImageException {std::string {"Magick read warning: "} + e.what()};
}
catch (Magick::Exception& e)
{
LMS_LOG(COVER, ERROR) << "Caught Magick exception: " << e.what();
LMS_LOG(COVER, ERROR, "Caught Magick exception: " << e.what());
throw ImageException {std::string {"Magick read error: "} + e.what()};
}
}
@@ -90,16 +90,16 @@ RawImage::RawImage(const std::filesystem::path& p)
}
catch (Magick::WarningCoder& e)
{
LMS_LOG(COVER, WARNING) << "Caught Magick WarningCoder: " << e.what();
LMS_LOG(COVER, WARNING, "Caught Magick WarningCoder: " << e.what());
}
catch (Magick::Warning& e)
{
LMS_LOG(COVER, WARNING) << "Caught Magick warning: " << e.what();
LMS_LOG(COVER, WARNING, "Caught Magick warning: " << e.what());
throw ImageException {std::string {"Magick read warning: "} + e.what()};
}
catch (Magick::Exception& e)
{
LMS_LOG(COVER, ERROR) << "Caught Magick exception: " << e.what();
LMS_LOG(COVER, ERROR, "Caught Magick exception: " << e.what());
throw ImageException {std::string {"Magick read error: "} + e.what()};
}
}
@@ -113,7 +113,7 @@ RawImage::resize(ImageSize width)
}
catch (Magick::Exception& e)
{
LMS_LOG(COVER, ERROR) << "Caught Magick exception while resizing: " << e.what();
LMS_LOG(COVER, ERROR, "Caught Magick exception while resizing: " << e.what());
throw ImageException {std::string {"Magick resize error: "} + e.what()};
}
}
src/libs/metadata/impl/AvFormatParser.cpp
+1 -1
View File
@@ -23,7 +23,7 @@
#include <iostream>
#include "av/IAudioFile.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/String.hpp"
#include "Utils.hpp"
src/libs/metadata/impl/Factory.cpp
+3 -3
View File
@@ -20,7 +20,7 @@
#include "metadata/IParser.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "AvFormatParser.hpp"
#include "TagLibParser.hpp"
@@ -34,10 +34,10 @@ namespace MetaData
switch (parserType)
{
case ParserType::TagLib:
LMS_LOG(METADATA, INFO) << "Creating TagLib parser with read style = " << Utils::readStyleToString(parserReadStyle);
LMS_LOG(METADATA, INFO, "Creating TagLib parser with read style = " << Utils::readStyleToString(parserReadStyle));
return std::make_unique<TagLibParser>(parserReadStyle);
case ParserType::AvFormat:
LMS_LOG(METADATA, INFO) << "Creating AvFormat parser";
LMS_LOG(METADATA, INFO, "Creating AvFormat parser");
return std::make_unique<AvFormatParser>();
}
src/libs/metadata/impl/TagLibParser.cpp
+3 -3
View File
@@ -37,7 +37,7 @@
#include "utils/IConfig.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Service.hpp"
#include "utils/String.hpp"
#include "Utils.hpp"
@@ -391,7 +391,7 @@ namespace MetaData
if (f.isNull())
{
LMS_LOG(METADATA, ERROR) << "File '" << p.string() << "': parsing failed";
LMS_LOG(METADATA, ERROR, "File '" << p.string() << "': parsing failed");
return std::nullopt;
}
@@ -404,7 +404,7 @@ namespace MetaData
}
else
{
LMS_LOG(METADATA, INFO) << "File '" << p.string() << "': no audio properties";
LMS_LOG(METADATA, INFO, "File '" << p.string() << "': no audio properties");
return std::nullopt;
}
src/libs/services/auth/impl/AuthServiceBase.cpp
+2 -2
View File
@@ -22,7 +22,7 @@
#include "services/database/Db.hpp"
#include "services/database/Session.hpp"
#include "services/database/User.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Auth
{
@@ -43,7 +43,7 @@ namespace Auth
{
const UserType type {User::getCount(session) == 0 ? UserType::ADMIN : UserType::REGULAR};
LMS_LOG(AUTH, DEBUG) << "Creating user '" << loginName << "', admin = " << (type == UserType::ADMIN);
LMS_LOG(AUTH, DEBUG, "Creating user '" << loginName << "', admin = " << (type == UserType::ADMIN));
user = session.create<User>(loginName);
user.modify()->setType(type);
src/libs/services/auth/impl/AuthTokenService.cpp
+3 -3
View File
@@ -28,7 +28,7 @@
#include "services/database/Session.hpp"
#include "services/database/User.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Auth
{
@@ -62,7 +62,7 @@ namespace Auth
Database::AuthToken::pointer authToken {session.create<Database::AuthToken>(secretHash, expiry, user)};
LMS_LOG(UI, DEBUG) << "Created auth token for user '" << user->getLoginName() << "', expiry = " << expiry.toString();
LMS_LOG(UI, DEBUG, "Created auth token for user '" << user->getLoginName() << "', expiry = " << expiry.toString());
if (user->getAuthTokensCount() >= 50)
Database::AuthToken::removeExpiredTokens(session, Wt::WDateTime::currentDateTime());
@@ -88,7 +88,7 @@ namespace Auth
return std::nullopt;
}
LMS_LOG(UI, DEBUG) << "Found auth token for user '" << authToken->getUser()->getLoginName() << "'!";
LMS_LOG(UI, DEBUG, "Found auth token for user '" << authToken->getUser()->getLoginName() << "'!");
AuthTokenService::AuthTokenProcessResult::AuthTokenInfo res {authToken->getUser()->getId(), authToken->getExpiry()};
authToken.remove();
src/libs/services/auth/impl/LoginThrottler.cpp
+3 -3
View File
@@ -21,7 +21,7 @@
#include "LoginThrottler.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Random.hpp"
namespace Auth {
@@ -81,10 +81,10 @@ LoginThrottler::onBadClientAttempt(const boost::asio::ip::address& address)
attemptInfo.badConsecutiveAttemptCount += 1;
LMS_LOG(AUTH, DEBUG) << "Registering bad attempt for '" << clientAddress.to_string() << "', consecutive bad attempts count = " << attemptInfo.badConsecutiveAttemptCount;
LMS_LOG(AUTH, DEBUG, "Registering bad attempt for '" << clientAddress.to_string() << "', consecutive bad attempts count = " << attemptInfo.badConsecutiveAttemptCount);
if (attemptInfo.badConsecutiveAttemptCount >= _maxBadConsecutiveAttemptCount)
{
LMS_LOG(AUTH, DEBUG) << "Throttling '" << clientAddress.to_string() << "'";
LMS_LOG(AUTH, DEBUG, "Throttling '" << clientAddress.to_string() << "'");
attemptInfo.nextAttempt = now.addMSecs(std::chrono::duration_cast<std::chrono::milliseconds>(_throttlingDuration).count());
}
else
src/libs/services/auth/impl/PasswordServiceBase.cpp
+2 -2
View File
@@ -31,7 +31,7 @@
#include "services/database/Session.hpp"
#include "services/database/User.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Auth
{
@@ -61,7 +61,7 @@ namespace Auth
PasswordServiceBase::CheckResult
PasswordServiceBase::checkUserPassword(const boost::asio::ip::address& clientAddress, std::string_view loginName, std::string_view password)
{
LMS_LOG(AUTH, DEBUG) << "Checking password for user '" << loginName << "'";
LMS_LOG(AUTH, DEBUG, "Checking password for user '" << loginName << "'");
// Do not waste too much resource on brute force attacks (optim)
{
src/libs/services/auth/impl/http-headers/HttpHeadersEnvService.cpp
+27 -32
View File
@@ -22,46 +22,41 @@
#include <Wt/WEnvironment.h>
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Service.hpp"
namespace Auth
{
HttpHeadersEnvService::HttpHeadersEnvService(Database::Db& db)
: AuthServiceBase{ db }
, _fieldName{ Service<IConfig>::get()->getString("http-headers-login-field", "X-Forwarded-User") }
{
LMS_LOG(AUTH, INFO, "Using http header field = '" << _fieldName << "'");
}
HttpHeadersEnvService::HttpHeadersEnvService(Database::Db& db)
: AuthServiceBase {db}
, _fieldName {Service<IConfig>::get()->getString("http-headers-login-field", "X-Forwarded-User")}
{
LMS_LOG(AUTH, INFO) << "Using http header field = '" << _fieldName << "'";
}
HttpHeadersEnvService::CheckResult HttpHeadersEnvService::processEnv(const Wt::WEnvironment& env)
{
const std::string loginName{ env.headerValue(_fieldName) };
if (loginName.empty())
return { CheckResult::State::Denied };
HttpHeadersEnvService::CheckResult
HttpHeadersEnvService::processEnv(const Wt::WEnvironment& env)
{
const std::string loginName {env.headerValue(_fieldName)};
if (loginName.empty())
return {CheckResult::State::Denied};
LMS_LOG(AUTH, DEBUG, "Extracted login name = '" << loginName << "' from HTTP header");
LMS_LOG(AUTH, DEBUG) << "Extracted login name = '" << loginName << "' from HTTP header";
const Database::UserId userId{ getOrCreateUser(loginName) };
onUserAuthenticated(userId);
return { CheckResult::State::Granted, userId };
}
const Database::UserId userId {getOrCreateUser(loginName)};
onUserAuthenticated(userId);
return {CheckResult::State::Granted, userId};
}
HttpHeadersEnvService::CheckResult HttpHeadersEnvService::processRequest(const Wt::Http::Request& request)
{
const std::string loginName{ request.headerValue(_fieldName) };
if (loginName.empty())
return { CheckResult::State::Denied };
HttpHeadersEnvService::CheckResult
HttpHeadersEnvService::processRequest(const Wt::Http::Request& request)
{
const std::string loginName {request.headerValue(_fieldName)};
if (loginName.empty())
return {CheckResult::State::Denied};
LMS_LOG(AUTH, DEBUG) << "Extracted login name = '" << loginName << "' from HTTP header";
const Database::UserId userId {getOrCreateUser(loginName)};
onUserAuthenticated(userId);
return {CheckResult::State::Granted, userId};
}
LMS_LOG(AUTH, DEBUG, "Extracted login name = '" << loginName << "' from HTTP header");
const Database::UserId userId{ getOrCreateUser(loginName) };
onUserAuthenticated(userId);
return { CheckResult::State::Granted, userId };
}
} // namespace Auth
src/libs/services/auth/impl/internal/InternalPasswordService.cpp
+3 -3
View File
@@ -25,7 +25,7 @@
#include "services/database/Session.hpp"
#include "services/database/User.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Auth
{
@@ -44,7 +44,7 @@ namespace Auth
bool
InternalPasswordService::checkUserPassword(std::string_view loginName, std::string_view password)
{
LMS_LOG(AUTH, DEBUG) << "Checking internal password for user '" << loginName << "'";
LMS_LOG(AUTH, DEBUG, "Checking internal password for user '" << loginName << "'");
Database::User::PasswordHash passwordHash;
{
@@ -54,7 +54,7 @@ namespace Auth
const Database::User::pointer user {Database::User::find(session, loginName)};
if (!user)
{
LMS_LOG(AUTH, DEBUG) << "hashing random stuff";
LMS_LOG(AUTH, DEBUG, "hashing random stuff");
// hash random stuff here to waste some time
hashRandomPassword();
return false;
src/libs/services/auth/impl/pam/PAMPasswordService.cpp
+135 -137
View File
@@ -28,175 +28,173 @@
#include "services/auth/Types.hpp"
#include "services/database/Session.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Auth
{
class PAMError
{
public:
PAMError(std::string_view msg, pam_handle_t *pamh, int err)
{
_errorMsg = std::string {msg} + ": " + pam_strerror(pamh, err);
}
namespace
{
class PAMError
{
public:
PAMError(std::string_view msg, pam_handle_t* pamh, int err)
{
_errorMsg = std::string{ msg } + ": " + pam_strerror(pamh, err);
}
std::string_view message() const { return _errorMsg; }
std::string_view message() const { return _errorMsg; }
private:
std::string _errorMsg;
};
private:
std::string _errorMsg;
};
class PAMContext
{
public:
PAMContext(std::string_view loginName)
{
int err {pam_start("lms", std::string {loginName}.c_str(), &_conv, &_pamh)};
if (err != PAM_SUCCESS)
throw PAMError {"start failed", _pamh, err};
}
class PAMContext
{
public:
PAMContext(std::string_view loginName)
{
int err{ pam_start("lms", std::string {loginName}.c_str(), &_conv, &_pamh) };
if (err != PAM_SUCCESS)
throw PAMError{ "start failed", _pamh, err };
}
~PAMContext()
{
int err {pam_end(_pamh, 0)};
if (err != PAM_SUCCESS)
LMS_LOG(AUTH, ERROR) << "end failed: " << pam_strerror(_pamh, err);
}
~PAMContext()
{
int err{ pam_end(_pamh, 0) };
if (err != PAM_SUCCESS)
LMS_LOG(AUTH, ERROR, "end failed: " << pam_strerror(_pamh, err));
}
void authenticate(std::string_view password)
{
AuthenticateConvContext authContext {password};
ScopedConvContextSetter scopedContext {*this, authContext};
void authenticate(std::string_view password)
{
AuthenticateConvContext authContext{ password };
ScopedConvContextSetter scopedContext{ *this, authContext };
int err {pam_authenticate(_pamh, 0)};
if (err != PAM_SUCCESS)
throw PAMError {"authenticate failed", _pamh, err};
}
int err{ pam_authenticate(_pamh, 0) };
if (err != PAM_SUCCESS)
throw PAMError{ "authenticate failed", _pamh, err };
}
void validateAccount()
{
int err {pam_acct_mgmt(_pamh, PAM_SILENT)};
if (err != PAM_SUCCESS)
throw PAMError {"acct_mgmt failed", _pamh, err};
}
void validateAccount()
{
int err{ pam_acct_mgmt(_pamh, PAM_SILENT) };
if (err != PAM_SUCCESS)
throw PAMError{ "acct_mgmt failed", _pamh, err };
}
private:
class ConvContext
{
public:
virtual ~ConvContext() = default;
};
private:
class ConvContext
{
public:
virtual ~ConvContext() = default;
};
class AuthenticateConvContext final : public ConvContext
{
public:
AuthenticateConvContext(std::string_view password) : _password {password} {}
class AuthenticateConvContext final : public ConvContext
{
public:
AuthenticateConvContext(std::string_view password) : _password{ password } {}
std::string_view getPassword() const { return _password; }
std::string_view getPassword() const { return _password; }
private:
std::string_view _password;
};
private:
std::string_view _password;
};
class ScopedConvContextSetter
{
public:
ScopedConvContextSetter(PAMContext& pamContext, ConvContext& convContext)
: _pamContext {pamContext}
{
_pamContext._convContext = &convContext;
}
class ScopedConvContextSetter
{
public:
ScopedConvContextSetter(PAMContext& pamContext, ConvContext& convContext)
: _pamContext{ pamContext }
{
_pamContext._convContext = &convContext;
}
~ScopedConvContextSetter()
{
_pamContext._convContext = nullptr;
}
~ScopedConvContextSetter()
{
_pamContext._convContext = nullptr;
}
ScopedConvContextSetter(const ScopedConvContextSetter&) = delete;
ScopedConvContextSetter(ScopedConvContextSetter&&) = delete;
ScopedConvContextSetter& operator=(const ScopedConvContextSetter&) = delete;
ScopedConvContextSetter& operator=(ScopedConvContextSetter&&) = delete;
ScopedConvContextSetter(const ScopedConvContextSetter&) = delete;
ScopedConvContextSetter(ScopedConvContextSetter&&) = delete;
ScopedConvContextSetter& operator=(const ScopedConvContextSetter&) = delete;
ScopedConvContextSetter& operator=(ScopedConvContextSetter&&) = delete;
private:
PAMContext& _pamContext;
};
private:
PAMContext& _pamContext;
};
static int conv(int msgCount, const pam_message** msgs, pam_response** resps, void* userData)
{
if (msgCount < 1)
return PAM_CONV_ERR;
if (!resps || !msgs || !userData)
return PAM_CONV_ERR;
static int conv(int msgCount, const pam_message** msgs, pam_response** resps, void* userData)
{
if (msgCount < 1)
return PAM_CONV_ERR;
if (!resps || !msgs || !userData)
return PAM_CONV_ERR;
PAMContext& context{ *static_cast<PAMContext*>(userData) };
PAMContext& context {*static_cast<PAMContext*>(userData)};
AuthenticateConvContext* authenticateContext = dynamic_cast<AuthenticateConvContext*>(context._convContext);
if (!authenticateContext)
{
LMS_LOG(AUTH, ERROR, "Unexpected conv!");
return PAM_CONV_ERR;
}
AuthenticateConvContext* authenticateContext = dynamic_cast<AuthenticateConvContext*>(context._convContext);
if (!authenticateContext)
{
LMS_LOG(AUTH, ERROR) << "Unexpected conv!";
return PAM_CONV_ERR;
}
// Only expect a PAM_PROMPT_ECHO_OFF msg
if (msgCount != 1 || msgs[0]->msg_style != PAM_PROMPT_ECHO_OFF)
{
LMS_LOG(AUTH, ERROR, "Unexpected conv message. Count = " << msgCount);
return PAM_CONV_ERR;
}
// Only expect a PAM_PROMPT_ECHO_OFF msg
if (msgCount != 1 || msgs[0]->msg_style != PAM_PROMPT_ECHO_OFF)
{
LMS_LOG(AUTH, ERROR) << "Unexpected conv message. Count = " << msgCount;
return PAM_CONV_ERR;
}
pam_response* response{ static_cast<pam_response*>(malloc(sizeof(pam_response))) };
if (!response)
return PAM_CONV_ERR;
pam_response* response {static_cast<pam_response*>(malloc(sizeof(pam_response)))};
if (!response)
return PAM_CONV_ERR;
response->resp = strdup(std::string{ authenticateContext->getPassword() }.c_str());
response->resp = strdup(std::string {authenticateContext->getPassword()}.c_str());
*resps = response;
return PAM_SUCCESS;
}
*resps = response;
return PAM_SUCCESS;
}
ConvContext* _convContext{};
pam_conv _conv{ &PAMContext::conv, this };
pam_handle_t* _pamh{};
};
}
ConvContext* _convContext {};
pam_conv _conv {&PAMContext::conv, this};
pam_handle_t *_pamh {};
};
bool PAMPasswordService::checkUserPassword(std::string_view loginName, std::string_view password)
{
try
{
LMS_LOG(AUTH, DEBUG, "Checking PAM password for user '" << loginName << "'");
PAMContext pamContext{ loginName };
bool
PAMPasswordService::checkUserPassword(std::string_view loginName, std::string_view password)
{
try
{
LMS_LOG(AUTH, DEBUG) << "Checking PAM password for user '" << loginName << "'";
PAMContext pamContext {loginName};
pamContext.authenticate(password);
pamContext.validateAccount();
pamContext.authenticate(password);
pamContext.validateAccount();
return true;
}
catch (const PAMError& error)
{
LMS_LOG(AUTH, ERROR, "PAM error: " << error.message());
return false;
}
}
return true;
}
catch (const PAMError& error)
{
LMS_LOG(AUTH, ERROR) << "PAM error: " << error.message();
return false;
}
}
bool PAMPasswordService::canSetPasswords() const
{
return false;
}
bool
PAMPasswordService::canSetPasswords() const
{
return false;
}
IPasswordService::PasswordAcceptabilityResult PAMPasswordService::checkPasswordAcceptability(std::string_view, const PasswordValidationContext&) const
{
throw NotImplementedException{};
}
IPasswordService::PasswordAcceptabilityResult
PAMPasswordService::checkPasswordAcceptability(std::string_view, const PasswordValidationContext&) const
{
throw NotImplementedException {};
}
void
PAMPasswordService::setPassword(Database::UserId, std::string_view)
{
throw NotImplementedException {};
}
void PAMPasswordService::setPassword(Database::UserId, std::string_view)
{
throw NotImplementedException{};
}
} // namespace Auth
src/libs/services/cover/impl/CoverService.cpp
+12 -12
View File
@@ -29,7 +29,7 @@
#include "image/Exception.hpp"
#include "image/IRawImage.hpp"
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Random.hpp"
#include "utils/String.hpp"
#include "utils/Utils.hpp"
@@ -110,10 +110,10 @@ namespace Cover
{
setJpegQuality(Service<IConfig>::get()->getULong("cover-jpeg-quality", 75));
LMS_LOG(COVER, INFO) << "Default cover path = '" << _defaultCoverPath.string() << "'";
LMS_LOG(COVER, INFO) << "Max cache size = " << _maxCacheSize;
LMS_LOG(COVER, INFO) << "Max file size = " << _maxFileSize;
LMS_LOG(COVER, INFO) << "Preferred file names: " << StringUtils::joinStrings(_preferredFileNames, ",");
LMS_LOG(COVER, INFO, "Default cover path = '" << _defaultCoverPath.string() << "'");
LMS_LOG(COVER, INFO, "Max cache size = " << _maxCacheSize);
LMS_LOG(COVER, INFO, "Max file size = " << _maxFileSize);
LMS_LOG(COVER, INFO, "Preferred file names: " << StringUtils::joinStrings(_preferredFileNames, ","));
#if LMS_SUPPORT_IMAGE_GM
GraphicsMagick::init(execPath);
@@ -148,7 +148,7 @@ namespace Cover
}
catch (const Image::ImageException& e)
{
LMS_LOG(COVER, ERROR) << "Cannot read embedded cover: " << e.what();
LMS_LOG(COVER, ERROR, "Cannot read embedded cover: " << e.what());
}
});
@@ -167,7 +167,7 @@ namespace Cover
}
catch (const ImageException& e)
{
LMS_LOG(COVER, ERROR) << "Cannot read cover in file '" << p.string() << "': " << e.what();
LMS_LOG(COVER, ERROR, "Cannot read cover in file '" << p.string() << "': " << e.what());
}
return image;
@@ -190,7 +190,7 @@ namespace Cover
std::shared_ptr<IEncodedImage> image{ getFromCoverFile(_defaultCoverPath, width) };
_defaultCoverCache[width] = image;
LMS_LOG(COVER, DEBUG) << "Default cache entries = " << _defaultCoverCache.size();
LMS_LOG(COVER, DEBUG, "Default cache entries = " << _defaultCoverCache.size());
return image;
}
@@ -269,7 +269,7 @@ namespace Cover
if (std::filesystem::file_size(filePath, ec) > _maxFileSize && !ec)
{
LMS_LOG(COVER, INFO) << "Cover file '" << filePath.string() << " is too big (" << std::filesystem::file_size(filePath, ec) << "), limit is " << _maxFileSize;
LMS_LOG(COVER, INFO, "Cover file '" << filePath.string() << " is too big (" << std::filesystem::file_size(filePath, ec) << "), limit is " << _maxFileSize);
return false;
}
@@ -306,7 +306,7 @@ namespace Cover
}
catch (Av::Exception& e)
{
LMS_LOG(COVER, ERROR) << "Cannot get covers from track " << p.string() << ": " << e.what();
LMS_LOG(COVER, ERROR, "Cannot get covers from track " << p.string() << ": " << e.what());
}
return image;
@@ -404,7 +404,7 @@ namespace Cover
{
std::unique_lock lock{ _cacheMutex };
LMS_LOG(COVER, DEBUG) << "Cache stats: hits = " << _cacheHits << ", misses = " << _cacheMisses << ", nb entries = " << _cache.size() << ", size = " << _cacheSize;
LMS_LOG(COVER, DEBUG, "Cache stats: hits = " << _cacheHits << ", misses = " << _cacheMisses << ", nb entries = " << _cache.size() << ", size = " << _cacheSize);
_cacheHits = 0;
_cacheMisses = 0;
_cacheSize = 0;
@@ -415,7 +415,7 @@ namespace Cover
{
_jpegQuality = Utils::clamp<unsigned>(quality, 1, 100);
LMS_LOG(COVER, INFO) << "JPEG export quality = " << _jpegQuality;
LMS_LOG(COVER, INFO, "JPEG export quality = " << _jpegQuality);
}
void CoverService::saveToCache(const CacheEntryDesc& entryDesc, std::shared_ptr<IEncodedImage> image)
src/libs/services/database/impl/Artist.cpp
+1 -1
View File
@@ -25,7 +25,7 @@
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "services/database/User.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "SqlQuery.hpp"
#include "Utils.hpp"
#include "EnumSetTraits.hpp"
src/libs/services/database/impl/Cluster.cpp
+1 -1
View File
@@ -157,7 +157,7 @@ namespace Database
{
}
ClusterType::pointer ClusterType::create(Session& session, const std::string& name)
ClusterType::pointer ClusterType::create(Session& session, std::string_view name)
{
return session.getDboSession().add(std::unique_ptr<ClusterType> {new ClusterType{ name }});
}
src/libs/services/database/impl/Db.cpp
+6 -6
View File
@@ -26,7 +26,7 @@
#include "services/database/User.hpp"
#include "utils/IConfig.hpp"
#include "utils/Service.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Database
{
@@ -65,18 +65,18 @@ namespace Database
void prepare()
{
LMS_LOG(DB, DEBUG) << "Setting per-connection settings...";
LMS_LOG(DB, DEBUG, "Setting per-connection settings...");
executeSql("pragma journal_mode=WAL");
executeSql("pragma synchronous=normal");
executeSql("pragma analysis_limit=2000"); // to help make analyze command faster, 1000 does not seem to be enough to speed up all queries
LMS_LOG(DB, DEBUG) << "Setting per-connection settings done!";
LMS_LOG(DB, DEBUG, "Setting per-connection settings done!");
}
void optimize()
{
LMS_LOG(DB, DEBUG) << "connection close: Running pragma optimize...";
LMS_LOG(DB, DEBUG, "connection close: Running pragma optimize...");
executeSql("pragma optimize");
LMS_LOG(DB, DEBUG) << "connection close: pragma optimize complete";
LMS_LOG(DB, DEBUG, "connection close: pragma optimize complete");
}
std::filesystem::path _dbPath;
@@ -86,7 +86,7 @@ namespace Database
// Session living class handling the database and the login
Db::Db(const std::filesystem::path& dbPath, std::size_t connectionCount)
{
LMS_LOG(DB, INFO) << "Creating connection pool on file " << dbPath.string();
LMS_LOG(DB, INFO, "Creating connection pool on file " << dbPath.string());
auto connection{ std::make_unique<Connection>(dbPath.string()) };
if (IConfig * config{ Service<IConfig>::get() })// may not be here on testU
src/libs/services/database/impl/Migration.cpp
+5 -5
View File
@@ -26,7 +26,7 @@
#include "services/database/Session.hpp"
#include "services/database/User.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Database
{
@@ -282,11 +282,11 @@ CREATE TABLE IF NOT EXISTS "track_backup" (
try
{
version = VersionInfo::getOrCreate(session)->getVersion();
LMS_LOG(DB, INFO) << "Database version = " << version << ", LMS binary version = " << LMS_DATABASE_VERSION;
LMS_LOG(DB, INFO, "Database version = " << version << ", LMS binary version = " << LMS_DATABASE_VERSION);
}
catch (std::exception& e)
{
LMS_LOG(DB, ERROR) << "Cannot get database version info: " << e.what();
LMS_LOG(DB, ERROR, "Cannot get database version info: " << e.what());
throw LmsException{ outdatedMsg };
}
@@ -298,7 +298,7 @@ CREATE TABLE IF NOT EXISTS "track_backup" (
while (version < LMS_DATABASE_VERSION)
{
LMS_LOG(DB, INFO) << "Migrating database from version " << version << " to " << version + 1 << "...";
LMS_LOG(DB, INFO, "Migrating database from version " << version << " to " << version + 1 << "...");
auto itMigrationFunc{ migrationFunctions.find(version) };
assert(itMigrationFunc != std::cend(migrationFunctions));
@@ -306,7 +306,7 @@ CREATE TABLE IF NOT EXISTS "track_backup" (
VersionInfo::get(session).modify()->setVersion(++version);
LMS_LOG(DB, INFO) << "Migration complete to version " << version;
LMS_LOG(DB, INFO, "Migration complete to version " << version);
}
}
}
src/libs/services/database/impl/Release.cpp
+1 -1
View File
@@ -26,7 +26,7 @@
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "services/database/User.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "SqlQuery.hpp"
#include "EnumSetTraits.hpp"
#include "IdTypeTraits.hpp"
src/libs/services/database/impl/ScanSettings.cpp
+91 -99
View File
@@ -22,131 +22,123 @@
#include <Wt/Dbo/WtSqlTraits.h>
#include "utils/Path.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/String.hpp"
#include "services/database/Cluster.hpp"
#include "services/database/Session.hpp"
namespace {
const std::set<std::string> defaultClusterTypeNames =
namespace Database
{
"GENRE",
"ALBUMGROUPING",
"MOOD",
"ALBUMMOOD",
};
namespace
{
}
const std::set<std::string_view> defaultClusterTypeNames =
{
"GENRE",
"ALBUMGROUPING",
"MOOD",
"ALBUMMOOD",
};
namespace Database {
}
void ScanSettings::init(Session& session)
{
session.checkWriteTransaction();
void
ScanSettings::init(Session& session)
{
session.checkWriteTransaction();
pointer settings{ get(session) };
if (settings)
return;
pointer settings {get(session)};
if (settings)
return;
settings = session.getDboSession().add(std::make_unique<ScanSettings>());
settings.modify()->setClusterTypes(session, defaultClusterTypeNames);
}
settings = session.getDboSession().add(std::make_unique<ScanSettings>());
settings.modify()->setClusterTypes(session, defaultClusterTypeNames );
}
ScanSettings::pointer ScanSettings::get(Session& session)
{
session.checkReadTransaction();
ScanSettings::pointer
ScanSettings::get(Session& session)
{
session.checkReadTransaction();
return session.getDboSession().find<ScanSettings>().resultValue();
}
return session.getDboSession().find<ScanSettings>().resultValue();
}
std::vector<std::filesystem::path> ScanSettings::getAudioFileExtensions() const
{
const auto extensions{ StringUtils::splitString(_audioFileExtensions, " ") };
std::vector<std::filesystem::path>
ScanSettings::getAudioFileExtensions() const
{
const auto extensions {StringUtils::splitString(_audioFileExtensions, " ")};
std::vector<std::filesystem::path> res(std::cbegin(extensions), std::cend(extensions));
std::sort(std::begin(res), std::end(res));
res.erase(std::unique(std::begin(res), std::end(res)), std::end(res));
std::vector<std::filesystem::path> res (std::cbegin(extensions), std::cend(extensions));
std::sort(std::begin(res), std::end(res));
res.erase(std::unique( std::begin(res), std::end(res)), std::end(res));
return res;
}
return res;
}
void ScanSettings::addAudioFileExtension(const std::filesystem::path& ext)
{
_audioFileExtensions += " " + ext.string();
}
void
ScanSettings::addAudioFileExtension(const std::filesystem::path& ext)
{
_audioFileExtensions += " " + ext.string();
}
std::vector<ClusterType::pointer> ScanSettings::getClusterTypes() const
{
return std::vector<ClusterType::pointer>(std::cbegin(_clusterTypes), std::cend(_clusterTypes));
}
std::vector<ClusterType::pointer>
ScanSettings::getClusterTypes() const
{
return std::vector<ClusterType::pointer>(std::cbegin(_clusterTypes), std::cend(_clusterTypes));
}
void ScanSettings::setMediaDirectory(const std::filesystem::path& p)
{
_mediaDirectory = StringUtils::stringTrimEnd(p.string(), "/\\");
}
void
ScanSettings::setMediaDirectory(const std::filesystem::path& p)
{
_mediaDirectory = StringUtils::stringTrimEnd(p.string(), "/\\");
}
template <typename It>
std::set<std::string> getNames(It begin, It end)
{
std::set<std::string> names;
std::transform(begin, end, std::inserter(names, std::cbegin(names)),
[](const ClusterType::pointer& clusterType)
{
return clusterType->getName();
});
template <typename It>
std::set<std::string> getNames(It begin, It end)
{
std::set<std::string> names;
std::transform(begin, end, std::inserter(names, std::cbegin(names)),
[](const ClusterType::pointer& clusterType)
{
return clusterType->getName();
});
return names;
}
return names;
}
void ScanSettings::setClusterTypes(Session& session, const std::set<std::string_view>& clusterTypeNames)
{
session.checkWriteTransaction();
void
ScanSettings::setClusterTypes(Session& session, const std::set<std::string>& clusterTypeNames)
{
session.checkWriteTransaction();
bool needRescan{};
bool needRescan {};
// Create any missing cluster type
for (const std::string_view clusterTypeName : clusterTypeNames)
{
ClusterType::pointer clusterType{ ClusterType::find(session, clusterTypeName) };
if (!clusterType)
{
LMS_LOG(DB, INFO, "Creating cluster type " << clusterTypeName);
clusterType = session.create<ClusterType>(clusterTypeName);
_clusterTypes.insert(getDboPtr(clusterType));
// Create any missing cluster type
for (const std::string& clusterTypeName : clusterTypeNames)
{
ClusterType::pointer clusterType {ClusterType::find(session, clusterTypeName)};
if (!clusterType)
{
LMS_LOG(DB, INFO) << "Creating cluster type " << clusterTypeName;
clusterType = session.create<ClusterType>(clusterTypeName);
_clusterTypes.insert(getDboPtr(clusterType));
needRescan = true;
}
}
needRescan = true;
}
}
// Delete no longer existing cluster types
for (Wt::Dbo::ptr<ClusterType> clusterType : _clusterTypes)
{
if (std::none_of(clusterTypeNames.begin(), clusterTypeNames.end(),
[clusterType](std::string_view name) { return name == clusterType->getName(); }))
{
LMS_LOG(DB, INFO, "Deleting cluster type " << clusterType->getName());
clusterType.remove();
}
}
// Delete no longer existing cluster types
for (Wt::Dbo::ptr<ClusterType> clusterType : _clusterTypes)
{
if (std::none_of(clusterTypeNames.begin(), clusterTypeNames.end(),
[clusterType](const std::string& name) { return name == clusterType->getName(); }))
{
LMS_LOG(DB, INFO) << "Deleting cluster type " << clusterType->getName();
clusterType.remove();
}
}
if (needRescan)
_scanVersion += 1;
}
if (needRescan)
_scanVersion += 1;
}
void
ScanSettings::incScanVersion()
{
_scanVersion += 1;
}
void
ScanSettings::incScanVersion()
{
_scanVersion += 1;
}
} // namespace Database
src/libs/services/database/impl/Session.cpp
+9 -9
View File
@@ -22,7 +22,7 @@
#include <cassert>
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "services/database/Artist.hpp"
#include "services/database/AuthToken.hpp"
@@ -107,20 +107,20 @@ namespace Database
void Session::prepareTables()
{
LMS_LOG(DB, INFO) << "Preparing tables...";
LMS_LOG(DB, INFO, "Preparing tables...");
// Initial creation case
try
{
_session.createTables();
LMS_LOG(DB, INFO) << "Tables created";
LMS_LOG(DB, INFO, "Tables created");
}
catch (Wt::Dbo::Exception& e)
{
LMS_LOG(DB, DEBUG) << "Cannot create tables: " << e.what();
LMS_LOG(DB, DEBUG, "Cannot create tables: " << e.what());
if (std::string_view{ e.what() }.find("already exists") == std::string_view::npos)
{
LMS_LOG(DB, ERROR) << "Cannot create tables: " << e.what();
LMS_LOG(DB, ERROR, "Cannot create tables: " << e.what());
throw e;
}
}
@@ -182,22 +182,22 @@ namespace Database
void Session::analyze()
{
LMS_LOG(DB, INFO) << "Analyzing database...";
LMS_LOG(DB, INFO, "Analyzing database...");
{
auto transaction{ createWriteTransaction() };
_session.execute("ANALYZE");
}
LMS_LOG(DB, INFO) << "Database Analyze complete";
LMS_LOG(DB, INFO, "Database Analyze complete");
}
void Session::optimize()
{
LMS_LOG(DB, INFO) << "Optimizing database...";
LMS_LOG(DB, INFO, "Optimizing database...");
{
auto transaction{ createWriteTransaction() };
_session.execute("PRAGMA optimize");
}
LMS_LOG(DB, INFO) << "Database optimizing complete";
LMS_LOG(DB, INFO, "Database optimizing complete");
}
} // namespace Database
src/libs/services/database/impl/Track.cpp
+2 -2
View File
@@ -28,7 +28,7 @@
#include "services/database/TrackFeatures.hpp"
#include "services/database/Session.hpp"
#include "services/database/User.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "IdTypeTraits.hpp"
#include "SqlQuery.hpp"
@@ -536,7 +536,7 @@ namespace Database
for (auto artist : track->getArtists({ TrackArtistLinkType::Artist }))
os << " - " << artist->getName();
for (auto cluster : track->getClusters())
os << " {" + cluster->getType()->getName() << "-" << cluster->getName() << "}";
os << " {" << cluster->getType()->getName() << "-" << cluster->getName() << "}";
}
else
{
src/libs/services/database/impl/TrackFeatures.cpp
+2 -2
View File
@@ -24,7 +24,7 @@
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "IdTypeTraits.hpp"
#include "Utils.hpp"
@@ -111,7 +111,7 @@ namespace Database {
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(DB, ERROR) << "Track " << _track.id() << ": ptree exception: " << error.what();
LMS_LOG(DB, ERROR, "Track " << _track.id() << ": ptree exception: " << error.what());
res.clear();
}
src/libs/services/database/impl/TrackList.cpp
+1 -1
View File
@@ -20,7 +20,7 @@
#include <cassert>
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "services/database/Artist.hpp"
#include "services/database/Cluster.hpp"
src/libs/services/database/impl/User.cpp
+1 -1
View File
@@ -23,7 +23,7 @@
#include "services/database/Release.hpp"
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "IdTypeTraits.hpp"
#include "StringViewTraits.hpp"
#include "Utils.hpp"
src/libs/services/database/include/services/database/Cluster.hpp
+2 -2
View File
@@ -126,7 +126,7 @@ namespace Database {
static void remove(Session& session, const std::string& name);
// Accessors
const std::string& getName() const { return _name; }
std::string_view getName() const { return _name; }
std::vector<Cluster::pointer> getClusters() const;
Cluster::pointer getCluster(const std::string& name) const;
@@ -141,7 +141,7 @@ namespace Database {
private:
friend class Session;
ClusterType(std::string_view name);
static pointer create(Session& session, const std::string& name);
static pointer create(Session& session, std::string_view name);
static const std::size_t _maxNameLength = 128;
src/libs/services/database/include/services/database/ScanSettings.hpp
+3 -1
View File
@@ -20,6 +20,8 @@
#pragma once
#include <filesystem>
#include <string>
#include <string_view>
#include <vector>
#include <Wt/Dbo/Dbo.h>
@@ -74,7 +76,7 @@ namespace Database {
void setMediaDirectory(const std::filesystem::path& p);
void setUpdateStartTime(Wt::WTime t) { _startTime = t; }
void setUpdatePeriod(UpdatePeriod p) { _updatePeriod = p; }
void setClusterTypes(Session& session, const std::set<std::string>& clusterTypeNames);
void setClusterTypes(Session& session, const std::set<std::string_view>& clusterTypeNames);
void setSimilarityEngineType(SimilarityEngineType type) { _similarityEngineType = type; }
void incScanVersion();
src/libs/services/feedback/impl/FeedbackService.cpp
+4 -4
View File
@@ -29,7 +29,7 @@
#include "services/database/StarredTrack.hpp"
#include "services/database/Track.hpp"
#include "services/database/User.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "internal/InternalBackend.hpp"
#include "listenbrainz/ListenBrainzBackend.hpp"
@@ -44,15 +44,15 @@ namespace Feedback
FeedbackService::FeedbackService(boost::asio::io_context& ioContext, Db& db)
: _db{ db }
{
LMS_LOG(SCROBBLING, INFO) << "Starting service...";
LMS_LOG(SCROBBLING, INFO, "Starting service...");
_backends.emplace(Database::FeedbackBackend::Internal, std::make_unique<InternalBackend>(_db));
_backends.emplace(Database::FeedbackBackend::ListenBrainz, std::make_unique<ListenBrainz::ListenBrainzBackend>(ioContext, _db));
LMS_LOG(SCROBBLING, INFO) << "Service started!";
LMS_LOG(SCROBBLING, INFO, "Service started!");
}
FeedbackService::~FeedbackService()
{
LMS_LOG(SCROBBLING, INFO) << "Service stopped!";
LMS_LOG(SCROBBLING, INFO, "Service stopped!");
}
std::optional<Database::FeedbackBackend> FeedbackService::getUserFeedbackBackend(UserId userId)
src/libs/services/feedback/impl/listenbrainz/FeedbacksParser.cpp
+4 -4
View File
@@ -57,7 +57,7 @@ namespace Feedback::ListenBrainz
const Wt::Json::Array& feedbacks = root.get("feedback");
LOG(DEBUG) << "Got " << feedbacks.size() << " feedbacks";
LOG(DEBUG, "Got " << feedbacks.size() << " feedbacks");
if (feedbacks.empty())
return res;
@@ -72,17 +72,17 @@ namespace Feedback::ListenBrainz
}
catch (const Exception& e)
{
LOG(DEBUG) << "Cannot parse feedback: " << e.what() << ", skipping";
LOG(DEBUG, "Cannot parse feedback: " << e.what() << ", skipping");
}
catch (const Wt::WException& e)
{
LOG(DEBUG) << "Cannot parse feedback: " << e.what() << ", skipping";
LOG(DEBUG, "Cannot parse feedback: " << e.what() << ", skipping");
}
}
}
catch (const Wt::WException& error)
{
LOG(ERROR) << "Cannot parse 'feedback' result: " << error.what();
LOG(ERROR, "Cannot parse 'feedback' result: " << error.what());
}
return res;
src/libs/services/feedback/impl/listenbrainz/FeedbacksSynchronizer.cpp
+22 -22
View File
@@ -53,7 +53,7 @@ namespace Feedback::ListenBrainz
}
catch (const Wt::WException& e)
{
LOG(ERROR) << "Cannot parse listen count response: " << e.what();
LOG(ERROR, "Cannot parse listen count response: " << e.what());
return std::nullopt;
}
}
@@ -66,7 +66,7 @@ namespace Feedback::ListenBrainz
, _maxSyncFeedbackCount{ Service<IConfig>::get()->getULong("listenbrainz-max-sync-feedback-count", 1000) }
, _syncFeedbacksPeriod{ Service<IConfig>::get()->getULong("listenbrainz-sync-feedbacks-period-hours", 1) }
{
LOG(INFO) << "Starting Feedbacks synchronizer, maxSyncFeedbackCount = " << _maxSyncFeedbackCount << ", _syncFeedbacksPeriod = " << _syncFeedbacksPeriod.count() << " hours";
LOG(INFO, "Starting Feedbacks synchronizer, maxSyncFeedbackCount = " << _maxSyncFeedbackCount << ", _syncFeedbacksPeriod = " << _syncFeedbacksPeriod.count() << " hours");
scheduleSync(std::chrono::seconds{ 30 });
}
@@ -95,7 +95,7 @@ namespace Feedback::ListenBrainz
case FeedbackType::Erase:
if (!recordingMBID)
{
LOG(DEBUG) << "Track has no recording MBID: erasing star";
LOG(DEBUG, "Track has no recording MBID: erasing star");
starredTrack.remove();
}
else
@@ -112,7 +112,7 @@ namespace Feedback::ListenBrainz
if (!recordingMBID)
{
LOG(DEBUG) << "Track has no recording MBID: skipping";
LOG(DEBUG, "Track has no recording MBID: skipping");
return;
}
@@ -142,7 +142,7 @@ namespace Feedback::ListenBrainz
}
catch (Exception& e)
{
LOG(DEBUG) << "Cannot send feedback: " << e.what();
LOG(DEBUG, "Cannot send feedback: " << e.what());
}
}
@@ -156,7 +156,7 @@ namespace Feedback::ListenBrainz
Database::StarredTrack::pointer starredTrack{ Database::StarredTrack::find(session, starredTrackId) };
if (!starredTrack)
{
LOG(DEBUG) << "Starred track not found. deleted?";
LOG(DEBUG, "Starred track not found. deleted?");
return;
}
@@ -166,23 +166,23 @@ namespace Feedback::ListenBrainz
{
case FeedbackType::Love:
starredTrack.modify()->setSyncState(Database::SyncState::Synchronized);
LOG(DEBUG) << "State set to synchronized";
LOG(DEBUG, "State set to synchronized");
if (userContext.feedbackCount)
{
(*userContext.feedbackCount)++;
LOG(DEBUG) << "Feedback count set to " << *userContext.feedbackCount << " for user '" << userContext.listenBrainzUserName << "'";
LOG(DEBUG, "Feedback count set to " << *userContext.feedbackCount << " for user '" << userContext.listenBrainzUserName << "'");
}
break;
case FeedbackType::Erase:
starredTrack.remove();
LOG(DEBUG) << "Removed starred track";
LOG(DEBUG, "Removed starred track");
if (userContext.feedbackCount && *userContext.feedbackCount > 0)
{
(*userContext.feedbackCount)--;
LOG(DEBUG) << "Feedback count set to " << *userContext.feedbackCount << " for user '" << userContext.listenBrainzUserName << "'";
LOG(DEBUG, "Feedback count set to " << *userContext.feedbackCount << " for user '" << userContext.listenBrainzUserName << "'");
}
break;
@@ -211,7 +211,7 @@ namespace Feedback::ListenBrainz
pendingFeedbacks = StarredTrack::find(session, params);
}
LOG(DEBUG) << "Queing " << pendingFeedbacks.results.size() << " pending '" << (feedbackType == FeedbackType::Love ? "love" : "erase") << "' feedbacks";
LOG(DEBUG, "Queing " << pendingFeedbacks.results.size() << " pending '" << (feedbackType == FeedbackType::Love ? "love" : "erase") << "' feedbacks");
for (const StarredTrackId starredTrackId : pendingFeedbacks.results)
enqueFeedback(feedbackType, starredTrackId);
@@ -247,13 +247,13 @@ namespace Feedback::ListenBrainz
if (_syncFeedbacksPeriod.count() == 0 || _maxSyncFeedbackCount == 0)
return;
LOG(DEBUG) << "Scheduled sync in " << fromNow.count() << " seconds...";
LOG(DEBUG, "Scheduled sync in " << fromNow.count() << " seconds...");
_syncTimer.expires_after(fromNow);
_syncTimer.async_wait(boost::asio::bind_executor(_strand, [this](const boost::system::error_code& ec)
{
if (ec == boost::asio::error::operation_aborted)
{
LOG(DEBUG) << "getFeedbacks aborted";
LOG(DEBUG, "getFeedbacks aborted");
return;
}
else if (ec)
@@ -267,7 +267,7 @@ namespace Feedback::ListenBrainz
void FeedbacksSynchronizer::startSync()
{
LOG(DEBUG) << "Starting sync!";
LOG(DEBUG, "Starting sync!");
assert(!isSyncing());
assert(_strand.running_in_this_thread());
@@ -303,7 +303,7 @@ namespace Feedback::ListenBrainz
{
_strand.dispatch([this, &context]
{
LOG(INFO) << "Feedback sync done for user '" << context.listenBrainzUserName << "', fetched: " << context.fetchedFeedbackCount << ", matched: " << context.matchedFeedbackCount << ", imported: " << context.importedFeedbackCount;
LOG(INFO, "Feedback sync done for user '" << context.listenBrainzUserName << "', fetched: " << context.fetchedFeedbackCount << ", matched: " << context.matchedFeedbackCount << ", imported: " << context.importedFeedbackCount);
context.syncing = false;
if (!isSyncing())
@@ -356,11 +356,11 @@ namespace Feedback::ListenBrainz
std::string msgBodyCopy{ msgBody };
_strand.dispatch([this, msgBodyCopy, &context]
{
LOG(DEBUG) << "Current feedback count = " << (context.feedbackCount ? *context.feedbackCount : 0) << " for user '" << context.listenBrainzUserName << "'";
LOG(DEBUG, "Current feedback count = " << (context.feedbackCount ? *context.feedbackCount : 0) << " for user '" << context.listenBrainzUserName << "'");
const auto totalFeedbackCount = parseTotalFeedbackCount(msgBodyCopy);
if (totalFeedbackCount)
LOG(DEBUG) << "Feedback count for listenbrainz user '" << context.listenBrainzUserName << "' = " << *totalFeedbackCount;
LOG(DEBUG, "Feedback count for listenbrainz user '" << context.listenBrainzUserName << "' = " << *totalFeedbackCount);
bool needSync{ totalFeedbackCount && (!context.feedbackCount || *context.feedbackCount != *totalFeedbackCount) };
context.feedbackCount = totalFeedbackCount;
@@ -416,7 +416,7 @@ namespace Feedback::ListenBrainz
{
const FeedbacksParser::Result parseResult{ FeedbacksParser::parse(msgBody) };
LOG(DEBUG) << "Parsed " << parseResult.feedbackCount << " feedbacks, found " << parseResult.feedbacks.size() << " usable entries";
LOG(DEBUG, "Parsed " << parseResult.feedbackCount << " feedbacks, found " << parseResult.feedbacks.size() << " usable entries");
context.fetchedFeedbackCount += parseResult.feedbackCount;
for (const Feedback& feedback : parseResult.feedbacks)
@@ -441,12 +441,12 @@ namespace Feedback::ListenBrainz
const std::vector<Track::pointer> tracks{ Track::findByRecordingMBID(session, feedback.recordingMBID) };
if (tracks.size() > 1)
{
LOG(DEBUG) << "Too many matches for feedback '" << feedback << "': duplicate recording MBIDs found";
LOG(DEBUG, "Too many matches for feedback '" << feedback << "': duplicate recording MBIDs found");
return;
}
else if (tracks.empty())
{
LOG(DEBUG) << "Cannot match feedback '" << feedback << "': no track found for this recording MBID";
LOG(DEBUG, "Cannot match feedback '" << feedback << "': no track found for this recording MBID");
return;
}
@@ -461,7 +461,7 @@ namespace Feedback::ListenBrainz
if (needImport)
{
LOG(DEBUG) << "Importing feedback '" << feedback << "'";
LOG(DEBUG, "Importing feedback '" << feedback << "'");
auto transaction{ session.createWriteTransaction() };
@@ -481,7 +481,7 @@ namespace Feedback::ListenBrainz
}
else
{
LOG(DEBUG) << "No need to import feedback '" << feedback << "', already imported";
LOG(DEBUG, "No need to import feedback '" << feedback << "', already imported");
context.matchedFeedbackCount++;
}
}
src/libs/services/feedback/impl/listenbrainz/ListenBrainzBackend.cpp
+3 -3
View File
@@ -26,7 +26,7 @@
#include "services/database/Track.hpp"
#include "utils/IConfig.hpp"
#include "utils/http/IClient.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Service.hpp"
#include "Utils.hpp"
@@ -63,12 +63,12 @@ namespace Feedback::ListenBrainz
, _client{ Http::createClient(_ioContext, _baseAPIUrl) }
, _feedbacksSynchronizer{ _ioContext, db, *_client }
{
LOG(INFO) << "Starting ListenBrainz feedback backend... API endpoint = '" << _baseAPIUrl << "'";
LOG(INFO, "Starting ListenBrainz feedback backend... API endpoint = '" << _baseAPIUrl << "'");
}
ListenBrainzBackend::~ListenBrainzBackend()
{
LOG(INFO) << "Stopped ListenBrainz feedback backend!";
LOG(INFO, "Stopped ListenBrainz feedback backend!");
}
void ListenBrainzBackend::onStarred(Database::StarredArtistId starredArtistId)
src/libs/services/feedback/impl/listenbrainz/Utils.cpp
+2 -2
View File
@@ -46,13 +46,13 @@ namespace Feedback::ListenBrainz::Utils
Wt::Json::Object root;
if (!Wt::Json::parse(std::string{ msgBody }, root, error))
{
LOG(ERROR) << "Cannot parse 'validate-token' result: " << error.what();
LOG(ERROR, "Cannot parse 'validate-token' result: " << error.what());
return listenBrainzUserName;
}
if (!root.get("valid").orIfNull(false))
{
LOG(INFO) << "Invalid listenbrainz user";
LOG(INFO, "Invalid listenbrainz user");
return listenBrainzUserName;
}
src/libs/services/feedback/impl/listenbrainz/Utils.hpp
+2 -2
View File
@@ -20,10 +20,10 @@
#pragma once
#include "services/database/UserId.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/UUID.hpp"
#define LOG(sev) LMS_LOG(FEEDBACK, sev) << "[listenbrainz] "
#define LOG(sev, message) LMS_LOG(FEEDBACK, sev, "[listenbrainz] " << message)
namespace Database
{
src/libs/services/recommendation/impl/PlaylistGeneratorService.cpp
+2 -2
View File
@@ -26,7 +26,7 @@
#include "playlist-constraints/ConsecutiveArtists.hpp"
#include "playlist-constraints/ConsecutiveReleases.hpp"
#include "playlist-constraints/DuplicateTracks.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Recommendation
{
@@ -48,7 +48,7 @@ namespace Recommendation
std::vector<TrackId> PlaylistGeneratorService::extendPlaylist(TrackListId tracklistId, std::size_t maxCount) const
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Requested to extend playlist by " << maxCount << " similar tracks";
LMS_LOG(RECOMMENDATION, DEBUG, "Requested to extend playlist by " << maxCount << " similar tracks");
// supposed to be ordered from most similar to least similar
std::vector<TrackId> similarTracks{ _recommendationService.findSimilarTracks(tracklistId, maxCount * 2) }; // ask for more tracks than we need as it will be easier to respect constraints
src/libs/services/recommendation/impl/RecommendationService.cpp
+1 -1
View File
@@ -29,7 +29,7 @@
#include "services/database/Session.hpp"
#include "services/database/ScanSettings.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Recommendation
{
src/libs/services/recommendation/impl/features/FeaturesEngine.cpp
+368 -381
View File
@@ -30,397 +30,384 @@
#include "services/database/TrackFeatures.hpp"
#include "services/database/TrackList.hpp"
#include "som/DataNormalizer.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Random.hpp"
namespace Recommendation {
using namespace Database;
std::unique_ptr<IEngine> createFeaturesEngine(Db& db)
namespace Recommendation
{
return std::make_unique<FeaturesEngine>(db);
}
const FeatureSettingsMap&
FeaturesEngine::getDefaultTrainFeatureSettings()
{
static const FeatureSettingsMap defaultTrainFeatureSettings
{
{ "lowlevel.spectral_energyband_high.mean", {1}},
{ "lowlevel.spectral_rolloff.median", {1}},
{ "lowlevel.spectral_contrast_valleys.var", {1}},
{ "lowlevel.erbbands.mean", {1}},
{ "lowlevel.gfcc.mean", {1}},
};
return defaultTrainFeatureSettings;
}
static
std::optional<SOM::InputVector>
convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions)
{
std::size_t i {};
std::optional<SOM::InputVector> res {SOM::InputVector {nbDimensions}};
for (const auto& [featureName, values] : featureValuesMap)
{
if (values.size() != getFeatureDef(featureName).nbDimensions)
{
LMS_LOG(RECOMMENDATION, WARNING) << "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size();
res.reset();
break;
}
for (double val : values)
(*res)[i++] = val;
}
return res;
}
static
SOM::InputVector
getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions)
{
SOM::InputVector weights {nbDimensions};
std::size_t index {};
for (const auto& [featureName, featureSettings] : featureSettingsMap)
{
const std::size_t featureNbDimensions {getFeatureDef(featureName).nbDimensions};
for (std::size_t i {}; i < featureNbDimensions; ++i)
weights[index++] = (1. / featureNbDimensions * featureSettings.weight);
}
assert(index == nbDimensions);
return weights;
}
void
FeaturesEngine::loadFromTraining(const TrainSettings& trainSettings, const ProgressCallback& progressCallback)
{
LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier...";
std::unordered_set<FeatureName> featureNames;
std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)),
[](const auto& itFeatureSetting) { return itFeatureSetting.first; });
const std::size_t nbDimensions {std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0},
[](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; })};
LMS_LOG(RECOMMENDATION, DEBUG) << "Features dimension = " << nbDimensions;
Session& session {_db.getTLSSession()};
RangeResults<TrackFeaturesId> trackFeaturesIds;
{
auto transaction {session.createReadTransaction()};
LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Track features...";
trackFeaturesIds = TrackFeatures::find(session);
LMS_LOG(RECOMMENDATION, DEBUG) << "Getting Track features DONE (found " << trackFeaturesIds.results.size() << " track features)";
}
std::vector<SOM::InputVector> samples;
std::vector<TrackId> samplesTrackIds;
samples.reserve(trackFeaturesIds.results.size());
samplesTrackIds.reserve(trackFeaturesIds.results.size());
LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features...";
// TODO handle errors using exceptions
for (const TrackFeaturesId trackFeaturesId : trackFeaturesIds.results)
{
if (_loadCancelled)
return;
auto transaction {session.createReadTransaction()};
TrackFeatures::pointer trackFeatures {TrackFeatures::find(session, trackFeaturesId)};
if (!trackFeatures)
continue;
FeatureValuesMap featureValuesMap {trackFeatures->getFeatureValuesMap(featureNames)};
if (featureValuesMap.empty())
continue;
std::optional<SOM::InputVector> inputVector {convertFeatureValuesMapToInputVector(featureValuesMap, nbDimensions)};
if (!inputVector)
continue;
samples.emplace_back(std::move(*inputVector));
samplesTrackIds.emplace_back(trackFeatures->getTrack()->getId());
}
LMS_LOG(RECOMMENDATION, DEBUG) << "Extracting features DONE";
if (samples.empty())
{
LMS_LOG(RECOMMENDATION, INFO) << "Nothing to classify!";
return;
}
LMS_LOG(RECOMMENDATION, DEBUG) << "Normalizing data...";
SOM::DataNormalizer dataNormalizer {nbDimensions};
dataNormalizer.computeNormalizationFactors(samples);
for (auto& sample : samples)
dataNormalizer.normalizeData(sample);
using namespace Database;
std::unique_ptr<IEngine> createFeaturesEngine(Db& db)
{
return std::make_unique<FeaturesEngine>(db);
}
namespace
{
std::optional<SOM::InputVector> convertFeatureValuesMapToInputVector(const FeatureValuesMap& featureValuesMap, std::size_t nbDimensions)
{
std::size_t i{};
std::optional<SOM::InputVector> res{ SOM::InputVector {nbDimensions} };
for (const auto& [featureName, values] : featureValuesMap)
{
if (values.size() != getFeatureDef(featureName).nbDimensions)
{
LMS_LOG(RECOMMENDATION, WARNING, "Dimension mismatch for feature '" << featureName << "'. Expected " << getFeatureDef(featureName).nbDimensions << ", got " << values.size());
res.reset();
break;
}
for (double val : values)
(*res)[i++] = val;
}
return res;
}
SOM::InputVector getInputVectorWeights(const FeatureSettingsMap& featureSettingsMap, std::size_t nbDimensions)
{
SOM::InputVector weights{ nbDimensions };
std::size_t index{};
for (const auto& [featureName, featureSettings] : featureSettingsMap)
{
const std::size_t featureNbDimensions{ getFeatureDef(featureName).nbDimensions };
for (std::size_t i{}; i < featureNbDimensions; ++i)
weights[index++] = (1. / featureNbDimensions * featureSettings.weight);
}
assert(index == nbDimensions);
return weights;
}
}
const FeatureSettingsMap& FeaturesEngine::getDefaultTrainFeatureSettings()
{
static const FeatureSettingsMap defaultTrainFeatureSettings
{
{ "lowlevel.spectral_energyband_high.mean", {1}},
{ "lowlevel.spectral_rolloff.median", {1}},
{ "lowlevel.spectral_contrast_valleys.var", {1}},
{ "lowlevel.erbbands.mean", {1}},
{ "lowlevel.gfcc.mean", {1}},
};
return defaultTrainFeatureSettings;
}
void FeaturesEngine::loadFromTraining(const TrainSettings& trainSettings, const ProgressCallback& progressCallback)
{
LMS_LOG(RECOMMENDATION, INFO, "Constructing features classifier...");
std::unordered_set<FeatureName> featureNames;
std::transform(std::cbegin(trainSettings.featureSettingsMap), std::cend(trainSettings.featureSettingsMap), std::inserter(featureNames, std::begin(featureNames)),
[](const auto& itFeatureSetting) { return itFeatureSetting.first; });
const std::size_t nbDimensions{ std::accumulate(std::cbegin(featureNames), std::cend(featureNames), std::size_t {0},
[](std::size_t sum, const FeatureName& featureName) { return sum + getFeatureDef(featureName).nbDimensions; }) };
LMS_LOG(RECOMMENDATION, DEBUG, "Features dimension = " << nbDimensions);
Session & session{ _db.getTLSSession() };
RangeResults<TrackFeaturesId> trackFeaturesIds;
{
auto transaction{ session.createReadTransaction() };
LMS_LOG(RECOMMENDATION, DEBUG, "Getting Track features...");
trackFeaturesIds = TrackFeatures::find(session);
LMS_LOG(RECOMMENDATION, DEBUG, "Getting Track features DONE (found " << trackFeaturesIds.results.size() << " track features)");
}
std::vector<SOM::InputVector> samples;
std::vector<TrackId> samplesTrackIds;
samples.reserve(trackFeaturesIds.results.size());
samplesTrackIds.reserve(trackFeaturesIds.results.size());
LMS_LOG(RECOMMENDATION, DEBUG, "Extracting features...");
// TODO handle errors using exceptions
for (const TrackFeaturesId trackFeaturesId : trackFeaturesIds.results)
{
if (_loadCancelled)
return;
auto transaction{ session.createReadTransaction() };
TrackFeatures::pointer trackFeatures{ TrackFeatures::find(session, trackFeaturesId) };
if (!trackFeatures)
continue;
FeatureValuesMap featureValuesMap{ trackFeatures->getFeatureValuesMap(featureNames) };
if (featureValuesMap.empty())
continue;
std::optional<SOM::InputVector> inputVector{ convertFeatureValuesMapToInputVector(featureValuesMap, nbDimensions) };
if (!inputVector)
continue;
samples.emplace_back(std::move(*inputVector));
samplesTrackIds.emplace_back(trackFeatures->getTrack()->getId());
}
LMS_LOG(RECOMMENDATION, DEBUG, "Extracting features DONE");
if (samples.empty())
{
LMS_LOG(RECOMMENDATION, INFO, "Nothing to classify!");
return;
}
SOM::Coordinate size {static_cast<SOM::Coordinate>(std::sqrt(samples.size() / trainSettings.sampleCountPerNeuron))};
if (size < 2)
{
LMS_LOG(RECOMMENDATION, WARNING) << "Very few tracks (" << samples.size() << ") are being used by the features engine, expect bad behaviors";
size = 2;
}
LMS_LOG(RECOMMENDATION, INFO) << "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network";
LMS_LOG(RECOMMENDATION, DEBUG, "Normalizing data...");
SOM::DataNormalizer dataNormalizer{ nbDimensions };
SOM::Network network {size, size, nbDimensions};
SOM::InputVector weights {getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions)};
network.setDataWeights(weights);
auto somProgressCallback{[&](const SOM::Network::CurrentIteration& iter)
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Current pass = " << iter.idIteration << " / " << iter.iterationCount;
progressCallback(Progress {iter.idIteration, iter.iterationCount});
}};
dataNormalizer.computeNormalizationFactors(samples);
for (auto& sample : samples)
dataNormalizer.normalizeData(sample);
LMS_LOG(RECOMMENDATION, DEBUG) << "Training network...";
network.train(samples, trainSettings.iterationCount,
progressCallback ? somProgressCallback : SOM::Network::ProgressCallback {},
[this] { return _loadCancelled; });
LMS_LOG(RECOMMENDATION, DEBUG) << "Training network DONE";
SOM::Coordinate size{ static_cast<SOM::Coordinate>(std::sqrt(samples.size() / trainSettings.sampleCountPerNeuron)) };
if (size < 2)
{
LMS_LOG(RECOMMENDATION, WARNING, "Very few tracks (" << samples.size() << ") are being used by the features engine, expect bad behaviors");
size = 2;
}
LMS_LOG(RECOMMENDATION, INFO, "Found " << samples.size() << " tracks, constructing a " << size << "*" << size << " network");
LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks...";
TrackPositions trackPositions;
for (std::size_t i {}; i < samples.size(); ++i)
{
if (_loadCancelled)
return;
SOM::Network network{ size, size, nbDimensions };
const SOM::Position position {network.getClosestRefVectorPosition(samples[i])};
SOM::InputVector weights{ getInputVectorWeights(trainSettings.featureSettingsMap, nbDimensions) };
network.setDataWeights(weights);
trackPositions[samplesTrackIds[i]].push_back(position);
}
LMS_LOG(RECOMMENDATION, DEBUG) << "Classifying tracks DONE";
auto somProgressCallback{ [&](const SOM::Network::CurrentIteration& iter)
{
LMS_LOG(RECOMMENDATION, DEBUG, "Current pass = " << iter.idIteration << " / " << iter.iterationCount);
progressCallback(Progress {iter.idIteration, iter.iterationCount});
} };
LMS_LOG(RECOMMENDATION, DEBUG, "Training network...");
network.train(samples, trainSettings.iterationCount,
progressCallback ? somProgressCallback : SOM::Network::ProgressCallback{},
[this] { return _loadCancelled; });
LMS_LOG(RECOMMENDATION, DEBUG, "Training network DONE");
load(std::move(network), std::move(trackPositions));
}
LMS_LOG(RECOMMENDATION, DEBUG, "Classifying tracks...");
TrackPositions trackPositions;
for (std::size_t i{}; i < samples.size(); ++i)
{
if (_loadCancelled)
return;
void
FeaturesEngine::loadFromCache(FeaturesEngineCache&& cache)
{
LMS_LOG(RECOMMENDATION, INFO) << "Constructing features classifier from cache...";
load(std::move(cache._network), cache._trackPositions);
}
TrackContainer
FeaturesEngine::findSimilarTracksFromTrackList(TrackListId trackListId, std::size_t maxCount) const
{
const TrackContainer trackIds {[&]
{
TrackContainer res;
Session& session {_db.getTLSSession()};
auto transaction {session.createReadTransaction()};
const TrackList::pointer trackList {TrackList::find(session, trackListId)};
if (trackList)
res = trackList->getTrackIds();
return res;
}()};
return findSimilarTracks(trackIds, maxCount);
}
TrackContainer
FeaturesEngine::findSimilarTracks(const std::vector<TrackId>& tracksIds, std::size_t maxCount) const
{
auto similarTrackIds {getSimilarObjects(tracksIds, _trackMatrix, _trackPositions, maxCount)};
Session& session {_db.getTLSSession()};
{
// Report only existing ids, as tracks may have been removed a long time ago (refreshing the SOM takes some time)
auto transaction {session.createReadTransaction()};
similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds),
[&](TrackId trackId)
{
return !Track::exists(session, trackId);
}), std::end(similarTrackIds));
}
return similarTrackIds;
}
ReleaseContainer
FeaturesEngine::getSimilarReleases(ReleaseId releaseId, std::size_t maxCount) const
{
auto similarReleaseIds {getSimilarObjects({releaseId}, _releaseMatrix, _releasePositions, maxCount)};
Session& session {_db.getTLSSession()};
if (!similarReleaseIds.empty())
{
// Report only existing ids
auto transaction {session.createReadTransaction()};
similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds),
[&](ReleaseId releaseId)
{
return !Release::exists(session, releaseId);
}), std::end(similarReleaseIds));
}
return similarReleaseIds;
}
ArtistContainer
FeaturesEngine::getSimilarArtists(ArtistId artistId, EnumSet<TrackArtistLinkType> linkTypes, std::size_t maxCount) const
{
auto getSimilarArtistIdsForLinkType {[&] (TrackArtistLinkType linkType)
{
ArtistContainer similarArtistIds;
const auto itArtists {_artistMatrix.find(linkType)};
if (itArtists == std::cend(_artistMatrix))
{
return similarArtistIds;
}
return getSimilarObjects({artistId}, itArtists->second, _artistPositions, maxCount);
}};
std::unordered_set<ArtistId> similarArtistIds;
for (TrackArtistLinkType linkType : linkTypes)
{
const auto similarArtistIdsForLinkType {getSimilarArtistIdsForLinkType(linkType)};
similarArtistIds.insert(std::begin(similarArtistIdsForLinkType), std::end(similarArtistIdsForLinkType));
}
ArtistContainer res(std::cbegin(similarArtistIds), std::cend(similarArtistIds));
Session& session {_db.getTLSSession()};
{
// Report only existing ids
auto transaction {session.createReadTransaction()};
res.erase(std::remove_if(std::begin(res), std::end(res),
[&](ArtistId artistId)
{
return !Artist::exists(session, artistId);
}), std::end(res));
}
while (res.size() > maxCount)
res.erase(Random::pickRandom(res));
return res;
}
FeaturesEngineCache
FeaturesEngine::toCache() const
{
return FeaturesEngineCache {*_network, _trackPositions};
}
void
FeaturesEngine::load(bool forceReload, const ProgressCallback& progressCallback)
{
if (forceReload)
{
FeaturesEngineCache::invalidate();
}
else if (std::optional<FeaturesEngineCache> cache {FeaturesEngineCache::read()})
{
loadFromCache(std::move(*cache));
return;
}
TrainSettings trainSettings;
trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings();
loadFromTraining(trainSettings, progressCallback);
if (!_loadCancelled && _network)
toCache().write();
}
void
FeaturesEngine::requestCancelLoad()
{
LMS_LOG(RECOMMENDATION, DEBUG) << "Requesting init cancellation";
_loadCancelled = true;
}
void
FeaturesEngine::load(const SOM::Network& network, const TrackPositions& trackPositions)
{
using namespace Database;
_networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian();
LMS_LOG(RECOMMENDATION, DEBUG) << "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian;
const SOM::Coordinate width {network.getWidth()};
const SOM::Coordinate height {network.getHeight()};
_releaseMatrix = ReleaseMatrix {width, height};
_trackMatrix = TrackMatrix {width, height};
LMS_LOG(RECOMMENDATION, DEBUG) << "Constructing maps...";
Session& session {_db.getTLSSession()};
for (const auto& [trackId, positions] : trackPositions)
{
if (_loadCancelled)
return;
auto transaction {session.createReadTransaction()};
const Track::pointer track {Track::find(session, trackId)};
if (!track)
continue;
for (const SOM::Position& position : positions)
{
Utils::push_back_if_not_present(_trackPositions[trackId], position);
Utils::push_back_if_not_present(_trackMatrix[position], trackId);
if (Release::pointer release {track->getRelease()})
{
const ReleaseId releaseId {release->getId()};
Utils::push_back_if_not_present(_releasePositions[releaseId], position);
Utils::push_back_if_not_present(_releaseMatrix[position], releaseId);
}
for (const TrackArtistLink::pointer& artistLink : track->getArtistLinks())
{
const ArtistId artistId {artistLink->getArtist()->getId()};
Utils::push_back_if_not_present(_artistPositions[artistId], position);
auto itArtists {_artistMatrix.find(artistLink->getType())};
if (itArtists == std::cend(_artistMatrix))
{
[[maybe_unused]] auto [it, inserted] = _artistMatrix.try_emplace(artistLink->getType(), ArtistMatrix {width, height});
assert(inserted);
itArtists = it;
}
Utils::push_back_if_not_present(itArtists->second[position], artistId);
}
}
}
_network = std::make_unique<SOM::Network>(network);
LMS_LOG(RECOMMENDATION, INFO) << "Classifier successfully loaded!";
}
const SOM::Position position{ network.getClosestRefVectorPosition(samples[i]) };
trackPositions[samplesTrackIds[i]].push_back(position);
}
LMS_LOG(RECOMMENDATION, DEBUG, "Classifying tracks DONE");
load(std::move(network), std::move(trackPositions));
}
void FeaturesEngine::loadFromCache(FeaturesEngineCache&& cache)
{
LMS_LOG(RECOMMENDATION, INFO, "Constructing features classifier from cache...");
load(std::move(cache._network), cache._trackPositions);
}
TrackContainer FeaturesEngine::findSimilarTracksFromTrackList(TrackListId trackListId, std::size_t maxCount) const
{
const TrackContainer trackIds{ [&]
{
TrackContainer res;
Session& session {_db.getTLSSession()};
auto transaction {session.createReadTransaction()};
const TrackList::pointer trackList {TrackList::find(session, trackListId)};
if (trackList)
res = trackList->getTrackIds();
return res;
}() };
return findSimilarTracks(trackIds, maxCount);
}
TrackContainer FeaturesEngine::findSimilarTracks(const std::vector<TrackId>& tracksIds, std::size_t maxCount) const
{
auto similarTrackIds{ getSimilarObjects(tracksIds, _trackMatrix, _trackPositions, maxCount) };
Session& session{ _db.getTLSSession() };
{
// Report only existing ids, as tracks may have been removed a long time ago (refreshing the SOM takes some time)
auto transaction{ session.createReadTransaction() };
similarTrackIds.erase(std::remove_if(std::begin(similarTrackIds), std::end(similarTrackIds),
[&](TrackId trackId)
{
return !Track::exists(session, trackId);
}), std::end(similarTrackIds));
}
return similarTrackIds;
}
ReleaseContainer FeaturesEngine::getSimilarReleases(ReleaseId releaseId, std::size_t maxCount) const
{
auto similarReleaseIds{ getSimilarObjects({releaseId}, _releaseMatrix, _releasePositions, maxCount) };
Session& session{ _db.getTLSSession() };
if (!similarReleaseIds.empty())
{
// Report only existing ids
auto transaction{ session.createReadTransaction() };
similarReleaseIds.erase(std::remove_if(std::begin(similarReleaseIds), std::end(similarReleaseIds),
[&](ReleaseId releaseId)
{
return !Release::exists(session, releaseId);
}), std::end(similarReleaseIds));
}
return similarReleaseIds;
}
ArtistContainer FeaturesEngine::getSimilarArtists(ArtistId artistId, EnumSet<TrackArtistLinkType> linkTypes, std::size_t maxCount) const
{
auto getSimilarArtistIdsForLinkType{ [&](TrackArtistLinkType linkType)
{
ArtistContainer similarArtistIds;
const auto itArtists {_artistMatrix.find(linkType)};
if (itArtists == std::cend(_artistMatrix))
{
return similarArtistIds;
}
return getSimilarObjects({artistId}, itArtists->second, _artistPositions, maxCount);
} };
std::unordered_set<ArtistId> similarArtistIds;
for (TrackArtistLinkType linkType : linkTypes)
{
const auto similarArtistIdsForLinkType{ getSimilarArtistIdsForLinkType(linkType) };
similarArtistIds.insert(std::begin(similarArtistIdsForLinkType), std::end(similarArtistIdsForLinkType));
}
ArtistContainer res(std::cbegin(similarArtistIds), std::cend(similarArtistIds));
Session& session{ _db.getTLSSession() };
{
// Report only existing ids
auto transaction{ session.createReadTransaction() };
res.erase(std::remove_if(std::begin(res), std::end(res),
[&](ArtistId artistId)
{
return !Artist::exists(session, artistId);
}), std::end(res));
}
while (res.size() > maxCount)
res.erase(Random::pickRandom(res));
return res;
}
FeaturesEngineCache FeaturesEngine::toCache() const
{
return FeaturesEngineCache{ *_network, _trackPositions };
}
void FeaturesEngine::load(bool forceReload, const ProgressCallback& progressCallback)
{
if (forceReload)
{
FeaturesEngineCache::invalidate();
}
else if (std::optional<FeaturesEngineCache> cache{ FeaturesEngineCache::read() })
{
loadFromCache(std::move(*cache));
return;
}
TrainSettings trainSettings;
trainSettings.featureSettingsMap = getDefaultTrainFeatureSettings();
loadFromTraining(trainSettings, progressCallback);
if (!_loadCancelled && _network)
toCache().write();
}
void FeaturesEngine::requestCancelLoad()
{
LMS_LOG(RECOMMENDATION, DEBUG, "Requesting init cancellation");
_loadCancelled = true;
}
void FeaturesEngine::load(const SOM::Network& network, const TrackPositions& trackPositions)
{
using namespace Database;
_networkRefVectorsDistanceMedian = network.computeRefVectorsDistanceMedian();
LMS_LOG(RECOMMENDATION, DEBUG, "Median distance betweend ref vectors = " << _networkRefVectorsDistanceMedian);
const SOM::Coordinate width{ network.getWidth() };
const SOM::Coordinate height{ network.getHeight() };
_releaseMatrix = ReleaseMatrix{ width, height };
_trackMatrix = TrackMatrix{ width, height };
LMS_LOG(RECOMMENDATION, DEBUG, "Constructing maps...");
Session & session{ _db.getTLSSession() };
for (const auto& [trackId, positions] : trackPositions)
{
if (_loadCancelled)
return;
auto transaction{ session.createReadTransaction() };
const Track::pointer track{ Track::find(session, trackId) };
if (!track)
continue;
for (const SOM::Position& position : positions)
{
Utils::push_back_if_not_present(_trackPositions[trackId], position);
Utils::push_back_if_not_present(_trackMatrix[position], trackId);
if (Release::pointer release{ track->getRelease() })
{
const ReleaseId releaseId{ release->getId() };
Utils::push_back_if_not_present(_releasePositions[releaseId], position);
Utils::push_back_if_not_present(_releaseMatrix[position], releaseId);
}
for (const TrackArtistLink::pointer& artistLink : track->getArtistLinks())
{
const ArtistId artistId{ artistLink->getArtist()->getId() };
Utils::push_back_if_not_present(_artistPositions[artistId], position);
auto itArtists{ _artistMatrix.find(artistLink->getType()) };
if (itArtists == std::cend(_artistMatrix))
{
[[maybe_unused]] auto [it, inserted] = _artistMatrix.try_emplace(artistLink->getType(), ArtistMatrix{ width, height });
assert(inserted);
itArtists = it;
}
Utils::push_back_if_not_present(itArtists->second[position], artistId);
}
}
}
_network = std::make_unique<SOM::Network>(network);
LMS_LOG(RECOMMENDATION, INFO, "Classifier successfully loaded!");
}
} // ns Recommendation
src/libs/services/recommendation/impl/features/FeaturesEngineCache.cpp
+172 -179
View File
@@ -23,233 +23,226 @@
#include <boost/property_tree/xml_parser.hpp>
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Service.hpp"
namespace Recommendation {
static
std::filesystem::path getCacheDirectory()
namespace Recommendation
{
return Service<IConfig>::get()->getPath("working-dir") / "cache" / "features";
}
namespace
{
std::filesystem::path getCacheDirectory()
{
return Service<IConfig>::get()->getPath("working-dir") / "cache" / "features";
}
static std::filesystem::path getCacheNetworkFilePath()
{
return getCacheDirectory() / "network";
}
std::filesystem::path getCacheNetworkFilePath()
{
return getCacheDirectory() / "network";
}
static std::filesystem::path getCacheTrackPositionsFilePath()
{
return getCacheDirectory() / "track_positions";
}
std::filesystem::path getCacheTrackPositionsFilePath()
{
return getCacheDirectory() / "track_positions";
}
static
bool
networkToCacheFile(const SOM::Network& network, std::filesystem::path path)
{
try
{
boost::property_tree::ptree root;
bool networkToCacheFile(const SOM::Network& network, std::filesystem::path path)
{
try
{
boost::property_tree::ptree root;
root.put("width", network.getWidth());
root.put("height", network.getHeight());
root.put("dim_count", network.getInputDimCount());
root.put("width", network.getWidth());
root.put("height", network.getHeight());
root.put("dim_count", network.getInputDimCount());
for (SOM::InputVector::value_type weight : network.getDataWeights())
root.add("weights.weight", weight);
for (SOM::InputVector::value_type weight : network.getDataWeights())
root.add("weights.weight", weight);
for (SOM::Coordinate x = 0; x < network.getWidth(); ++x)
{
for (SOM::Coordinate y = 0; y < network.getWidth(); ++y)
{
const auto& refVector = network.getRefVector({x, y});
for (SOM::Coordinate x = 0; x < network.getWidth(); ++x)
{
for (SOM::Coordinate y = 0; y < network.getWidth(); ++y)
{
const auto& refVector = network.getRefVector({ x, y });
boost::property_tree::ptree node;
for (auto value : refVector)
node.add("values.value", value);
boost::property_tree::ptree node;
for (auto value : refVector)
node.add("values.value", value);
node.put("coord_x", x);
node.put("coord_y", y);
node.put("coord_x", x);
node.put("coord_y", y);
root.add_child("ref_vectors.ref_vector", node);
}
}
root.add_child("ref_vectors.ref_vector", node);
}
}
boost::property_tree::write_xml(path.string(), root);
boost::property_tree::write_xml(path.string(), root);
LMS_LOG(RECOMMENDATION, DEBUG) << "Created network cache";
return true;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create network cache: " << error.what();
return false;
}
}
LMS_LOG(RECOMMENDATION, DEBUG, "Created network cache");
return true;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(RECOMMENDATION, ERROR, "Cannot create network cache: " << error.what());
return false;
}
}
}
std::optional<SOM::Network>
FeaturesEngineCache::createNetworkFromCacheFile(const std::filesystem::path& path)
{
if (!std::filesystem::exists(path))
return std::nullopt;
std::optional<SOM::Network> FeaturesEngineCache::createNetworkFromCacheFile(const std::filesystem::path& path)
{
if (!std::filesystem::exists(path))
return std::nullopt;
try
{
LMS_LOG(RECOMMENDATION, INFO) << "Reading network from cache...";
try
{
LMS_LOG(RECOMMENDATION, INFO, "Reading network from cache...");
boost::property_tree::ptree root;
boost::property_tree::ptree root;
boost::property_tree::read_xml(path.string(), root);
boost::property_tree::read_xml(path.string(), root);
SOM::Coordinate width {root.get<SOM::Coordinate>("width")};
SOM::Coordinate height {root.get<SOM::Coordinate>("height")};
std::size_t dimCount {root.get<std::size_t>("dim_count")};
SOM::Coordinate width{ root.get<SOM::Coordinate>("width") };
SOM::Coordinate height{ root.get<SOM::Coordinate>("height") };
std::size_t dimCount{ root.get<std::size_t>("dim_count") };
SOM::Network res {width, height, dimCount};
SOM::Network res{ width, height, dimCount };
{
SOM::InputVector weights {dimCount};
std::size_t i {};
for (const auto& val : root.get_child("weights"))
weights[i++] = val.second.get_value<double>();
{
SOM::InputVector weights{ dimCount };
std::size_t i{};
for (const auto& val : root.get_child("weights"))
weights[i++] = val.second.get_value<double>();
res.setDataWeights(weights);
}
res.setDataWeights(weights);
}
for (const auto& node : root.get_child("ref_vectors"))
{
SOM::Coordinate x {node.second.get<SOM::Coordinate>("coord_x")};
SOM::Coordinate y {node.second.get<SOM::Coordinate>("coord_y")};
for (const auto& node : root.get_child("ref_vectors"))
{
SOM::Coordinate x{ node.second.get<SOM::Coordinate>("coord_x") };
SOM::Coordinate y{ node.second.get<SOM::Coordinate>("coord_y") };
SOM::InputVector refVector {dimCount};
std::size_t i {};
for (const auto& val : node.second.get_child("values"))
refVector[i++] = val.second.get_value<SOM::InputVector::value_type>();
SOM::InputVector refVector{ dimCount };
std::size_t i{};
for (const auto& val : node.second.get_child("values"))
refVector[i++] = val.second.get_value<SOM::InputVector::value_type>();
res.setRefVector({x, y}, refVector);
}
res.setRefVector({ x, y }, refVector);
}
LMS_LOG(RECOMMENDATION, INFO) << "Successfully read network from cache";
LMS_LOG(RECOMMENDATION, INFO, "Successfully read network from cache");
return res;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot read network cache: " << error.what();
return std::nullopt;
}
}
return res;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(RECOMMENDATION, ERROR, "Cannot read network cache: " << error.what());
return std::nullopt;
}
}
bool
FeaturesEngineCache::objectPositionToCacheFile(const TrackPositions& trackPositions, const std::filesystem::path& path)
{
try
{
boost::property_tree::ptree root;
bool FeaturesEngineCache::objectPositionToCacheFile(const TrackPositions& trackPositions, const std::filesystem::path& path)
{
try
{
boost::property_tree::ptree root;
for (const auto& [id, positions] : trackPositions)
{
boost::property_tree::ptree node;
for (const auto& [id, positions] : trackPositions)
{
boost::property_tree::ptree node;
node.put("id", id.getValue());
node.put("id", id.getValue());
for (const SOM::Position& position : positions)
{
boost::property_tree::ptree positionNode;
positionNode.put("x", position.x);
positionNode.put("y", position.y);
for (const SOM::Position& position : positions)
{
boost::property_tree::ptree positionNode;
positionNode.put("x", position.x);
positionNode.put("y", position.y);
node.add_child("position.position", positionNode);
}
node.add_child("position.position", positionNode);
}
root.add_child("objects.object", node);
}
root.add_child("objects.object", node);
}
boost::property_tree::write_xml(path.string(), root);
return true;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot cache object position: " << error.what();
return false;
}
}
boost::property_tree::write_xml(path.string(), root);
return true;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(RECOMMENDATION, ERROR, "Cannot cache object position: " << error.what());
return false;
}
}
std::optional<FeaturesEngineCache::TrackPositions>
FeaturesEngineCache::createObjectPositionsFromCacheFile(const std::filesystem::path& path)
{
try
{
LMS_LOG(RECOMMENDATION, INFO) << "Reading object position from cache...";
std::optional<FeaturesEngineCache::TrackPositions> FeaturesEngineCache::createObjectPositionsFromCacheFile(const std::filesystem::path& path)
{
try
{
LMS_LOG(RECOMMENDATION, INFO, "Reading object position from cache...");
boost::property_tree::ptree root;
boost::property_tree::ptree root;
boost::property_tree::read_xml(path.string(), root);
boost::property_tree::read_xml(path.string(), root);
TrackPositions res;
TrackPositions res;
for (const auto& object : root.get_child("objects"))
{
const Database::TrackId id {object.second.get<Database::IdType::ValueType>("id")};
for (const auto& position : object.second.get_child("position"))
{
auto x = position.second.get<SOM::Coordinate>("x");
auto y = position.second.get<SOM::Coordinate>("y");
for (const auto& object : root.get_child("objects"))
{
const Database::TrackId id{ object.second.get<Database::IdType::ValueType>("id") };
for (const auto& position : object.second.get_child("position"))
{
auto x = position.second.get<SOM::Coordinate>("x");
auto y = position.second.get<SOM::Coordinate>("y");
res[id].push_back({x, y});
}
}
res[id].push_back({ x, y });
}
}
LMS_LOG(RECOMMENDATION, INFO) << "Successfully read object position from cache";
LMS_LOG(RECOMMENDATION, INFO, "Successfully read object position from cache");
return res;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(RECOMMENDATION, ERROR) << "Cannot create object position from cache file: " << error.what();
return std::nullopt;
}
}
return res;
}
catch (boost::property_tree::ptree_error& error)
{
LMS_LOG(RECOMMENDATION, ERROR, "Cannot create object position from cache file: " << error.what());
return std::nullopt;
}
}
void
FeaturesEngineCache::invalidate()
{
std::filesystem::remove(getCacheNetworkFilePath());
std::filesystem::remove(getCacheTrackPositionsFilePath());
}
void FeaturesEngineCache::invalidate()
{
std::filesystem::remove(getCacheNetworkFilePath());
std::filesystem::remove(getCacheTrackPositionsFilePath());
}
std::optional<FeaturesEngineCache>
FeaturesEngineCache::read()
{
auto network{createNetworkFromCacheFile(getCacheNetworkFilePath())};
if (!network)
return std::nullopt;
std::optional<FeaturesEngineCache> FeaturesEngineCache::read()
{
auto network{ createNetworkFromCacheFile(getCacheNetworkFilePath()) };
if (!network)
return std::nullopt;
auto trackPositions{createObjectPositionsFromCacheFile(getCacheTrackPositionsFilePath())};
if (!trackPositions)
return std::nullopt;
auto trackPositions{ createObjectPositionsFromCacheFile(getCacheTrackPositionsFilePath()) };
if (!trackPositions)
return std::nullopt;
return FeaturesEngineCache {std::move(*network), std::move(*trackPositions)};
}
return FeaturesEngineCache{ std::move(*network), std::move(*trackPositions) };
}
void
FeaturesEngineCache::write() const
{
std::filesystem::create_directories(Service<IConfig>::get()->getPath("working-dir") / "cache" / "features");
void FeaturesEngineCache::write() const
{
std::filesystem::create_directories(Service<IConfig>::get()->getPath("working-dir") / "cache" / "features");
if (!networkToCacheFile(_network, getCacheNetworkFilePath())
|| !objectPositionToCacheFile(_trackPositions, getCacheTrackPositionsFilePath()))
{
invalidate();
}
}
if (!networkToCacheFile(_network, getCacheNetworkFilePath())
|| !objectPositionToCacheFile(_trackPositions, getCacheTrackPositionsFilePath()))
{
invalidate();
}
}
FeaturesEngineCache::FeaturesEngineCache(SOM::Network network, TrackPositions trackPositions)
: _network {std::move(network)},
_trackPositions {std::move(trackPositions)}
{
}
FeaturesEngineCache::FeaturesEngineCache(SOM::Network network, TrackPositions trackPositions)
: _network{ std::move(network) },
_trackPositions{ std::move(trackPositions) }
{
}
} // namespace Recommendation
src/libs/services/recommendation/impl/playlist-constraints/ConsecutiveArtists.cpp
+1 -1
View File
@@ -25,7 +25,7 @@
#include "services/database/Release.hpp"
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Recommendation::PlaylistGeneratorConstraint
{
src/libs/services/recommendation/impl/playlist-constraints/ConsecutiveReleases.cpp
+1 -1
View File
@@ -23,7 +23,7 @@
#include "services/database/Release.hpp"
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Recommendation::PlaylistGeneratorConstraint
{
src/libs/services/scanner/impl/ScanStepCheckDuplicatedDbFiles.cpp
+3 -3
View File
@@ -22,7 +22,7 @@
#include "services/database/Db.hpp"
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Scanner
{
@@ -45,13 +45,13 @@ namespace Scanner
const Track::pointer track{ Track::find(session, trackId) };
if (auto trackMBID{ track->getTrackMBID() })
{
LMS_LOG(DBUPDATER, INFO) << "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName();
LMS_LOG(DBUPDATER, INFO, "Found duplicated track MBID [" << trackMBID->getAsString() << "], file: " << track->getPath().string() << " - " << track->getName());
context.stats.duplicates.emplace_back(ScanDuplicate{ track->getId(), DuplicateReason::SameTrackMBID });
context.currentStepStats.processedElems++;
_progressCallback(context.currentStepStats);
}
}
LMS_LOG(DBUPDATER, DEBUG) << "Found " << context.currentStepStats.processedElems << " duplicated audio files";
LMS_LOG(DBUPDATER, DEBUG, "Found " << context.currentStepStats.processedElems << " duplicated audio files");
}
}
src/libs/services/scanner/impl/ScanStepComputeClusterStats.cpp
+2 -2
View File
@@ -21,7 +21,7 @@
#include "services/database/Db.hpp"
#include "services/database/Cluster.hpp"
#include "services/database/Session.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Path.hpp"
namespace Scanner
@@ -84,6 +84,6 @@ namespace Scanner
return true;
});
LMS_LOG(DBUPDATER, DEBUG) << "Recomputed stats for " << context.currentStepStats.processedElems << " clusters!";
LMS_LOG(DBUPDATER, DEBUG, "Recomputed stats for " << context.currentStepStats.processedElems << " clusters!");
}
}
src/libs/services/scanner/impl/ScanStepDiscoverFiles.cpp
+2 -2
View File
@@ -18,7 +18,7 @@
*/
#include "ScanStepDiscoverFiles.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Path.hpp"
namespace Scanner
@@ -42,6 +42,6 @@ namespace Scanner
context.stats.filesScanned = context.currentStepStats.processedElems;
LMS_LOG(DBUPDATER, DEBUG) << "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'";
LMS_LOG(DBUPDATER, DEBUG, "Discovered " << context.stats.filesScanned << " files in '" << context.directory << "'");
}
}
src/libs/services/scanner/impl/ScanStepRemoveOrphanDbFiles.cpp
+11 -11
View File
@@ -25,7 +25,7 @@
#include "services/database/Release.hpp"
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Path.hpp"
namespace Scanner
@@ -87,14 +87,14 @@ namespace Scanner
Session& session{ _db.getTLSSession() };
LMS_LOG(DBUPDATER, DEBUG) << "Checking tracks to be removed...";
LMS_LOG(DBUPDATER, DEBUG, "Checking tracks to be removed...");
std::size_t trackCount{};
{
auto transaction{ session.createReadTransaction() };
trackCount = Track::getCount(session);
}
LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks to be checked...";
LMS_LOG(DBUPDATER, DEBUG, trackCount << " tracks to be checked...");
context.currentStepStats.totalElems = trackCount;
@@ -143,24 +143,24 @@ namespace Scanner
break;
}
LMS_LOG(DBUPDATER, DEBUG) << trackCount << " tracks checked!";
LMS_LOG(DBUPDATER, DEBUG, trackCount << " tracks checked!");
}
void ScanStepRemoveOrphanDbFiles::removeOrphanClusters()
{
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan clusters...";
LMS_LOG(DBUPDATER, DEBUG, "Checking orphan clusters...");
removeOrphanEntries<Database::Cluster>(_db.getTLSSession(), _abortScan);
}
void ScanStepRemoveOrphanDbFiles::removeOrphanArtists()
{
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan artists...";
LMS_LOG(DBUPDATER, DEBUG, "Checking orphan artists...");
removeOrphanEntries<Database::Artist>(_db.getTLSSession(), _abortScan);
}
void ScanStepRemoveOrphanDbFiles::removeOrphanReleases()
{
LMS_LOG(DBUPDATER, DEBUG) << "Checking orphan releases...";
LMS_LOG(DBUPDATER, DEBUG, "Checking orphan releases...");
removeOrphanEntries<Database::Release>(_db.getTLSSession(), _abortScan);
}
@@ -172,19 +172,19 @@ namespace Scanner
// and still belongs to a media directory
if (!std::filesystem::exists(p) || !std::filesystem::is_regular_file(p))
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': missing";
LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': missing");
return false;
}
if (!PathUtils::isPathInRootPath(p, _settings.mediaDirectory, &excludeDirFileName))
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': out of media directory";
LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': out of media directory");
return false;
}
if (!PathUtils::hasFileAnyExtension(p, _settings.supportedExtensions))
{
LMS_LOG(DBUPDATER, INFO) << "Removing '" << p.string() << "': file format no longer handled";
LMS_LOG(DBUPDATER, INFO, "Removing '" << p.string() << "': file format no longer handled");
return false;
}
@@ -192,7 +192,7 @@ namespace Scanner
}
catch (std::filesystem::filesystem_error& e)
{
LMS_LOG(DBUPDATER, ERROR) << "Caught exception while checking file '" << p.string() << "': " << e.what();
LMS_LOG(DBUPDATER, ERROR, "Caught exception while checking file '" << p.string() << "': " << e.what());
return false;
}
}
src/libs/services/scanner/impl/ScanStepScanFiles.cpp
+9 -9
View File
@@ -30,7 +30,7 @@
#include "services/database/TrackArtistLink.hpp"
#include "utils/Exception.hpp"
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Path.hpp"
using namespace Database;
@@ -258,7 +258,7 @@ namespace Scanner
MetaData::ParserReadStyle getParserReadStyle()
{
std::string_view readStyle{ Service<IConfig>::get()->getString("scanner-parser-read-style", "accurate") };
std::string_view readStyle{ Service<IConfig>::get()->getString("scanner-parser-read-style", "average") };
if (readStyle == "fast")
return MetaData::ParserReadStyle::Fast;
@@ -290,7 +290,7 @@ namespace Scanner
if (ec)
{
LMS_LOG(DBUPDATER, ERROR) << "Cannot process entry '" << path.string() << "': " << ec.message();
LMS_LOG(DBUPDATER, ERROR, "Cannot process entry '" << path.string() << "': " << ec.message());
context.stats.errors.emplace_back(ScanError{ path, ScanErrorType::CannotReadFile, ec.message() });
}
else if (PathUtils::hasFileAnyExtension(path, _settings.supportedExtensions))
@@ -319,7 +319,7 @@ namespace Scanner
}
catch (LmsException& e)
{
LMS_LOG(DBUPDATER, ERROR) << e.what();
LMS_LOG(DBUPDATER, ERROR, e.what());
stats.skips++;
return;
}
@@ -365,7 +365,7 @@ namespace Scanner
std::error_code ec;
if (!std::filesystem::exists(otherTrack->getPath(), ec))
{
LMS_LOG(DBUPDATER, DEBUG) << "Considering track '" << file.string() << "' moved from '" << otherTrack->getPath() << "'";
LMS_LOG(DBUPDATER, DEBUG, "Considering track '" << file.string() << "' moved from '" << otherTrack->getPath() << "'");
track = otherTrack;
track.modify()->setPath(file);
}
@@ -384,7 +384,7 @@ namespace Scanner
if (!PathUtils::isPathInRootPath(file, _settings.mediaDirectory, &excludeDirFileName))
continue;
LMS_LOG(DBUPDATER, DEBUG) << "Skipped '" << file.string() << "' (similar MBID in '" << otherTrack->getPath().string() << "')";
LMS_LOG(DBUPDATER, DEBUG, "Skipped '" << file.string() << "' (similar MBID in '" << otherTrack->getPath().string() << "')");
// As this MBID already exists, just remove what we just scanned
if (track)
{
@@ -399,7 +399,7 @@ namespace Scanner
// We estimate this is an audio file if the duration is not null
if (trackInfo->duration == std::chrono::milliseconds::zero())
{
LMS_LOG(DBUPDATER, DEBUG) << "Skipped '" << file.string() << "' (duration is 0)";
LMS_LOG(DBUPDATER, DEBUG, "Skipped '" << file.string() << "' (duration is 0)");
// If Track exists here, delete it!
if (track)
@@ -427,12 +427,12 @@ namespace Scanner
if (!track)
{
track = dbSession.create<Track>(file);
LMS_LOG(DBUPDATER, DEBUG) << "Adding '" << file.string() << "'";
LMS_LOG(DBUPDATER, DEBUG, "Adding '" << file.string() << "'");
stats.additions++;
}
else
{
LMS_LOG(DBUPDATER, DEBUG) << "Updating '" << file.string() << "'";
LMS_LOG(DBUPDATER, DEBUG, "Updating '" << file.string() << "'");
stats.updates++;
}
src/libs/services/scanner/impl/ScannerService.cpp
+20 -20
View File
@@ -27,7 +27,7 @@
#include "services/database/ScanSettings.hpp"
#include "utils/Exception.hpp"
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Path.hpp"
#include "utils/Tuple.hpp"
@@ -82,9 +82,9 @@ namespace Scanner
ScannerService::~ScannerService()
{
LMS_LOG(DBUPDATER, INFO) << "Stopping service...";
LMS_LOG(DBUPDATER, INFO, "Stopping service...");
stop();
LMS_LOG(DBUPDATER, INFO) << "Service stopped!";
LMS_LOG(DBUPDATER, INFO, "Service stopped!");
}
void ScannerService::start()
@@ -113,15 +113,15 @@ namespace Scanner
void ScannerService::abortScan()
{
LMS_LOG(DBUPDATER, DEBUG) << "Aborting scan...";
LMS_LOG(DBUPDATER, DEBUG, "Aborting scan...");
std::scoped_lock lock{ _controlMutex };
LMS_LOG(DBUPDATER, DEBUG) << "Waiting for the scan to abort...";
LMS_LOG(DBUPDATER, DEBUG, "Waiting for the scan to abort...");
_abortScan = true;
_scheduleTimer.cancel();
_ioService.stop();
LMS_LOG(DBUPDATER, DEBUG) << "Scan abort done!";
LMS_LOG(DBUPDATER, DEBUG, "Scan abort done!");
_abortScan = false;
_ioService.start();
@@ -167,7 +167,7 @@ namespace Scanner
void ScannerService::scheduleNextScan()
{
LMS_LOG(DBUPDATER, DEBUG) << "Scheduling next scan";
LMS_LOG(DBUPDATER, DEBUG, "Scheduling next scan");
refreshScanSettings();
@@ -202,7 +202,7 @@ namespace Scanner
break;
case ScanSettings::UpdatePeriod::Never:
LMS_LOG(DBUPDATER, INFO) << "Auto scan disabled!";
LMS_LOG(DBUPDATER, INFO, "Auto scan disabled!");
break;
}
@@ -230,7 +230,7 @@ namespace Scanner
if (dateTime.isNull())
{
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan right now";
LMS_LOG(DBUPDATER, INFO, "Scheduling next scan right now");
_scheduleTimer.expires_from_now(std::chrono::seconds{ 0 });
_scheduleTimer.async_wait(cb);
}
@@ -240,7 +240,7 @@ namespace Scanner
std::time_t t{ std::chrono::system_clock::to_time_t(timePoint) };
char ctimeStr[26];
LMS_LOG(DBUPDATER, INFO) << "Scheduling next scan at " << std::string(::ctime_r(&t, ctimeStr));
LMS_LOG(DBUPDATER, INFO, "Scheduling next scan at " << std::string(::ctime_r(&t, ctimeStr)));
_scheduleTimer.expires_at(timePoint);
_scheduleTimer.async_wait(cb);
}
@@ -257,7 +257,7 @@ namespace Scanner
}
LMS_LOG(UI, INFO) << "New scan started!";
LMS_LOG(UI, INFO, "New scan started!");
refreshScanSettings();
@@ -267,16 +267,16 @@ namespace Scanner
for (auto& scanStep : _scanSteps)
{
LMS_LOG(DBUPDATER, DEBUG) << "Starting scan step '" << scanStep->getStepName() << "'";
LMS_LOG(DBUPDATER, DEBUG, "Starting scan step '" << scanStep->getStepName() << "'");
scanContext.currentStepStats = ScanStepStats{ Wt::WDateTime::currentDateTime(), scanStep->getStep() };
notifyInProgress(scanContext.currentStepStats);
scanStep->process(scanContext);
notifyInProgress(scanContext.currentStepStats);
LMS_LOG(DBUPDATER, DEBUG) << "Completed scan step '" << scanStep->getStepName() << "'";
LMS_LOG(DBUPDATER, DEBUG, "Completed scan step '" << scanStep->getStepName() << "'");
}
LMS_LOG(DBUPDATER, INFO) << "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size();
LMS_LOG(DBUPDATER, INFO, "Scan " << (_abortScan ? "aborted" : "complete") << ". Changes = " << stats.nbChanges() << " (added = " << stats.additions << ", removed = " << stats.deletions << ", updated = " << stats.updates << "), Not changed = " << stats.skips << ", Scanned = " << stats.scans << " (errors = " << stats.errors.size() << "), features fetched = " << stats.featuresFetched << ", duplicates = " << stats.duplicates.size());
_dbSession.analyze();
@@ -290,14 +290,14 @@ namespace Scanner
_currentScanStepStats.reset();
}
LMS_LOG(DBUPDATER, DEBUG) << "Scan not aborted, scheduling next scan!";
LMS_LOG(DBUPDATER, DEBUG, "Scan not aborted, scheduling next scan!");
scheduleNextScan();
_events.scanComplete.emit(stats);
}
else
{
LMS_LOG(DBUPDATER, DEBUG) << "Scan aborted, not scheduling next scan!";
LMS_LOG(DBUPDATER, DEBUG, "Scan aborted, not scheduling next scan!");
std::unique_lock lock{ _statusMutex };
@@ -312,9 +312,9 @@ namespace Scanner
if (_settings == newSettings)
return;
LMS_LOG(DBUPDATER, DEBUG) << "Scanner settings updated";
LMS_LOG(DBUPDATER, DEBUG) << "skipDuplicateMBID = " << newSettings.skipDuplicateMBID;
LMS_LOG(DBUPDATER, DEBUG) << "Using scan settings version " << newSettings.scanVersion;
LMS_LOG(DBUPDATER, DEBUG, "Scanner settings updated");
LMS_LOG(DBUPDATER, DEBUG, "skipDuplicateMBID = " << newSettings.skipDuplicateMBID);
LMS_LOG(DBUPDATER, DEBUG, "Using scan settings version " << newSettings.scanVersion);
_settings = std::move(newSettings);
@@ -366,7 +366,7 @@ namespace Scanner
std::transform(std::cbegin(clusterTypes), std::cend(clusterTypes),
std::inserter(clusterTypeNames, clusterTypeNames.begin()),
[](ClusterType::pointer clusterType) { return clusterType->getName(); });
[](const ClusterType::pointer& clusterType) { return std::string{ clusterType->getName() }; });
newSettings.clusterTypeNames = std::move(clusterTypeNames);
}
src/libs/services/scrobbling/impl/ScrobblingService.cpp
+4 -4
View File
@@ -26,7 +26,7 @@
#include "services/database/Session.hpp"
#include "services/database/Track.hpp"
#include "services/database/User.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "internal/InternalBackend.hpp"
#include "listenbrainz/ListenBrainzBackend.hpp"
@@ -43,15 +43,15 @@ namespace Scrobbling
ScrobblingService::ScrobblingService(boost::asio::io_context& ioContext, Db& db)
: _db{ db }
{
LMS_LOG(SCROBBLING, INFO) << "Starting service...";
LMS_LOG(SCROBBLING, INFO, "Starting service...");
_scrobblingBackends.emplace(ScrobblingBackend::Internal, std::make_unique<InternalBackend>(_db));
_scrobblingBackends.emplace(ScrobblingBackend::ListenBrainz, std::make_unique<ListenBrainz::ListenBrainzBackend>(ioContext, _db));
LMS_LOG(SCROBBLING, INFO) << "Service started!";
LMS_LOG(SCROBBLING, INFO, "Service started!");
}
ScrobblingService::~ScrobblingService()
{
LMS_LOG(SCROBBLING, INFO) << "Service stopped!";
LMS_LOG(SCROBBLING, INFO, "Service stopped!");
}
void ScrobblingService::listenStarted(const Listen& listen)
src/libs/services/scrobbling/impl/listenbrainz/ListenBrainzBackend.cpp
+4 -4
View File
@@ -24,7 +24,7 @@
#include "services/database/Track.hpp"
#include "utils/IConfig.hpp"
#include "utils/http/IClient.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Service.hpp"
#include "Utils.hpp"
@@ -44,7 +44,7 @@ namespace Scrobbling::ListenBrainz
const bool res{ duration >= std::chrono::minutes(4) || (duration >= track->getDuration() / 2) };
if (!res)
LOG(DEBUG) << "Track cannot be scrobbled since played duration is too short: " << duration.count() << "s, total duration = " << std::chrono::duration_cast<std::chrono::seconds>(track->getDuration()).count() << "s";
LOG(DEBUG, "Track cannot be scrobbled since played duration is too short: " << duration.count() << "s, total duration = " << std::chrono::duration_cast<std::chrono::seconds>(track->getDuration()).count() << "s");
return res;
}
@@ -57,12 +57,12 @@ namespace Scrobbling::ListenBrainz
, _client{ Http::createClient(_ioContext, _baseAPIUrl) }
, _listensSynchronizer{ _ioContext, db, *_client }
{
LOG(INFO) << "Starting ListenBrainz backend... API endpoint = '" << _baseAPIUrl << "'";
LOG(INFO, "Starting ListenBrainz backend... API endpoint = '" << _baseAPIUrl << "'");
}
ListenBrainzBackend::~ListenBrainzBackend()
{
LOG(INFO) << "Stopped ListenBrainz backend!";
LOG(INFO, "Stopped ListenBrainz backend!");
}
void ListenBrainzBackend::listenStarted(const Listen& listen)
src/libs/services/scrobbling/impl/listenbrainz/ListensParser.cpp
+68 -72
View File
@@ -24,86 +24,82 @@
#include <Wt/Json/Value.h>
#include <Wt/Json/Parser.h>
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "Utils.hpp"
namespace
{
using namespace Scrobbling::ListenBrainz;
Listen
parseListen(const Wt::Json::Object& listenObject)
{
Listen listen;
// Mandatory fields
const Wt::Json::Object& metadata = listenObject.get("track_metadata");
listen.trackName = static_cast<std::string>(metadata.get("track_name"));
listen.artistName = static_cast<std::string>(metadata.get("artist_name"));
// Optional fields
listen.releaseName = static_cast<std::string>(metadata.get("release_name").orIfNull(""));
if (listenObject.type("listened_at") == Wt::Json::Type::Number)
listen.listenedAt = Wt::WDateTime::fromTime_t(static_cast<int>(listenObject.get("listened_at")));
if (!listen.listenedAt.isValid())
LOG(ERROR) << "Invalid or missing 'listened_at' field!";
if (metadata.type("additional_info") == Wt::Json::Type::Object)
{
const Wt::Json::Object& additionalInfo = metadata.get("additional_info");
listen.trackMBID = UUID::fromString(additionalInfo.get("track_mbid").orIfNull(""));
listen.recordingMBID = UUID::fromString(additionalInfo.get("recording_mbid").orIfNull(""));
listen.releaseMBID = UUID::fromString(additionalInfo.get("release_mbid").orIfNull(""));
// tracknumber should be an integer but some players encode as strings
int trackNumber {additionalInfo.get("tracknumber").toNumber().orIfNull(-1)};
if (trackNumber > 0)
listen.trackNumber = trackNumber;
}
return listen;
}
} // namespace
namespace Scrobbling::ListenBrainz
{
ListensParser::Result
ListensParser::parse(std::string_view msgBody)
{
Result result;
namespace
{
Listen parseListen(const Wt::Json::Object& listenObject)
{
Listen listen;
try
{
Wt::Json::Object root;
Wt::Json::parse(std::string {msgBody}, root);
// Mandatory fields
const Wt::Json::Object& metadata = listenObject.get("track_metadata");
listen.trackName = static_cast<std::string>(metadata.get("track_name"));
listen.artistName = static_cast<std::string>(metadata.get("artist_name"));
const Wt::Json::Object& payload = root.get("payload");
const Wt::Json::Array& listens = payload.get("listens");
// Optional fields
listen.releaseName = static_cast<std::string>(metadata.get("release_name").orIfNull(""));
if (listenObject.type("listened_at") == Wt::Json::Type::Number)
listen.listenedAt = Wt::WDateTime::fromTime_t(static_cast<int>(listenObject.get("listened_at")));
if (!listen.listenedAt.isValid())
LOG(ERROR, "Invalid or missing 'listened_at' field!");
LOG(DEBUG) << "Parsing " << listens.size() << " listens...";
result.listenCount = listens.size();
if (metadata.type("additional_info") == Wt::Json::Type::Object)
{
const Wt::Json::Object& additionalInfo = metadata.get("additional_info");
listen.trackMBID = UUID::fromString(additionalInfo.get("track_mbid").orIfNull(""));
listen.recordingMBID = UUID::fromString(additionalInfo.get("recording_mbid").orIfNull(""));
listen.releaseMBID = UUID::fromString(additionalInfo.get("release_mbid").orIfNull(""));
if (listens.empty())
return result;
// tracknumber should be an integer but some players encode as strings
int trackNumber{ additionalInfo.get("tracknumber").toNumber().orIfNull(-1) };
if (trackNumber > 0)
listen.trackNumber = trackNumber;
}
for (const Wt::Json::Value& value : listens)
{
try
{
const Wt::Json::Object& listen = value;
result.listens.push_back(parseListen(listen));
}
catch (const Wt::WException& error)
{
LOG(ERROR) << "Cannot parse 'listen': " << error.what();
}
}
}
catch (const Wt::WException& error)
{
LOG(ERROR) << "Cannot parse 'listens': " << error.what();
}
return listen;
}
} // namespace
return result;
}
ListensParser::Result ListensParser::parse(std::string_view msgBody)
{
Result result;
try
{
Wt::Json::Object root;
Wt::Json::parse(std::string{ msgBody }, root);
const Wt::Json::Object& payload = root.get("payload");
const Wt::Json::Array& listens = payload.get("listens");
LOG(DEBUG, "Parsing " << listens.size() << " listens...");
result.listenCount = listens.size();
if (listens.empty())
return result;
for (const Wt::Json::Value& value : listens)
{
try
{
const Wt::Json::Object& listen = value;
result.listens.push_back(parseListen(listen));
}
catch (const Wt::WException& error)
{
LOG(ERROR, "Cannot parse 'listen': " << error.what());
}
}
}
catch (const Wt::WException& error)
{
LOG(ERROR, "Cannot parse 'listens': " << error.what());
}
return result;
}
} // Scrobbling::ListenBrainz
src/libs/services/scrobbling/impl/listenbrainz/ListensSynchronizer.cpp
+20 -20
View File
@@ -58,7 +58,7 @@ namespace
if (artists.empty())
{
LOG(DEBUG) << "Track cannot be scrobbled since it does not have any artist";
LOG(DEBUG, "Track cannot be scrobbled since it does not have any artist");
return std::nullopt;
}
@@ -134,7 +134,7 @@ namespace
}
catch (const Wt::WException& e)
{
LOG(ERROR) << "Cannot parse listen count response: " << e.what();
LOG(ERROR, "Cannot parse listen count response: " << e.what());
return std::nullopt;
}
}
@@ -152,12 +152,12 @@ namespace
// if duplicated files, do not record it (let the user correct its database)
if (tracks.size() == 1)
{
LOG(DEBUG) << "Matched listen '" << listen << "' using track MBID";
LOG(DEBUG, "Matched listen '" << listen << "' using track MBID");
return tracks.front()->getId();
}
else if (tracks.size() > 1)
{
LOG(DEBUG) << "Too many matches for listen '" << listen << "' using track MBID!";
LOG(DEBUG, "Too many matches for listen '" << listen << "' using track MBID!");
return {};
}
}
@@ -168,12 +168,12 @@ namespace
// if duplicated files, do not record it (let the user correct its database)
if (tracks.size() == 1)
{
LOG(DEBUG) << "Matched listen '" << listen << "' using recording MBID";
LOG(DEBUG, "Matched listen '" << listen << "' using recording MBID");
return tracks.front()->getId();
}
else if (tracks.size() > 1)
{
LOG(DEBUG) << "Too many matches for listen '" << listen << "' using recording MBID!";
LOG(DEBUG, "Too many matches for listen '" << listen << "' using recording MBID!");
return {};
}
}
@@ -192,16 +192,16 @@ namespace
// conservative behavior: in case of multiple matches: reject
if (tracks.results.size() == 1)
{
LOG(DEBUG) << "Matched listen '" << listen << "' using metadata";
LOG(DEBUG, "Matched listen '" << listen << "' using metadata");
return tracks.results.front();
}
else if (tracks.results.size() > 1)
{
LOG(DEBUG) << "Too many matches for listen '" << listen << "' using metadata";
LOG(DEBUG, "Too many matches for listen '" << listen << "' using metadata");
return {};
}
LOG(DEBUG) << "No match for listen '" << listen << "'";
LOG(DEBUG, "No match for listen '" << listen << "'");
return {};
}
}
@@ -215,7 +215,7 @@ namespace Scrobbling::ListenBrainz
, _maxSyncListenCount{ Service<IConfig>::get()->getULong("listenbrainz-max-sync-listen-count", 1000) }
, _syncListensPeriod{ Service<IConfig>::get()->getULong("listenbrainz-sync-listens-period-hours", 1) }
{
LOG(INFO) << "Starting Listens synchronizer, maxSyncListenCount = " << _maxSyncListenCount << ", _syncListensPeriod = " << _syncListensPeriod.count() << " hours";
LOG(INFO, "Starting Listens synchronizer, maxSyncListenCount = " << _maxSyncListenCount << ", _syncListensPeriod = " << _syncListensPeriod.count() << " hours");
scheduleSync(std::chrono::seconds{ 30 });
}
@@ -267,14 +267,14 @@ namespace Scrobbling::ListenBrainz
std::string bodyText{ listenToJsonString(_db.getTLSSession(), listen, timePoint, timePoint.isValid() ? "single" : "playing_now") };
if (bodyText.empty())
{
LOG(DEBUG) << "Cannot convert listen to json: skipping";
LOG(DEBUG, "Cannot convert listen to json: skipping");
return;
}
const std::optional<UUID> listenBrainzToken{ Utils::getListenBrainzToken(_db.getTLSSession(), listen.userId) };
if (!listenBrainzToken)
{
LOG(DEBUG) << "No listenbrainz token found: skipping";
LOG(DEBUG, "No listenbrainz token found: skipping");
return;
}
@@ -305,7 +305,7 @@ namespace Scrobbling::ListenBrainz
dbListen = session.create<Database::Listen>(user, track, Database::ScrobblingBackend::ListenBrainz, listen.listenedAt);
dbListen.modify()->setSyncState(scrobblingState);
LOG(DEBUG) << "LISTEN CREATED for user " << user->getLoginName() << ", track '" << track->getName() << "' AT " << listen.listenedAt.toString();
LOG(DEBUG, "LISTEN CREATED for user " << user->getLoginName() << ", track '" << track->getName() << "' AT " << listen.listenedAt.toString());
return true;
}
@@ -347,7 +347,7 @@ namespace Scrobbling::ListenBrainz
}
}
LOG(DEBUG) << "Queing " << pendingListens.size() << " pending listen";
LOG(DEBUG, "Queing " << pendingListens.size() << " pending listen");
for (const TimedListen& pendingListen : pendingListens)
enqueListen(pendingListen);
@@ -379,13 +379,13 @@ namespace Scrobbling::ListenBrainz
if (_syncListensPeriod.count() == 0 || _maxSyncListenCount == 0)
return;
LOG(DEBUG) << "Scheduled sync in " << fromNow.count() << " seconds...";
LOG(DEBUG, "Scheduled sync in " << fromNow.count() << " seconds...");
_syncTimer.expires_after(fromNow);
_syncTimer.async_wait(boost::asio::bind_executor(_strand, [this](const boost::system::error_code& ec)
{
if (ec == boost::asio::error::operation_aborted)
{
LOG(DEBUG) << "getListens aborted";
LOG(DEBUG, "getListens aborted");
return;
}
else if (ec)
@@ -399,7 +399,7 @@ namespace Scrobbling::ListenBrainz
void ListensSynchronizer::startSync()
{
LOG(DEBUG) << "Starting sync!";
LOG(DEBUG, "Starting sync!");
assert(!isSyncing());
@@ -435,7 +435,7 @@ namespace Scrobbling::ListenBrainz
{
_strand.dispatch([this, &context]
{
LOG(INFO) << "Sync done for user '" << context.listenBrainzUserName << "', fetched: " << context.fetchedListenCount << ", matched: " << context.matchedListenCount << ", imported: " << context.importedListenCount;
LOG(INFO, "Sync done for user '" << context.listenBrainzUserName << "', fetched: " << context.fetchedListenCount << ", matched: " << context.matchedListenCount << ", imported: " << context.importedListenCount);
context.syncing = false;
if (!isSyncing())
@@ -489,7 +489,7 @@ namespace Scrobbling::ListenBrainz
{
const auto listenCount = parseListenCount(msgBody);
if (listenCount)
LOG(DEBUG) << "Listen count for listenbrainz user '" << context.listenBrainzUserName << "' = " << *listenCount;
LOG(DEBUG, "Listen count for listenbrainz user '" << context.listenBrainzUserName << "' = " << *listenCount);
bool needSync{ listenCount && (!context.listenCount || *context.listenCount != *listenCount) };
context.listenCount = listenCount;
@@ -551,7 +551,7 @@ namespace Scrobbling::ListenBrainz
// update oldest listen for the next query
if (!parsedListen.listenedAt.isValid())
{
LOG(DEBUG) << "Skipping entry due to invalid listenedAt";
LOG(DEBUG, "Skipping entry due to invalid listenedAt");
continue;
}
src/libs/services/scrobbling/impl/listenbrainz/Utils.cpp
+26 -28
View File
@@ -27,38 +27,36 @@
namespace Scrobbling::ListenBrainz::Utils
{
std::optional<UUID>
getListenBrainzToken(Database::Session& session, Database::UserId userId)
{
auto transaction {session.createReadTransaction()};
std::optional<UUID> getListenBrainzToken(Database::Session& session, Database::UserId userId)
{
auto transaction{ session.createReadTransaction() };
const Database::User::pointer user {Database::User::find(session, userId)};
if (!user)
return std::nullopt;
const Database::User::pointer user{ Database::User::find(session, userId) };
if (!user)
return std::nullopt;
return user->getListenBrainzToken();
}
return user->getListenBrainzToken();
}
std::string
parseValidateToken(std::string_view msgBody)
{
std::string listenBrainzUserName;
std::string parseValidateToken(std::string_view msgBody)
{
std::string listenBrainzUserName;
Wt::Json::ParseError error;
Wt::Json::Object root;
if (!Wt::Json::parse(std::string {msgBody}, root, error))
{
LOG(ERROR) << "Cannot parse 'validate-token' result: " << error.what();
return listenBrainzUserName;
}
Wt::Json::ParseError error;
Wt::Json::Object root;
if (!Wt::Json::parse(std::string{ msgBody }, root, error))
{
LOG(ERROR, "Cannot parse 'validate-token' result: " << error.what());
return listenBrainzUserName;
}
if (!root.get("valid").orIfNull(false))
{
LOG(INFO) << "Invalid listenbrainz user";
return listenBrainzUserName;
}
if (!root.get("valid").orIfNull(false))
{
LOG(INFO, "Invalid listenbrainz user");
return listenBrainzUserName;
}
listenBrainzUserName = root.get("user_name").orIfNull("");
return listenBrainzUserName;
}
listenBrainzUserName = root.get("user_name").orIfNull("");
return listenBrainzUserName;
}
}
src/libs/services/scrobbling/impl/listenbrainz/Utils.hpp
+2 -2
View File
@@ -20,10 +20,10 @@
#pragma once
#include "services/database/UserId.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/UUID.hpp"
#define LOG(sev) LMS_LOG(SCROBBLING, sev) << "[listenbrainz] "
#define LOG(sev, message) LMS_LOG(SCROBBLING, sev, message << "[listenbrainz] ")
namespace Database
{
src/libs/services/scrobbling/test/Scrobbling.cpp
+6 -6
View File
@@ -19,16 +19,16 @@
#include <gtest/gtest.h>
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Service.hpp"
#include "utils/StreamLogger.hpp"
int main(int argc, char **argv)
int main(int argc, char** argv)
{
// log to stdout
Service<Logger> logger {std::make_unique<StreamLogger>(std::cout, EnumSet<Severity> {Severity::FATAL, Severity::ERROR})};
// log to stdout
Service<ILogger> logger{ std::make_unique<StreamLogger>(std::cout, EnumSet<Severity> {Severity::FATAL, Severity::ERROR}) };
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
src/libs/som/impl/Network.cpp
+1 -1
View File
@@ -26,7 +26,7 @@
#include <sstream>
#include <unordered_set>
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Random.hpp"
namespace SOM
src/libs/subsonic/impl/SubsonicId.cpp
+1 -1
View File
@@ -21,7 +21,7 @@
#include "SubsonicResponse.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/String.hpp"
namespace API::Subsonic
src/libs/subsonic/impl/SubsonicResource.cpp
+7 -7
View File
@@ -29,7 +29,7 @@
#include "services/database/User.hpp"
#include "utils/EnumSet.hpp"
#include "utils/IConfig.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Service.hpp"
#include "utils/String.hpp"
#include "utils/Utils.hpp"
@@ -290,7 +290,7 @@ namespace API::Subsonic
const std::size_t requestId{ curRequestId++ };
LMS_LOG(API_SUBSONIC, DEBUG) << "Handling request " << requestId << " '" << request.pathInfo() << "', continuation = " << (request.continuation() ? "true" : "false") << ", params = " << parameterMapToDebugString(request.getParameterMap());
LMS_LOG(API_SUBSONIC, DEBUG, "Handling request " << requestId << " '" << request.pathInfo() << "', continuation = " << (request.continuation() ? "true" : "false") << ", params = " << parameterMapToDebugString(request.getParameterMap()));
std::string requestPath{ request.pathInfo() };
if (StringUtils::stringEndsWith(requestPath, ".view"))
@@ -319,7 +319,7 @@ namespace API::Subsonic
resp.write(response.out(), format);
response.setMimeType(std::string{ ResponseFormatToMimeType(format) });
LMS_LOG(API_SUBSONIC, DEBUG) << "Request " << requestId << " '" << requestPath << "' handled!";
LMS_LOG(API_SUBSONIC, DEBUG, "Request " << requestId << " '" << requestPath << "' handled!");
return;
}
@@ -328,18 +328,18 @@ namespace API::Subsonic
if (itStreamHandler != mediaRetrievalHandlers.end())
{
itStreamHandler->second(requestContext, request, response);
LMS_LOG(API_SUBSONIC, DEBUG) << "Request " << requestId << " '" << requestPath << "' handled!";
LMS_LOG(API_SUBSONIC, DEBUG, "Request " << requestId << " '" << requestPath << "' handled!");
return;
}
LMS_LOG(API_SUBSONIC, ERROR) << "Unhandled command '" << requestPath << "'";
LMS_LOG(API_SUBSONIC, ERROR, "Unhandled command '" << requestPath << "'");
throw UnknownEntryPointGenericError{};
}
catch (const Error& e)
{
LMS_LOG(API_SUBSONIC, ERROR) << "Error while processing request '" << requestPath << "'"
LMS_LOG(API_SUBSONIC, ERROR, "Error while processing request '" << requestPath << "'"
<< ", params = [" << parameterMapToDebugString(request.getParameterMap()) << "]"
<< ", code = " << static_cast<int>(e.getCode()) << ", msg = '" << e.getMessage() << "'";
<< ", code = " << static_cast<int>(e.getCode()) << ", msg = '" << e.getMessage() << "'");
Response resp{ Response::createFailedResponse(protocolVersion, e) };
resp.write(response.out(), format);
response.setMimeType(std::string{ ResponseFormatToMimeType(format) });
src/libs/subsonic/impl/entrypoints/Browsing.cpp
+3 -3
View File
@@ -26,7 +26,7 @@
#include "services/database/Track.hpp"
#include "services/database/User.hpp"
#include "services/recommendation/IRecommendationService.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/Random.hpp"
#include "utils/Service.hpp"
#include "responses/Album.hpp"
@@ -122,7 +122,7 @@ namespace API::Subsonic
// This endpoint does not scale: make sort lived transactions in order not to block the whole application
// first pass: dispatch the artists by first letter
LMS_LOG(API_SUBSONIC, DEBUG) << "GetArtists: fetching all artists...";
LMS_LOG(API_SUBSONIC, DEBUG, "GetArtists: fetching all artists...");
std::map<char, std::vector<ArtistId>> artistsSortedByFirstChar;
std::size_t currentArtistOffset{ 0 };
constexpr std::size_t batchSize{ 100 };
@@ -151,7 +151,7 @@ namespace API::Subsonic
}
// second pass: add each artist
LMS_LOG(API_SUBSONIC, DEBUG) << "GetArtists: constructing response...";
LMS_LOG(API_SUBSONIC, DEBUG, "GetArtists: constructing response...");
for (const auto& [sortChar, artistIds] : artistsSortedByFirstChar)
{
Response::Node& indexNode{ artistsNode.createArrayChild("index") };
src/libs/subsonic/impl/entrypoints/MediaRetrieval.cpp
+4 -4
View File
@@ -29,7 +29,7 @@
#include "services/database/Track.hpp"
#include "services/database/User.hpp"
#include "utils/IResourceHandler.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/FileResourceHandlerCreator.hpp"
#include "utils/Utils.hpp"
#include "utils/String.hpp"
@@ -153,7 +153,7 @@ namespace API::Subsonic
if (!requestedFormat && (maxBitRate == 0 || track->getBitrate() <= maxBitRate ))
{
LMS_LOG(API_SUBSONIC, DEBUG) << "File's bitrate is compatible with parameters => no transcoding";
LMS_LOG(API_SUBSONIC, DEBUG, "File's bitrate is compatible with parameters => no transcoding");
return parameters; // no transcoding needed
}
@@ -165,7 +165,7 @@ namespace API::Subsonic
{
if (maxBitRate == 0 || track->getBitrate() <= maxBitRate)
{
LMS_LOG(API_SUBSONIC, DEBUG) << "File's bitrate and format are compatible with parameters => no transcoding";
LMS_LOG(API_SUBSONIC, DEBUG, "File's bitrate and format are compatible with parameters => no transcoding");
return parameters; // no transcoding needed
}
bitrate = maxBitRate;
@@ -246,7 +246,7 @@ namespace API::Subsonic
}
catch (const Av::Exception& e)
{
LMS_LOG(API_SUBSONIC, ERROR) << "Caught Av exception: " << e.what();
LMS_LOG(API_SUBSONIC, ERROR, "Caught Av exception: " << e.what());
}
}
src/libs/utils/impl/ArchiveZipper.cpp
+3 -3
View File
@@ -26,7 +26,7 @@
#include <archive.h>
#include <archive_entry.h>
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace Zip
{
@@ -73,7 +73,7 @@ namespace Zip
{
const int res {::archive_write_free(arch)};
if (res != ARCHIVE_OK)
LMS_LOG(UTILS, ERROR) << "Failure while freeing archive control struct: " << std::string {::strerror(res)};
LMS_LOG(UTILS, ERROR, "Failure while freeing archive control struct: " << std::string {::strerror(res)});
}
void
@@ -176,7 +176,7 @@ namespace Zip
void
ArchiveZipper::abort()
{
LMS_LOG(UTILS, DEBUG) << "Aborting zip creation";
LMS_LOG(UTILS, DEBUG, "Aborting zip creation");
if (_archive)
{
::archive_write_fail(_archive.get());
src/libs/utils/impl/ChildProcess.cpp
+116 -121
View File
@@ -36,177 +36,172 @@
#include <boost/asio/buffer.hpp>
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
namespace
{
class SystemException : public ChildProcessException
{
public:
SystemException(int err, const std::string& errMsg)
: ChildProcessException {errMsg + ": " + ::strerror(err)}
{}
class SystemException : public ChildProcessException
{
public:
SystemException(int err, const std::string& errMsg)
: ChildProcessException{ errMsg + ": " + ::strerror(err) }
{}
SystemException(boost::system::error_code ec, const std::string& errMsg)
: ChildProcessException {errMsg + ": " + ec.message()}
{}
};
SystemException(boost::system::error_code ec, const std::string& errMsg)
: ChildProcessException{ errMsg + ": " + ec.message() }
{}
};
}
ChildProcess::ChildProcess(boost::asio::io_context& ioContext, const std::filesystem::path& path, const Args& args)
: _ioContext {ioContext}
, _childStdout {_ioContext}
: _ioContext{ ioContext }
, _childStdout{ _ioContext }
{
// make sure only one thread is executing this part of code
static std::mutex mutex;
std::unique_lock<std::mutex> lock {mutex};
// make sure only one thread is executing this part of code
static std::mutex mutex;
std::unique_lock<std::mutex> lock{ mutex };
int pipe[2];
int pipe[2];
int res {pipe2(pipe, O_NONBLOCK | O_CLOEXEC)};
if (res < 0)
throw SystemException {errno, "pipe2 failed!"};
int res{ pipe2(pipe, O_NONBLOCK | O_CLOEXEC) };
if (res < 0)
throw SystemException{ errno, "pipe2 failed!" };
{
{
#if defined(__linux__) && defined(F_SETPIPE_SZ)
// Just a hint here to prevent the writer from writing too many bytes ahead of the reader
constexpr std::size_t pipeSize {65536*4};
// Just a hint here to prevent the writer from writing too many bytes ahead of the reader
constexpr std::size_t pipeSize{ 65536 * 4 };
if (fcntl(pipe[0], F_SETPIPE_SZ, pipeSize) == -1)
throw SystemException {errno, "fcntl failed!"};
if (fcntl(pipe[1], F_SETPIPE_SZ, pipeSize) == -1)
throw SystemException {errno, "fcntl failed!"};
if (fcntl(pipe[0], F_SETPIPE_SZ, pipeSize) == -1)
throw SystemException{ errno, "fcntl failed!" };
if (fcntl(pipe[1], F_SETPIPE_SZ, pipeSize) == -1)
throw SystemException{ errno, "fcntl failed!" };
#endif
}
}
res = fork();
if (res == -1)
throw SystemException {errno, "fork failed!"};
res = fork();
if (res == -1)
throw SystemException{ errno, "fork failed!" };
if (res == 0) // CHILD
{
close(pipe[0]);
close(STDIN_FILENO);
close(STDERR_FILENO);
if (res == 0) // CHILD
{
close(pipe[0]);
close(STDIN_FILENO);
close(STDERR_FILENO);
// Replace stdout with pipe write
if (dup2(pipe[1], STDOUT_FILENO) == -1)
exit(-1);
// Replace stdout with pipe write
if (dup2(pipe[1], STDOUT_FILENO) == -1)
exit(-1);
std::vector<const char*> execArgs;
std::transform(std::cbegin(args), std::cend(args), std::back_inserter(execArgs), [](const std::string& arg) { return arg.c_str(); });
execArgs.push_back(nullptr);
std::vector<const char*> execArgs;
std::transform(std::cbegin(args), std::cend(args), std::back_inserter(execArgs), [](const std::string& arg) { return arg.c_str(); });
execArgs.push_back(nullptr);
res = execv(path.string().c_str(), (char *const*)&execArgs[0]);
if (res == -1)
exit(-1);
}
else // PARENT
{
close(pipe[1]);
{
boost::system::error_code assignError;
_childStdout.assign(pipe[0], assignError);
if (assignError)
throw SystemException {assignError, "fork failed!"};
}
_childPID = res;
}
res = execv(path.string().c_str(), (char* const*)&execArgs[0]);
if (res == -1)
exit(-1);
}
else // PARENT
{
close(pipe[1]);
{
boost::system::error_code assignError;
_childStdout.assign(pipe[0], assignError);
if (assignError)
throw SystemException{ assignError, "fork failed!" };
}
_childPID = res;
}
}
ChildProcess::~ChildProcess()
{
LMS_LOG(CHILDPROCESS, DEBUG) << "Closing child process...";
{
boost::system::error_code closeError;
_childStdout.close(closeError);
if (closeError)
LMS_LOG(CHILDPROCESS, ERROR) << "Closed failed: " << closeError.message();
}
LMS_LOG(CHILDPROCESS, DEBUG, "Closing child process...");
{
boost::system::error_code closeError;
_childStdout.close(closeError);
if (closeError)
LMS_LOG(CHILDPROCESS, ERROR, "Closed failed: " << closeError.message());
}
if (!_finished)
kill();
if (!_finished)
kill();
wait(true);
wait(true);
}
void
ChildProcess::kill()
void ChildProcess::kill()
{
// process may already have finished
LMS_LOG(CHILDPROCESS, DEBUG) << "Killing child process...";
if (::kill(_childPID, SIGKILL) == -1)
LMS_LOG(CHILDPROCESS, DEBUG) << "Kill failed: " << ::strerror(errno);
// process may already have finished
LMS_LOG(CHILDPROCESS, DEBUG, "Killing child process...");
if (::kill(_childPID, SIGKILL) == -1)
LMS_LOG(CHILDPROCESS, DEBUG, "Kill failed: " << ::strerror(errno));
}
bool
ChildProcess::wait(bool block)
bool ChildProcess::wait(bool block)
{
assert(!_waited);
assert(!_waited);
int wstatus {};
const pid_t pid {waitpid(_childPID, &wstatus, block ? 0 : WNOHANG)};
int wstatus{};
const pid_t pid{ waitpid(_childPID, &wstatus, block ? 0 : WNOHANG) };
if (pid == -1)
throw SystemException {errno, "waitpid failed!"};
else if (pid == 0)
return false;
if (pid == -1)
throw SystemException{ errno, "waitpid failed!" };
else if (pid == 0)
return false;
if (WIFEXITED(wstatus))
{
_exitCode = WEXITSTATUS(wstatus);
LMS_LOG(CHILDPROCESS, DEBUG) << "Exit code = " << *_exitCode;
}
if (WIFEXITED(wstatus))
{
_exitCode = WEXITSTATUS(wstatus);
LMS_LOG(CHILDPROCESS, DEBUG, "Exit code = " << *_exitCode);
}
_waited = true;
return true;
_waited = true;
return true;
}
void
ChildProcess::asyncRead(std::byte* data, std::size_t bufferSize, ReadCallback callback)
void ChildProcess::asyncRead(std::byte* data, std::size_t bufferSize, ReadCallback callback)
{
assert(!finished());
assert(!finished());
LMS_LOG(CHILDPROCESS, DEBUG) << "Async read, bufferSize = " << bufferSize;
LMS_LOG(CHILDPROCESS, DEBUG, "Async read, bufferSize = " << bufferSize);
boost::asio::async_read(_childStdout, boost::asio::buffer(data, bufferSize),
[this, callback {std::move(callback)}](const boost::system::error_code& error, std::size_t bytesTransferred)
{
LMS_LOG(CHILDPROCESS, DEBUG) << "Async read cb - ec = '" << error.message() << "' (" << error.value() << "), bytesTransferred = " << bytesTransferred;
boost::asio::async_read(_childStdout, boost::asio::buffer(data, bufferSize),
[this, callback{ std::move(callback) }](const boost::system::error_code& error, std::size_t bytesTransferred)
{
LMS_LOG(CHILDPROCESS, DEBUG, "Async read cb - ec = '" << error.message() << "' (" << error.value() << "), bytesTransferred = " << bytesTransferred);
ReadResult readResult {ReadResult::Success};
if (error)
{
if (error != boost::asio::error::eof)
{
// forbidden to read any captured param here as the ChildProcess instance may already have been killed
return;
}
ReadResult readResult{ ReadResult::Success };
if (error)
{
if (error != boost::asio::error::eof)
{
// forbidden to read any captured param here as the ChildProcess instance may already have been killed
return;
}
readResult = ReadResult::EndOfFile;
_finished = true;
}
readResult = ReadResult::EndOfFile;
_finished = true;
}
callback(readResult, bytesTransferred);
});
callback(readResult, bytesTransferred);
});
}
std::size_t
ChildProcess::readSome(std::byte* data, std::size_t bufferSize)
std::size_t ChildProcess::readSome(std::byte* data, std::size_t bufferSize)
{
boost::system::error_code ec;
const std::size_t res {_childStdout.read_some(boost::asio::buffer(data, bufferSize), ec)};
LMS_LOG(CHILDPROCESS, DEBUG) << "read some " << res << " bytes, ec = " << ec.message();
if (ec)
_childStdout.close(ec);
boost::system::error_code ec;
const std::size_t res{ _childStdout.read_some(boost::asio::buffer(data, bufferSize), ec) };
LMS_LOG(CHILDPROCESS, DEBUG, "read some " << res << " bytes, ec = " << ec.message());
if (ec)
_childStdout.close(ec);
return res;
return res;
}
bool
ChildProcess::finished() const
bool ChildProcess::finished() const
{
return _finished;
return _finished;
}
src/libs/utils/impl/ChildProcessManager.cpp
+1 -1
View File
@@ -19,7 +19,7 @@
#include "ChildProcessManager.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "ChildProcess.hpp"
src/libs/utils/impl/Config.cpp
+1 -1
View File
@@ -20,7 +20,7 @@
#include "Config.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
std::unique_ptr<IConfig> createConfig(const std::filesystem::path& p)
{
src/libs/utils/impl/FileResourceHandler.cpp
+19 -14
View File
@@ -21,7 +21,7 @@
#include <fstream>
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
std::unique_ptr<IResourceHandler>
createFileResourceHandler(const std::filesystem::path& path, std::string_view mimeType)
@@ -45,7 +45,7 @@ FileResourceHandler::processRequest(const Wt::Http::Request& request, Wt::Http::
{
if (!ifs)
{
LMS_LOG(UTILS, ERROR) << "Cannot open file stream for '" << _path.string() << "'";
LMS_LOG(UTILS, ERROR, "Cannot open file stream for '" << _path.string() << "'");
response.setStatus(404);
return {};
}
@@ -54,7 +54,7 @@ FileResourceHandler::processRequest(const Wt::Http::Request& request, Wt::Http::
const ::uint64_t fileSize{ static_cast<::uint64_t>(ifs.tellg()) };
ifs.seekg(0, std::ios::beg);
LMS_LOG(UTILS, DEBUG) << "File '" << _path.string() << "', fileSize = " << fileSize;
LMS_LOG(UTILS, DEBUG, "File '" << _path.string() << "', fileSize = " << fileSize);
response.addHeader("Accept-Ranges", "bytes");
@@ -66,13 +66,13 @@ FileResourceHandler::processRequest(const Wt::Http::Request& request, Wt::Http::
response.setStatus(416); // Requested range not satisfiable
response.addHeader("Content-Range", contentRange.str());
LMS_LOG(UTILS, DEBUG) << "Range not satisfiable";
LMS_LOG(UTILS, DEBUG, "Range not satisfiable");
return {};
}
if (ranges.size() == 1)
{
LMS_LOG(UTILS, DEBUG) << "Range requested = " << ranges[0].firstByte() << "/" << ranges[0].lastByte();
LMS_LOG(UTILS, DEBUG, "Range requested = " << ranges[0].firstByte() << "-" << ranges[0].lastByte());
response.setStatus(206);
startByte = ranges[0].firstByte();
@@ -87,19 +87,19 @@ FileResourceHandler::processRequest(const Wt::Http::Request& request, Wt::Http::
}
else
{
LMS_LOG(UTILS, DEBUG) << "No range requested";
LMS_LOG(UTILS, DEBUG, "No range requested");
response.setStatus(200);
_beyondLastByte = fileSize;
response.setContentLength(_beyondLastByte);
}
LMS_LOG(UTILS, DEBUG) << "Mimetype set to '" << _mimeType << "'";
LMS_LOG(UTILS, DEBUG, "Mimetype set to '" << _mimeType << "'");
response.setMimeType(_mimeType);
}
else if (!ifs)
{
LMS_LOG(UTILS, ERROR) << "Cannot reopen file stream for '" << _path.string() << "'";
LMS_LOG(UTILS, ERROR, "Cannot reopen file stream for '" << _path.string() << "'");
return {};
}
@@ -113,19 +113,24 @@ FileResourceHandler::processRequest(const Wt::Http::Request& request, Wt::Http::
ifs.read(&buf[0], pieceSize);
const ::uint64_t actualPieceSize{ static_cast<::uint64_t>(ifs.gcount()) };
response.out().write(&buf[0], actualPieceSize);
if (actualPieceSize > 0)
{
response.out().write(&buf[0], actualPieceSize);
LMS_LOG(UTILS, DEBUG, "Written " << actualPieceSize << " bytes, range = " << startByte << "-" << startByte + actualPieceSize - 1 << "");
}
else
{
LMS_LOG(UTILS, DEBUG, "Written 0 byte");
}
LMS_LOG(UTILS, DEBUG) << "Written " << actualPieceSize << " bytes";
LMS_LOG(UTILS, DEBUG) << "Progress: " << actualPieceSize << "/" << restSize;
if (ifs.good() && actualPieceSize < restSize)
{
_offset = startByte + actualPieceSize;
LMS_LOG(UTILS, DEBUG) << "Job not complete! Next chunk offset = " << _offset;
LMS_LOG(UTILS, DEBUG, "Job not complete! Remaining range: " << _offset << "-" << _beyondLastByte - 1);
return response.createContinuation();
}
LMS_LOG(UTILS, DEBUG) << "Job complete!";
LMS_LOG(UTILS, DEBUG, "Job complete!");
return nullptr;
}
src/libs/utils/impl/IOContextRunner.cpp
+5 -5
View File
@@ -21,13 +21,13 @@
#include <cstdlib>
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
IOContextRunner::IOContextRunner(boost::asio::io_service& ioService, std::size_t threadCount)
: _ioService {ioService}
, _work {ioService}
{
LMS_LOG(UTILS, INFO) << "Starting IO context with " << threadCount << " threads...";
LMS_LOG(UTILS, INFO, "Starting IO context with " << threadCount << " threads...");
for (std::size_t i {}; i < threadCount; ++i)
{
_threads.emplace_back([&]
@@ -38,7 +38,7 @@ IOContextRunner::IOContextRunner(boost::asio::io_service& ioService, std::size_t
}
catch (const std::exception& e)
{
LMS_LOG(UTILS, FATAL) << "Exception caught in IO context: " << e.what();
LMS_LOG(UTILS, FATAL, "Exception caught in IO context: " << e.what());
std::abort();
}
});
@@ -48,10 +48,10 @@ IOContextRunner::IOContextRunner(boost::asio::io_service& ioService, std::size_t
void
IOContextRunner::stop()
{
LMS_LOG(UTILS, DEBUG) << "Stopping IO context...";
LMS_LOG(UTILS, DEBUG, "Stopping IO context...");
_work.reset();
_ioService.stop();
LMS_LOG(UTILS, DEBUG) << "IO context stopped!";
LMS_LOG(UTILS, DEBUG, "IO context stopped!");
}
IOContextRunner::~IOContextRunner()
src/libs/utils/impl/Logger.cpp
+8 -9
View File
@@ -17,7 +17,7 @@
* along with LMS. If not, see <http://www.gnu.org/licenses/>.
*/
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
const char* getModuleName(Module mod)
{
@@ -59,20 +59,19 @@ const char* getSeverityName(Severity sev)
return "";
}
Log::Log(Logger* logger, Module module, Severity severity)
: _module{ module },
_severity{ severity },
_logger{ logger }
Log::Log(ILogger& logger, Module module, Severity severity)
: _logger{ logger }
, _module{ module }
, _severity{ severity }
{}
Log::~Log()
{
if (_logger)
_logger->processLog(*this);
_logger.processLog(*this);
}
std::string
Log::getMessage() const
std::string Log::getMessage() const
{
return _oss.str();
}
src/libs/utils/impl/Path.cpp
+104 -114
View File
@@ -30,146 +30,136 @@
#include "utils/Crc32Calculator.hpp"
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/String.hpp"
namespace PathUtils
{
std::uint32_t computeCrc32(const std::filesystem::path& p)
{
Utils::Crc32Calculator crc32;
std::uint32_t
computeCrc32(const std::filesystem::path& p)
{
Utils::Crc32Calculator crc32;
std::ifstream ifs{ p.string().c_str(), std::ios_base::binary };
if (ifs)
{
do
{
std::array<char, 1024> buffer;
std::ifstream ifs {p.string().c_str(), std::ios_base::binary};
if (ifs)
{
do
{
std::array<char,1024> buffer;
ifs.read(buffer.data(), buffer.size());
crc32.processBytes(reinterpret_cast<const std::byte*>(buffer.data()), ifs.gcount());
} while (ifs);
}
else
{
LMS_LOG(DBUPDATER, ERROR, "Failed to open file '" << p.string() << "'");
throw LmsException("Failed to open file '" + p.string() + "'");
}
ifs.read( buffer.data(), buffer.size() );
crc32.processBytes( reinterpret_cast<const std::byte*>(buffer.data()), ifs.gcount() );
}
while (ifs);
}
else
{
LMS_LOG(DBUPDATER, ERROR) << "Failed to open file '" << p.string() << "'";
throw LmsException("Failed to open file '" + p.string() + "'" );
}
return crc32.getResult();
}
return crc32.getResult();
}
bool ensureDirectory(const std::filesystem::path& dir)
{
if (std::filesystem::exists(dir))
return std::filesystem::is_directory(dir);
else
return std::filesystem::create_directory(dir);
}
bool
ensureDirectory(const std::filesystem::path& dir)
{
if (std::filesystem::exists(dir))
return std::filesystem::is_directory(dir);
else
return std::filesystem::create_directory(dir);
}
Wt::WDateTime getLastWriteTime(const std::filesystem::path& file)
{
struct stat sb {};
Wt::WDateTime
getLastWriteTime(const std::filesystem::path& file)
{
struct stat sb {};
if (stat(file.string().c_str(), &sb) == -1)
throw LmsException("Failed to get stats on file '" + file.string() + "'");
if (stat(file.string().c_str(), &sb) == -1)
throw LmsException("Failed to get stats on file '" + file.string() + "'" );
return Wt::WDateTime::fromTime_t(sb.st_mtime);
}
return Wt::WDateTime::fromTime_t(sb.st_mtime);
}
bool exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path* excludeDirFileName)
{
std::error_code ec;
std::filesystem::directory_iterator itPath{ directory, std::filesystem::directory_options::follow_directory_symlink, ec };
bool
exploreFilesRecursive(const std::filesystem::path& directory, std::function<bool(std::error_code, const std::filesystem::path&)> cb, const std::filesystem::path* excludeDirFileName)
{
std::error_code ec;
std::filesystem::directory_iterator itPath {directory, std::filesystem::directory_options::follow_directory_symlink, ec};
if (ec)
{
cb(ec, directory);
return true; // try to continue exploring anyway
}
if (ec)
{
cb(ec, directory);
return true; // try to continue exploring anyway
}
if (excludeDirFileName && !excludeDirFileName->empty())
{
const std::filesystem::path excludePath{ directory / *excludeDirFileName };
if (excludeDirFileName && !excludeDirFileName->empty())
{
const std::filesystem::path excludePath {directory / *excludeDirFileName};
if (std::filesystem::exists(excludePath, ec))
{
LMS_LOG(DBUPDATER, DEBUG, "Found '" << excludePath.string() << "': skipping directory");
return true;
}
}
if (std::filesystem::exists(excludePath, ec))
{
LMS_LOG(DBUPDATER, DEBUG) << "Found '" << excludePath.string() << "': skipping directory";
return true;
}
}
std::filesystem::directory_iterator itEnd;
while (itPath != itEnd)
{
bool continueExploring{ true };
std::filesystem::directory_iterator itEnd;
while (itPath != itEnd)
{
bool continueExploring {true};
if (ec)
{
continueExploring = cb(ec, *itPath);
}
else
{
if (std::filesystem::is_regular_file(*itPath, ec))
{
continueExploring = cb(ec, *itPath);
}
else if (std::filesystem::is_directory(*itPath, ec))
{
if (!ec)
continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName);
else
continueExploring = cb(ec, *itPath);
}
}
if (ec)
{
continueExploring = cb(ec, *itPath);
}
else
{
if (std::filesystem::is_regular_file(*itPath, ec))
{
continueExploring = cb(ec, *itPath);
}
else if (std::filesystem::is_directory(*itPath, ec))
{
if (!ec)
continueExploring = exploreFilesRecursive(*itPath, cb, excludeDirFileName);
else
continueExploring = cb(ec, *itPath);
}
}
if (!continueExploring)
return false;
if (!continueExploring)
return false;
itPath.increment(ec);
}
itPath.increment(ec);
}
return true;
}
return true;
}
bool hasFileAnyExtension(const std::filesystem::path& file, const std::vector<std::filesystem::path>& supportedExtensions)
{
const std::filesystem::path extension{ StringUtils::stringToLower(file.extension().string()) };
bool
hasFileAnyExtension(const std::filesystem::path& file, const std::vector<std::filesystem::path>& supportedExtensions)
{
const std::filesystem::path extension {StringUtils::stringToLower(file.extension().string())};
return (std::find(std::cbegin(supportedExtensions), std::cend(supportedExtensions), extension) != std::cend(supportedExtensions));
}
return (std::find(std::cbegin(supportedExtensions), std::cend(supportedExtensions), extension) != std::cend(supportedExtensions));
}
bool isPathInRootPath(const std::filesystem::path& path, const std::filesystem::path& rootPath, const std::filesystem::path* excludeDirFileName)
{
std::filesystem::path curPath = path;
bool
isPathInRootPath(const std::filesystem::path& path, const std::filesystem::path& rootPath, const std::filesystem::path* excludeDirFileName)
{
std::filesystem::path curPath = path;
while (curPath.parent_path() != curPath)
{
curPath = curPath.parent_path();
while (curPath.parent_path() != curPath)
{
curPath = curPath.parent_path();
if (excludeDirFileName && !excludeDirFileName->empty())
{
assert(!excludeDirFileName->has_parent_path());
if (excludeDirFileName && !excludeDirFileName->empty())
{
assert(!excludeDirFileName->has_parent_path());
std::error_code ec;
if (std::filesystem::exists(curPath / *excludeDirFileName, ec))
return false;
}
if (curPath == rootPath)
return true;
}
return false;
}
std::error_code ec;
if (std::filesystem::exists(curPath / *excludeDirFileName, ec))
return false;
}
if (curPath == rootPath)
return true;
}
return false;
}
} // ns PathUtils
src/libs/utils/impl/String.cpp
+16
View File
@@ -170,6 +170,22 @@ namespace StringUtils
return res;
}
std::string joinStrings(const std::vector<std::string_view>& strings, std::string_view delimiter)
{
std::string res;
bool first{ true };
for (std::string_view str : strings)
{
if (!first)
res += delimiter;
res += str;
first = false;
}
return res;
}
std::string joinStrings(const std::vector<std::string>& strings, const std::string& delimiter)
{
return boost::algorithm::join(strings, delimiter);
src/libs/utils/impl/WtLogger.cpp
+26 -2
View File
@@ -21,10 +21,10 @@
#include <thread>
#include <sstream>
#include <Wt/WApplication.h>
#include <Wt/WServer.h>
#include <Wt/WLogger.h>
#include "utils/Logger.hpp"
#include "utils/Exception.hpp"
namespace
{
@@ -36,6 +36,30 @@ namespace
}
}
WtLogger::WtLogger(Severity minSeverity)
: _minSeverity{ minSeverity }
{
}
std::string WtLogger::computeLogConfig(Severity minSeverity)
{
switch (minSeverity)
{
case Severity::DEBUG: return "*";
case Severity::INFO: return "* -debug";
case Severity::WARNING: return "* -debug -info";
case Severity::ERROR: return "* -debug -info -warning";
case Severity::FATAL: return "* -debug -info -warning -error";
}
throw LmsException{ "Unhandled severity" };
}
bool WtLogger::isSeverityActive(Severity severity) const
{
return static_cast<int>(severity) <= static_cast<int>(_minSeverity);
}
void WtLogger::processLog(const Log& log)
{
Wt::log(getSeverityName(log.getSeverity())) << Wt::WLogger::sep << to_string(std::this_thread::get_id()) << Wt::WLogger::sep << "[" << getModuleName(log.getModule()) << "]" << Wt::WLogger::sep << log.getMessage();
src/libs/utils/impl/http/SendQueue.cpp
+13 -13
View File
@@ -23,10 +23,10 @@
#include <boost/asio/bind_executor.hpp>
#include "utils/Exception.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
#include "utils/String.hpp"
#define LOG(sev) LMS_LOG(SCROBBLING, sev) << "[Http SendQueue] - "
#define LOG(sev, message) LMS_LOG(SCROBBLING, sev, "[Http SendQueue] - " << message)
namespace StringUtils
{
@@ -98,7 +98,7 @@ namespace Http
for (auto& [prio, requests] : _sendQueue)
{
LOG(DEBUG) << "Processing prio " << static_cast<int>(prio) << ", request count = " << requests.size();
LOG(DEBUG, "Processing prio " << static_cast<int>(prio) << ", request count = " << requests.size());
while (!requests.empty())
{
std::unique_ptr<ClientRequest> request {std::move(requests.front())};
@@ -118,7 +118,7 @@ namespace Http
SendQueue::sendRequest(const ClientRequest& request)
{
std::string url {_baseUrl + request.getParameters().relativeUrl};
LOG(DEBUG) << "Sending request to url '" << url << "'";
LOG(DEBUG, "Sending request to url '" << url << "'");
bool res {};
switch (request.getType())
@@ -133,7 +133,7 @@ namespace Http
}
if (!res)
LOG(ERROR) << "Send failed, bad url or unsupported scheme?";
LOG(ERROR, "Send failed, bad url or unsupported scheme?");
return res;
}
@@ -143,14 +143,14 @@ namespace Http
{
if (ec == boost::asio::error::operation_aborted)
{
LOG(DEBUG) << "Client aborted";
LOG(DEBUG, "Client aborted");
return;
}
assert(_currentRequest);
_state = State::Idle;
LOG(DEBUG) << "Client done. status = " << msg.status();
LOG(DEBUG, "Client done. status = " << msg.status());
if (ec)
onClientDoneError(std::move(_currentRequest), ec);
else
@@ -160,7 +160,7 @@ namespace Http
void
SendQueue::onClientDoneError(std::unique_ptr<ClientRequest> request, Wt::AsioWrapper::error_code ec)
{
LOG(ERROR) << "Retry " << request->retryCount << ", client error: '" << ec.message() << "'";
LOG(ERROR, "Retry " << request->retryCount << ", client error: '" << ec.message() << "'");
// may be a network error, try again later
throttle(_defaultRetryWaitDuration);
@@ -171,7 +171,7 @@ namespace Http
}
else
{
LOG(ERROR) << "Too many retries, giving up operation and throttle";
LOG(ERROR, "Too many retries, giving up operation and throttle");
if (request->getParameters().onFailureFunc)
request->getParameters().onFailureFunc();
}
@@ -189,7 +189,7 @@ namespace Http
}
const auto remainingCount {headerReadAs<std::size_t>(msg, "X-RateLimit-Remaining")};
LOG(DEBUG) << "Remaining messages = " << (remainingCount ? *remainingCount : 0);
LOG(DEBUG, "Remaining messages = " << (remainingCount ? *remainingCount : 0));
if (mustThrottle || (remainingCount && *remainingCount == 0))
{
const auto waitDuration {headerReadAs<std::chrono::seconds>(msg, "X-RateLimit-Reset-In")};
@@ -205,7 +205,7 @@ namespace Http
}
else
{
LOG(ERROR) << "Send error: '" << msg.body() << "'";
LOG(ERROR, "Send error: '" << msg.body() << "'");
if (requestParameters.onFailureFunc)
requestParameters.onFailureFunc();
}
@@ -221,14 +221,14 @@ namespace Http
assert(_state == State::Idle);
const std::chrono::seconds duration {clamp(requestedDuration, _minRetryWaitDuration, _maxRetryWaitDuration)};
LOG(DEBUG) << "Throttling for " << duration.count() << " seconds";
LOG(DEBUG, "Throttling for " << duration.count() << " seconds");
_throttleTimer.expires_after(duration);
_throttleTimer.async_wait([this](const boost::system::error_code& ec)
{
if (ec == boost::asio::error::operation_aborted)
{
LOG(DEBUG) << "Throttle aborted";
LOG(DEBUG, "Throttle aborted");
return;
}
else if (ec)
src/libs/utils/include/utils/Logger.hpp → src/libs/utils/include/utils/ILogger.hpp
+14 -7
View File
@@ -22,6 +22,7 @@
#include <string>
#include <sstream>
#include "utils/String.hpp"
#include "Service.hpp"
enum class Severity
@@ -59,11 +60,11 @@ enum class Module
const char* getModuleName(Module mod);
const char* getSeverityName(Severity sev);
class Logger;
class ILogger;
class Log
{
public:
Log(Logger* logger, Module module, Severity severity);
Log(ILogger& logger, Module module, Severity severity);
~Log();
Module getModule() const { return _module; }
@@ -76,18 +77,24 @@ private:
Log(const Log&) = delete;
Log& operator=(const Log&) = delete;
ILogger& _logger;
Module _module;
Severity _severity;
std::ostringstream _oss;
Logger* _logger{};
};
class Logger
class ILogger
{
public:
virtual ~Logger() = default;
virtual ~ILogger() = default;
virtual bool isSeverityActive(Severity severity) const = 0;
virtual void processLog(const Log& log) = 0;
};
#define LMS_LOG(module, severity) Log{Service<Logger>::get(), Module::module, Severity::severity}.getOstream()
#define LMS_LOG_EX(module, severity) Log{Service<Logger>::get(), module, severity}.getOstream()
#define LMS_LOG(module, severity, message) \
do \
{ \
if (auto* logger {::Service<::ILogger>::get()}; logger && logger->isSeverityActive(::Severity::severity)) \
::Log{ *logger, ::Module::module, ::Severity::severity }.getOstream() << message; \
} while(0)
src/libs/utils/include/utils/StreamLogger.hpp
+10 -9
View File
@@ -20,19 +20,20 @@
#pragma once
#include "utils/EnumSet.hpp"
#include "utils/Logger.hpp"
#include "utils/ILogger.hpp"
class StreamLogger final : public Logger
class StreamLogger final : public ILogger
{
public:
static constexpr EnumSet<Severity> defaultSeverities {Severity::FATAL, Severity::ERROR, Severity::WARNING, Severity::INFO};
public:
static constexpr EnumSet<Severity> defaultSeverities{ Severity::FATAL, Severity::ERROR, Severity::WARNING, Severity::INFO };
StreamLogger(std::ostream& oss, EnumSet<Severity> severities = defaultSeverities);
StreamLogger(std::ostream& oss, EnumSet<Severity> severities = defaultSeverities);
void processLog(const Log& log);
bool isSeverityActive(Severity) const override { return true; }
void processLog(const Log& log) override;
private:
std::ostream& _os;
const EnumSet<Severity> _severities;
private:
std::ostream& _os;
const EnumSet<Severity> _severities;
};
src/libs/utils/include/utils/String.hpp
+1 -4
View File
@@ -38,19 +38,16 @@ namespace Wt
namespace StringUtils {
[[nodiscard]] std::vector<std::string> splitStringCopy(std::string_view string, std::string_view separators);
[[nodiscard]] std::vector<std::string_view> splitString(std::string_view string, std::string_view separators);
[[nodiscard]] std::string joinStrings(const std::vector<std::string_view>& strings, std::string_view delimiter);
[[nodiscard]] std::string joinStrings(const std::vector<std::string>& strings, const std::string& delimiter);
[[nodiscard]] std::string_view stringTrim(std::string_view str, std::string_view whitespaces = " \t");
[[nodiscard]] std::string_view stringTrimEnd(std::string_view str, std::string_view whitespaces = " \t");
[[nodiscard]] std::string stringToLower(std::string_view str);
void stringToLower(std::string& str);
[[nodiscard]] std::string stringToUpper(const std::string& str);
[[nodiscard]] std::string bufferToString(const std::vector<unsigned char>& data);
src/libs/utils/include/utils/WtLogger.hpp
+13 -5
View File
@@ -19,11 +19,19 @@
#pragma once
#include "Logger.hpp"
#include <string>
class WtLogger final : public Logger
#include "utils/ILogger.hpp"
class WtLogger final : public ILogger
{
public:
void processLog(const Log& log) override;
};
public:
WtLogger(Severity minSeverity);
static std::string computeLogConfig(Severity minSeverity);
private:
bool isSeverityActive(Severity severity) const override;
void processLog(const Log& log) override;
const Severity _minSeverity;
};
src/libs/utils/test/String.cpp
+129 -102
View File
@@ -26,155 +26,182 @@
TEST(StringUtils, splitString)
{
{
const std::string test{ "a" };
{
const std::string test{ "a" };
const std::vector<std::string_view> strings{ StringUtils::splitString(test, "") };
ASSERT_EQ(strings.size(), 1);
EXPECT_EQ(strings.front(), "a");
}
const std::vector<std::string_view> strings{ StringUtils::splitString(test, "") };
ASSERT_EQ(strings.size(), 1);
EXPECT_EQ(strings.front(), "a");
}
{
const std::string test{ "a b" };
{
const std::string test{ "a b" };
const std::vector<std::string_view> strings{ StringUtils::splitString(test, "|") };
ASSERT_EQ(strings.size(), 1);
EXPECT_EQ(strings.front(), "a b");
}
const std::vector<std::string_view> strings{ StringUtils::splitString(test, "|") };
ASSERT_EQ(strings.size(), 1);
EXPECT_EQ(strings.front(), "a b");
}
{
const std::string test{ " a" };
{
const std::string test{ " a" };
const std::vector<std::string_view> strings{ StringUtils::splitString(test, " ") };
ASSERT_EQ(strings.size(), 1);
EXPECT_EQ(strings.front(), "a");
}
const std::vector<std::string_view> strings{ StringUtils::splitString(test, " ") };
ASSERT_EQ(strings.size(), 1);
EXPECT_EQ(strings.front(), "a");
}
{
const std::string test{ "a " };
{
const std::string test{ "a " };
const std::vector<std::string_view> strings{ StringUtils::splitString(test, " ") };
ASSERT_EQ(strings.size(), 1);
EXPECT_EQ(strings.front(), "a");
}
const std::vector<std::string_view> strings{ StringUtils::splitString(test, " ") };
ASSERT_EQ(strings.size(), 1);
EXPECT_EQ(strings.front(), "a");
}
{
const std::string test{ "a b" };
{
const std::string test{ "a b" };
const std::vector<std::string_view> strings{ StringUtils::splitString(test, " ") };
ASSERT_EQ(strings.size(), 2);
EXPECT_EQ(strings.front(), "a");
EXPECT_EQ(strings.back(), "b");
}
const std::vector<std::string_view> strings{ StringUtils::splitString(test, " ") };
ASSERT_EQ(strings.size(), 2);
EXPECT_EQ(strings.front(), "a");
EXPECT_EQ(strings.back(), "b");
}
{
const std::string test{ "a b,c|defgh " };
{
const std::string test{ "a b,c|defgh " };
const std::vector<std::string_view> strings{ StringUtils::splitString(test, " ,|") };
ASSERT_EQ(strings.size(), 4);
EXPECT_EQ(strings[0], "a");
EXPECT_EQ(strings[1], "b");
EXPECT_EQ(strings[2], "c");
EXPECT_EQ(strings[3], "defgh");
}
const std::vector<std::string_view> strings{ StringUtils::splitString(test, " ,|") };
ASSERT_EQ(strings.size(), 4);
EXPECT_EQ(strings[0], "a");
EXPECT_EQ(strings[1], "b");
EXPECT_EQ(strings[2], "c");
EXPECT_EQ(strings[3], "defgh");
}
}
TEST(StringUtils, splitStringCopy)
{
{
const std::string test{ "test=foo" };
{
const std::string test{ "test=foo" };
const std::vector<std::string> strings{ StringUtils::splitStringCopy(test, "=") };
ASSERT_EQ(strings.size(), 2);
EXPECT_EQ(strings[0], "test");
EXPECT_EQ(strings[1], "foo");
}
const std::vector<std::string> strings{ StringUtils::splitStringCopy(test, "=") };
ASSERT_EQ(strings.size(), 2);
EXPECT_EQ(strings[0], "test");
EXPECT_EQ(strings[1], "foo");
}
{
const std::string test{ "test=foo bar" };
{
const std::string test{ "test=foo bar" };
const std::vector<std::string> strings{ StringUtils::splitStringCopy(test, "=") };
ASSERT_EQ(strings.size(), 2);
EXPECT_EQ(strings[0], "test");
EXPECT_EQ(strings[1], "foo bar");
}
const std::vector<std::string> strings{ StringUtils::splitStringCopy(test, "=") };
ASSERT_EQ(strings.size(), 2);
EXPECT_EQ(strings[0], "test");
EXPECT_EQ(strings[1], "foo bar");
}
}
TEST(StringUtils, joinStrings)
{
struct TestCase
{
std::vector<std::string_view> input;
std::string delimiter;
std::string expectedOutput;
};
TestCase tests[]
{
{{"a", "b", "c"}, "-", "a-b-c"},
{{"a", "b", "c"}, ",", "a,b,c"},
{{"a", "b", "c"}, "***", "a***b***c"},
{{"a", "", "c"}, "-", "a--c"},
{{"", "b", "c"}, "-", "-b-c"},
{{"a"}, "-", "a"},
{{"a"}, ",", "a"},
};
for (const TestCase& test : tests)
{
const std::string str{ StringUtils::joinStrings(test.input, test.delimiter) };
EXPECT_EQ(str, test.expectedOutput);
}
}
TEST(StringUtils, escapeJSString)
{
EXPECT_EQ(StringUtils::jsEscape(""), "");
EXPECT_EQ(StringUtils::jsEscape(R"(Test'.mp3)"), R"(Test\'.mp3)");
EXPECT_EQ(StringUtils::jsEscape(R"(Test"".mp3)"), R"(Test\"\".mp3)");
EXPECT_EQ(StringUtils::jsEscape(R"(\Test\.mp3)"), R"(\\Test\\.mp3)");
EXPECT_EQ(StringUtils::jsEscape(""), "");
EXPECT_EQ(StringUtils::jsEscape(R"(Test'.mp3)"), R"(Test\'.mp3)");
EXPECT_EQ(StringUtils::jsEscape(R"(Test"".mp3)"), R"(Test\"\".mp3)");
EXPECT_EQ(StringUtils::jsEscape(R"(\Test\.mp3)"), R"(\\Test\\.mp3)");
}
TEST(StringUtils, escapeJsonString)
{
EXPECT_EQ(StringUtils::jsonEscape(""), "");
EXPECT_EQ(StringUtils::jsonEscape(R"(Test'.mp3)"), R"(Test'.mp3)");
EXPECT_EQ(StringUtils::jsonEscape(R"(Test"".mp3)"), R"(Test\"\".mp3)");
EXPECT_EQ(StringUtils::jsonEscape(R"(\Test\.mp3)"), R"(\\Test\\.mp3)");
EXPECT_EQ(StringUtils::jsonEscape(""), "");
EXPECT_EQ(StringUtils::jsonEscape(R"(Test'.mp3)"), R"(Test'.mp3)");
EXPECT_EQ(StringUtils::jsonEscape(R"(Test"".mp3)"), R"(Test\"\".mp3)");
EXPECT_EQ(StringUtils::jsonEscape(R"(\Test\.mp3)"), R"(\\Test\\.mp3)");
}
TEST(StringUtils, escapeString)
{
EXPECT_EQ(StringUtils::escapeString("", "*", ' '), "");
EXPECT_EQ(StringUtils::escapeString("", "", ' '), "");
EXPECT_EQ(StringUtils::escapeString("a", "", ' '), "a");
EXPECT_EQ(StringUtils::escapeString("*", "*", '_'), "_*");
EXPECT_EQ(StringUtils::escapeString("*a*", "*", '_'), "_*a_*");
EXPECT_EQ(StringUtils::escapeString("*a|", "*|", '_'), "_*a_|");
EXPECT_EQ(StringUtils::escapeString("**||", "*|", '_'), "_*_*_|_|");
EXPECT_EQ(StringUtils::escapeString("", "*", ' '), "");
EXPECT_EQ(StringUtils::escapeString("", "", ' '), "");
EXPECT_EQ(StringUtils::escapeString("a", "", ' '), "a");
EXPECT_EQ(StringUtils::escapeString("*", "*", '_'), "_*");
EXPECT_EQ(StringUtils::escapeString("*a*", "*", '_'), "_*a_*");
EXPECT_EQ(StringUtils::escapeString("*a|", "*|", '_'), "_*a_|");
EXPECT_EQ(StringUtils::escapeString("**||", "*|", '_'), "_*_*_|_|");
}
TEST(StringUtils, readAs)
{
EXPECT_EQ(StringUtils::readAs<bool>("true"), true);
EXPECT_EQ(StringUtils::readAs<bool>("1"), true);
EXPECT_EQ(StringUtils::readAs<bool>("false"), false);
EXPECT_EQ(StringUtils::readAs<bool>("0"), false);
EXPECT_EQ(StringUtils::readAs<bool>("foo"), std::nullopt);
EXPECT_EQ(StringUtils::readAs<bool>(""), std::nullopt);
EXPECT_EQ(StringUtils::readAs<bool>("true"), true);
EXPECT_EQ(StringUtils::readAs<bool>("1"), true);
EXPECT_EQ(StringUtils::readAs<bool>("false"), false);
EXPECT_EQ(StringUtils::readAs<bool>("0"), false);
EXPECT_EQ(StringUtils::readAs<bool>("foo"), std::nullopt);
EXPECT_EQ(StringUtils::readAs<bool>(""), std::nullopt);
}
TEST(StringUtils, capitalize)
{
struct TestCase
{
std::string input;
std::string expectedOutput;
};
struct TestCase
{
std::string input;
std::string expectedOutput;
};
TestCase tests[]
{
{"", ""},
{"C", "C"},
{"c", "C"},
{" c", " C"},
{" cc", " Cc"},
{"(c", "(c"},
{"1c", "1c"},
{"&c", "&c"},
{"c c", "C c"}
};
TestCase tests[]
{
{"", ""},
{"C", "C"},
{"c", "C"},
{" c", " C"},
{" cc", " Cc"},
{"(c", "(c"},
{"1c", "1c"},
{"&c", "&c"},
{"c c", "C c"}
};
for (const TestCase& test : tests)
{
std::string str{ test.input };
StringUtils::capitalize(str);
EXPECT_EQ(str, test.expectedOutput) << " str was '" << test.input << "'";
}
for (const TestCase& test : tests)
{
std::string str{ test.input };
StringUtils::capitalize(str);
EXPECT_EQ(str, test.expectedOutput) << " str was '" << test.input << "'";
}
}
TEST(Stringutils, date)
{
const Wt::WDate date{ 2020, 01, 03 };
EXPECT_EQ(StringUtils::toISO8601String(date), "2020-01-03");
const Wt::WDate date{ 2020, 01, 03 };
EXPECT_EQ(StringUtils::toISO8601String(date), "2020-01-03");
}
TEST(Stringutils, dateTime)
{
const Wt::WDateTime dateTime{ Wt::WDate {2020, 01, 03 }, Wt::WTime{9, 8, 11, 75} };
EXPECT_EQ(StringUtils::toISO8601String(dateTime), "2020-01-03T09:08:11.075");
const Wt::WDateTime dateTime{ Wt::WDate {2020, 01, 03 }, Wt::WTime{9, 8, 11, 75} };
EXPECT_EQ(StringUtils::toISO8601String(dateTime), "2020-01-03T09:08:11.075");
}