diff --git a/src/libs/core/impl/String.cpp b/src/libs/core/impl/String.cpp index 04edadb5..f6c11c01 100644 --- a/src/libs/core/impl/String.cpp +++ b/src/libs/core/impl/String.cpp @@ -341,6 +341,15 @@ namespace lms::core::stringUtils return str.substr(0, str.find_last_not_of(whitespaces) + 1); } + std::string_view utf8Truncate(std::string_view str, std::size_t maxBytes) + { + std::size_t len{ std::min(maxBytes, str.size()) }; + while (len > 0 && len < str.size() && (static_cast(str[len]) & 0xC0) == 0x80) + --len; + + return str.substr(0, len); + } + std::string stringToLower(std::string_view str) { std::string res; diff --git a/src/libs/core/include/core/String.hpp b/src/libs/core/include/core/String.hpp index 8ac81d8a..2fc564f5 100644 --- a/src/libs/core/include/core/String.hpp +++ b/src/libs/core/include/core/String.hpp @@ -54,6 +54,9 @@ namespace lms::core::stringUtils [[nodiscard]] std::string_view stringTrim(std::string_view str, std::string_view whitespaces = " \t\r"); [[nodiscard]] std::string_view stringTrimEnd(std::string_view str, std::string_view whitespaces = " \t\r"); + // Like str.substr(0, maxBytes), but takes utf8 into account to properly truncate + [[nodiscard]] std::string_view utf8Truncate(std::string_view str, std::size_t maxBytes); + [[nodiscard]] std::string stringToLower(std::string_view str); void stringToLower(std::string& str); [[nodiscard]] std::string stringToUpper(const std::string& str); diff --git a/src/libs/core/test/String.cpp b/src/libs/core/test/String.cpp index bbc7ee58..815e456f 100644 --- a/src/libs/core/test/String.cpp +++ b/src/libs/core/test/String.cpp @@ -387,6 +387,31 @@ namespace lms::core::stringUtils::tests EXPECT_FALSE(stringEndsWith("FooBar", "R")); } + TEST(StringUtils, utf8Truncate) + { + EXPECT_EQ(utf8Truncate("abc", 10), "abc"); + EXPECT_EQ(utf8Truncate("abcdef", 6), "abcdef"); + EXPECT_EQ(utf8Truncate("", 10), ""); + EXPECT_EQ(utf8Truncate("abc", 0), ""); + + EXPECT_EQ(utf8Truncate("caf\xC3\xA9", 4), "caf"); + EXPECT_EQ(utf8Truncate("caf\xC3\xA9", 5), "caf\xC3\xA9"); + + EXPECT_EQ(utf8Truncate("\xE2\x82\xAC", 1), ""); + EXPECT_EQ(utf8Truncate("\xE2\x82\xAC", 2), ""); + EXPECT_EQ(utf8Truncate("\xE2\x82\xAC", 3), "\xE2\x82\xAC"); + + EXPECT_EQ(utf8Truncate("\xF0\x9F\x98\x80", 1), ""); + EXPECT_EQ(utf8Truncate("\xF0\x9F\x98\x80", 2), ""); + EXPECT_EQ(utf8Truncate("\xF0\x9F\x98\x80", 3), ""); + EXPECT_EQ(utf8Truncate("\xF0\x9F\x98\x80", 4), "\xF0\x9F\x98\x80"); + + EXPECT_EQ(utf8Truncate("e\xCC\x81", 2), "e"); + EXPECT_EQ(utf8Truncate("e\xCC\x81", 3), "e\xCC\x81"); + + EXPECT_EQ(utf8Truncate("\x80\x80\x80", 2), ""); + } + TEST(StringUtils, stringCaseInsensitiveContains) { EXPECT_TRUE(stringCaseInsensitiveContains("FooBar", "Bar")); diff --git a/src/libs/database/impl/objects/Artist.cpp b/src/libs/database/impl/objects/Artist.cpp index ffa512f8..61da64e7 100644 --- a/src/libs/database/impl/objects/Artist.cpp +++ b/src/libs/database/impl/objects/Artist.cpp @@ -22,6 +22,8 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" + #include "database/Session.hpp" #include "database/objects/Artwork.hpp" #include "database/objects/Cluster.hpp" @@ -304,8 +306,7 @@ namespace lms::db { session.checkReadTransaction(); - if (name.size() > maxNameLength) - name = name.substr(0, maxNameLength); + name = core::stringUtils::utf8Truncate(name, maxNameLength); return utils::fetchQueryResults(session.getDboSession()->query>("SELECT a FROM artist a").where("a.name = ?").bind(name).orderBy("LENGTH(a.mbid) DESC")); // put mbid entries first } @@ -473,13 +474,13 @@ AND NOT EXISTS ( void Artist::setName(std::string_view name) { - _name.assign(name, 0, maxNameLength); + _name = core::stringUtils::utf8Truncate(name, maxNameLength); LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Artist name too long, truncated to '" << _name << "'"); } void Artist::setSortName(std::string_view sortName) { - _sortName.assign(sortName, 0, maxNameLength); + _sortName = core::stringUtils::utf8Truncate(sortName, maxNameLength); LMS_LOG_IF(DB, WARNING, sortName.size() > maxNameLength, "Artist sort name too long, truncated to '" << _sortName << "'"); } diff --git a/src/libs/database/impl/objects/Genre.cpp b/src/libs/database/impl/objects/Genre.cpp index aa74a78c..85417101 100644 --- a/src/libs/database/impl/objects/Genre.cpp +++ b/src/libs/database/impl/objects/Genre.cpp @@ -22,6 +22,7 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" #include "database/Session.hpp" #include "database/objects/Artist.hpp" @@ -114,7 +115,7 @@ namespace lms::db } // namespace Genre::Genre(std::string_view name) - : _name{ name.substr(0, maxNameLength) } + : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) } { LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Genre name too long, truncated to '" << _name << "'"); } @@ -161,8 +162,7 @@ namespace lms::db { session.checkReadTransaction(); - if (name.size() > maxNameLength) - name = name.substr(0, maxNameLength); + name = core::stringUtils::utf8Truncate(name, maxNameLength); return utils::fetchQuerySingleResult(session.getDboSession()->find().where("name = ?").bind(name)); } diff --git a/src/libs/database/impl/objects/Grouping.cpp b/src/libs/database/impl/objects/Grouping.cpp index 22612765..de74f0b5 100644 --- a/src/libs/database/impl/objects/Grouping.cpp +++ b/src/libs/database/impl/objects/Grouping.cpp @@ -22,6 +22,7 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" #include "database/Session.hpp" #include "database/objects/Artist.hpp" @@ -114,7 +115,7 @@ namespace lms::db } // namespace Grouping::Grouping(std::string_view name) - : _name{ name.substr(0, maxNameLength) } + : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) } { LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Grouping name too long, truncated to '" << _name << "'"); } @@ -161,8 +162,7 @@ namespace lms::db { session.checkReadTransaction(); - if (name.size() > maxNameLength) - name = name.substr(0, maxNameLength); + name = core::stringUtils::utf8Truncate(name, maxNameLength); return utils::fetchQuerySingleResult(session.getDboSession()->find().where("name = ?").bind(name)); } diff --git a/src/libs/database/impl/objects/Language.cpp b/src/libs/database/impl/objects/Language.cpp index 6a5c4ad9..60a32ee9 100644 --- a/src/libs/database/impl/objects/Language.cpp +++ b/src/libs/database/impl/objects/Language.cpp @@ -22,6 +22,7 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" #include "database/Session.hpp" #include "database/objects/Artist.hpp" @@ -114,7 +115,7 @@ namespace lms::db } // namespace Language::Language(std::string_view name) - : _name{ name.substr(0, maxNameLength) } + : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) } { LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Language name too long, truncated to '" << _name << "'"); } @@ -161,8 +162,7 @@ namespace lms::db { session.checkReadTransaction(); - if (name.size() > maxNameLength) - name = name.substr(0, maxNameLength); + name = core::stringUtils::utf8Truncate(name, maxNameLength); return utils::fetchQuerySingleResult(session.getDboSession()->find().where("name = ?").bind(name)); } diff --git a/src/libs/database/impl/objects/MediaLibrary.cpp b/src/libs/database/impl/objects/MediaLibrary.cpp index 853658ca..11520ebc 100644 --- a/src/libs/database/impl/objects/MediaLibrary.cpp +++ b/src/libs/database/impl/objects/MediaLibrary.cpp @@ -21,6 +21,8 @@ #include +#include "core/String.hpp" + #include "database/Session.hpp" #include "database/objects/Genre.hpp" #include "database/objects/Grouping.hpp" @@ -38,7 +40,7 @@ DBO_INSTANTIATE_TEMPLATES(lms::db::MediaLibrary) namespace lms::db { MediaLibrary::MediaLibrary(std::string_view name, const std::filesystem::path& p) - : _name{ std::string{ name, 0, maxNameLength } } + : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) } { setPath(p); } diff --git a/src/libs/database/impl/objects/Mood.cpp b/src/libs/database/impl/objects/Mood.cpp index cfd9bd37..b338b2be 100644 --- a/src/libs/database/impl/objects/Mood.cpp +++ b/src/libs/database/impl/objects/Mood.cpp @@ -22,6 +22,7 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" #include "database/Session.hpp" #include "database/objects/Artist.hpp" @@ -114,7 +115,7 @@ namespace lms::db } // namespace Mood::Mood(std::string_view name) - : _name{ name.substr(0, maxNameLength) } + : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) } { LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Mood name too long, truncated to '" << _name << "'"); } @@ -161,8 +162,7 @@ namespace lms::db { session.checkReadTransaction(); - if (name.size() > maxNameLength) - name = name.substr(0, maxNameLength); + name = core::stringUtils::utf8Truncate(name, maxNameLength); return utils::fetchQuerySingleResult(session.getDboSession()->find().where("name = ?").bind(name)); } diff --git a/src/libs/database/impl/objects/Movement.cpp b/src/libs/database/impl/objects/Movement.cpp index ab96550e..5afe983c 100644 --- a/src/libs/database/impl/objects/Movement.cpp +++ b/src/libs/database/impl/objects/Movement.cpp @@ -22,6 +22,7 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" #include "database/Session.hpp" #include "database/objects/Artist.hpp" @@ -48,7 +49,7 @@ DBO_INSTANTIATE_TEMPLATES(lms::db::Movement) namespace lms::db { Movement::Movement(std::string_view name, std::optional number, std::optional count, const ObjectPtr& track) - : _name{ name.substr(0, maxNameLength) } + : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) } , _number{ number } , _count{ count } , _track{ getDboPtr(track) } diff --git a/src/libs/database/impl/objects/PlayListFile.cpp b/src/libs/database/impl/objects/PlayListFile.cpp index 702e6918..a09cf1d0 100644 --- a/src/libs/database/impl/objects/PlayListFile.cpp +++ b/src/libs/database/impl/objects/PlayListFile.cpp @@ -23,6 +23,8 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" + #include "database/Session.hpp" #include "database/objects/Artwork.hpp" #include "database/objects/Directory.hpp" @@ -171,7 +173,7 @@ namespace lms::db void PlayListFile::setName(std::string_view name) { - _name = std::string{ name, 0, _maxNameLength }; + _name = core::stringUtils::utf8Truncate(name, _maxNameLength); LMS_LOG_IF(DB, WARNING, name.size() > _maxNameLength, "PlaylistFile name too long, truncated to '" << _name << "'"); } diff --git a/src/libs/database/impl/objects/Release.cpp b/src/libs/database/impl/objects/Release.cpp index 9b7be1b9..8bc1fef4 100644 --- a/src/libs/database/impl/objects/Release.cpp +++ b/src/libs/database/impl/objects/Release.cpp @@ -23,6 +23,8 @@ #include #include "core/PartialDateTime.hpp" +#include "core/String.hpp" + #include "database/Session.hpp" #include "database/Types.hpp" #include "database/objects/Artist.hpp" @@ -491,7 +493,7 @@ namespace lms::db } Release::Release(const std::string& name, const std::optional& MBID) - : _name{ std::string(name, 0, _maxNameLength) } + : _name{ core::stringUtils::utf8Truncate(name, _maxNameLength) } , _MBID{ MBID } { } diff --git a/src/libs/database/impl/objects/ReleaseArtistLink.cpp b/src/libs/database/impl/objects/ReleaseArtistLink.cpp index b6616f9b..00e49d10 100644 --- a/src/libs/database/impl/objects/ReleaseArtistLink.cpp +++ b/src/libs/database/impl/objects/ReleaseArtistLink.cpp @@ -22,6 +22,8 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" + #include "database/Session.hpp" #include "database/objects/Artist.hpp" #include "database/objects/Genre.hpp" @@ -149,13 +151,13 @@ namespace lms::db void ReleaseArtistLink::setArtistName(std::string_view artistName) { - _artistName.assign(artistName, 0, Artist::maxNameLength); + _artistName = core::stringUtils::utf8Truncate(artistName, Artist::maxNameLength); LMS_LOG_IF(DB, WARNING, artistName.size() > Artist::maxNameLength, "Artist link name too long, truncated to '" << _artistName << "'"); } void ReleaseArtistLink::setArtistSortName(std::string_view artistSortName) { - _artistSortName.assign(artistSortName, 0, Artist::maxNameLength); + _artistSortName = core::stringUtils::utf8Truncate(artistSortName, Artist::maxNameLength); LMS_LOG_IF(DB, WARNING, artistSortName.size() > Artist::maxNameLength, "Artist link sort name too long, truncated to '" << _artistSortName << "'"); } } // namespace lms::db diff --git a/src/libs/database/impl/objects/Track.cpp b/src/libs/database/impl/objects/Track.cpp index a3d23394..d1d2e7f0 100644 --- a/src/libs/database/impl/objects/Track.cpp +++ b/src/libs/database/impl/objects/Track.cpp @@ -23,6 +23,7 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" #include "database/Session.hpp" #include "database/Types.hpp" @@ -614,19 +615,19 @@ namespace lms::db void Track::setName(std::string_view name) { - _name = std::string{ name, 0, _maxNameLength }; + _name = core::stringUtils::utf8Truncate(name, _maxNameLength); LMS_LOG_IF(DB, WARNING, name.size() > _maxNameLength, "Track name too long, truncated to '" << _name << "'"); } void Track::setCopyright(std::string_view copyright) { - _copyright = std::string{ copyright, 0, _maxCopyrightLength }; + _copyright = core::stringUtils::utf8Truncate(copyright, _maxCopyrightLength); LMS_LOG_IF(DB, WARNING, copyright.size() > _maxCopyrightLength, "Track copyright too long, truncated to '" << _copyright << "'"); } void Track::setCopyrightURL(std::string_view copyrightURL) { - _copyrightURL = std::string{ copyrightURL, 0, _maxCopyrightURLLength }; + _copyrightURL = core::stringUtils::utf8Truncate(copyrightURL, _maxCopyrightURLLength); LMS_LOG_IF(DB, WARNING, copyrightURL.size() > _maxCopyrightURLLength, "Track copyright URL too long, truncated to '" << _copyrightURL << "'"); } diff --git a/src/libs/database/impl/objects/TrackArtistLink.cpp b/src/libs/database/impl/objects/TrackArtistLink.cpp index e4e1a238..6b32305f 100644 --- a/src/libs/database/impl/objects/TrackArtistLink.cpp +++ b/src/libs/database/impl/objects/TrackArtistLink.cpp @@ -22,6 +22,8 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" + #include "database/Session.hpp" #include "database/objects/Artist.hpp" #include "database/objects/Genre.hpp" @@ -191,13 +193,13 @@ namespace lms::db void TrackArtistLink::setArtistName(std::string_view artistName) { - _artistName.assign(artistName, 0, Artist::maxNameLength); + _artistName = core::stringUtils::utf8Truncate(artistName, Artist::maxNameLength); LMS_LOG_IF(DB, WARNING, artistName.size() > Artist::maxNameLength, "Artist link name too long, truncated to '" << _artistName << "'"); } void TrackArtistLink::setArtistSortName(std::string_view artistSortName) { - _artistSortName.assign(artistSortName, 0, Artist::maxNameLength); + _artistSortName = core::stringUtils::utf8Truncate(artistSortName, Artist::maxNameLength); LMS_LOG_IF(DB, WARNING, artistSortName.size() > Artist::maxNameLength, "Artist link sort name too long, truncated to '" << _artistSortName << "'"); } } // namespace lms::db diff --git a/src/libs/database/impl/objects/Work.cpp b/src/libs/database/impl/objects/Work.cpp index e032c890..9cb7a8a7 100644 --- a/src/libs/database/impl/objects/Work.cpp +++ b/src/libs/database/impl/objects/Work.cpp @@ -22,6 +22,7 @@ #include #include "core/ILogger.hpp" +#include "core/String.hpp" #include "database/Session.hpp" #include "database/objects/Artist.hpp" @@ -58,7 +59,7 @@ namespace lms::db void Work::setName(std::string_view name) { - _name = name.substr(0, maxNameLength); + _name = core::stringUtils::utf8Truncate(name, maxNameLength); LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Work name too long, truncated to '" << _name << "'"); } @@ -84,8 +85,7 @@ namespace lms::db { session.checkReadTransaction(); - if (name.size() > maxNameLength) - name = name.substr(0, maxNameLength); + name = core::stringUtils::utf8Truncate(name, maxNameLength); auto query{ session.getDboSession()->query>("SELECT w FROM work w")