Fixed bad name truncations, fixes #861

src/libs/core/impl/String.cpp

@@ -341,6 +341,15 @@ namespace lms::core::stringUtils
         return str.substr(0, str.find_last_not_of(whitespaces) + 1);
     }
 
+    std::string_view utf8Truncate(std::string_view str, std::size_t maxBytes)
+    {
+        std::size_t len{ std::min(maxBytes, str.size()) };
+        while (len > 0 && len < str.size() && (static_cast<unsigned char>(str[len]) & 0xC0) == 0x80)
+            --len;
+
+        return str.substr(0, len);
+    }
+
     std::string stringToLower(std::string_view str)
     {
         std::string res;

src/libs/core/include/core/String.hpp

@@ -54,6 +54,9 @@ namespace lms::core::stringUtils
     [[nodiscard]] std::string_view stringTrim(std::string_view str, std::string_view whitespaces = " \t\r");
     [[nodiscard]] std::string_view stringTrimEnd(std::string_view str, std::string_view whitespaces = " \t\r");
 
+    // Like str.substr(0, maxBytes), but takes utf8 into account to properly truncate
+    [[nodiscard]] std::string_view utf8Truncate(std::string_view str, std::size_t maxBytes);
+
     [[nodiscard]] std::string stringToLower(std::string_view str);
     void stringToLower(std::string& str);
     [[nodiscard]] std::string stringToUpper(const std::string& str);

src/libs/core/test/String.cpp

@@ -387,6 +387,31 @@ namespace lms::core::stringUtils::tests
         EXPECT_FALSE(stringEndsWith("FooBar", "R"));
     }
 
+    TEST(StringUtils, utf8Truncate)
+    {
+        EXPECT_EQ(utf8Truncate("abc", 10), "abc");
+        EXPECT_EQ(utf8Truncate("abcdef", 6), "abcdef");
+        EXPECT_EQ(utf8Truncate("", 10), "");
+        EXPECT_EQ(utf8Truncate("abc", 0), "");
+
+        EXPECT_EQ(utf8Truncate("caf\xC3\xA9", 4), "caf");
+        EXPECT_EQ(utf8Truncate("caf\xC3\xA9", 5), "caf\xC3\xA9");
+
+        EXPECT_EQ(utf8Truncate("\xE2\x82\xAC", 1), "");
+        EXPECT_EQ(utf8Truncate("\xE2\x82\xAC", 2), "");
+        EXPECT_EQ(utf8Truncate("\xE2\x82\xAC", 3), "\xE2\x82\xAC");
+
+        EXPECT_EQ(utf8Truncate("\xF0\x9F\x98\x80", 1), "");
+        EXPECT_EQ(utf8Truncate("\xF0\x9F\x98\x80", 2), "");
+        EXPECT_EQ(utf8Truncate("\xF0\x9F\x98\x80", 3), "");
+        EXPECT_EQ(utf8Truncate("\xF0\x9F\x98\x80", 4), "\xF0\x9F\x98\x80");
+
+        EXPECT_EQ(utf8Truncate("e\xCC\x81", 2), "e");
+        EXPECT_EQ(utf8Truncate("e\xCC\x81", 3), "e\xCC\x81");
+
+        EXPECT_EQ(utf8Truncate("\x80\x80\x80", 2), "");
+    }
+
     TEST(StringUtils, stringCaseInsensitiveContains)
     {
         EXPECT_TRUE(stringCaseInsensitiveContains("FooBar", "Bar"));

src/libs/database/impl/objects/Artist.cpp

@@ -22,6 +22,8 @@
 #include <Wt/Dbo/WtSqlTraits.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
+
 #include "database/Session.hpp"
 #include "database/objects/Artwork.hpp"
 #include "database/objects/Cluster.hpp"
@@ -304,8 +306,7 @@ namespace lms::db
     {
         session.checkReadTransaction();
 
-        if (name.size() > maxNameLength)
+        name = core::stringUtils::utf8Truncate(name, maxNameLength);
-            name = name.substr(0, maxNameLength);
 
         return utils::fetchQueryResults<Artist::pointer>(session.getDboSession()->query<Wt::Dbo::ptr<Artist>>("SELECT a FROM artist a").where("a.name = ?").bind(name).orderBy("LENGTH(a.mbid) DESC")); // put mbid entries first
     }
@@ -473,13 +474,13 @@ AND NOT EXISTS (
 
     void Artist::setName(std::string_view name)
     {
-        _name.assign(name, 0, maxNameLength);
+        _name = core::stringUtils::utf8Truncate(name, maxNameLength);
         LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Artist name too long, truncated to '" << _name << "'");
     }
 
     void Artist::setSortName(std::string_view sortName)
     {
-        _sortName.assign(sortName, 0, maxNameLength);
+        _sortName = core::stringUtils::utf8Truncate(sortName, maxNameLength);
         LMS_LOG_IF(DB, WARNING, sortName.size() > maxNameLength, "Artist sort name too long, truncated to '" << _sortName << "'");
     }
 

src/libs/database/impl/objects/Genre.cpp

@@ -22,6 +22,7 @@
 #include <Wt/Dbo/Impl.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
 
 #include "database/Session.hpp"
 #include "database/objects/Artist.hpp"
@@ -114,7 +115,7 @@ namespace lms::db
     } // namespace
 
     Genre::Genre(std::string_view name)
-        : _name{ name.substr(0, maxNameLength) }
+        : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) }
     {
         LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Genre name too long, truncated to '" << _name << "'");
     }
@@ -161,8 +162,7 @@ namespace lms::db
     {
         session.checkReadTransaction();
 
-        if (name.size() > maxNameLength)
+        name = core::stringUtils::utf8Truncate(name, maxNameLength);
-            name = name.substr(0, maxNameLength);
 
         return utils::fetchQuerySingleResult(session.getDboSession()->find<Genre>().where("name = ?").bind(name));
     }

src/libs/database/impl/objects/Grouping.cpp

@@ -22,6 +22,7 @@
 #include <Wt/Dbo/Impl.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
 
 #include "database/Session.hpp"
 #include "database/objects/Artist.hpp"
@@ -114,7 +115,7 @@ namespace lms::db
     } // namespace
 
     Grouping::Grouping(std::string_view name)
-        : _name{ name.substr(0, maxNameLength) }
+        : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) }
     {
         LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Grouping name too long, truncated to '" << _name << "'");
     }
@@ -161,8 +162,7 @@ namespace lms::db
     {
         session.checkReadTransaction();
 
-        if (name.size() > maxNameLength)
+        name = core::stringUtils::utf8Truncate(name, maxNameLength);
-            name = name.substr(0, maxNameLength);
 
         return utils::fetchQuerySingleResult(session.getDboSession()->find<Grouping>().where("name = ?").bind(name));
     }

src/libs/database/impl/objects/Language.cpp

@@ -22,6 +22,7 @@
 #include <Wt/Dbo/Impl.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
 
 #include "database/Session.hpp"
 #include "database/objects/Artist.hpp"
@@ -114,7 +115,7 @@ namespace lms::db
     } // namespace
 
     Language::Language(std::string_view name)
-        : _name{ name.substr(0, maxNameLength) }
+        : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) }
     {
         LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Language name too long, truncated to '" << _name << "'");
     }
@@ -161,8 +162,7 @@ namespace lms::db
     {
         session.checkReadTransaction();
 
-        if (name.size() > maxNameLength)
+        name = core::stringUtils::utf8Truncate(name, maxNameLength);
-            name = name.substr(0, maxNameLength);
 
         return utils::fetchQuerySingleResult(session.getDboSession()->find<Language>().where("name = ?").bind(name));
     }

src/libs/database/impl/objects/MediaLibrary.cpp

@@ -21,6 +21,8 @@
 
 #include <Wt/Dbo/Impl.h>
 
+#include "core/String.hpp"
+
 #include "database/Session.hpp"
 #include "database/objects/Genre.hpp"
 #include "database/objects/Grouping.hpp"
@@ -38,7 +40,7 @@ DBO_INSTANTIATE_TEMPLATES(lms::db::MediaLibrary)
 namespace lms::db
 {
     MediaLibrary::MediaLibrary(std::string_view name, const std::filesystem::path& p)
-        : _name{ std::string{ name, 0, maxNameLength } }
+        : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) }
     {
         setPath(p);
     }

src/libs/database/impl/objects/Mood.cpp

@@ -22,6 +22,7 @@
 #include <Wt/Dbo/Impl.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
 
 #include "database/Session.hpp"
 #include "database/objects/Artist.hpp"
@@ -114,7 +115,7 @@ namespace lms::db
     } // namespace
 
     Mood::Mood(std::string_view name)
-        : _name{ name.substr(0, maxNameLength) }
+        : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) }
     {
         LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Mood name too long, truncated to '" << _name << "'");
     }
@@ -161,8 +162,7 @@ namespace lms::db
     {
         session.checkReadTransaction();
 
-        if (name.size() > maxNameLength)
+        name = core::stringUtils::utf8Truncate(name, maxNameLength);
-            name = name.substr(0, maxNameLength);
 
         return utils::fetchQuerySingleResult(session.getDboSession()->find<Mood>().where("name = ?").bind(name));
     }

src/libs/database/impl/objects/Movement.cpp

@@ -22,6 +22,7 @@
 #include <Wt/Dbo/Impl.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
 
 #include "database/Session.hpp"
 #include "database/objects/Artist.hpp"
@@ -48,7 +49,7 @@ DBO_INSTANTIATE_TEMPLATES(lms::db::Movement)
 namespace lms::db
 {
     Movement::Movement(std::string_view name, std::optional<std::size_t> number, std::optional<std::size_t> count, const ObjectPtr<Track>& track)
-        : _name{ name.substr(0, maxNameLength) }
+        : _name{ core::stringUtils::utf8Truncate(name, maxNameLength) }
         , _number{ number }
         , _count{ count }
         , _track{ getDboPtr(track) }

src/libs/database/impl/objects/PlayListFile.cpp

@@ -23,6 +23,8 @@
 #include <Wt/Dbo/WtSqlTraits.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
+
 #include "database/Session.hpp"
 #include "database/objects/Artwork.hpp"
 #include "database/objects/Directory.hpp"
@@ -171,7 +173,7 @@ namespace lms::db
 
     void PlayListFile::setName(std::string_view name)
     {
-        _name = std::string{ name, 0, _maxNameLength };
+        _name = core::stringUtils::utf8Truncate(name, _maxNameLength);
         LMS_LOG_IF(DB, WARNING, name.size() > _maxNameLength, "PlaylistFile name too long, truncated to '" << _name << "'");
     }
 

src/libs/database/impl/objects/Release.cpp

@@ -23,6 +23,8 @@
 #include <Wt/Dbo/WtSqlTraits.h>
 
 #include "core/PartialDateTime.hpp"
+#include "core/String.hpp"
+
 #include "database/Session.hpp"
 #include "database/Types.hpp"
 #include "database/objects/Artist.hpp"
@@ -491,7 +493,7 @@ namespace lms::db
     }
 
     Release::Release(const std::string& name, const std::optional<core::UUID>& MBID)
-        : _name{ std::string(name, 0, _maxNameLength) }
+        : _name{ core::stringUtils::utf8Truncate(name, _maxNameLength) }
         , _MBID{ MBID }
     {
     }

src/libs/database/impl/objects/ReleaseArtistLink.cpp

@@ -22,6 +22,8 @@
 #include <Wt/Dbo/Impl.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
+
 #include "database/Session.hpp"
 #include "database/objects/Artist.hpp"
 #include "database/objects/Genre.hpp"
@@ -149,13 +151,13 @@ namespace lms::db
 
     void ReleaseArtistLink::setArtistName(std::string_view artistName)
     {
-        _artistName.assign(artistName, 0, Artist::maxNameLength);
+        _artistName = core::stringUtils::utf8Truncate(artistName, Artist::maxNameLength);
         LMS_LOG_IF(DB, WARNING, artistName.size() > Artist::maxNameLength, "Artist link name too long, truncated to '" << _artistName << "'");
     }
 
     void ReleaseArtistLink::setArtistSortName(std::string_view artistSortName)
     {
-        _artistSortName.assign(artistSortName, 0, Artist::maxNameLength);
+        _artistSortName = core::stringUtils::utf8Truncate(artistSortName, Artist::maxNameLength);
         LMS_LOG_IF(DB, WARNING, artistSortName.size() > Artist::maxNameLength, "Artist link sort name too long, truncated to '" << _artistSortName << "'");
     }
 } // namespace lms::db

src/libs/database/impl/objects/Track.cpp

@@ -23,6 +23,7 @@
 #include <Wt/Dbo/WtSqlTraits.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
 
 #include "database/Session.hpp"
 #include "database/Types.hpp"
@@ -614,19 +615,19 @@ namespace lms::db
 
     void Track::setName(std::string_view name)
     {
-        _name = std::string{ name, 0, _maxNameLength };
+        _name = core::stringUtils::utf8Truncate(name, _maxNameLength);
         LMS_LOG_IF(DB, WARNING, name.size() > _maxNameLength, "Track name too long, truncated to '" << _name << "'");
     }
 
     void Track::setCopyright(std::string_view copyright)
     {
-        _copyright = std::string{ copyright, 0, _maxCopyrightLength };
+        _copyright = core::stringUtils::utf8Truncate(copyright, _maxCopyrightLength);
         LMS_LOG_IF(DB, WARNING, copyright.size() > _maxCopyrightLength, "Track copyright too long, truncated to '" << _copyright << "'");
     }
 
     void Track::setCopyrightURL(std::string_view copyrightURL)
     {
-        _copyrightURL = std::string{ copyrightURL, 0, _maxCopyrightURLLength };
+        _copyrightURL = core::stringUtils::utf8Truncate(copyrightURL, _maxCopyrightURLLength);
         LMS_LOG_IF(DB, WARNING, copyrightURL.size() > _maxCopyrightURLLength, "Track copyright URL too long, truncated to '" << _copyrightURL << "'");
     }
 

src/libs/database/impl/objects/TrackArtistLink.cpp

@@ -22,6 +22,8 @@
 #include <Wt/Dbo/Impl.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
+
 #include "database/Session.hpp"
 #include "database/objects/Artist.hpp"
 #include "database/objects/Genre.hpp"
@@ -191,13 +193,13 @@ namespace lms::db
 
     void TrackArtistLink::setArtistName(std::string_view artistName)
     {
-        _artistName.assign(artistName, 0, Artist::maxNameLength);
+        _artistName = core::stringUtils::utf8Truncate(artistName, Artist::maxNameLength);
         LMS_LOG_IF(DB, WARNING, artistName.size() > Artist::maxNameLength, "Artist link name too long, truncated to '" << _artistName << "'");
     }
 
     void TrackArtistLink::setArtistSortName(std::string_view artistSortName)
     {
-        _artistSortName.assign(artistSortName, 0, Artist::maxNameLength);
+        _artistSortName = core::stringUtils::utf8Truncate(artistSortName, Artist::maxNameLength);
         LMS_LOG_IF(DB, WARNING, artistSortName.size() > Artist::maxNameLength, "Artist link sort name too long, truncated to '" << _artistSortName << "'");
     }
 } // namespace lms::db

src/libs/database/impl/objects/Work.cpp

@@ -22,6 +22,7 @@
 #include <Wt/Dbo/Impl.h>
 
 #include "core/ILogger.hpp"
+#include "core/String.hpp"
 
 #include "database/Session.hpp"
 #include "database/objects/Artist.hpp"
@@ -58,7 +59,7 @@ namespace lms::db
 
     void Work::setName(std::string_view name)
     {
-        _name = name.substr(0, maxNameLength);
+        _name = core::stringUtils::utf8Truncate(name, maxNameLength);
         LMS_LOG_IF(DB, WARNING, name.size() > maxNameLength, "Work name too long, truncated to '" << _name << "'");
     }
 
@@ -84,8 +85,7 @@ namespace lms::db
     {
         session.checkReadTransaction();
 
-        if (name.size() > maxNameLength)
+        name = core::stringUtils::utf8Truncate(name, maxNameLength);
-            name = name.substr(0, maxNameLength);
 
         auto query{
             session.getDboSession()->query<Wt::Dbo::ptr<Work>>("SELECT w FROM work w")