QOffsetStringArray: add support for UTF-8 and UTF-16 arrays

std::byte not tested and not included in the concept requires {} clause,
but would work otherwise.

Fixes: QTBUG-100556
Change-Id: I4878533dcb2d4b3e8efefffd17e4f87d96a2d126
Reviewed-by: Ahmad Samir <a.samirh78@gmail.com>
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

src/corelib/tools/qoffsetstringarray_p.h

@@ -26,6 +26,10 @@
 #include <string_view>
 #include <tuple>
 
+#ifdef __cpp_concepts
+#  include <concepts>
+#endif
+
 class tst_QOffsetStringArray;
 
 QT_BEGIN_NAMESPACE
@@ -40,22 +44,39 @@ QT_WARNING_DISABLE_GCC("-Wstringop-overread")
 template <typename StaticString, typename OffsetList>
 class QOffsetStringArray
 {
+    static auto viewType_helper()
+    {
+        // returning std::type_identity here to avoid having to #include
+        if constexpr (sizeof(Char) == 2) {
+            return q20::type_identity<QStringView>();
+#ifdef __cpp_char8_t
+        } else if constexpr (std::is_same_v<Char, char8_t>) {
+            return q20::type_identity<QUtf8StringView>();
+#endif
+        } else {
+            return q20::type_identity<QByteArrayView>();
+        }
+    }
+
 public:
+    using Char = typename StaticString::value_type;
+    using View = typename decltype(viewType_helper())::type;
+
     constexpr QOffsetStringArray(const StaticString &string, const OffsetList &offsets)
         : m_string(string), m_offsets(offsets)
     {}
 
-    constexpr const char *operator[](const int index) const noexcept
+    constexpr const Char *operator[](const int index) const noexcept
     {
         return m_string.data() + m_offsets[qBound(int(0), index, count())];
     }
 
-    constexpr const char *at(const int index) const noexcept
+    constexpr const Char *at(const int index) const noexcept
     {
         return m_string.data() + m_offsets[index];
     }
 
-    constexpr QByteArrayView viewAt(qsizetype index) const noexcept
+    constexpr View viewAt(qsizetype index) const noexcept
     {
         return { m_string.data() + m_offsets[index],
                     qsizetype(m_offsets[index + 1]) - qsizetype(m_offsets[index]) - 1 };
@@ -63,7 +84,7 @@ public:
 
     constexpr int count() const { return int(m_offsets.size()) - 1; }
 
-    bool contains(QByteArrayView needle, Qt::CaseSensitivity cs = Qt::CaseSensitive) const noexcept
+    bool contains(View needle, Qt::CaseSensitivity cs = Qt::CaseSensitive) const noexcept
     {
         for (qsizetype i = 0; i < count(); ++i) {
             if (viewAt(i).compare(needle, cs) == 0)
@@ -93,15 +114,15 @@ template <size_t Highest> constexpr auto minifyValue()
     }
 }
 
-template <size_t StringLength, typename Extractor, typename... T>
+template <typename Char, size_t StringLength, typename Extractor, typename... T>
 constexpr auto makeStaticString(Extractor extract, const T &... entries)
 {
     // append an extra null terminator
-    std::array<char, StringLength + 1> result = {};
+    std::array<Char, StringLength + 1> result = {};
     qptrdiff offset = 0;
 
-    const char *strings[] = { extract(entries).operator const char *()... };
+    const Char *strings[] = { extract(entries).operator const Char *()... };
-    size_t lengths[] = { sizeof(extract(T{}))... };
+    size_t lengths[] = { (sizeof(extract(T{})) / sizeof(Char))... };
     for (size_t i = 0; i < std::size(strings); ++i) {
         q20::copy_n(strings[i], lengths[i], result.begin() + offset);
         offset += lengths[i];
@@ -109,40 +130,46 @@ constexpr auto makeStaticString(Extractor extract, const T &... entries)
     return result;
 }
 
-template <size_t N> struct StaticString
+template <typename Char, size_t N> struct StaticString
 {
-    char value[N] = {};
+    Char value[N] = {};
     constexpr StaticString() = default;
-    constexpr StaticString(const char (&s)[N])  { q20::copy_n(s, N, value); }
+    constexpr StaticString(const Char (&s)[N])  { q20::copy_n(s, N, value); }
-    constexpr operator const char *() const     { return value; }
+    constexpr operator const Char *() const     { return value; }
 };
 
-template <typename StringExtractor, typename... T>
+template <typename Char, typename StringExtractor, typename... T>
 constexpr auto makeOffsetStringArray(StringExtractor extractString, const T &... entries)
 {
     constexpr size_t Count = sizeof...(T);
-    constexpr size_t StringLength = (sizeof(extractString(T{})) + ...);
+    constexpr size_t StringLength = (sizeof(extractString(T{})) + ...) / sizeof(Char);
     using MinifiedOffsetType = decltype(QtPrivate::minifyValue<StringLength>());
 
     size_t offset = 0;
-    std::array fullOffsetList = { offset += sizeof(extractString(T{}))... };
+    std::array fullOffsetList = { offset += (sizeof(extractString(T{})) / sizeof(Char))... };
 
-    // prepend zero
+    // prepend the first offset (zero) pointing to the *start* of the first element
     std::array<MinifiedOffsetType, Count + 1> minifiedOffsetList = {};
     q20::transform(fullOffsetList.begin(), fullOffsetList.end(),
                    minifiedOffsetList.begin() + 1,
                    [] (auto e) { return MinifiedOffsetType(e); });
 
-    std::array staticString = QtPrivate::makeStaticString<StringLength>(extractString, entries...);
+    std::array staticString = QtPrivate::makeStaticString<Char, StringLength>(extractString, entries...);
     return QOffsetStringArray(staticString, minifiedOffsetList);
 }
 } // namespace QtPrivate
 
-template<int ... Nx>
+template<typename Char, int ... Nx>
-constexpr auto qOffsetStringArray(const char (&...strings)[Nx]) noexcept
+#ifdef __cpp_concepts
+requires std::is_same_v<Char, char> || std::is_same_v<Char, char16_t>
+#  ifdef __cpp_char8_t
+    || std::is_same_v<Char, char8_t>
+#  endif
+#endif
+constexpr auto qOffsetStringArray(const Char (&...strings)[Nx]) noexcept
 {
     auto extractString = [](const auto &s) -> decltype(auto) { return s; };
-    return QtPrivate::makeOffsetStringArray(extractString, QtPrivate::StaticString(strings)...);
+    return QtPrivate::makeOffsetStringArray<Char>(extractString, QtPrivate::StaticString(strings)...);
 }
 
 QT_WARNING_POP

tests/auto/corelib/tools/qoffsetstringarray/CMakeLists.txt

@@ -17,6 +17,9 @@ qt_internal_add_test(tst_qoffsetstringarray
     LIBRARIES
         Qt::CorePrivate
 )
+if (NOT VXWORKS)
+    set_property(TARGET tst_qoffsetstringarray PROPERTY CXX_STANDARD 20)
+endif()
 
 if (CLANG)
     target_compile_options(tst_qoffsetstringarray

tests/auto/corelib/tools/qoffsetstringarray/tst_qoffsetstringarray.cpp

@@ -25,6 +25,23 @@ constexpr const auto messages = qOffsetStringArray(
     "level - 4"
 );
 
+// When compiled with C++20, this is using the native char8_t
+constexpr auto utf8Messages = qOffsetStringArray(
+    u8"level - 0",
+    u8"level - 1",
+    u8"level - 2",
+    u8"level - 3",
+    u8"level - 4"
+);
+
+constexpr auto utf16Messages = qOffsetStringArray(
+    u"level - 0",
+    u"level - 1",
+    u"level - 2",
+    u"level - 3",
+    u"level - 4"
+);
+
 constexpr const auto messages257 = qOffsetStringArray(
     "", "", "", "", "", "", "", "", "", "",
     "", "", "", "", "", "", "", "", "", "",
@@ -69,6 +86,14 @@ void tst_QOffsetStringArray::init()
     static_assert(messages.m_offsets.size() == 6);
     static_assert(std::is_same_v<decltype(messages.m_offsets)::value_type, quint8>);
 
+    static_assert(utf8Messages.m_string.size() == 51);
+    static_assert(utf8Messages.m_offsets.size() == 6);
+    static_assert(std::is_same_v<decltype(utf8Messages.m_offsets)::value_type, quint8>);
+
+    static_assert(utf16Messages.m_string.size() == 51);
+    static_assert(utf16Messages.m_offsets.size() == 6);
+    static_assert(std::is_same_v<decltype(utf16Messages.m_offsets)::value_type, quint8>);
+
     static_assert(messages257.m_offsets.size() == 258);
     static_assert(messages257.m_string.size() == 261);
     static_assert(std::is_same_v<decltype(messages257.m_offsets)::value_type, quint16>);
@@ -88,6 +113,45 @@ void tst_QOffsetStringArray::access()
     // out of bounds returns empty strings:
     QCOMPARE(messages[5], "");
     QCOMPARE(messages[6], "");
+
+    auto view0 = messages.viewAt(0);
+    static_assert(std::is_same_v<decltype(view0), QByteArrayView>);
+    QCOMPARE(view0, "level - 0");
+
+    QCOMPARE(utf8Messages[0], QUtf8StringView(u8"level - 0"));
+    QCOMPARE(utf8Messages[1], QUtf8StringView(u8"level - 1"));
+    QCOMPARE(utf8Messages[2], QUtf8StringView(u8"level - 2"));
+    QCOMPARE(utf8Messages[3], QUtf8StringView(u8"level - 3"));
+    QCOMPARE(utf8Messages[4], QUtf8StringView(u8"level - 4"));
+    QCOMPARE(utf8Messages[5], QUtf8StringView(u8""));
+    QCOMPARE(utf8Messages[6], QUtf8StringView(u8""));
+
+    auto u8view0 = utf8Messages.viewAt(0);
+#ifdef __cpp_char8_t
+    static_assert(std::is_same_v<decltype(u8view0), QUtf8StringView>);
+#endif
+    QCOMPARE(u8view0, u8"level - 0");
+    QCOMPARE(utf8Messages.viewAt(1), u8"level - 1");
+    QCOMPARE(utf8Messages.viewAt(2), u8"level - 2");
+    QCOMPARE(utf8Messages.viewAt(3), u8"level - 3");
+    QCOMPARE(utf8Messages.viewAt(4), u8"level - 4");
+    // viewAt has no size checking!
+
+    QCOMPARE(utf16Messages[0], QStringView(u"level - 0"));
+    QCOMPARE(utf16Messages[1], QStringView(u"level - 1"));
+    QCOMPARE(utf16Messages[2], QStringView(u"level - 2"));
+    QCOMPARE(utf16Messages[3], QStringView(u"level - 3"));
+    QCOMPARE(utf16Messages[4], QStringView(u"level - 4"));
+    QCOMPARE(utf16Messages[5], QStringView(u""));
+    QCOMPARE(utf16Messages[6], QStringView(u""));
+
+    auto uview0 = utf16Messages.viewAt(0);
+    static_assert(std::is_same_v<decltype(uview0), QStringView>);
+    QCOMPARE(uview0, u"level - 0");
+    QCOMPARE(utf16Messages.viewAt(1), u"level - 1");
+    QCOMPARE(utf16Messages.viewAt(2), u"level - 2");
+    QCOMPARE(utf16Messages.viewAt(3), u"level - 3");
+    QCOMPARE(utf16Messages.viewAt(4), u"level - 4");
 }
 
 void tst_QOffsetStringArray::contains()
@@ -99,6 +163,24 @@ void tst_QOffsetStringArray::contains()
     QByteArray L4 = "Level - 4";
     QVERIFY( messages.contains(L4, Qt::CaseInsensitive));
     QVERIFY(!messages.contains(L4, Qt::CaseSensitive));
+
+    QVERIFY(!utf8Messages.contains(u8""));
+    QVERIFY( utf8Messages.contains(u8"level - 0"));
+#ifdef __cpp_lib_char8_t
+    std::u8string u8l2 = u8"level - 2"; // make sure we don't compare pointer values
+    QVERIFY( utf8Messages.contains(u8l2));
+#endif
+    QUtf8StringView u8l4 = u8"Level - 4";
+    QVERIFY( utf8Messages.contains(u8l4, Qt::CaseInsensitive));
+    QVERIFY(!utf8Messages.contains(u8l4, Qt::CaseSensitive));
+
+    QVERIFY(!utf16Messages.contains(u""));
+    QVERIFY( utf16Messages.contains(u"level - 0"));
+    std::u16string ul2 = u"level - 2"; // make sure we don't compare pointer values
+    QVERIFY( utf16Messages.contains(ul2));
+    QString ul4 = "Level - 4";
+    QVERIFY( utf16Messages.contains(ul4, Qt::CaseInsensitive));
+    QVERIFY(!utf16Messages.contains(ul4, Qt::CaseSensitive));
 }
 
 QTEST_APPLESS_MAIN(tst_QOffsetStringArray)