tests: Add test for UTS #46 implementation using Unicode data

The new test is called tst_qurluts46. It verifies QUrl::{to,from}Ace()
functionality using the data from IdnaTestV2.txt supplied by Unicode.

The file was downloaded from
https://www.unicode.org/Public/idna/13.0.0/IdnaTestV2.txt

Task-Id: QTBUG-85371
Change-Id: I4c6a4942ef6018dafc90cb84ef73f6b2614566d7
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

tests/auto/corelib/io/CMakeLists.txt

@@ -22,6 +22,7 @@ add_subdirectory(qstorageinfo)
 add_subdirectory(qtemporarydir)
 add_subdirectory(qtemporaryfile)
 add_subdirectory(qurlquery)
+add_subdirectory(qurluts46)
 if(TARGET Qt::Concurrent)
     add_subdirectory(qdebug)
     add_subdirectory(qlockfile)

tests/auto/corelib/io/qurluts46/CMakeLists.txt

@@ -0,0 +1,10 @@
+#####################################################################
+## tst_qurluts46 Test:
+#####################################################################
+
+qt_internal_add_test(tst_qurluts46
+    SOURCES
+        tst_qurluts46.cpp
+    TESTDATA
+        testdata/IdnaTestV2.txt
+)

tests/auto/corelib/io/qurluts46/tst_qurluts46.cpp

@@ -0,0 +1,161 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QtCore/QUrl>
+#include <QtCore/QFile>
+#include <QTest>
+#include <algorithm>
+
+class tst_QUrlUts46 : public QObject
+{
+    Q_OBJECT
+private Q_SLOTS:
+    void idnaTestV2_data();
+    void idnaTestV2();
+
+private:
+    // All error codes:
+    //      A3, A4_1, A4_2,
+    //      B1, B2, B3, B4, B5, B6,
+    //      C1, C2,
+    //      P1, P4,
+    //      V1, V2, V3, V5, V6,
+    //      X4_2
+    //
+    // NOTE: moving this inside idnaTestV2_data() results in ICE with MSVC 2019
+    static const QSet<QByteArray> fatalErrors;
+};
+
+const QSet<QByteArray> tst_QUrlUts46::fatalErrors = { "A3", "A4_2", "P1", "X4_2" };
+
+void tst_QUrlUts46::idnaTestV2_data()
+{
+    QTest::addColumn<QString>("source");
+    QTest::addColumn<QString>("toUnicode");
+    QTest::addColumn<bool>("toUnicodeOk");
+    QTest::addColumn<QString>("toAsciiN");
+    QTest::addColumn<bool>("toAsciiNOk");
+    QTest::addColumn<QString>("toAsciiT");
+    QTest::addColumn<bool>("toAsciiTOk");
+
+    QFile dataFile(QFINDTESTDATA("testdata/IdnaTestV2.txt"));
+    qDebug() << "Data file:" << dataFile.fileName();
+    QVERIFY(dataFile.open(QFile::ReadOnly));
+
+    auto isToAsciiOk = [](const QByteArray &s, bool ifEmpty) {
+        if (s.isEmpty())
+            return ifEmpty;
+
+        Q_ASSERT(s.startsWith('[') && s.endsWith(']'));
+
+        const auto errors = s.sliced(1, s.length() - 2).split(',');
+        // NOTE: empty string is not in fatalErrors and it's ok
+        return std::all_of(errors.begin(), errors.end(),
+                           [](auto &e) { return !fatalErrors.contains(e.trimmed()); });
+    };
+
+    for (unsigned int lineNo = 1; !dataFile.atEnd(); lineNo++) {
+        auto line = dataFile.readLine().trimmed();
+
+        int commentIdx = line.indexOf('#');
+        if (commentIdx != -1)
+            line = line.left(commentIdx).trimmed();
+        if (line.isEmpty())
+            continue;
+
+        auto fields = line.split(';');
+        Q_ASSERT(fields.size() == 7);
+
+        for (auto &field : fields)
+            field = field.trimmed();
+
+        const QString &source = fields[0];
+        QString toUnicode = fields[1].isEmpty() ? source : fields[1];
+        bool toUnicodeOk = fields[2].isEmpty();
+        bool toUnicodeOkForAscii = isToAsciiOk(fields[2], true);
+        QString toAsciiN = fields[3].isEmpty() ? toUnicode : fields[3];
+        bool toAsciiNOk = isToAsciiOk(fields[4], toUnicodeOkForAscii);
+        QString toAsciiT = fields[5].isEmpty() ? toAsciiN : fields[5];
+        bool toAsciiTOk = isToAsciiOk(fields[6], toAsciiNOk);
+
+        QTest::addRow("line %u", lineNo) << source << toUnicode << toUnicodeOk << toAsciiN
+                                         << toAsciiNOk << toAsciiT << toAsciiTOk;
+    }
+}
+
+void tst_QUrlUts46::idnaTestV2()
+{
+    QFETCH(QString, source);
+    QFETCH(QString, toUnicode);
+    QFETCH(bool, toUnicodeOk);
+    QFETCH(QString, toAsciiN);
+    QFETCH(bool, toAsciiNOk);
+    QFETCH(QString, toAsciiT);
+    QFETCH(bool, toAsciiTOk);
+
+    auto dashesOk = [](const QString &domain) {
+        const auto labels = domain.split(u'.');
+        return std::all_of(labels.begin(), labels.end(), [](const QString &label) {
+            return label.isEmpty() || !(label.startsWith(u'-') || label.endsWith(u'-'));
+        });
+    };
+
+    // Some input file entries claim P1 error when none can be found.
+    // This is fixed in Unicode 14.0.0
+    QEXPECT_FAIL("line 1076", "Invalid P1 in the input file", Continue);
+    QEXPECT_FAIL("line 1077", "Invalid P1 in the input file", Continue);
+    QEXPECT_FAIL("line 4453", "Invalid P1 in the input file", Continue);
+    QEXPECT_FAIL("line 4454", "Invalid P1 in the input file", Continue);
+    QEXPECT_FAIL("line 5667", "Invalid P1 in the input file", Continue);
+    QString toAceN = QUrl::toAce(source);
+    if (toAsciiNOk && dashesOk(toAsciiN))
+        QCOMPARE(toAceN, toAsciiN);
+    else
+        QCOMPARE(toAceN, QString());
+
+    QEXPECT_FAIL("line 1076", "Invalid P1 in the input file", Continue);
+    QEXPECT_FAIL("line 1077", "Invalid P1 in the input file", Continue);
+    QEXPECT_FAIL("line 4453", "Invalid P1 in the input file", Continue);
+    QEXPECT_FAIL("line 4454", "Invalid P1 in the input file", Continue);
+    QEXPECT_FAIL("line 5667", "Invalid P1 in the input file", Continue);
+    QString toAceT = QUrl::toAce(source, QUrl::AceTransitionalProcessing);
+    if (toAsciiTOk && dashesOk(toAsciiT))
+        QCOMPARE(toAceT, toAsciiT);
+    else
+        QCOMPARE(toAceT, QString());
+
+    QString normalized = QUrl::fromAce(toAceN.toUtf8(), QUrl::IgnoreIDNWhitelist);
+    if (toUnicodeOk && !toAceN.isEmpty())
+        QCOMPARE(normalized, toUnicode);
+    else
+        QCOMPARE(normalized, toAceN);
+}
+
+QTEST_APPLESS_MAIN(tst_QUrlUts46)
+
+#include "tst_qurluts46.moc"

util/unicode/README

@@ -14,6 +14,8 @@ To update:
   to match
 * Download https://www.unicode.org/Public/idna/$Version/IdnaMappingTable.txt
   and put it into data/.
+* Download https://www.unicode.org/Public/idna/$Version/IdnaTestV2.txt
+  and put it into tests/auto/corelib/io/qurluts46/testdata.
 * Build this project. Its binary, unicode, ignores command-line
   options and assumes it is being run from this directory. When run,
   it produces lots of output. If it gets as far as updating