Network: Use system publicsuffix database copy when available

[ChangeLog][Network][QNetworkCookieJar] It is possible to use
system's copy of publicsuffix database when it is available.
This behavior is enabled by default on Linux and can be
controlled using new command line switches -system-publicsuffix,
-qt-publicsuffix, -no-publicsuffix, and -publicsuffix=all.

Fixes: QTBUG-95889
Change-Id: I911e1a13c1422cdc35851953309fff064e7c5f26
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>

src/network/configure.cmake

@@ -400,7 +400,22 @@ qt_feature("topleveldomain" PUBLIC
     SECTION "Networking"
     LABEL "qTopLevelDomain()"
     PURPOSE "Provides support for extracting the top level domain from URLs.  If enabled, a binary dump of the Public Suffix List (http://www.publicsuffix.org, Mozilla License) is included. The data is then also used in QNetworkCookieJar::validateCookie."
+    DISABLE INPUT_publicsuffix STREQUAL "no"
 )
+qt_feature("publicsuffix-qt" PRIVATE
+    LABEL "  Built-in publicsuffix database"
+    CONDITION QT_FEATURE_topleveldomain
+    ENABLE INPUT_publicsuffix STREQUAL "qt" OR INPUT_publicsuffix STREQUAL "all"
+    DISABLE INPUT_publicsuffix STREQUAL "system"
+)
+qt_feature("publicsuffix-system" PRIVATE
+    LABEL "  System publicsuffix database"
+    CONDITION QT_FEATURE_topleveldomain
+    AUTODETECT LINUX
+    ENABLE INPUT_publicsuffix STREQUAL "system" OR INPUT_publicsuffix STREQUAL "all"
+    DISABLE INPUT_publicsuffix STREQUAL "qt"
+)
+
 qt_configure_add_summary_section(NAME "Qt Network")
 qt_configure_add_summary_entry(ARGS "getifaddrs")
 qt_configure_add_summary_entry(ARGS "ipv6ifname")
@@ -426,6 +441,9 @@ qt_configure_add_summary_entry(ARGS "sctp")
 qt_configure_add_summary_entry(ARGS "system-proxies")
 qt_configure_add_summary_entry(ARGS "gssapi")
 qt_configure_add_summary_entry(ARGS "brotli")
+qt_configure_add_summary_entry(ARGS "topleveldomain")
+qt_configure_add_summary_entry(ARGS "publicsuffix-qt")
+qt_configure_add_summary_entry(ARGS "publicsuffix-system")
 qt_configure_end_summary_section() # end of "Qt Network" section
 # special case begin
 qt_configure_add_report_entry(

src/network/kernel/qtldurl.cpp

@@ -44,10 +44,19 @@
 #if QT_CONFIG(topleveldomain)
 
 #include "qurl.h"
-#include "private/qtldurl_p.h"
+#include "QtCore/qfile.h"
+#include "QtCore/qfileinfo.h"
+#include "QtCore/qloggingcategory.h"
+#include "QtCore/qstandardpaths.h"
 #include "QtCore/qstring.h"
 
-#include "qurltlds_p.h"
+#if !QT_CONFIG(publicsuffix_qt) && !QT_CONFIG(publicsuffix_system)
+#   error Enable at least one feature: publicsuffix-qt, publicsuffix-system
+#endif
+
+#if QT_CONFIG(publicsuffix_qt)
+#   include "qurltlds_p.h"
+#endif
 
 // Defined in src/3rdparty/libpsl/src/lookup_string_in_fixed_set.c
 extern "C" int LookupStringInFixedSet(const unsigned char *graph, std::size_t length,
@@ -55,15 +64,146 @@ extern "C" int LookupStringInFixedSet(const unsigned char *graph, std::size_t le
 
 QT_BEGIN_NAMESPACE
 
+Q_LOGGING_CATEGORY(lcTld, "qt.network.tld")
+
 static constexpr int PSL_NOT_FOUND = -1;
 static constexpr int PSL_FLAG_EXCEPTION = 1 << 0;
 static constexpr int PSL_FLAG_WILDCARD = 1 << 1;
 
-static int lookupDomain(QByteArrayView domain)
+class QPublicSuffixDatabase final
 {
-    return LookupStringInFixedSet(kDafsa, sizeof(kDafsa), domain.data(), domain.size());
+public:
+#if QT_CONFIG(publicsuffix_system)
+    QPublicSuffixDatabase();
+#endif // QT_CONFIG(publicsuffix_system)
+
+    int lookupDomain(QByteArrayView domain) const;
+
+private:
+    QByteArrayView m_data
+#if QT_CONFIG(publicsuffix_qt)
+    {
+        kDafsa, sizeof(kDafsa)
+    }
+#endif // QT_CONFIG(publicsuffix_qt)
+    ;
+
+#if QT_CONFIG(publicsuffix_system)
+    std::unique_ptr<QFile> m_dev;
+    QByteArray m_storage;
+    bool loadFile(const QString &fileName);
+#endif // QT_CONFIG(publicsuffix_system)
+};
+
+int QPublicSuffixDatabase::lookupDomain(QByteArrayView domain) const
+{
+    return LookupStringInFixedSet(reinterpret_cast<const unsigned char *>(m_data.constData()),
+                                     m_data.size(), domain.data(), domain.size());
 }
 
+#if QT_CONFIG(publicsuffix_system)
+
+static QStringList locatePublicSuffixFiles()
+{
+    return QStandardPaths::locateAll(QStandardPaths::GenericDataLocation,
+                                     u"publicsuffix/public_suffix_list.dafsa"_qs);
+}
+
+QPublicSuffixDatabase::QPublicSuffixDatabase()
+{
+    for (auto &&fileName : locatePublicSuffixFiles()) {
+        if (loadFile(fileName))
+            return;
+    }
+
+#if QT_CONFIG(publicsuffix_qt)
+    qCDebug(lcTld, "Using builtin publicsuffix list");
+#else
+    qCWarning(lcTld, "No usable publicsuffix file found");
+#endif
+}
+
+bool QPublicSuffixDatabase::loadFile(const QString &fileName)
+{
+    static const QByteArrayView DafsaFileHeader = ".DAFSA@PSL_0   \n";
+
+    qCDebug(lcTld, "Loading publicsuffix file: %s", qUtf8Printable(fileName));
+
+    auto systemFile = std::make_unique<QFile>(fileName);
+
+    if (!systemFile->open(QIODevice::ReadOnly)) {
+        qCDebug(lcTld, "Failed to open publicsuffix file: %s",
+                qUtf8Printable(systemFile->errorString()));
+        return false;
+    }
+
+    auto fileSize = systemFile->size();
+    // Check if there is enough data for header, version byte and some data
+    if (fileSize < DafsaFileHeader.size() + 2) {
+        qCWarning(lcTld, "publicsuffix file is too small: %zu", std::size_t(fileSize));
+        return false;
+    }
+
+    auto header = systemFile->read(DafsaFileHeader.size());
+    if (header != DafsaFileHeader) {
+        qCWarning(lcTld, "Invalid publicsuffix file header: %s", qUtf8Printable(header.toHex()));
+        return false;
+    }
+
+    // Check if the file is UTF-8 compatible
+    if (!systemFile->seek(fileSize - 1)) {
+        qCWarning(lcTld, "Failed to seek to the end of file: %s",
+                  qUtf8Printable(systemFile->errorString()));
+        return false;
+    }
+
+    char version;
+    if (systemFile->read(&version, 1) != 1) {
+        qCWarning(lcTld, "Failed to read publicsuffix version");
+        return false;
+    }
+
+    if (version != 0x01) {
+        qCWarning(lcTld, "Unsupported publicsuffix version: %d", int(version));
+        return false;
+    }
+
+    const auto dataSize = fileSize - DafsaFileHeader.size() - 1;
+    // Try to map the file first
+    auto mappedData = systemFile->map(DafsaFileHeader.size(), dataSize);
+    if (mappedData) {
+        qCDebug(lcTld, "Using mapped system publicsuffix data");
+        systemFile->close();
+        m_data = QByteArrayView(mappedData, dataSize);
+        m_dev = std::move(systemFile);
+        return true;
+    }
+
+    qCDebug(lcTld, "Failed to map publicsuffix file: %s",
+            qUtf8Printable(systemFile->errorString()));
+
+    systemFile->seek(DafsaFileHeader.size());
+    m_storage = systemFile->read(dataSize);
+    if (m_storage.size() != dataSize) {
+        qCWarning(lcTld, "Failed to read publicsuffix file");
+        m_storage.clear();
+        return false;
+    }
+
+    qCDebug(lcTld, "Using system publicsuffix data");
+    m_data = m_storage;
+
+    return true;
+}
+
+Q_GLOBAL_STATIC(QPublicSuffixDatabase, publicSuffix);
+
+#else
+
+static const QPublicSuffixDatabase m_publicSuffix;
+
+#endif // QT_CONFIG(publicsuffix_system)
+
 /*!
     \internal
 
@@ -82,7 +222,14 @@ Q_NETWORK_EXPORT bool qIsEffectiveTLD(QStringView domain)
     QByteArray decodedDomain = domain.toUtf8();
     QByteArrayView domainView(decodedDomain);
 
-    auto ret = lookupDomain(domainView);
+#if QT_CONFIG(publicsuffix_system)
+    if (publicSuffix.isDestroyed())
+        return false;
+#else
+    auto publicSuffix = &m_publicSuffix;
+#endif // QT_CONFIG(publicsuffix_system)
+
+    auto ret = publicSuffix->lookupDomain(domainView);
     if (ret != PSL_NOT_FOUND) {
         if (ret & PSL_FLAG_EXCEPTION) // 1
             return false;
@@ -93,7 +240,7 @@ Q_NETWORK_EXPORT bool qIsEffectiveTLD(QStringView domain)
     const auto dot = domainView.indexOf('.');
     if (dot < 0) // Actual TLD: may be effective if the subject of a wildcard rule:
         return ret != PSL_NOT_FOUND;
-    ret = lookupDomain(domainView.sliced(dot + 1)); // 3
+    ret = publicSuffix->lookupDomain(domainView.sliced(dot + 1)); // 3
     if (ret == PSL_NOT_FOUND)
         return false;
     return (ret & PSL_FLAG_WILDCARD) != 0;

src/network/kernel/qurltlds_p.h.INFO

@@ -5,6 +5,8 @@ qtbase/src/3rdparty/libpsl/src/psl-make-dafsa in the Qt source tree.
 To regenerate the file, run the following command from qtbase tree:
 
     src/3rdparty/libpsl/src/psl-make-dafsa public_suffix_list.dat src/network/kernel/qurltlds_p.h
+    src/3rdparty/libpsl/src/psl-make-dafsa --output-format=binary public_suffix_list.dat \
+        tests/auto/network/access/qnetworkcookiejar/testdata/publicsuffix/public_suffix_list.dafsa
 
 Those arrays in qurltlds_p.h are derived from the Public
 Suffix List ([2]), which was originally provided by

src/network/qt_cmdline.cmake

@@ -9,3 +9,4 @@ qt_commandline_option(securetransport TYPE boolean)
 qt_commandline_option(schannel TYPE boolean)
 qt_commandline_option(ssl TYPE boolean)
 qt_commandline_option(system-proxies TYPE boolean)
+qt_commandline_option(publicsuffix TYPE optionalString VALUES system qt no all)

tests/auto/network/access/qnetworkcookiejar/CMakeLists.txt

@@ -5,7 +5,7 @@
 #####################################################################
 
 # Collect test data
-list(APPEND test_data "parser.json")
+list(APPEND test_data "parser.json" "testdata/publicsuffix/public_suffix_list.dafsa")
 
 qt_internal_add_test(tst_qnetworkcookiejar
     SOURCES

tests/auto/network/access/qnetworkcookiejar/tst_qnetworkcookiejar.cpp

@@ -44,6 +44,8 @@ class tst_QNetworkCookieJar: public QObject
     Q_OBJECT
 
 private slots:
+    void initTestCase();
+
     void getterSetter();
     void setCookiesFromUrl_data();
     void setCookiesFromUrl();
@@ -55,6 +57,8 @@ private slots:
 #endif
     void rfc6265_data();
     void rfc6265();
+private:
+    QSharedPointer<QTemporaryDir> m_dataDir;
 };
 
 class MyCookieJar: public QNetworkCookieJar
@@ -82,6 +86,22 @@ void tst_QNetworkCookieJar::getterSetter()
     QCOMPARE(jar.allCookies(), list);
 }
 
+void tst_QNetworkCookieJar::initTestCase()
+{
+#if QT_CONFIG(topleveldomain) && QT_CONFIG(publicsuffix_system)
+    QString testDataDir;
+#ifdef BUILTIN_TESTDATA
+    m_dataDir = QEXTRACTTESTDATA("/testdata");
+    QVERIFY(m_dataDir);
+    testDataDir = m_dataDir->path() + "/testdata";
+#else
+    testDataDir = QFINDTESTDATA("testdata");
+#endif
+    qDebug() << "Test data dir:" << testDataDir;
+    qputenv("XDG_DATA_DIRS", QFile::encodeName(testDataDir));
+#endif
+}
+
 void tst_QNetworkCookieJar::setCookiesFromUrl_data()
 {
     QTest::addColumn<QList<QNetworkCookie> >("preset");