QDecompressHelper: Add brotli support

Task-number: QTBUG-83269
Change-Id: If23b098ee76a4892e4c2c6ce5c635688d8d9138d
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

cmake/FindWrapBrotli.cmake

@@ -0,0 +1,43 @@
+if(TARGET WrapBrotli::WrapBrotliDec)
+    set(WrapBrotli_FOUND ON)
+    return()
+endif()
+
+# From VCPKG
+find_package(unofficial-brotli CONFIG QUIET)
+if (unofficial-brotli_FOUND)
+    add_library(WrapBrotli::WrapBrotliDec INTERFACE IMPORTED)
+    target_link_libraries(WrapBrotli::WrapBrotliDec INTERFACE unofficial::brotli::brotlidec)
+
+    add_library(WrapBrotli::WrapBrotliEnc INTERFACE IMPORTED)
+    target_link_libraries(WrapBrotli::WrapBrotliEnc INTERFACE unofficial::brotli::brotlienc)
+
+    add_library(WrapBrotli::WrapBrotliCommon INTERFACE IMPORTED)
+    target_link_libraries(WrapBrotli::WrapBrotliCommon INTERFACE unofficial::brotli::brotlicommon)
+
+    set(WrapBrotli_FOUND ON)
+else()
+    find_package(PkgConfig QUIET)
+    if (PKG_CONFIG_FOUND)
+        pkg_check_modules(libbrotlidec QUIET libbrotlidec IMPORTED_TARGET)
+        if (libbrotlidec_FOUND)
+            add_library(WrapBrotli::WrapBrotliDec INTERFACE IMPORTED)
+            target_link_libraries(WrapBrotli::WrapBrotliDec INTERFACE PkgConfig::libbrotlidec)
+            set(WrapBrotli_FOUND ON)
+        endif()
+
+        pkg_check_modules(libbrotlienc QUIET libbrotlienc IMPORTED_TARGET)
+        if (libbrotlienc_FOUND)
+            add_library(WrapBrotli::WrapBrotliEnc INTERFACE IMPORTED)
+            target_link_libraries(WrapBrotli::WrapBrotliEnc INTERFACE PkgConfig::libbrotlienc)
+            set(WrapBrotli_FOUND ON)
+        endif()
+
+        pkg_check_modules(libbrotlicommon QUIET libbrotlicommon IMPORTED_TARGET)
+        if (libbrotlicommon_FOUND)
+            add_library(WrapBrotli::WrapBrotliCommon INTERFACE IMPORTED)
+            target_link_libraries(WrapBrotli::WrapBrotliCommon INTERFACE PkgConfig::libbrotlicommon)
+            set(WrapBrotli_FOUND ON)
+        endif()
+    endif()
+endif()

src/network/.prev_CMakeLists.txt

@@ -123,6 +123,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_http
         socket/qhttpsocketengine.cpp socket/qhttpsocketengine_p.h
 )
 
+qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http
+    LIBRARIES
+        WrapBrotli::WrapBrotliDec
+)
+
 qt_extend_target(Network CONDITION QT_FEATURE_system_zlib
     LIBRARIES
         ZLIB::ZLIB

src/network/CMakeLists.txt

@@ -123,6 +123,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_http
         socket/qhttpsocketengine.cpp socket/qhttpsocketengine_p.h
 )
 
+qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http
+    LIBRARIES
+        WrapBrotli::WrapBrotliDec
+)
+
 qt_extend_target(Network CONDITION QT_FEATURE_system_zlib
     LIBRARIES
         ZLIB::ZLIB

src/network/access/access.pri

@@ -116,4 +116,8 @@ qtConfig(http) {
         access/qhttpthreaddelegate_p.h \
         access/qnetworkreplyhttpimpl_p.h \
         access/qhttp2configuration.h
+
+    qtConfig(brotli) {
+        QMAKE_USE_PRIVATE += brotli
+    }
 }

src/network/access/qdecompresshelper.cpp

@@ -44,6 +44,10 @@
 
 #include <zlib.h>
 
+#if QT_CONFIG(brotli)
+#    include <brotli/decode.h>
+#endif
+
 #include <array>
 
 QT_BEGIN_NAMESPACE
@@ -57,6 +61,9 @@ struct ContentEncodingMapping
 constexpr ContentEncodingMapping contentEncodingMapping[] {
     { "deflate", QDecompressHelper::Deflate },
     { "gzip", QDecompressHelper::GZip },
+#if QT_CONFIG(brotli)
+    { "br", QDecompressHelper::Brotli },
+#endif
 };
 
 QDecompressHelper::ContentEncoding encodingFromByteArray(const QByteArray &ce) noexcept
@@ -72,6 +79,13 @@ z_stream *toZlibPointer(void *ptr)
 {
     return static_cast<z_stream_s *>(ptr);
 }
+
+#if QT_CONFIG(brotli)
+BrotliDecoderState *toBrotliPointer(void *ptr)
+{
+    return static_cast<BrotliDecoderState *>(ptr);
+}
+#endif
 }
 
 bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding)
@@ -134,6 +148,13 @@ bool QDecompressHelper::setEncoding(ContentEncoding ce)
         decoderPointer = inflateStream;
         break;
     }
+    case Brotli:
+#if QT_CONFIG(brotli)
+        decoderPointer = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
+#else
+        Q_UNREACHABLE();
+#endif
+        break;
     }
     if (!decoderPointer) {
         qWarning("Failed to initialize the decoder.");
@@ -325,6 +346,9 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize)
     case GZip:
         bytesRead = readZLib(data, maxSize);
         break;
+    case Brotli:
+        bytesRead = readBrotli(data, maxSize);
+        break;
     }
     if (bytesRead == -1)
         clear();
@@ -369,6 +393,14 @@ void QDecompressHelper::clear()
         delete inflateStream;
         break;
     }
+    case Brotli: {
+#if QT_CONFIG(brotli)
+        BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
+        if (brotliDecoderState)
+            BrotliDecoderDestroyInstance(brotliDecoderState);
+#endif
+        break;
+    }
     }
     decoderPointer = nullptr;
     contentEncoding = None;
@@ -489,4 +521,87 @@ qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize)
     return bytesDecoded;
 }
 
+qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize)
+{
+#if !QT_CONFIG(brotli)
+    Q_UNUSED(data);
+    Q_UNUSED(maxSize);
+    Q_UNREACHABLE();
+#else
+    qint64 bytesDecoded = 0;
+
+    BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
+
+    while (decoderHasData && bytesDecoded < maxSize) {
+        Q_ASSERT(brotliUnconsumedDataPtr || BrotliDecoderHasMoreOutput(brotliDecoderState));
+        if (brotliUnconsumedDataPtr) {
+            Q_ASSERT(brotliUnconsumedAmount);
+            size_t toRead = std::min(size_t(maxSize - bytesDecoded), brotliUnconsumedAmount);
+            memcpy(data + bytesDecoded, brotliUnconsumedDataPtr, toRead);
+            bytesDecoded += toRead;
+            brotliUnconsumedAmount -= toRead;
+            brotliUnconsumedDataPtr += toRead;
+            if (brotliUnconsumedAmount == 0) {
+                brotliUnconsumedDataPtr = nullptr;
+                decoderHasData = false;
+            }
+        }
+        if (BrotliDecoderHasMoreOutput(brotliDecoderState) == BROTLI_TRUE) {
+            brotliUnconsumedDataPtr =
+                    BrotliDecoderTakeOutput(brotliDecoderState, &brotliUnconsumedAmount);
+            decoderHasData = true;
+        }
+    }
+    if (bytesDecoded == maxSize)
+        return bytesDecoded;
+    Q_ASSERT(bytesDecoded < maxSize);
+
+    QByteArray input;
+    if (!compressedDataBuffer.isEmpty())
+        input = compressedDataBuffer.read();
+    const uint8_t *encodedPtr = reinterpret_cast<const uint8_t *>(input.constData());
+    size_t encodedBytesRemaining = input.size();
+
+    uint8_t *decodedPtr = reinterpret_cast<uint8_t *>(data + bytesDecoded);
+    size_t unusedDecodedSize = size_t(maxSize - bytesDecoded);
+    while (unusedDecodedSize > 0) {
+        auto previousUnusedDecodedSize = unusedDecodedSize;
+        BrotliDecoderResult result = BrotliDecoderDecompressStream(
+                brotliDecoderState, &encodedBytesRemaining, &encodedPtr, &unusedDecodedSize,
+                &decodedPtr, nullptr);
+        bytesDecoded += previousUnusedDecodedSize - unusedDecodedSize;
+
+        switch (result) {
+        case BROTLI_DECODER_RESULT_ERROR:
+            qWarning("Brotli error: %s",
+                     BrotliDecoderErrorString(BrotliDecoderGetErrorCode(brotliDecoderState)));
+            return -1;
+        case BROTLI_DECODER_RESULT_SUCCESS:
+            BrotliDecoderDestroyInstance(brotliDecoderState);
+            decoderPointer = nullptr;
+            return bytesDecoded;
+        case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
+            if (!compressedDataBuffer.isEmpty()) {
+                input = compressedDataBuffer.read();
+                encodedPtr = reinterpret_cast<const uint8_t *>(input.constData());
+                encodedBytesRemaining = input.size();
+                break;
+            }
+            return bytesDecoded;
+        case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
+            // Some data is leftover inside the brotli decoder, remember for next time
+            decoderHasData = BrotliDecoderHasMoreOutput(brotliDecoderState);
+            Q_ASSERT(unusedDecodedSize == 0);
+            break;
+        }
+    }
+    if (encodedBytesRemaining) {
+        // Some input was left unused; move back to the buffer
+        input = input.right(QByteArray::size_type(encodedBytesRemaining));
+        compressedDataBuffer.prepend(input);
+    }
+    return bytesDecoded;
+#endif
+}
+
 QT_END_NAMESPACE

src/network/access/qdecompresshelper_p.h

@@ -66,6 +66,7 @@ public:
         None,
         Deflate,
         GZip,
+        Brotli,
     };
 
     QDecompressHelper() = default;
@@ -101,6 +102,7 @@ private:
     qint64 encodedBytesAvailable() const;
 
     qsizetype readZLib(char *data, qsizetype maxSize);
+    qsizetype readBrotli(char *data, qsizetype maxSize);
 
     QByteDataBuffer compressedDataBuffer;
     bool decoderHasData = false;
@@ -112,6 +114,10 @@ private:
     ContentEncoding contentEncoding = None;
 
     void *decoderPointer = nullptr;
+#if QT_CONFIG(brotli)
+    const uint8_t *brotliUnconsumedDataPtr = nullptr;
+    size_t brotliUnconsumedAmount = 0;
+#endif
 };
 
 QT_END_NAMESPACE

src/network/configure.cmake

@@ -6,6 +6,7 @@
 
 #### Libraries
 
+qt_find_package(WrapBrotli PROVIDED_TARGETS WrapBrotli::WrapBrotliDec)
 qt_find_package(Libproxy PROVIDED_TARGETS PkgConfig::Libproxy MODULE_NAME network QMAKE_LIB libproxy)
 qt_find_package(WrapOpenSSLHeaders PROVIDED_TARGETS WrapOpenSSLHeaders::WrapOpenSSLHeaders MODULE_NAME network QMAKE_LIB openssl_headers)
 # openssl_headers
@@ -366,6 +367,13 @@ qt_feature("networkdiskcache" PUBLIC
     CONDITION QT_FEATURE_temporaryfile
 )
 qt_feature_definition("networkdiskcache" "QT_NO_NETWORKDISKCACHE" NEGATE VALUE "1")
+qt_feature("brotli" PUBLIC
+    SECTION "Networking"
+    LABEL "Brotli Decompression Support"
+    PURPOSE "Support for downloading and decompressing resources compressed with Brotli through QNetworkAccessManager."
+    CONDITION WrapBrotli_FOUND
+)
+qt_feature_definition("brotli" "QT_NO_BROTLI" NEGATE VALUE "1")
 qt_feature("localserver" PUBLIC
     SECTION "Networking"
     LABEL "QLocalServer"
@@ -432,6 +440,7 @@ qt_configure_add_summary_entry(ARGS "ftp")
 qt_configure_add_summary_entry(ARGS "sctp")
 qt_configure_add_summary_entry(ARGS "system-proxies")
 qt_configure_add_summary_entry(ARGS "gssapi")
+qt_configure_add_summary_entry(ARGS "brotli")
 qt_configure_end_summary_section() # end of "Qt Network" section
 qt_configure_add_report_entry(
     TYPE NOTE

src/network/configure.json

@@ -25,6 +25,21 @@
     },
 
     "libraries": {
+        "brotli": {
+            "label": "Brotli Decompression",
+            "test": {
+                "main": [
+                  "BrotliDecoderState *state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);"
+                ]
+            },
+            "headers": [
+                "brotli/decode.h"
+            ],
+            "sources": [
+                { "type": "pkgConfig", "args": "libbrotlidec" },
+                "-lbrotlidec"
+            ]
+        },
         "corewlan": {
             "label": "CoreWLan",
             "export": "",
@@ -387,6 +402,13 @@
             "condition": "features.temporaryfile",
             "output": [ "publicFeature", "feature" ]
         },
+        "brotli": {
+            "label": "Brotli Decompression Support",
+            "purpose": "Support for downloading and decompressing resources compressed with Brotli through QNetworkAccessManager.",
+            "section": "Networking",
+            "condition": "libs.brotli",
+            "output": [ "publicFeature", "feature" ]
+        },
         "localserver": {
             "label": "QLocalServer",
             "purpose": "Provides a local socket based server.",
@@ -479,7 +501,8 @@ For example:
                 "ftp",
                 "sctp",
                 "system-proxies",
-                "gssapi"
+                "gssapi",
+                "brotli"
             ]
         }
     ]

tests/auto/network/access/http2/tst_http2.cpp

@@ -796,6 +796,11 @@ void tst_Http2::contentEncoding_data()
     contentEncodingData.emplace_back(
             "deflate", QByteArray::fromBase64("eJzLSM3JyVcozy/KSQEAGgsEXQ=="), "hello world");
 
+#if QT_CONFIG(brotli)
+    contentEncodingData.emplace_back("br", QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD"),
+                                     "hello world");
+#endif
+
     // Loop through and add the data...
     for (const auto &data : contentEncodingData) {
         const char *name = data.contentEncoding.data();

tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp

@@ -90,6 +90,12 @@ void tst_QDecompressHelper::encodingSupported()
     QVERIFY(accepted.contains("gzip"));
     int expected = 2;
 
+#if QT_CONFIG(brotli)
+    QVERIFY(QDecompressHelper::isSupportedEncoding("br"));
+    QVERIFY(accepted.contains("br"));
+    ++expected;
+#endif
+
     QCOMPARE(expected, accepted.size());
 }
 
@@ -115,6 +121,12 @@ void tst_QDecompressHelper::sharedDecompress_data()
     QTest::newRow("deflate-hello-world")
             << QByteArray("deflate") << QByteArray::fromBase64("eJzLSM3JyVcozy/KSQEAGgsEXQ==")
             << QByteArray("hello world");
+
+#if QT_CONFIG(brotli)
+    QTest::newRow("brotli-hello-world")
+            << QByteArray("br") << QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD")
+            << QByteArray("hello world");
+#endif
 }
 
 void tst_QDecompressHelper::decompress_data()
@@ -322,6 +334,10 @@ void tst_QDecompressHelper::decompressBigData_data()
     QTest::newRow("gzip-4G") << QByteArray("gzip") << QString(":/4G.gz") << fourGiB;
     QTest::newRow("deflate-5G") << QByteArray("deflate") << QString(":/5GiB.txt.inflate")
                                 << fiveGiB;
+
+#if QT_CONFIG(brotli)
+    QTest::newRow("brotli-4G") << QByteArray("br") << (srcDir + "/4G.br") << fourGiB;
+#endif
 }
 
 void tst_QDecompressHelper::decompressBigData()

tests/auto/network/access/qnetworkreply/tst_qnetworkreply.cpp

@@ -9264,6 +9264,12 @@ void tst_QNetworkReply::contentEncoding_data()
     QTest::newRow("deflate-hello-world")
             << QByteArray("deflate") << QByteArray::fromBase64("eJzLSM3JyVcozy/KSQEAGgsEXQ==")
             << QByteArray("hello world");
+
+#if QT_CONFIG(brotli)
+    QTest::newRow("brotli-hello-world")
+            << QByteArray("br") << QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD")
+            << QByteArray("hello world");
+#endif
 }
 
 void tst_QNetworkReply::contentEncoding()

tests/benchmarks/network/access/qdecompresshelper/main.cpp

@@ -34,7 +34,6 @@ class tst_QDecompressHelper : public QObject
 {
     Q_OBJECT
 private slots:
-
     void decompress_data();
     void decompress();
 };
@@ -53,6 +52,10 @@ void tst_QDecompressHelper::decompress_data()
 #ifndef QT_NO_COMPRESS
     QTest::addRow("gzip") << QByteArray("gzip") << srcDir + QString("50mb.txt.gz");
     dataAdded = true;
+#endif
+#if QT_CONFIG(brotli)
+    QTest::addRow("brotli") << QByteArray("br") << srcDir + QString("50mb.txt.br");
+    dataAdded = true;
 #endif
     if (!dataAdded)
         QSKIP("There's no decompression support");

util/cmake/helper.py

@@ -372,6 +372,7 @@ _library_map = [
     # 3rd party:
     LibraryMapping("atspi", "ATSPI2", "PkgConfig::ATSPI2"),
     LibraryMapping("bluez", "BlueZ", "PkgConfig::BlueZ"),
+    LibraryMapping("brotli", "WrapBrotli", "WrapBrotli::WrapBrotliDec"),
     LibraryMapping("corewlan", None, None),
     LibraryMapping("cups", "Cups", "Cups::Cups"),
     LibraryMapping("directfb", "DirectFB", "PkgConfig::DirectFB"),