Add streaming text support to QRestReply

Provides the possibility to read text data as it arrives, instead
of needing to wait until the whole body is received.

Pick-to: 6.7
Task-number: QTBUG-119002
Change-Id: I64f90148fd41a77c4ae2d5dbd6194a924a9f3a86
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>

src/network/access/qrestreply.cpp

@@ -222,7 +222,7 @@ QByteArray QRestReply::body()
 }
 
 /*!
-    Returns the received data as a QString. Requires the reply to be finished.
+    Returns the received data as a QString.
 
     The received data is decoded into a QString (UTF-16). The decoding
     uses the \e Content-Type header's \e charset parameter to determine the
@@ -230,33 +230,36 @@ QByteArray QRestReply::body()
     available or not supported by \l QStringConverter, UTF-8 is used as a
     default.
 
-    Calling this function consumes the received data, and any further calls
+    Calling this function consumes the data received so far. Returns
-    to get response data will return empty.
+    a default constructed value if no new data is available, or if the
-
+    decoding is not supported by \l QStringConverter, or if the decoding
-    This function returns a default-constructed value and will not consume
+    has errors (for example invalid characters).
-    any data if the reply is not finished.
 
     \sa json(), body(), isFinished(), finished()
 */
 QString QRestReply::text()
 {
     Q_D(QRestReply);
-    if (!isFinished()) {
+    QString result;
-        qCWarning(lcQrest, "Attempt to read text() of an unfinished reply, ignoring.");
+
-        return {};
-    }
     QByteArray data = d->networkReply->readAll();
     if (data.isEmpty())
-        return {};
+        return result;
 
-    const QByteArray charset = d->contentCharset();
+    if (!d->decoder) {
-    QStringDecoder decoder(charset);
+        const QByteArray charset = d->contentCharset();
-    if (!decoder.isValid()) { // the decoder may not support the mimetype's charset
+        d->decoder = QStringDecoder(charset);
-        qCWarning(lcQrest, "Charset \"%s\" is not supported, defaulting to UTF-8",
+        if (!d->decoder->isValid()) { // the decoder may not support the mimetype's charset
-                  charset.constData());
+            qCWarning(lcQrest, "text(): Charset \"%s\" is not supported", charset.constData());
-        decoder = QStringDecoder(QStringDecoder::Utf8);
+            return result;
+        }
     }
-    return decoder(data);
+    // Check if the decoder already had an error, or has errors after decoding current data chunk
+    if (d->decoder->hasError() || (result = (*d->decoder)(data), d->decoder->hasError())) {
+        qCWarning(lcQrest, "text() Decoding error occurred");
+        return {};
+    }
+    return result;
 }
 
 /*!

src/network/access/qrestreply_p.h

@@ -19,8 +19,12 @@
 #include <QtNetwork/qnetworkreply.h>
 #include <QtCore/qjsondocument.h>
 
+#include <optional>
+
 QT_BEGIN_NAMESPACE
 
+class QStringDecoder;
+
 class QRestReplyPrivate : public QObjectPrivate
 {
 public:
@@ -28,6 +32,7 @@ public:
     ~QRestReplyPrivate() override;
 
     QNetworkReply *networkReply = nullptr;
+    std::optional<QStringDecoder> decoder;
 
     QByteArray contentCharset() const;
     bool hasNonHttpError() const;

tests/auto/network/access/qrestaccessmanager/tst_qrestaccessmanager.cpp

@@ -41,6 +41,7 @@ private slots:
     void body();
     void json();
     void text();
+    void textStreaming();
     void download();
     void upload();
     void timeout();
@@ -788,12 +789,22 @@ void tst_QRestAccessManager::json()
 }
 
 #define VERIFY_TEXT_REPLY_OK \
+    manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; }); \
     QTRY_VERIFY(replyFromServer); \
     responseString = replyFromServer->text(); \
     QCOMPARE(responseString, sourceString); \
     replyFromServer->deleteLater(); \
     replyFromServer = nullptr; \
 
+#define VERIFY_TEXT_REPLY_ERROR(WARNING_MESSAGE) \
+    manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; }); \
+    QTRY_VERIFY(replyFromServer); \
+    QTest::ignoreMessage(QtWarningMsg, WARNING_MESSAGE); \
+    responseString = replyFromServer->text(); \
+    QVERIFY(responseString.isEmpty()); \
+    replyFromServer->deleteLater(); \
+    replyFromServer = nullptr; \
+
 void tst_QRestAccessManager::text()
 {
     // Test using QRestReply::text() data accessor with various text encodings
@@ -830,32 +841,27 @@ void tst_QRestAccessManager::text()
     // Successful UTF-8
     serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-8"_ba);
     serverSideResponse.body = encUTF8(sourceString);
-    manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
     VERIFY_TEXT_REPLY_OK;
 
     // Successful UTF-16
     serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-16"_ba);
     serverSideResponse.body = encUTF16(sourceString);
-    manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
     VERIFY_TEXT_REPLY_OK;
 
     // Successful UTF-16, parameter case insensitivity
     serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; chARset=uTf-16"_ba);
     serverSideResponse.body = encUTF16(sourceString);
-    manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
     VERIFY_TEXT_REPLY_OK;
 
     // Successful UTF-32
     serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-32"_ba);
     serverSideResponse.body = encUTF32(sourceString);
-    manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
     VERIFY_TEXT_REPLY_OK;
 
     // Successful UTF-32 with spec-wise allowed extra content in the Content-Type header value
     serverSideResponse.headers.insert("Content-Type:"_ba,
                                       "text/plain; charset = \"UTF-32\";extraparameter=bar"_ba);
     serverSideResponse.body = encUTF32(sourceString);
-    manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
     VERIFY_TEXT_REPLY_OK;
 
     // Unsuccessful UTF-32, wrong encoding indicated (indicated charset UTF-32 but data is UTF-8)
@@ -868,16 +874,68 @@ void tst_QRestAccessManager::text()
     replyFromServer->deleteLater();
     replyFromServer = nullptr;
 
-    // Unsupported encoding, defaults to UTF-8
+    // Unsupported encoding
     serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=foo"_ba);
     serverSideResponse.body = encUTF8(sourceString);
-    manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
+    VERIFY_TEXT_REPLY_ERROR("text(): Charset \"foo\" is not supported")
-    QTRY_VERIFY(replyFromServer);
+
-    QTest::ignoreMessage(QtWarningMsg, "Charset \"foo\" is not supported, defaulting to UTF-8");
+    // Broken UTF-8
-    responseString = replyFromServer->text();
+    serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-8"_ba);
-    QCOMPARE(responseString, sourceString);
+    serverSideResponse.body = "\xF0\x28\x8C\x28\xA0\xB0\xC0\xD0"; // invalid characters
-    replyFromServer->deleteLater();
+    VERIFY_TEXT_REPLY_ERROR("text() Decoding error occurred");
-    replyFromServer = nullptr;
+}
+
+void tst_QRestAccessManager::textStreaming()
+{
+    // Tests textual data received in chunks
+    QRestAccessManager manager;
+    manager.setDeletesRepliesOnFinished(false);
+    HttpTestServer server;
+    QTRY_VERIFY(server.isListening());
+
+    // Create long text data
+    const QString expectedData = u"사랑abcd€fghiklmnΩpqrstuvwx愛사랑A사랑BCD€FGHIJKLMNΩPQRsTUVWXYZ愛"_s;
+    QString cumulativeReceivedText;
+    QStringEncoder encUTF8("UTF-8");
+    ResponseControl *responseControl = nullptr;
+
+    HttpData serverSideResponse; // The response data the server responds with
+    serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-8"_ba);
+    serverSideResponse.body = encUTF8(expectedData);
+    serverSideResponse.status = 200;
+
+    server.setHandler([&](HttpData, HttpData &response, ResponseControl &control) {
+        response = serverSideResponse;
+        responseControl = &control; // store for later
+        control.responseChunkSize = 5; // tell testserver to send data in chunks of this size
+    });
+
+    QNetworkRequest request(server.url());
+    QRestReply *reply = manager.get(request);
+    QObject::connect(reply, &QRestReply::readyRead, this, [&](QRestReply *reply) {
+        cumulativeReceivedText += reply->text();
+        // Tell testserver that test is ready for next chunk
+        responseControl->readyForNextChunk = true;
+    });
+    QTRY_VERIFY(reply->isFinished());
+    QCOMPARE(cumulativeReceivedText, expectedData);
+
+    cumulativeReceivedText.clear();
+    // Broken UTF-8 characters after first five ok characters
+    serverSideResponse.body =
+            "12345"_ba + "\xF0\x28\x8C\x28\xA0\xB0\xC0\xD0" + "abcde"_ba;
+    reply = manager.get(request);
+    QObject::connect(reply, &QRestReply::readyRead, this, [&](QRestReply *reply) {
+        static bool firstTime = true;
+        if (!firstTime) // First text part is without warnings
+            QTest::ignoreMessage(QtWarningMsg, "text() Decoding error occurred");
+        firstTime = false;
+        cumulativeReceivedText += reply->text();
+        // Tell testserver that test is ready for next chunk
+        responseControl->readyForNextChunk = true;
+    });
+    QTRY_VERIFY(reply->isFinished());
+    QCOMPARE(cumulativeReceivedText, "12345"_ba);
 }
 
 void tst_QRestAccessManager::download()
@@ -915,14 +973,11 @@ void tst_QRestAccessManager::download()
     QObject::connect(reply, &QRestReply::readyRead, this, [&](QRestReply *reply) {
         static bool testOnce = true;
         if (!reply->isFinished() && testOnce) {
-            // Test once that reading jsonObject or text of an unfinished reply will not work
+            // Test once that reading json of an unfinished reply will not work
             testOnce = false;
             QTest::ignoreMessage(QtWarningMsg, "Attempt to read json() of an unfinished"
                                                " reply, ignoring.");
             reply->json();
-            QTest::ignoreMessage(QtWarningMsg, "Attempt to read text() of an unfinished reply,"
-                                               " ignoring.");
-            (void)reply->text();
         }
 
         cumulativeReceivedBytesAvailable += reply->bytesAvailable();