Add streaming text support to QRestReply

Provides the possibility to read text data as it arrives, instead
of needing to wait until the whole body is received.

Pick-to: 6.7
Task-number: QTBUG-119002
Change-Id: I64f90148fd41a77c4ae2d5dbd6194a924a9f3a86
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
This commit is contained in:
Juha Vuolle 2023-12-12 14:08:04 +02:00
parent 9aaf1a031b
commit 4da14a67a6
3 changed files with 98 additions and 35 deletions

View File

@ -222,7 +222,7 @@ QByteArray QRestReply::body()
} }
/*! /*!
Returns the received data as a QString. Requires the reply to be finished. Returns the received data as a QString.
The received data is decoded into a QString (UTF-16). The decoding The received data is decoded into a QString (UTF-16). The decoding
uses the \e Content-Type header's \e charset parameter to determine the uses the \e Content-Type header's \e charset parameter to determine the
@ -230,33 +230,36 @@ QByteArray QRestReply::body()
available or not supported by \l QStringConverter, UTF-8 is used as a available or not supported by \l QStringConverter, UTF-8 is used as a
default. default.
Calling this function consumes the received data, and any further calls Calling this function consumes the data received so far. Returns
to get response data will return empty. a default constructed value if no new data is available, or if the
decoding is not supported by \l QStringConverter, or if the decoding
This function returns a default-constructed value and will not consume has errors (for example invalid characters).
any data if the reply is not finished.
\sa json(), body(), isFinished(), finished() \sa json(), body(), isFinished(), finished()
*/ */
QString QRestReply::text() QString QRestReply::text()
{ {
Q_D(QRestReply); Q_D(QRestReply);
if (!isFinished()) { QString result;
qCWarning(lcQrest, "Attempt to read text() of an unfinished reply, ignoring.");
return {};
}
QByteArray data = d->networkReply->readAll(); QByteArray data = d->networkReply->readAll();
if (data.isEmpty()) if (data.isEmpty())
return {}; return result;
const QByteArray charset = d->contentCharset(); if (!d->decoder) {
QStringDecoder decoder(charset); const QByteArray charset = d->contentCharset();
if (!decoder.isValid()) { // the decoder may not support the mimetype's charset d->decoder = QStringDecoder(charset);
qCWarning(lcQrest, "Charset \"%s\" is not supported, defaulting to UTF-8", if (!d->decoder->isValid()) { // the decoder may not support the mimetype's charset
charset.constData()); qCWarning(lcQrest, "text(): Charset \"%s\" is not supported", charset.constData());
decoder = QStringDecoder(QStringDecoder::Utf8); return result;
}
} }
return decoder(data); // Check if the decoder already had an error, or has errors after decoding current data chunk
if (d->decoder->hasError() || (result = (*d->decoder)(data), d->decoder->hasError())) {
qCWarning(lcQrest, "text() Decoding error occurred");
return {};
}
return result;
} }
/*! /*!

View File

@ -19,8 +19,12 @@
#include <QtNetwork/qnetworkreply.h> #include <QtNetwork/qnetworkreply.h>
#include <QtCore/qjsondocument.h> #include <QtCore/qjsondocument.h>
#include <optional>
QT_BEGIN_NAMESPACE QT_BEGIN_NAMESPACE
class QStringDecoder;
class QRestReplyPrivate : public QObjectPrivate class QRestReplyPrivate : public QObjectPrivate
{ {
public: public:
@ -28,6 +32,7 @@ public:
~QRestReplyPrivate() override; ~QRestReplyPrivate() override;
QNetworkReply *networkReply = nullptr; QNetworkReply *networkReply = nullptr;
std::optional<QStringDecoder> decoder;
QByteArray contentCharset() const; QByteArray contentCharset() const;
bool hasNonHttpError() const; bool hasNonHttpError() const;

View File

@ -41,6 +41,7 @@ private slots:
void body(); void body();
void json(); void json();
void text(); void text();
void textStreaming();
void download(); void download();
void upload(); void upload();
void timeout(); void timeout();
@ -788,12 +789,22 @@ void tst_QRestAccessManager::json()
} }
#define VERIFY_TEXT_REPLY_OK \ #define VERIFY_TEXT_REPLY_OK \
manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; }); \
QTRY_VERIFY(replyFromServer); \ QTRY_VERIFY(replyFromServer); \
responseString = replyFromServer->text(); \ responseString = replyFromServer->text(); \
QCOMPARE(responseString, sourceString); \ QCOMPARE(responseString, sourceString); \
replyFromServer->deleteLater(); \ replyFromServer->deleteLater(); \
replyFromServer = nullptr; \ replyFromServer = nullptr; \
#define VERIFY_TEXT_REPLY_ERROR(WARNING_MESSAGE) \
manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; }); \
QTRY_VERIFY(replyFromServer); \
QTest::ignoreMessage(QtWarningMsg, WARNING_MESSAGE); \
responseString = replyFromServer->text(); \
QVERIFY(responseString.isEmpty()); \
replyFromServer->deleteLater(); \
replyFromServer = nullptr; \
void tst_QRestAccessManager::text() void tst_QRestAccessManager::text()
{ {
// Test using QRestReply::text() data accessor with various text encodings // Test using QRestReply::text() data accessor with various text encodings
@ -830,32 +841,27 @@ void tst_QRestAccessManager::text()
// Successful UTF-8 // Successful UTF-8
serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-8"_ba); serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-8"_ba);
serverSideResponse.body = encUTF8(sourceString); serverSideResponse.body = encUTF8(sourceString);
manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
VERIFY_TEXT_REPLY_OK; VERIFY_TEXT_REPLY_OK;
// Successful UTF-16 // Successful UTF-16
serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-16"_ba); serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-16"_ba);
serverSideResponse.body = encUTF16(sourceString); serverSideResponse.body = encUTF16(sourceString);
manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
VERIFY_TEXT_REPLY_OK; VERIFY_TEXT_REPLY_OK;
// Successful UTF-16, parameter case insensitivity // Successful UTF-16, parameter case insensitivity
serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; chARset=uTf-16"_ba); serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; chARset=uTf-16"_ba);
serverSideResponse.body = encUTF16(sourceString); serverSideResponse.body = encUTF16(sourceString);
manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
VERIFY_TEXT_REPLY_OK; VERIFY_TEXT_REPLY_OK;
// Successful UTF-32 // Successful UTF-32
serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-32"_ba); serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-32"_ba);
serverSideResponse.body = encUTF32(sourceString); serverSideResponse.body = encUTF32(sourceString);
manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
VERIFY_TEXT_REPLY_OK; VERIFY_TEXT_REPLY_OK;
// Successful UTF-32 with spec-wise allowed extra content in the Content-Type header value // Successful UTF-32 with spec-wise allowed extra content in the Content-Type header value
serverSideResponse.headers.insert("Content-Type:"_ba, serverSideResponse.headers.insert("Content-Type:"_ba,
"text/plain; charset = \"UTF-32\";extraparameter=bar"_ba); "text/plain; charset = \"UTF-32\";extraparameter=bar"_ba);
serverSideResponse.body = encUTF32(sourceString); serverSideResponse.body = encUTF32(sourceString);
manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; });
VERIFY_TEXT_REPLY_OK; VERIFY_TEXT_REPLY_OK;
// Unsuccessful UTF-32, wrong encoding indicated (indicated charset UTF-32 but data is UTF-8) // Unsuccessful UTF-32, wrong encoding indicated (indicated charset UTF-32 but data is UTF-8)
@ -868,16 +874,68 @@ void tst_QRestAccessManager::text()
replyFromServer->deleteLater(); replyFromServer->deleteLater();
replyFromServer = nullptr; replyFromServer = nullptr;
// Unsupported encoding, defaults to UTF-8 // Unsupported encoding
serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=foo"_ba); serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=foo"_ba);
serverSideResponse.body = encUTF8(sourceString); serverSideResponse.body = encUTF8(sourceString);
manager.get(request, this, [&](QRestReply *reply) { replyFromServer = reply; }); VERIFY_TEXT_REPLY_ERROR("text(): Charset \"foo\" is not supported")
QTRY_VERIFY(replyFromServer);
QTest::ignoreMessage(QtWarningMsg, "Charset \"foo\" is not supported, defaulting to UTF-8"); // Broken UTF-8
responseString = replyFromServer->text(); serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-8"_ba);
QCOMPARE(responseString, sourceString); serverSideResponse.body = "\xF0\x28\x8C\x28\xA0\xB0\xC0\xD0"; // invalid characters
replyFromServer->deleteLater(); VERIFY_TEXT_REPLY_ERROR("text() Decoding error occurred");
replyFromServer = nullptr; }
void tst_QRestAccessManager::textStreaming()
{
// Tests textual data received in chunks
QRestAccessManager manager;
manager.setDeletesRepliesOnFinished(false);
HttpTestServer server;
QTRY_VERIFY(server.isListening());
// Create long text data
const QString expectedData = u"사랑abcd€fghiklmnΩpqrstuvwx愛사랑A사랑BCD€FGHIJKLMNΩPQRsTUVWXYZ愛"_s;
QString cumulativeReceivedText;
QStringEncoder encUTF8("UTF-8");
ResponseControl *responseControl = nullptr;
HttpData serverSideResponse; // The response data the server responds with
serverSideResponse.headers.insert("Content-Type:"_ba, "text/plain; charset=UTF-8"_ba);
serverSideResponse.body = encUTF8(expectedData);
serverSideResponse.status = 200;
server.setHandler([&](HttpData, HttpData &response, ResponseControl &control) {
response = serverSideResponse;
responseControl = &control; // store for later
control.responseChunkSize = 5; // tell testserver to send data in chunks of this size
});
QNetworkRequest request(server.url());
QRestReply *reply = manager.get(request);
QObject::connect(reply, &QRestReply::readyRead, this, [&](QRestReply *reply) {
cumulativeReceivedText += reply->text();
// Tell testserver that test is ready for next chunk
responseControl->readyForNextChunk = true;
});
QTRY_VERIFY(reply->isFinished());
QCOMPARE(cumulativeReceivedText, expectedData);
cumulativeReceivedText.clear();
// Broken UTF-8 characters after first five ok characters
serverSideResponse.body =
"12345"_ba + "\xF0\x28\x8C\x28\xA0\xB0\xC0\xD0" + "abcde"_ba;
reply = manager.get(request);
QObject::connect(reply, &QRestReply::readyRead, this, [&](QRestReply *reply) {
static bool firstTime = true;
if (!firstTime) // First text part is without warnings
QTest::ignoreMessage(QtWarningMsg, "text() Decoding error occurred");
firstTime = false;
cumulativeReceivedText += reply->text();
// Tell testserver that test is ready for next chunk
responseControl->readyForNextChunk = true;
});
QTRY_VERIFY(reply->isFinished());
QCOMPARE(cumulativeReceivedText, "12345"_ba);
} }
void tst_QRestAccessManager::download() void tst_QRestAccessManager::download()
@ -915,14 +973,11 @@ void tst_QRestAccessManager::download()
QObject::connect(reply, &QRestReply::readyRead, this, [&](QRestReply *reply) { QObject::connect(reply, &QRestReply::readyRead, this, [&](QRestReply *reply) {
static bool testOnce = true; static bool testOnce = true;
if (!reply->isFinished() && testOnce) { if (!reply->isFinished() && testOnce) {
// Test once that reading jsonObject or text of an unfinished reply will not work // Test once that reading json of an unfinished reply will not work
testOnce = false; testOnce = false;
QTest::ignoreMessage(QtWarningMsg, "Attempt to read json() of an unfinished" QTest::ignoreMessage(QtWarningMsg, "Attempt to read json() of an unfinished"
" reply, ignoring."); " reply, ignoring.");
reply->json(); reply->json();
QTest::ignoreMessage(QtWarningMsg, "Attempt to read text() of an unfinished reply,"
" ignoring.");
(void)reply->text();
} }
cumulativeReceivedBytesAvailable += reply->bytesAvailable(); cumulativeReceivedBytesAvailable += reply->bytesAvailable();