From e9f703ed3b5ff8a9987c6c4e3b658b41244919e3 Mon Sep 17 00:00:00 2001 From: Juha Vuolle Date: Thu, 6 Jul 2023 11:59:04 +0300 Subject: [PATCH] Add signals and methods for QRestReply download progress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These include: - readyRead(), signal for indicating new data availability - bytesAvailable(), function for checking available data amount - downloadProgress(), signal for monitoring download progress Task-number: QTBUG-114717 Change-Id: Id6c49530d7857f5c76bd111eba84525137294ea7 Reviewed-by: Marc Mutz Reviewed-by: Mårten Nordheim --- src/network/access/qrestreply.cpp | 48 +++++++++- src/network/access/qrestreply.h | 3 + .../qrestaccessmanager/httptestserver.cpp | 63 ++++++++----- .../qrestaccessmanager/httptestserver_p.h | 11 ++- .../tst_qrestaccessmanager.cpp | 88 +++++++++++++++++-- 5 files changed, 181 insertions(+), 32 deletions(-) diff --git a/src/network/access/qrestreply.cpp b/src/network/access/qrestreply.cpp index efff394f832..77707377598 100644 --- a/src/network/access/qrestreply.cpp +++ b/src/network/access/qrestreply.cpp @@ -33,6 +33,33 @@ Q_DECLARE_LOGGING_CATEGORY(lcQrest) \sa QRestAccessManager, QNetworkReply */ +/*! + \fn void QRestReply::readyRead(QRestReply *reply) + + This signal is emitted when \a reply has received new data. + + \sa body(), bytesAvailable(), isFinished() +*/ + +/*! + \fn void QRestReply::downloadProgress(qint64 bytesReceived, + qint64 bytesTotal, + QRestReply* reply) + + This signal is emitted to indicate the progress of the download part of + this network \a reply. + + The \a bytesReceived parameter indicates the number of bytes received, + while \a bytesTotal indicates the total number of bytes expected to be + downloaded. If the number of bytes to be downloaded is not known, for + instance due to a missing \c Content-Length header, \a bytesTotal + will be -1. + + See \l QNetworkReply::downloadProgress() documentation for more details. + + \sa bytesAvailable(), readyRead() +*/ + /*! \fn void QRestReply::finished(QRestReply *reply) @@ -63,6 +90,14 @@ QRestReply::QRestReply(QNetworkReply *reply, QObject *parent) d->networkReply = reply; // Reparent so that destruction of QRestReply destroys QNetworkReply reply->setParent(this); + + QObject::connect(reply, &QNetworkReply::readyRead, this, [this] { + emit readyRead(this); + }); + QObject::connect(reply, &QNetworkReply::downloadProgress, this, + [this](qint64 bytesReceived, qint64 bytesTotal) { + emit downloadProgress(bytesReceived, bytesTotal, this); + }); } /*! @@ -155,7 +190,7 @@ std::optional QRestReply::jsonArray() calls to get response data will return empty until further data has been received. - \sa json(), text() + \sa json(), text(), bytesAvailable(), readyRead() */ QByteArray QRestReply::body() { @@ -294,6 +329,17 @@ bool QRestReply::isFinished() const return d->networkReply->isFinished(); } +/*! + Returns the number of bytes available. + + \sa body +*/ +qint64 QRestReply::bytesAvailable() const +{ + Q_D(const QRestReply); + return d->networkReply->bytesAvailable(); +} + QRestReplyPrivate::QRestReplyPrivate() = default; diff --git a/src/network/access/qrestreply.h b/src/network/access/qrestreply.h index 6e35b45003a..91e8d17cc3a 100644 --- a/src/network/access/qrestreply.h +++ b/src/network/access/qrestreply.h @@ -35,6 +35,7 @@ public: QString errorString() const; bool isFinished() const; + qint64 bytesAvailable() const; public Q_SLOTS: void abort(); @@ -42,6 +43,8 @@ public Q_SLOTS: Q_SIGNALS: void finished(QRestReply *reply); void errorOccurred(QRestReply *reply); + void readyRead(QRestReply *reply); + void downloadProgress(qint64 bytesReceived, qint64 bytesTotal, QRestReply *reply); private: friend class QRestAccessManagerPrivate; diff --git a/tests/auto/network/access/qrestaccessmanager/httptestserver.cpp b/tests/auto/network/access/qrestaccessmanager/httptestserver.cpp index ff222dbfb2c..6e2f82a217b 100644 --- a/tests/auto/network/access/qrestaccessmanager/httptestserver.cpp +++ b/tests/auto/network/access/qrestaccessmanager/httptestserver.cpp @@ -73,31 +73,52 @@ void HttpTestServer::handleDataAvailable() m_request.url.setPort(parts.at(1).toUInt()); } HttpData response; + ResponseControl control; // Inform the testcase about request and ask for response data - m_handler(m_request, response); + m_handler(m_request, response, control); - if (response.respond) { - QByteArray responseMessage; - responseMessage += "HTTP/1.1 "; - responseMessage += QByteArray::number(response.status); + QByteArray responseMessage; + responseMessage += "HTTP/1.1 "; + responseMessage += QByteArray::number(response.status); + responseMessage += CRLF; + // Insert headers if any + for (const auto &[name,value] : response.headers.asKeyValueRange()) { + responseMessage += name; + responseMessage += value; responseMessage += CRLF; - // Insert headers if any - for (const auto &[name,value] : response.headers.asKeyValueRange()) { - responseMessage += name; - responseMessage += value; - responseMessage += CRLF; + } + responseMessage += CRLF; + /* + qDebug() << "HTTPTestServer received request" + << "\nMethod:" << m_request.method + << "\nHeaders:" << m_request.headers + << "\nBody:" << m_request.body; + */ + if (control.respond) { + if (control.responseChunkSize <= 0) { + responseMessage += response.body; + // qDebug() << "HTTPTestServer response:" << responseMessage; + m_socket->write(responseMessage); + } else { + // Respond in chunks, first write the headers + // qDebug() << "HTTPTestServer response:" << responseMessage; + m_socket->write(responseMessage); + // Then write bodydata in chunks, while allowing the testcase to process as well + QByteArray chunk; + while (!response.body.isEmpty()) { + chunk = response.body.left(control.responseChunkSize); + response.body.remove(0, control.responseChunkSize); + // qDebug() << "SERVER writing chunk" << chunk; + m_socket->write(chunk); + m_socket->flush(); + m_socket->waitForBytesWritten(); + // Process events until testcase indicates it's ready for next chunk. + // This way we can control the bytes the testcase gets in each chunk + control.readyForNextChunk = false; + while (!control.readyForNextChunk) + QCoreApplication::processEvents(); + } } - responseMessage += CRLF; - responseMessage += response.body; - - /* - qDebug() << "HTTPTestServer received request" - << "\nMethod:" << m_request.method - << "\nHeaders:" << m_request.headers - << "\nBody:" << m_request.body; - qDebug() << "HTTPTestServer sends response:" << responseMessage; - */ - m_socket->write(responseMessage); } m_socket->disconnectFromHost(); m_request = {}; diff --git a/tests/auto/network/access/qrestaccessmanager/httptestserver_p.h b/tests/auto/network/access/qrestaccessmanager/httptestserver_p.h index 3af34dd0f08..dd4b9024a0d 100644 --- a/tests/auto/network/access/qrestaccessmanager/httptestserver_p.h +++ b/tests/auto/network/access/qrestaccessmanager/httptestserver_p.h @@ -14,7 +14,6 @@ struct HttpData { QUrl url; int status = 0; - bool respond = true; QByteArray body; QByteArray method; quint16 port = 0; @@ -22,6 +21,13 @@ struct HttpData { QMap headers; }; +struct ResponseControl +{ + bool respond = true; + qsizetype responseChunkSize = -1; + bool readyForNextChunk = true; +}; + // Simple HTTP server. Currently supports only one concurrent connection class HttpTestServer : public QTcpServer { @@ -62,7 +68,8 @@ public: QByteArray fragment; // Settable callback for testcase. Gives the received request data, and takes in response data - using Handler = std::function; + using Handler = std::function; void setHandler(const Handler &handler); private slots: diff --git a/tests/auto/network/access/qrestaccessmanager/tst_qrestaccessmanager.cpp b/tests/auto/network/access/qrestaccessmanager/tst_qrestaccessmanager.cpp index 0dd6bdd9c17..a49161359e3 100644 --- a/tests/auto/network/access/qrestaccessmanager/tst_qrestaccessmanager.cpp +++ b/tests/auto/network/access/qrestaccessmanager/tst_qrestaccessmanager.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ private slots: void body(); void json(); void text(); + void download(); private: void memberHandler(QRestReply *reply); @@ -86,7 +88,7 @@ void tst_QRestAccessManager::networkRequestReply() HttpData serverSideRequest; // The request data the server received HttpData serverSideResponse; // The response data the server responds with serverSideResponse.status = 200; - server.setHandler([&](HttpData request, HttpData &response) { + server.setHandler([&](HttpData request, HttpData &response, ResponseControl&) { serverSideRequest = request; response = serverSideResponse; @@ -227,8 +229,8 @@ void tst_QRestAccessManager::abort() QTRY_COMPARE(finishedSpy.size(), 1); // Abort after request has been sent out - server.setHandler([&](HttpData, HttpData &response) { - response.respond = false; + server.setHandler([&](HttpData, HttpData&, ResponseControl &control) { + control.respond = false; manager.abortRequests(); }); manager.get(request, this, callback); @@ -480,7 +482,7 @@ void tst_QRestAccessManager::authentication() QRestReply *replyFromServer = nullptr; HttpData serverSideRequest; - server.setHandler([&](HttpData request, HttpData &response) { + server.setHandler([&](HttpData request, HttpData &response, ResponseControl&) { if (!request.headers.contains("Authorization"_ba)) { response.status = 401; response.headers.insert("WWW-Authenticate: "_ba, "Basic realm=\"secret_place\""_ba); @@ -533,7 +535,7 @@ void tst_QRestAccessManager::errors() bool errorSignalReceived = false; HttpData serverSideResponse; // The response data the server responds with - server.setHandler([&](HttpData, HttpData &response) { + server.setHandler([&](HttpData, HttpData &response, ResponseControl &) { response = serverSideResponse; }); @@ -604,7 +606,7 @@ void tst_QRestAccessManager::body() HttpData serverSideRequest; // The request data the server received HttpData serverSideResponse; // The response data the server responds with - server.setHandler([&](HttpData request, HttpData &response) { + server.setHandler([&](HttpData request, HttpData &response, ResponseControl&) { serverSideRequest = request; response = serverSideResponse; }); @@ -654,7 +656,7 @@ void tst_QRestAccessManager::json() HttpData serverSideRequest; // The request data the server received HttpData serverSideResponse; // The response data the server responds with serverSideResponse.status = 200; - server.setHandler([&](HttpData request, HttpData &response) { + server.setHandler([&](HttpData request, HttpData &response, ResponseControl&) { serverSideRequest = request; response = serverSideResponse; }); @@ -723,7 +725,7 @@ void tst_QRestAccessManager::text() HttpData serverSideRequest; // The request data the server received HttpData serverSideResponse; // The response data the server responds with serverSideResponse.status = 200; - server.setHandler([&](HttpData request, HttpData &response) { + server.setHandler([&](HttpData request, HttpData &response, ResponseControl&) { serverSideRequest = request; response = serverSideResponse; }); @@ -790,5 +792,75 @@ void tst_QRestAccessManager::text() replyFromServer = nullptr; } +void tst_QRestAccessManager::download() +{ + // Test case where data is received in chunks. + QRestAccessManager manager; + manager.setDeletesRepliesOnFinished(false); + HttpTestServer server; + QTRY_VERIFY(server.isListening()); + QNetworkRequest request(server.url()); + HttpData serverSideResponse; // The response data the server responds with + constexpr qsizetype dataSize = 1 * 1024 * 1024; // 1 MB + QByteArray expectedData{dataSize, 0}; + for (qsizetype i = 0; i < dataSize; ++i) // initialize the data we download + expectedData[i] = i % 100; + QByteArray cumulativeReceivedData; + qsizetype cumulativeReceivedBytesAvailable = 0; + + serverSideResponse.body = expectedData; + ResponseControl *responseControl = nullptr; + serverSideResponse.status = 200; + // Set content-length header so that underlying QNAM is able to report bytesTotal correctly + serverSideResponse.headers.insert("Content-Length: ", + QString::number(expectedData.size()).toLatin1()); + server.setHandler([&](HttpData, HttpData &response, ResponseControl &control) { + response = serverSideResponse; + responseControl = &control; // store for later + control.responseChunkSize = 1024; // tell testserver to send data in chunks of this size + }); + + QRestReply* reply = manager.get(request, this, [&responseControl](QRestReply */*reply*/){ + responseControl = nullptr; // all finished, no more need for controlling the response + }); + + QObject::connect(reply, &QRestReply::readyRead, this, [&](QRestReply *reply) { + static bool testOnce = true; + if (!reply->isFinished() && testOnce) { + // Test once that reading jsonObject or text of an unfinished reply will not work + testOnce = false; + QTest::ignoreMessage(QtWarningMsg, "Attempt to read json() of an unfinished" + " reply, ignoring."); + reply->json(); + QTest::ignoreMessage(QtWarningMsg, "Attempt to read text() of an unfinished reply," + " ignoring."); + (void)reply->text(); + } + + cumulativeReceivedBytesAvailable += reply->bytesAvailable(); + cumulativeReceivedData += reply->body(); + // Tell testserver that test is ready for next chunk + responseControl->readyForNextChunk = true; + }); + + qint64 totalBytes = 0; + qint64 receivedBytes = 0; + QObject::connect(reply, &QRestReply::downloadProgress, this, + [&](qint64 bytesReceived, qint64 bytesTotal) { + if (totalBytes == 0 && bytesTotal > 0) + totalBytes = bytesTotal; + receivedBytes = bytesReceived; + }); + QTRY_VERIFY(reply->isFinished()); + reply->deleteLater(); + reply = nullptr; + // Checks specific for readyRead() and bytesAvailable() + QCOMPARE(cumulativeReceivedData, expectedData); + QCOMPARE(cumulativeReceivedBytesAvailable, expectedData.size()); + // Checks specific for downloadProgress() + QCOMPARE(totalBytes, expectedData.size()); + QCOMPARE(receivedBytes, expectedData.size()); +} + QTEST_MAIN(tst_QRestAccessManager) #include "tst_qrestaccessmanager.moc"