QNetworkReply: Add two new signals

These signals allow monitoring where in the HTTP1/HTTP2
flow a request is currently in.

Fixes: QTBUG-71698
Fixes: QTBUG-18766
Change-Id: Icc2fe435afc9f680fa7a76c32731e25fcdfeb4b4
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>

src/network/access/qhttp2protocolhandler.cpp

@@ -1369,6 +1369,8 @@ quint32 QHttp2ProtocolHandler::createNewStream(const HttpMessagePair &message, b
         }
     }
 
+    QMetaObject::invokeMethod(reply, "requestSent", Qt::QueuedConnection);
+
     activeStreams.insert(newStreamID, newStream);
 
     return newStreamID;

src/network/access/qhttpnetworkconnection.cpp

@@ -749,6 +749,15 @@ QHttpNetworkRequest QHttpNetworkConnectionPrivate::predictNextRequest() const
     return QHttpNetworkRequest();
 }
 
+QHttpNetworkReply* QHttpNetworkConnectionPrivate::predictNextRequestsReply() const
+{
+    if (!highPriorityQueue.isEmpty())
+        return highPriorityQueue.last().second;
+    if (!lowPriorityQueue.isEmpty())
+        return lowPriorityQueue.last().second;
+    return nullptr;
+}
+
 // this is called from _q_startNextRequest and when a request has been sent down a socket from the channel
 void QHttpNetworkConnectionPrivate::fillPipeline(QAbstractSocket *socket)
 {

src/network/access/qhttpnetworkconnection_p.h

@@ -214,6 +214,7 @@ public:
     void prepareRequest(HttpMessagePair &request);
     void updateChannel(int i, const HttpMessagePair &messagePair);
     QHttpNetworkRequest predictNextRequest() const;
+    QHttpNetworkReply* predictNextRequestsReply() const;
 
     void fillPipeline(QAbstractSocket *socket);
     bool fillPipeline(QList<HttpMessagePair> &queue, QHttpNetworkConnectionChannel &channel);

src/network/access/qhttpnetworkconnectionchannel.cpp

@@ -356,6 +356,13 @@ bool QHttpNetworkConnectionChannel::ensureConnection()
         QString connectHost = connection->d_func()->hostName;
         quint16 connectPort = connection->d_func()->port;
 
+        QHttpNetworkReply *potentialReply = connection->d_func()->predictNextRequestsReply();
+        if (potentialReply) {
+            QMetaObject::invokeMethod(potentialReply, "socketConnecting", Qt::QueuedConnection);
+        } else if (h2RequestsToSend.count() > 0) {
+            QMetaObject::invokeMethod(h2RequestsToSend.values().at(0).second, "socketConnecting", Qt::QueuedConnection);
+        }
+
 #ifndef QT_NO_NETWORKPROXY
         // HTTPS always use transparent proxy.
         if (connection->d_func()->networkProxy.type() != QNetworkProxy::NoProxy && !ssl) {

src/network/access/qhttpnetworkreply_p.h

@@ -170,6 +170,8 @@ Q_SIGNALS:
 #endif
 
 Q_SIGNALS:
+    void socketConnecting();
+    void requestSent();
     void readyRead();
     void finished();
     void finishedWithError(QNetworkReply::NetworkError errorCode, const QString &detail = QString());

src/network/access/qhttpprotocolhandler.cpp

@@ -319,6 +319,8 @@ bool QHttpProtocolHandler::sendRequest()
 #else
         m_header = QHttpNetworkRequestPrivate::header(m_channel->request, false);
 #endif
+        QMetaObject::invokeMethod(m_reply, "requestSent", Qt::QueuedConnection);
+
         // flushing is dangerous (QSslSocket calls transmit which might read or error)
 //        m_socket->flush();
         QNonContiguousByteDevice* uploadByteDevice = m_channel->request.uploadByteDevice();

src/network/access/qhttpthreaddelegate.cpp

@@ -377,6 +377,8 @@ void QHttpThreadDelegate::startRequest()
 
         // Don't care about ignored SSL errors for now in the synchronous HTTP case.
     } else if (!synchronous) {
+        connect(httpReply,SIGNAL(socketConnecting()), this, SIGNAL(socketConnecting()));
+        connect(httpReply,SIGNAL(requestSent()), this, SIGNAL(requestSent()));
         connect(httpReply,SIGNAL(headerChanged()), this, SLOT(headerChangedSlot()));
         connect(httpReply,SIGNAL(finished()), this, SLOT(finishedSlot()));
         connect(httpReply,SIGNAL(finishedWithError(QNetworkReply::NetworkError,QString)),

src/network/access/qhttpthreaddelegate_p.h

@@ -143,6 +143,8 @@ signals:
     void sslConfigurationChanged(const QSslConfiguration &);
     void preSharedKeyAuthenticationRequired(QSslPreSharedKeyAuthenticator *);
 #endif
+    void socketConnecting();
+    void requestSent();
     void downloadMetaData(const QList<QPair<QByteArray,QByteArray> > &, int, const QString &, bool,
                           QSharedPointer<char>, qint64, qint64, bool, bool);
     void downloadProgress(qint64, qint64);

src/network/access/qnetworkreply.cpp

@@ -321,6 +321,27 @@ QNetworkReplyPrivate::QNetworkReplyPrivate()
     QNetworkRequest::RedirectPolicyAttribute
 */
 
+/*!
+    \fn void QNetworkReply::socketConnecting()
+    \since 6.3
+
+    This signal is emitted 0 or more times, when the socket
+    is connecting, before sending the request. Useful for
+    custom progress or timeout handling.
+
+    \sa metaDataChanged(), requestSent()
+*/
+
+/*!
+    \fn void QNetworkReply::requestSent()
+    \since 6.3
+
+    This signal is emitted 1 or more times when the request was
+    sent. Useful for custom progress or timeout handling.
+
+    \sa metaDataChanged(), socketConnecting()
+*/
+
 /*!
     \fn void QNetworkReply::metaDataChanged()
 

src/network/access/qnetworkreply.h

@@ -154,6 +154,8 @@ public Q_SLOTS:
     virtual void ignoreSslErrors();
 
 Q_SIGNALS:
+    void socketConnecting();
+    void requestSent();
     void metaDataChanged();
     void finished();
     void errorOccurred(QNetworkReply::NetworkError);

src/network/access/qnetworkreplyhttpimpl.cpp

@@ -869,6 +869,10 @@ void QNetworkReplyHttpImplPrivate::postRequest(const QNetworkRequest &newHttpReq
         QObject::connect(delegate, SIGNAL(downloadFinished()),
                 q, SLOT(replyFinished()),
                 Qt::QueuedConnection);
+        QObject::connect(delegate, &QHttpThreadDelegate::socketConnecting,
+                q, &QNetworkReply::socketConnecting, Qt::QueuedConnection);
+        QObject::connect(delegate, &QHttpThreadDelegate::requestSent,
+                q, &QNetworkReply::requestSent, Qt::QueuedConnection);
         connect(delegate, &QHttpThreadDelegate::downloadMetaData, this,
                 &QNetworkReplyHttpImplPrivate::replyDownloadMetaData, Qt::QueuedConnection);
         QObject::connect(delegate, SIGNAL(downloadProgress(qint64,qint64)),

tests/auto/network/access/http2/tst_http2.cpp

@@ -105,6 +105,9 @@ private slots:
     void connectToHost();
     void maxFrameSize();
 
+    void moreActivitySignals_data();
+    void moreActivitySignals();
+
     void contentEncoding_data();
     void contentEncoding();
 
@@ -785,6 +788,77 @@ void tst_Http2::maxFrameSize()
     QVERIFY(serverGotSettingsACK);
 }
 
+void tst_Http2::moreActivitySignals_data()
+{
+    QTest::addColumn<QNetworkRequest::Attribute>("h2Attribute");
+    QTest::addColumn<H2Type>("connectionType");
+
+    QTest::addRow("h2c-upgrade")
+            << QNetworkRequest::Http2AllowedAttribute << H2Type::h2c;
+    QTest::addRow("h2c-direct")
+            << QNetworkRequest::Http2DirectAttribute << H2Type::h2cDirect;
+
+    if (!clearTextHTTP2)
+        QTest::addRow("h2-ALPN")
+                << QNetworkRequest::Http2AllowedAttribute << H2Type::h2Alpn;
+
+#if QT_CONFIG(ssl)
+    QTest::addRow("h2-direct")
+            << QNetworkRequest::Http2DirectAttribute << H2Type::h2Direct;
+#endif
+}
+
+void tst_Http2::moreActivitySignals()
+{
+    clearHTTP2State();
+
+#if QT_CONFIG(securetransport)
+    // Normally on macOS we use plain text only for SecureTransport
+    // does not support ALPN on the server side. With 'direct encrytped'
+    // we have to use TLS sockets (== private key) and thus suppress a
+    // keychain UI asking for permission to use a private key.
+    // Our CI has this, but somebody testing locally - will have a problem.
+    qputenv("QT_SSL_USE_TEMPORARY_KEYCHAIN", QByteArray("1"));
+    auto envRollback = qScopeGuard([]() { qunsetenv("QT_SSL_USE_TEMPORARY_KEYCHAIN"); });
+#endif
+
+    serverPort = 0;
+    QFETCH(H2Type, connectionType);
+    ServerPtr srv(newServer(defaultServerSettings, connectionType));
+    QMetaObject::invokeMethod(srv.data(), "startServer", Qt::QueuedConnection);
+    runEventLoop(100);
+    QVERIFY(serverPort != 0);
+    auto url = requestUrl(connectionType);
+    url.setPath(QString("/stream1.html"));
+    QNetworkRequest request(url);
+    QFETCH(const QNetworkRequest::Attribute, h2Attribute);
+    request.setAttribute(h2Attribute, QVariant(true));
+    request.setHeader(QNetworkRequest::ContentTypeHeader, QVariant("text/plain"));
+    QSharedPointer<QNetworkReply> reply(manager->get(request));
+    nRequests = 1;
+    connect(reply.data(), &QNetworkReply::finished, this, &tst_Http2::replyFinished);
+    QSignalSpy spy1(reply.data(), SIGNAL(socketConnecting()));
+    QSignalSpy spy2(reply.data(), SIGNAL(requestSent()));
+    QSignalSpy spy3(reply.data(), SIGNAL(metaDataChanged()));
+    // Since we're using self-signed certificates,
+    // ignore SSL errors:
+    reply->ignoreSslErrors();
+
+    spy1.wait();
+    spy2.wait();
+    spy3.wait();
+
+    runEventLoop();
+    STOP_ON_FAILURE
+
+    QVERIFY(nRequests == 0);
+    QVERIFY(prefaceOK);
+    QVERIFY(serverGotSettingsACK);
+
+    QVERIFY(reply->error() == QNetworkReply::NoError);
+    QVERIFY(reply->isFinished());
+}
+
 void tst_Http2::contentEncoding_data()
 {
     QTest::addColumn<QByteArray>("encoding");

tests/auto/network/access/qnetworkreply/tst_qnetworkreply.cpp

@@ -507,6 +507,9 @@ private Q_SLOTS:
     void getWithTimeout();
     void postWithTimeout();
 
+    void moreActivitySignals_data();
+    void moreActivitySignals();
+
     void contentEncoding_data();
     void contentEncoding();
     void contentEncodingBigPayload_data();
@@ -9370,6 +9373,58 @@ void tst_QNetworkReply::postWithTimeout()
     manager.setTransferTimeout(0);
 }
 
+void tst_QNetworkReply::moreActivitySignals_data()
+{
+    QTest::addColumn<QUrl>("url");
+    QTest::addColumn<bool>("useipv6");
+    QTest::addRow("local4") << QUrl("http://127.0.0.1") << false;
+    QTest::addRow("local6") << QUrl("http://[::1]") << true;
+    if (qEnvironmentVariable("QTEST_ENVIRONMENT").split(' ').contains("ci")) {
+        // On CI server
+        QTest::addRow("localDns") << QUrl("http://localhost") << false; // will find v6
+    } else {
+        // For manual testing
+        QTest::addRow("localDns4") << QUrl("http://localhost") << true; // will find both v4 and v6
+        QTest::addRow("localDns6") << QUrl("http://localhost") << false; // will find both v4 and v6
+    }
+}
+
+void tst_QNetworkReply::moreActivitySignals()
+{
+    QFETCH(QUrl, url);
+    QFETCH(bool, useipv6);
+    MiniHttpServer server(tst_QNetworkReply::httpEmpty200Response, false, nullptr/*thread*/, useipv6);
+    server.doClose = false;
+    url.setPort(server.serverPort());
+    QNetworkRequest request(url);
+    QNetworkReplyPtr reply(manager.get(request));
+    QSignalSpy spy1(reply.data(), SIGNAL(socketConnecting()));
+    QSignalSpy spy2(reply.data(), SIGNAL(requestSent()));
+    QSignalSpy spy3(reply.data(), SIGNAL(metaDataChanged()));
+    QSignalSpy spy4(reply.data(), SIGNAL(finished()));
+    spy1.wait();
+    QCOMPARE(spy1.count(), 1);
+    spy2.wait();
+    QCOMPARE(spy2.count(), 1);
+    spy3.wait();
+    QCOMPARE(spy3.count(), 1);
+    spy4.wait();
+    QCOMPARE(spy4.count(), 1);
+    QVERIFY(reply->error() == QNetworkReply::NoError);
+    // Second request will not send socketConnecting because of keep-alive, so don't check it.
+    QNetworkReplyPtr secondreply(manager.get(request));
+    QSignalSpy secondspy2(secondreply.data(), SIGNAL(requestSent()));
+    QSignalSpy secondspy3(secondreply.data(), SIGNAL(metaDataChanged()));
+    QSignalSpy secondspy4(secondreply.data(), SIGNAL(finished()));
+    secondspy2.wait();
+    QCOMPARE(secondspy2.count(), 1);
+    secondspy3.wait();
+    QCOMPARE(secondspy3.count(), 1);
+    secondspy4.wait();
+    QCOMPARE(secondspy4.count(), 1);
+    QVERIFY(secondreply->error() == QNetworkReply::NoError);
+}
+
 void tst_QNetworkReply::contentEncoding_data()
 {
     QTest::addColumn<QByteArray>("encoding");