QSslServer: Check that first byte is ClientHello

SecureTransport ignores any content that comes in until it is large
enough to be a handshake. So a plaintext client may be left hanging
while it is waiting for a response.

Pick-to: 6.4
Change-Id: I501ae61d89d516765c7ba5f0d916d9246fde5d4d
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

src/network/ssl/qsslserver.cpp

@@ -256,6 +256,8 @@ void QSslServer::incomingConnection(qintptr socket)
                         pSslSocket->deleteLater();
                 });
         connect(pSslSocket, &QSslSocket::encrypted, this, [this, pSslSocket]() {
+            Q_D(QSslServer);
+            d->removeSocketData(quintptr(pSslSocket));
             pSslSocket->disconnect(this);
             addPendingConnection(pSslSocket);
         });
@@ -278,10 +280,63 @@ void QSslServer::incomingConnection(qintptr socket)
                     Q_EMIT handshakeInterruptedOnError(pSslSocket, error);
                 });
 
-        Q_EMIT startedEncryptionHandshake(pSslSocket);
+        d_func()->initializeHandshakeProcess(pSslSocket);
-
-        pSslSocket->startServerEncryption();
     }
 }
 
+void QSslServerPrivate::initializeHandshakeProcess(QSslSocket *socket)
+{
+    Q_Q(QSslServer);
+    QMetaObject::Connection readyRead = QObject::connect(
+            socket, &QSslSocket::readyRead, q, [this]() { checkClientHelloAndContinue(); });
+
+    QMetaObject::Connection destroyed =
+            QObject::connect(socket, &QSslSocket::destroyed, q, [this](QObject *obj) {
+                // This cast is not safe to use since the socket is inside the
+                // QObject dtor, but we only use the pointer value!
+                removeSocketData(quintptr(obj));
+            });
+    socketData.emplace(quintptr(socket), readyRead, destroyed);
+}
+
+// This function may be called while in the socket's QObject dtor, __never__ use
+// the socket for anything other than a lookup!
+void QSslServerPrivate::removeSocketData(quintptr socket)
+{
+    auto it = socketData.find(socket);
+    if (it != socketData.end()) {
+        it->disconnectSignals();
+        socketData.erase(it);
+    }
+}
+
+void QSslServerPrivate::checkClientHelloAndContinue()
+{
+    Q_Q(QSslServer);
+    QSslSocket *socket = qobject_cast<QSslSocket *>(q->sender());
+    if (Q_UNLIKELY(!socket) || socket->bytesAvailable() <= 0)
+        return;
+
+    char byte = '\0';
+    if (socket->peek(&byte, 1) != 1) {
+        socket->deleteLater();
+        return;
+    }
+
+    auto it = socketData.find(quintptr(socket));
+    const bool foundData = it != socketData.end();
+    if (foundData && it->readyReadConnection)
+        QObject::disconnect(std::exchange(it->readyReadConnection, {}));
+
+    constexpr char CLIENT_HELLO = 0x16;
+    if (byte != CLIENT_HELLO) {
+        socket->disconnectFromHost();
+        socket->deleteLater();
+        return;
+    }
+
+    socket->startServerEncryption();
+    Q_EMIT q->startedEncryptionHandshake(socket);
+}
+
 QT_END_NAMESPACE

src/network/ssl/qsslserver_p.h

@@ -16,8 +16,13 @@
 // We mean it.
 //
 
+#include <QtNetwork/private/qtnetworkglobal_p.h>
+
+#include <QtCore/qhash.h>
+
 #include <QtNetwork/QSslConfiguration>
 #include <QtNetwork/private/qtcpserver_p.h>
+#include <utility>
 
 QT_BEGIN_NAMESPACE
 
@@ -27,6 +32,26 @@ public:
     Q_DECLARE_PUBLIC(QSslServer)
 
     QSslServerPrivate();
+    void checkClientHelloAndContinue();
+    void initializeHandshakeProcess(QSslSocket *socket);
+    void removeSocketData(quintptr socket);
+
+    struct SocketData {
+        QMetaObject::Connection readyReadConnection;
+        QMetaObject::Connection destroyedConnection;
+
+        SocketData(QMetaObject::Connection readyRead, QMetaObject::Connection destroyed)
+            : readyReadConnection(readyRead), destroyedConnection(destroyed)
+        {
+        }
+
+        void disconnectSignals()
+        {
+            QObject::disconnect(std::exchange(readyReadConnection, {}));
+            QObject::disconnect(std::exchange(destroyedConnection, {}));
+        }
+    };
+    QHash<quintptr, SocketData> socketData;
 
     QSslConfiguration sslConfiguration;
 };

tests/auto/network/ssl/qsslserver/tst_qsslserver.cpp

@@ -23,6 +23,7 @@ private slots:
     void testHandshakeInterruptedOnError();
     void testPreSharedKeyAuthenticationRequired();
 #endif
+    void plaintextClient();
 
 private:
     QString testDataDir;
@@ -436,6 +437,28 @@ void tst_QSslServer::testPreSharedKeyAuthenticationRequired()
 
 #endif
 
+void tst_QSslServer::plaintextClient()
+{
+    QSslConfiguration serverConfiguration = selfSignedServerQSslConfiguration();
+    SslServerSpy server(serverConfiguration);
+    QVERIFY(server.server.listen());
+
+    QTcpSocket socket;
+    QSignalSpy socketDisconnectedSpy(&socket, &QTcpSocket::disconnected);
+    socket.connectToHost(QHostAddress::LocalHost, server.server.serverPort());
+    QVERIFY(socket.waitForConnected());
+    QTest::qWait(100);
+    // No disconnect from short break...:
+    QCOMPARE(socket.state(), QAbstractSocket::SocketState::ConnectedState);
+
+    // ... but we write some plaintext data...:
+    socket.write("Hello World!");
+    socket.waitForBytesWritten();
+    // ... and quickly get disconnected:
+    QTRY_COMPARE_GT(socketDisconnectedSpy.count(), 0);
+    QCOMPARE(socket.state(), QAbstractSocket::SocketState::UnconnectedState);
+}
+
 QTEST_MAIN(tst_QSslServer)
 
 #include "tst_qsslserver.moc"