QNAM HTTP: Re-write compression code

This eliminates some code (header parsing) that can be done by
zlib already.
Add support for 'deflate' encoding.
Also do less memory copying while uncompressing.

Change-Id: I94de21e3c58b904dd91d004c375ed8cbea56cb0b
Task-Number: QTBUG-13191
Reviewed-on: http://codereview.qt.nokia.com/1314
Reviewed-by: Qt Sanity Bot <qt_sanity_bot@ovi.com>
Reviewed-by: Martin Petersson <Martin.Petersson@nokia.com>
Reviewed-by: Peter Hartmann <peter.hartmann@nokia.com>
Reviewed-by: Markus Goetz

src/network/access/qhttpnetworkconnection.cpp

@@ -271,7 +271,7 @@ void QHttpNetworkConnectionPrivate::prepareRequest(HttpMessagePair &messagePair)
     value = request.headerField("accept-encoding");
     if (value.isEmpty()) {
 #ifndef QT_NO_COMPRESS
-        request.setHeaderField("Accept-Encoding", "gzip");
+        request.setHeaderField("Accept-Encoding", "gzip, deflate");
         request.d->autoDecompress = true;
 #else
         // if zlib is not available set this to false always

src/network/access/qhttpnetworkconnection_p.h

@@ -202,9 +202,6 @@ public:
     QString errorDetail(QNetworkReply::NetworkError errorCode, QAbstractSocket *socket,
                         const QString &extraDetail = QString());
 
-#ifndef QT_NO_COMPRESS
-    bool expand(QAbstractSocket *socket, QHttpNetworkReply *reply, bool dataComplete);
-#endif
     void removeReply(QHttpNetworkReply *reply);
 
     QString hostName;

src/network/access/qhttpnetworkconnectionchannel.cpp

@@ -403,7 +403,7 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
             bytes += headerBytes;
             // If headers were parsed successfully now it is the ReadingDataState
             if (replyPrivate->state == QHttpNetworkReplyPrivate::ReadingDataState) {
-                if (replyPrivate->isGzipped() && replyPrivate->autoDecompress) {
+                if (replyPrivate->isCompressed() && replyPrivate->autoDecompress) {
                     // remove the Content-Length from header
                     replyPrivate->removeAutoDecompressHeader();
                 } else {
@@ -475,30 +475,18 @@ void QHttpNetworkConnectionChannel::_q_receiveReply()
             {
                 // use the traditional slower reading (for compressed encoding, chunked encoding,
                 // no content-length etc)
-                QByteDataBuffer byteDatas;
+                qint64 haveRead = replyPrivate->readBody(socket, &replyPrivate->responseData);
-                qint64 haveRead = replyPrivate->readBody(socket, &byteDatas);
+                if (haveRead > 0) {
-                if (haveRead) {
                     bytes += haveRead;
-                    if (replyPrivate->autoDecompress)
+                    replyPrivate->totalProgress += haveRead;
-                        replyPrivate->appendCompressedReplyData(byteDatas);
+                    if (replyPrivate->shouldEmitSignals()) {
-                    else
+                        emit reply->readyRead();
-                        replyPrivate->appendUncompressedReplyData(byteDatas);
+                        emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength);
-
-                    if (!replyPrivate->autoDecompress) {
-                        replyPrivate->totalProgress += bytes;
-                        if (replyPrivate->shouldEmitSignals()) {
-                            // important: At the point of this readyRead(), the byteDatas list must be empty,
-                            // else implicit sharing will trigger memcpy when the user is reading data!
-                            emit reply->readyRead();
-                            emit reply->dataReadProgress(replyPrivate->totalProgress, replyPrivate->bodyLength);
-                        }
                     }
-#ifndef QT_NO_COMPRESS
+                } else if (haveRead == -1) {
-                    else if (!expand(false)) { // expand a chunk if possible
+                    // Some error occured
-                        // If expand() failed we can just return, it had already called connection->emitReplyError()
+                    connection->d_func()->emitReplyError(socket, reply, QNetworkReply::ProtocolFailure);
-                        return;
+                    break;
-                    }
-#endif
                 }
             }
             // still in ReadingDataState? This function will be called again by the socket's readyRead
@@ -638,57 +626,9 @@ bool QHttpNetworkConnectionChannel::ensureConnection()
     return true;
 }
 
-
-#ifndef QT_NO_COMPRESS
-bool QHttpNetworkConnectionChannel::expand(bool dataComplete)
-{
-    Q_ASSERT(socket);
-    Q_ASSERT(reply);
-
-    qint64 total = reply->d_func()->compressedData.size();
-    if (total >= CHUNK || dataComplete) {
-         // uncompress the data
-        QByteArray content, inflated;
-        content = reply->d_func()->compressedData;
-        reply->d_func()->compressedData.clear();
-
-        int ret = Z_OK;
-        if (content.size())
-            ret = reply->d_func()->gunzipBodyPartially(content, inflated);
-        int retCheck = (dataComplete) ? Z_STREAM_END : Z_OK;
-        if (ret >= retCheck) {
-            if (inflated.size()) {
-                reply->d_func()->totalProgress += inflated.size();
-                reply->d_func()->appendUncompressedReplyData(inflated);
-                if (reply->d_func()->shouldEmitSignals()) {
-                    // important: At the point of this readyRead(), inflated must be cleared,
-                    // else implicit sharing will trigger memcpy when the user is reading data!
-                    emit reply->readyRead();
-                    emit reply->dataReadProgress(reply->d_func()->totalProgress, 0);
-                }
-            }
-        } else {
-            connection->d_func()->emitReplyError(socket, reply, QNetworkReply::ProtocolFailure);
-            return false;
-        }
-    }
-    return true;
-}
-#endif
-
-
 void QHttpNetworkConnectionChannel::allDone()
 {
     Q_ASSERT(reply);
-#ifndef QT_NO_COMPRESS
-    // expand the whole data.
-    if (reply->d_func()->expectContent() && reply->d_func()->autoDecompress && !reply->d_func()->streamEnd) {
-        bool expandResult = expand(true);
-        // If expand() failed we can just return, it had already called connection->emitReplyError()
-        if (!expandResult)
-            return;
-    }
-#endif
 
     if (!reply) {
         qWarning() << "QHttpNetworkConnectionChannel::allDone() called without reply. Please report at http://bugreports.qt.nokia.com/";

src/network/access/qhttpnetworkconnectionchannel_p.h

@@ -149,7 +149,6 @@ public:
 
     bool ensureConnection();
 
-    bool expand(bool dataComplete);
     void allDone(); // reply header + body have been read
     void handleStatus(); // called from allDone()
 

src/network/access/qhttpnetworkreply.cpp

@@ -65,6 +65,11 @@ QHttpNetworkReply::~QHttpNetworkReply()
     if (d->connection) {
         d->connection->d_func()->removeReply(this);
     }
+
+#ifndef QT_NO_COMPRESS
+    if (d->autoDecompress && d->isCompressed())
+        inflateEnd(&d->inflateStrm);
+#endif
 }
 
 QUrl QHttpNetworkReply::url() const
@@ -252,7 +257,7 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl)
       chunkedTransferEncoding(false),
       connectionCloseEnabled(true),
       forceConnectionCloseEnabled(false),
-      currentChunkSize(0), currentChunkRead(0), connection(0), initInflate(false),
+      currentChunkSize(0), currentChunkRead(0), connection(0),
       autoDecompress(false), responseData(), requestIsPrepared(false)
       ,pipeliningUsed(false), downstreamLimited(false)
       ,userProvidedDownloadBuffer(0)
@@ -274,11 +279,9 @@ void QHttpNetworkReplyPrivate::clearHttpLayerInformation()
     currentChunkRead = 0;
     connectionCloseEnabled = true;
 #ifndef QT_NO_COMPRESS
-    if (initInflate)
+    if (autoDecompress)
         inflateEnd(&inflateStrm);
 #endif
-    initInflate = false;
-    streamEnd = false;
     fields.clear();
 }
 
@@ -297,10 +300,10 @@ qint64 QHttpNetworkReplyPrivate::bytesAvailable() const
     return (state != ReadingDataState ? 0 : fragment.size());
 }
 
-bool QHttpNetworkReplyPrivate::isGzipped()
+bool QHttpNetworkReplyPrivate::isCompressed()
 {
     QByteArray encoding = headerField("content-encoding");
-    return qstricmp(encoding.constData(), "gzip") == 0;
+    return qstricmp(encoding.constData(), "gzip") == 0 || qstricmp(encoding.constData(), "deflate") == 0;
 }
 
 void QHttpNetworkReplyPrivate::removeAutoDecompressHeader()
@@ -358,120 +361,6 @@ QAuthenticatorPrivate::Method QHttpNetworkReplyPrivate::authenticationMethod(boo
     return method;
 }
 
-#ifndef QT_NO_COMPRESS
-bool QHttpNetworkReplyPrivate::gzipCheckHeader(QByteArray &content, int &pos)
-{
-    int method = 0; // method byte
-    int flags = 0;  // flags byte
-    bool ret = false;
-
-    // Assure two bytes in the buffer so we can peek ahead -- handle case
-    // where first byte of header is at the end of the buffer after the last
-    // gzip segment
-    pos = -1;
-    QByteArray &body = content;
-    int maxPos = body.size()-1;
-    if (maxPos < 1) {
-        return ret;
-    }
-
-    // Peek ahead to check the gzip magic header
-    if (body[0] != char(gz_magic[0]) ||
-        body[1] != char(gz_magic[1])) {
-        return ret;
-    }
-    pos += 2;
-    // Check the rest of the gzip header
-    if (++pos <= maxPos)
-        method = body[pos];
-    if (pos++ <= maxPos)
-        flags = body[pos];
-    if (method != Z_DEFLATED || (flags & RESERVED) != 0) {
-        return ret;
-    }
-
-    // Discard time, xflags and OS code:
-    pos += 6;
-    if (pos > maxPos)
-        return ret;
-    if ((flags & EXTRA_FIELD) && ((pos+2) <= maxPos)) { // skip the extra field
-        unsigned len =  (unsigned)body[++pos];
-        len += ((unsigned)body[++pos])<<8;
-        pos += len;
-        if (pos > maxPos)
-            return ret;
-    }
-    if ((flags & ORIG_NAME) != 0) { // skip the original file name
-        while(++pos <= maxPos && body[pos]) {}
-    }
-    if ((flags & COMMENT) != 0) {   // skip the .gz file comment
-        while(++pos <= maxPos && body[pos]) {}
-    }
-    if ((flags & HEAD_CRC) != 0) {  // skip the header crc
-        pos += 2;
-        if (pos > maxPos)
-            return ret;
-    }
-    ret = (pos < maxPos); // return failed, if no more bytes left
-    return ret;
-}
-
-int QHttpNetworkReplyPrivate::gunzipBodyPartially(QByteArray &compressed, QByteArray &inflated)
-{
-    int ret = Z_DATA_ERROR;
-    unsigned have;
-    unsigned char out[CHUNK];
-    int pos = -1;
-
-    if (!initInflate) {
-        // check the header
-        if (!gzipCheckHeader(compressed, pos))
-            return ret;
-        // allocate inflate state
-        inflateStrm.zalloc = Z_NULL;
-        inflateStrm.zfree = Z_NULL;
-        inflateStrm.opaque = Z_NULL;
-        inflateStrm.avail_in = 0;
-        inflateStrm.next_in = Z_NULL;
-        ret = inflateInit2(&inflateStrm, -MAX_WBITS);
-        if (ret != Z_OK)
-            return ret;
-        initInflate = true;
-        streamEnd = false;
-    }
-
-    //remove the header.
-    compressed.remove(0, pos+1);
-    // expand until deflate stream ends
-    inflateStrm.next_in = (unsigned char *)compressed.data();
-    inflateStrm.avail_in = compressed.size();
-    do {
-        inflateStrm.avail_out = sizeof(out);
-        inflateStrm.next_out = out;
-        ret = inflate(&inflateStrm, Z_NO_FLUSH);
-        switch (ret) {
-        case Z_NEED_DICT:
-            ret = Z_DATA_ERROR;
-            // and fall through
-        case Z_DATA_ERROR:
-        case Z_MEM_ERROR:
-            inflateEnd(&inflateStrm);
-            initInflate = false;
-            return ret;
-        }
-        have = sizeof(out) - inflateStrm.avail_out;
-        inflated.append(QByteArray((const char *)out, have));
-     } while (inflateStrm.avail_out == 0);
-    // clean up and return
-    if (ret <= Z_ERRNO || ret == Z_STREAM_END) {
-        inflateEnd(&inflateStrm);
-        initInflate = false;
-    }
-    streamEnd = (ret == Z_STREAM_END);
-    return ret;
-}
-#endif
-
 qint64 QHttpNetworkReplyPrivate::readStatus(QAbstractSocket *socket)
 {
     if (fragment.isEmpty()) {
@@ -616,6 +505,24 @@ qint64 QHttpNetworkReplyPrivate::readHeader(QAbstractSocket *socket)
         connectionCloseEnabled = (connectionHeaderField.toLower().contains("close") ||
             headerField("proxy-connection").toLower().contains("close")) ||
             (majorVersion == 1 && minorVersion == 0 && connectionHeaderField.isEmpty());
+
+#ifndef QT_NO_COMPRESS
+        if (autoDecompress && isCompressed()) {
+            // allocate inflate state
+            inflateStrm.zalloc = Z_NULL;
+            inflateStrm.zfree = Z_NULL;
+            inflateStrm.opaque = Z_NULL;
+            inflateStrm.avail_in = 0;
+            inflateStrm.next_in = Z_NULL;
+            // "windowBits can also be greater than 15 for optional gzip decoding.
+            // Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection"
+            // http://www.zlib.net/manual.html
+            int ret = inflateInit2(&inflateStrm, MAX_WBITS+32);
+            if (ret != Z_OK)
+                return -1;
+        }
+#endif
+
     }
     return bytes;
 }
@@ -712,22 +619,74 @@ qint64 QHttpNetworkReplyPrivate::readBodyFast(QAbstractSocket *socket, QByteData
 qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuffer *out)
 {
     qint64 bytes = 0;
+
+#ifndef QT_NO_COMPRESS
+    // for gzip we'll allocate a temporary one that we then decompress
+    QByteDataBuffer *tempOutDataBuffer = (autoDecompress ? new QByteDataBuffer : out);
+#else
+    QByteDataBuffer *tempOutDataBuffer = out;
+#endif
+
+
     if (isChunked()) {
         // chunked transfer encoding (rfc 2616, sec 3.6)
-        bytes += readReplyBodyChunked(socket, out);
+        bytes += readReplyBodyChunked(socket, tempOutDataBuffer);
     } else if (bodyLength > 0) {
         // we have a Content-Length
-        bytes += readReplyBodyRaw(socket, out, bodyLength - contentRead);
+        bytes += readReplyBodyRaw(socket, tempOutDataBuffer, bodyLength - contentRead);
         if (contentRead + bytes == bodyLength)
             state = AllDoneState;
     } else {
         // no content length. just read what's possible
-        bytes += readReplyBodyRaw(socket, out, socket->bytesAvailable());
+        bytes += readReplyBodyRaw(socket, tempOutDataBuffer, socket->bytesAvailable());
     }
+
+#ifndef QT_NO_COMPRESS
+    // This is true if there is compressed encoding and we're supposed to use it.
+    if (autoDecompress) {
+        qint64 uncompressRet = uncompressBodyData(tempOutDataBuffer, out);
+        delete tempOutDataBuffer;
+        if (uncompressRet < 0)
+            return -1;
+    }
+#endif
+
     contentRead += bytes;
     return bytes;
 }
 
+#ifndef QT_NO_COMPRESS
+qint64 QHttpNetworkReplyPrivate::uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out)
+{
+    for (int i = 0; i < in->bufferCount(); i++) {
+        QByteArray &bIn = (*in)[i];
+
+        inflateStrm.avail_in = bIn.size();
+        inflateStrm.next_in = reinterpret_cast<Bytef*>(bIn.data());
+
+        do {
+            QByteArray bOut;
+            // make a wild guess about the uncompressed size.
+            bOut.reserve(inflateStrm.avail_in * 3 + 512);
+            inflateStrm.avail_out = bOut.capacity();
+            inflateStrm.next_out = reinterpret_cast<Bytef*>(bOut.data());
+
+            int ret = inflate(&inflateStrm, Z_NO_FLUSH);
+            switch (ret) {
+            case Z_NEED_DICT:
+            case Z_DATA_ERROR:
+            case Z_MEM_ERROR:
+                return -1;
+            }
+            bOut.resize(bOut.capacity() - inflateStrm.avail_out);
+            out->append(bOut);
+        } while (inflateStrm.avail_in > 0);
+    }
+
+    return out->byteAmount();
+}
+#endif
+
 qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QAbstractSocket *socket, QByteDataBuffer *out, qint64 size)
 {
     // FIXME get rid of this function and just use readBodyFast and give it socket->bytesAvailable()
@@ -841,36 +800,6 @@ qint64 QHttpNetworkReplyPrivate::getChunkSize(QAbstractSocket *socket, qint64 *c
     return bytes;
 }
 
-void QHttpNetworkReplyPrivate::appendUncompressedReplyData(QByteArray &qba)
-{
-    responseData.append(qba);
-
-    // clear the original! helps with implicit sharing and
-    // avoiding memcpy when the user is reading the data
-    qba.clear();
-}
-
-void QHttpNetworkReplyPrivate::appendUncompressedReplyData(QByteDataBuffer &data)
-{
-    responseData.append(data);
-
-    // clear the original! helps with implicit sharing and
-    // avoiding memcpy when the user is reading the data
-    data.clear();
-}
-
-void QHttpNetworkReplyPrivate::appendCompressedReplyData(QByteDataBuffer &data)
-{
-    // Work in progress: Later we will directly use a list of QByteArray or a QRingBuffer
-    // instead of one QByteArray.
-    for(int i = 0; i < data.bufferCount(); i++) {
-        QByteArray &byteData = data[i];
-        compressedData.append(byteData.constData(), byteData.size());
-    }
-    data.clear();
-}
-
-
 bool QHttpNetworkReplyPrivate::shouldEmitSignals()
 {
     // for 401 & 407 don't emit the data signals. Content along with these

src/network/access/qhttpnetworkreply_p.h

@@ -56,15 +56,7 @@
 #ifndef QT_NO_HTTP
 
 #ifndef QT_NO_COMPRESS
-#    include <zlib.h>
+#include <zlib.h>
-static const unsigned char gz_magic[2] = {0x1f, 0x8b}; // gzip magic header
-// gzip flag byte
-#define HEAD_CRC     0x02 // bit 1 set: header CRC present
-#define EXTRA_FIELD  0x04 // bit 2 set: extra field present
-#define ORIG_NAME    0x08 // bit 3 set: original file name present
-#define COMMENT      0x10 // bit 4 set: file comment present
-#define RESERVED     0xE0 // bits 5..7: reserved
-#define CHUNK 16384
 #endif
 
 #include <QtNetwork/qtcpsocket.h>
@@ -192,10 +184,6 @@ public:
     qint64 readReplyBodyChunked(QAbstractSocket *in, QByteDataBuffer *out);
     qint64 getChunkSize(QAbstractSocket *in, qint64 *chunkSize);
 
-    void appendUncompressedReplyData(QByteArray &qba);
-    void appendUncompressedReplyData(QByteDataBuffer &data);
-    void appendCompressedReplyData(QByteDataBuffer &data);
-
     bool shouldEmitSignals();
     bool expectContent();
     void eraseData();
@@ -203,11 +191,8 @@ public:
     qint64 bytesAvailable() const;
     bool isChunked();
     bool isConnectionCloseEnabled();
-    bool isGzipped();
+
-#ifndef QT_NO_COMPRESS
+    bool isCompressed();
-    bool gzipCheckHeader(QByteArray &content, int &pos);
-    int gunzipBodyPartially(QByteArray &compressed, QByteArray &inflated);
-#endif
     void removeAutoDecompressHeader();
 
     enum ReplyState {
@@ -236,11 +221,12 @@ public:
     qint64 currentChunkRead;
     QPointer<QHttpNetworkConnection> connection;
     QPointer<QHttpNetworkConnectionChannel> connectionChannel;
-    bool initInflate;
+
-    bool streamEnd;
 #ifndef QT_NO_COMPRESS
     z_stream inflateStrm;
+    qint64 uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out);
 #endif
+
     bool autoDecompress;
 
     QByteDataBuffer responseData; // uncompressed body