QResource: Add API to get the decompressed content

[ChangeLog][QtCore][QResource] Added uncompressedSize() and
uncompressedData(), which will perform any required decompression on the
data, prior to returning (unlike data() and size()).

Change-Id: Ief874765cd7b43798de3fffd15aa053bc505dcb1
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>

src/corelib/io/qresource.cpp

@@ -1,7 +1,7 @@
 /****************************************************************************
 **
-** Copyright (C) 2018 The Qt Company Ltd.
+** Copyright (C) 2019 The Qt Company Ltd.
-** Copyright (C) 2019 Intel Corporation.
+** Copyright (C) 2020 Intel Corporation.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the QtCore module of the Qt Toolkit.
@@ -60,14 +60,14 @@
 #include "private/qtools_p.h"
 #include "private/qsystemerror_p.h"
 
+#ifndef QT_NO_COMPRESS
+#  include <zconf.h>
+#  include <zlib.h>
+#endif
 #if QT_CONFIG(zstd)
 #  include <zstd.h>
 #endif
 
-#ifdef Q_OS_UNIX
-# include "private/qcore_unix_p.h"
-#endif
-
 #if defined(Q_OS_UNIX) && !defined(Q_OS_NACL) && !defined(Q_OS_INTEGRITY)
 #  define QT_USE_MMAP
 #  include <sys/mman.h>
@@ -296,6 +296,8 @@ public:
 
     void ensureInitialized() const;
     void ensureChildren() const;
+    qint64 uncompressedSize() const Q_DECL_PURE_FUNCTION;
+    qsizetype decompress(char *buffer, qsizetype bufferSize) const;
 
     bool load(const QString &file);
     void clear();
@@ -440,6 +442,78 @@ QResourcePrivate::ensureChildren() const
     }
 }
 
+qint64 QResourcePrivate::uncompressedSize() const
+{
+    switch (compressionAlgo) {
+    case QResource::NoCompression:
+        return size;
+
+    case QResource::ZlibCompression:
+#ifndef QT_NO_COMPRESS
+        if (size_t(size) >= sizeof(quint32))
+            return qFromBigEndian<quint32>(data);
+#else
+        Q_ASSERT(!"QResource: Qt built without support for Zlib compression");
+        Q_UNREACHABLE();
+#endif
+        break;
+
+    case QResource::ZstdCompression: {
+#if QT_CONFIG(zstd)
+        size_t n = ZSTD_getFrameContentSize(data, size);
+        return ZSTD_isError(n) ? -1 : qint64(n);
+#else
+        // This should not happen because we've refused to load such resource
+        Q_ASSERT(!"QResource: Qt built without support for Zstd compression");
+        Q_UNREACHABLE();
+#endif
+    }
+
+    }
+    return -1;
+}
+
+qsizetype QResourcePrivate::decompress(char *buffer, qsizetype bufferSize) const
+{
+    Q_ASSERT(data);
+
+    switch (compressionAlgo) {
+    case QResource::NoCompression:
+        Q_UNREACHABLE();
+        break;
+
+    case QResource::ZlibCompression: {
+#ifndef QT_NO_COMPRESS
+        uLong len = uLong(bufferSize);
+        int res = ::uncompress(reinterpret_cast<Bytef *>(buffer), &len,
+                               data + sizeof(quint32), uLong(size - sizeof(quint32)));
+        if (res != Z_OK) {
+            qWarning("QResource: error decompressing zlib content (%d)", res);
+            return -1;
+        }
+        return len;
+#else
+        Q_UNREACHABLE();
+#endif
+    }
+
+    case QResource::ZstdCompression: {
+#if QT_CONFIG(zstd)
+        size_t usize = ZSTD_decompress(buffer, bufferSize, data, size);
+        if (ZSTD_isError(usize)) {
+            qWarning("QResource: error decompressing zstd content: %s", ZSTD_getErrorName(usize));
+            return -1;
+        }
+        return usize;
+#else
+        Q_UNREACHABLE();
+#endif
+    }
+    }
+
+    return -1;
+}
+
 /*!
     Constructs a QResource pointing to \a file. \a locale is used to
     load a specific localization of a resource data.
@@ -600,9 +674,12 @@ QResource::Compression QResource::compressionAlgorithm() const
 }
 
 /*!
-    Returns the size of the data backing the resource.
+    Returns the size of the stored data backing the resource.
 
-    \sa data(), isFile()
+    If the resource is compressed, this function returns the size of the
+    compressed data. See uncompressedSize() for the uncompressed size.
+
+    \sa data(), uncompressedSize(), isFile()
 */
 
 qint64 QResource::size() const
@@ -613,12 +690,29 @@ qint64 QResource::size() const
 }
 
 /*!
-    Returns direct access to a read only segment of data that this resource
+    \since 5.15
-    represents. If the resource is compressed the data returned is compressed
-    and the appropriate library functions must be used to access the data. If
-    the resource is a directory \nullptr is returned.
 
-    \sa size(), compressionAlgorithm(), isFile()
+    Returns the size of the data in this resource. If the data was not
+    compressed, this function returns the same as size(). If it was, then this
+    function extracts the size of the original uncompressed data from the
+    stored stream.
+
+    \sa size(), uncompressedData(), isFile()
+*/
+qint64 QResource::uncompressedSize() const
+{
+    Q_D(const QResource);
+    d->ensureInitialized();
+    return d->uncompressedSize();
+}
+
+/*!
+    Returns direct access to a segment of read-only data, that this resource
+    represents. If the resource is compressed, the data returned is also
+    compressed. The caller must then decompress the data or use
+    uncompressedData(). If the resource is a directory, \c nullptr is returned.
+
+    \sa uncompressedData(), size(), isFile()
 */
 
 const uchar *QResource::data() const
@@ -628,6 +722,42 @@ const uchar *QResource::data() const
     return d->data;
 }
 
+/*!
+    \since 5.15
+
+    Returns the resource data, decompressing it first, if the data was stored
+    compressed. If the resource is a directory or an error occurs while
+    decompressing, a null QByteArray is returned.
+
+    \note If the data was compressed, this function will decompress every time
+    it is called. The result is not cached between calls.
+
+    \sa uncompressedData(), size(), isCompressed(), isFile()
+*/
+
+QByteArray QResource::uncompressedData() const
+{
+    Q_D(const QResource);
+    qint64 n = uncompressedSize();
+    if (n < 0)
+        return QByteArray();
+    if (n > std::numeric_limits<QByteArray::size_type>::max()) {
+        qWarning("QResource: compressed content does not fit into a QByteArray; use QFile instead");
+        return QByteArray();
+    }
+    if (d->compressionAlgo == NoCompression)
+        return QByteArray::fromRawData(reinterpret_cast<const char *>(d->data), n);
+
+    // decompress
+    QByteArray result(n, Qt::Uninitialized);
+    n = d->decompress(result.data(), n);
+    if (n < 0)
+        result.clear();
+    else
+        result.truncate(n);
+    return result;
+}
+
 /*!
     \since 5.8
 
@@ -1451,13 +1581,7 @@ bool QResourceFileEngine::link(const QString &)
 qint64 QResourceFileEngine::size() const
 {
     Q_D(const QResourceFileEngine);
-    if (!d->resource.isValid())
+    return d->resource.isValid() ? d->resource.uncompressedSize() : 0;
-        return 0;
-    if (d->resource.compressionAlgorithm() != QResource::NoCompression) {
-        d->uncompress();
-        return d->uncompressed.size();
-    }
-    return d->resource.size();
 }
 
 qint64 QResourceFileEngine::pos() const
@@ -1615,12 +1739,7 @@ uchar *QResourceFileEnginePrivate::map(qint64 offset, qint64 size, QFile::Memory
     Q_Q(QResourceFileEngine);
     Q_UNUSED(flags);
 
-    qint64 max = resource.size();
+    qint64 max = resource.uncompressedSize();
-    if (resource.compressionAlgorithm() != QResource::NoCompression) {
-        uncompress();
-        max = uncompressed.size();
-    }
-
     qint64 end;
     if (offset < 0 || size <= 0 || !resource.isValid() ||
             add_overflow(offset, size, &end) || end > max) {
@@ -1629,8 +1748,12 @@ uchar *QResourceFileEnginePrivate::map(qint64 offset, qint64 size, QFile::Memory
     }
 
     const uchar *address = resource.data();
-    if (resource.compressionAlgorithm() != QResource::NoCompression)
+    if (resource.compressionAlgorithm() != QResource::NoCompression) {
+        uncompress();
+        if (uncompressed.isNull())
+            return nullptr;
         address = reinterpret_cast<const uchar *>(uncompressed.constData());
+    }
 
     return const_cast<uchar *>(address) + offset;
 }
@@ -1643,41 +1766,10 @@ bool QResourceFileEnginePrivate::unmap(uchar *ptr)
 
 void QResourceFileEnginePrivate::uncompress() const
 {
-    if (uncompressed.isEmpty() && resource.size()) {
+    if (resource.compressionAlgorithm() == QResource::NoCompression
-        quint64 size;
+            || !uncompressed.isEmpty() || resource.size() == 0)
-        switch (resource.compressionAlgorithm()) {
+        return;     // nothing to do
-        case QResource::NoCompression:
+    uncompressed = resource.uncompressedData();
-            return;     // nothing to do
-
-        case QResource::ZlibCompression:
-#ifndef QT_NO_COMPRESS
-            uncompressed = qUncompress(resource.data(), resource.size());
-#else
-            Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zlib compression");
-#endif
-            break;
-
-        case QResource::ZstdCompression:
-#if QT_CONFIG(zstd)
-            size = ZSTD_getFrameContentSize(resource.data(), resource.size());
-            if (!ZSTD_isError(size)) {
-                if (size >= MaxAllocSize) {
-                    qWarning("QResourceFileEngine::open: content bigger than memory (size %lld)", size);
-                } else {
-                    uncompressed = QByteArray(size, Qt::Uninitialized);
-                    size = ZSTD_decompress(const_cast<char *>(uncompressed.data()), size,
-                                           resource.data(), resource.size());
-                }
-            }
-            if (ZSTD_isError(size))
-                qWarning("QResourceFileEngine::open: error decoding: %s", ZSTD_getErrorName(size));
-#else
-            Q_UNUSED(size);
-            Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zstd compression");
-#endif
-            break;
-        }
-    }
 }
 
 #endif // !defined(QT_BOOTSTRAPPED)

src/corelib/io/qresource.h

@@ -1,6 +1,7 @@
 /****************************************************************************
 **
-** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2019 The Qt Company Ltd.
+** Copyright (C) 2019 Intel Corporation.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the QtCore module of the Qt Toolkit.
@@ -75,6 +76,8 @@ public:
     Compression compressionAlgorithm() const;
     qint64 size() const;
     const uchar *data() const;
+    qint64 uncompressedSize() const;
+    QByteArray uncompressedData() const;
     QDateTime lastModified() const;
 
 #if QT_DEPRECATED_SINCE(5, 13)

tests/auto/corelib/io/qresourceengine/compressed.qrc

@@ -0,0 +1,5 @@
+<RCC version="1.0">
+    <qresource>
+        <file>zero.txt</file>
+    </qresource>
+</RCC>

tests/auto/corelib/io/qresourceengine/generateResources.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+count=`awk '/ZERO_FILE_LEN/ { print $3 }' tst_qresourceengine.cpp`
+dd if=/dev/zero of=zero.txt bs=1 count=$count
+rcc --binary -o uncompressed.rcc --no-compress compressed.qrc
+rcc --binary -o zlib.rcc --compress-algo zlib --compress 9 compressed.qrc
+rcc --binary -o zstd.rcc --compress-algo zstd --compress 19 compressed.qrc
+rm zero.txt

tests/auto/corelib/io/qresourceengine/qresourceengine_test.pro

@@ -1,7 +1,7 @@
 CONFIG += testcase
 TARGET = tst_qresourceengine
 
-QT = core testlib
+QT = core-private testlib
 SOURCES = tst_qresourceengine.cpp
 RESOURCES += testqrc/test.qrc
 
@@ -15,7 +15,8 @@ QMAKE_DISTCLEAN += $${runtime_resource.target}
 
 TESTDATA += \
     parentdir.txt \
-    testqrc/*
+    testqrc/* \
+    *.rcc
 GENERATED_TESTDATA = $${runtime_resource.target}
 
 android:!android-embedded {

tests/auto/corelib/io/qresourceengine/tst_qresourceengine.cpp

@@ -27,9 +27,10 @@
 **
 ****************************************************************************/
 
-
 #include <QtTest/QtTest>
 #include <QtCore/QCoreApplication>
+#include <QtCore/QScopeGuard>
+#include <QtCore/private/qglobal_p.h>
 
 class tst_QResourceEngine: public QObject
 {
@@ -50,6 +51,8 @@ private slots:
 
     void checkUnregisterResource_data();
     void checkUnregisterResource();
+    void compressedResource_data();
+    void compressedResource();
     void checkStructure_data();
     void checkStructure();
     void searchPath_data();
@@ -105,6 +108,75 @@ void tst_QResourceEngine::cleanupTestCase()
     QVERIFY(QResource::unregisterResource(m_runtimeResourceRcc, "/secondary_root/"));
 }
 
+void tst_QResourceEngine::compressedResource_data()
+{
+    QTest::addColumn<QString>("fileName");
+    QTest::addColumn<int>("compressionAlgo");
+    QTest::addColumn<bool>("supported");
+
+    QTest::newRow("uncompressed")
+            << QFINDTESTDATA("uncompressed.rcc") << int(QResource::NoCompression) << true;
+    QTest::newRow("zlib")
+            << QFINDTESTDATA("zlib.rcc") << int(QResource::ZlibCompression) << true;
+    QTest::newRow("zstd")
+            << QFINDTESTDATA("zstd.rcc") << int(QResource::ZstdCompression) << QT_CONFIG(zstd);
+}
+
+// Note: generateResource.sh parses this line. Make sure it's a simple number.
+#define ZERO_FILE_LEN   16384
+// End note
+void tst_QResourceEngine::compressedResource()
+{
+    QFETCH(QString, fileName);
+    QFETCH(int, compressionAlgo);
+    QFETCH(bool, supported);
+    const QByteArray expectedData(ZERO_FILE_LEN, '\0');
+
+    QVERIFY(!QResource("zero.txt").isValid());
+    QCOMPARE(QResource::registerResource(fileName), supported);
+    if (!supported)
+        return;
+
+    auto unregister = qScopeGuard([=] { QResource::unregisterResource(fileName); });
+
+    QResource resource("zero.txt");
+    QVERIFY(resource.isValid());
+    QVERIFY(resource.size() > 0);
+    QVERIFY(resource.data());
+    QCOMPARE(resource.compressionAlgorithm(), QResource::Compression(compressionAlgo));
+
+    if (compressionAlgo == QResource::NoCompression) {
+        QCOMPARE(resource.size(), ZERO_FILE_LEN);
+        QCOMPARE(memcmp(resource.data(), expectedData.data(), ZERO_FILE_LEN), 0);
+
+        // API guarantees it will be QByteArray::fromRawData:
+        QCOMPARE(static_cast<const void *>(resource.uncompressedData().constData()),
+                 static_cast<const void *>(resource.data()));
+    } else {
+        // reasonable expectation:
+        QVERIFY(resource.size() < ZERO_FILE_LEN);
+    }
+
+    // using the engine
+    QFile f(":/zero.txt");
+    QVERIFY(f.exists());
+    QVERIFY(f.open(QIODevice::ReadOnly));
+
+    // verify that we can decompress correctly
+    QCOMPARE(resource.uncompressedSize(), ZERO_FILE_LEN);
+    QCOMPARE(f.size(), ZERO_FILE_LEN);
+
+    QByteArray data = resource.uncompressedData();
+    QCOMPARE(data.size(), expectedData.size());
+    QCOMPARE(data, expectedData);
+
+    // decompression through the engine
+    data = f.readAll();
+    QCOMPARE(data.size(), expectedData.size());
+    QCOMPARE(data, expectedData);
+}
+
+
 void tst_QResourceEngine::checkStructure_data()
 {
     QTest::addColumn<QString>("pathName");
@@ -140,7 +212,13 @@ void tst_QResourceEngine::checkStructure_data()
                                            << "parentdir.txt"
                                            << "runtime_resource.rcc"
 #endif
-                                           << "search_file.txt")
+                                           << "search_file.txt"
+#if defined(BUILTIN_TESTDATA)
+                                           << "uncompressed.rcc"
+                                           << "zlib.rcc"
+                                           << "zstd.rcc"
+#endif
+                                           )
                                        << rootContents
                                        << QLocale::c()
                                        << qlonglong(0);