1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QDecompressHelper: Introduce zstd support

Browse Source 
Also take this opportunity to reshuffle the content-encodings in the
intended ordering since the ordering is used to signify priority.

Task-number: QTBUG-83269
Change-Id: I022eecf1ba03b54dbd9c98a9d63d05fb05fd2124
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
This commit is contained in:
Mårten Nordheim 2020-05-06 18:30:43 +02:00
parent 94b7a4f9b8
commit d40f88e8d2
13 changed files with 252 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/network/.prev_CMakeLists.txt
View File

@ -128,6 +128,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http
WrapBrotli::WrapBrotliDec
)
qt_extend_target(Network CONDITION QT_FEATURE_http AND QT_FEATURE_zstd
LIBRARIES
ZSTD::ZSTD
)
qt_extend_target(Network CONDITION QT_FEATURE_system_zlib
LIBRARIES
ZLIB::ZLIB

5
src/network/CMakeLists.txt
View File

@ -128,6 +128,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http
WrapBrotli::WrapBrotliDec
)
qt_extend_target(Network CONDITION QT_FEATURE_http AND QT_FEATURE_zstd
LIBRARIES
ZSTD::ZSTD
)
qt_extend_target(Network CONDITION QT_FEATURE_system_zlib
LIBRARIES
ZLIB::ZLIB

4
src/network/access/access.pri
View File

@ -120,4 +120,8 @@ qtConfig(http) {
qtConfig(brotli) {
QMAKE_USE_PRIVATE += brotli
}
qtConfig(zstd) {
QMAKE_USE_PRIVATE += zstd
}
}

82
src/network/access/qdecompresshelper.cpp
View File

@ -48,6 +48,10 @@
# include <brotli/decode.h>
#endif
#if QT_CONFIG(zstd)
# include <zstd.h>
#endif
#include <array>
QT_BEGIN_NAMESPACE
@ -59,11 +63,14 @@ struct ContentEncodingMapping
};
constexpr ContentEncodingMapping contentEncodingMapping[] {
{ "gzip", QDecompressHelper::GZip },
{ "deflate", QDecompressHelper::Deflate },
#if QT_CONFIG(zstd)
{ "zstd", QDecompressHelper::Zstandard },
#endif
#if QT_CONFIG(brotli)
{ "br", QDecompressHelper::Brotli },
#endif
{ "gzip", QDecompressHelper::GZip },
{ "deflate", QDecompressHelper::Deflate },
};
QDecompressHelper::ContentEncoding encodingFromByteArray(const QByteArray &ce) noexcept
@ -86,6 +93,13 @@ BrotliDecoderState *toBrotliPointer(void *ptr)
return static_cast<BrotliDecoderState *>(ptr);
}
#endif
#if QT_CONFIG(zstd)
ZSTD_DStream *toZstandardPointer(void *ptr)
{
return static_cast<ZSTD_DStream *>(ptr);
}
#endif
}
bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding)
@ -153,6 +167,13 @@ bool QDecompressHelper::setEncoding(ContentEncoding ce)
decoderPointer = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
#else
Q_UNREACHABLE();
#endif
break;
case Zstandard:
#if QT_CONFIG(zstd)
decoderPointer = ZSTD_createDStream();
#else
Q_UNREACHABLE();
#endif
break;
}
@ -349,6 +370,9 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize)
case Brotli:
bytesRead = readBrotli(data, maxSize);
break;
case Zstandard:
bytesRead = readZstandard(data, maxSize);
break;
}
if (bytesRead == -1)
clear();
@ -398,6 +422,14 @@ void QDecompressHelper::clear()
BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
if (brotliDecoderState)
BrotliDecoderDestroyInstance(brotliDecoderState);
#endif
break;
}
case Zstandard: {
#if QT_CONFIG(zstd)
ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
if (zstdStream)
ZSTD_freeDStream(zstdStream);
#endif
break;
}
@ -604,4 +636,50 @@ qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize)
#endif
}
qsizetype QDecompressHelper::readZstandard(char *data, const qsizetype maxSize)
{
#if !QT_CONFIG(zstd)
Q_UNUSED(data);
Q_UNUSED(maxSize);
Q_UNREACHABLE();
#else
ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
QByteArray input;
if (!compressedDataBuffer.isEmpty())
input = compressedDataBuffer.read();
ZSTD_inBuffer inBuf { input.constData(), size_t(input.size()), 0 };
ZSTD_outBuffer outBuf { data, size_t(maxSize), 0 };
bool dataLeftover = false;
qsizetype bytesDecoded = 0;
while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) {
dataLeftover = false;
size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf);
if (ZSTD_isError(retValue)) {
qWarning("ZStandard error: %s", ZSTD_getErrorName(retValue));
return -1;
} else if (retValue >= 0) {
decoderHasData = false;
bytesDecoded = outBuf.pos;
// if pos == size then there may be data left over in internal buffers
if (outBuf.pos == outBuf.size) {
decoderHasData = true;
} else if (inBuf.pos == inBuf.size && !compressedDataBuffer.isEmpty()) {
input = compressedDataBuffer.read();
inBuf = { input.constData(), size_t(input.size()), 0 };
}
}
}
if (inBuf.pos < inBuf.size) {
// Some input was left unused; move back to the buffer
input = input.mid(QByteArray::size_type(inBuf.pos));
compressedDataBuffer.prepend(std::move(input));
}
return bytesDecoded;
#endif
}
QT_END_NAMESPACE

2
src/network/access/qdecompresshelper_p.h
View File

@ -67,6 +67,7 @@ public:
Deflate,
GZip,
Brotli,
Zstandard,
};
QDecompressHelper() = default;
@ -103,6 +104,7 @@ private:
qsizetype readZLib(char *data, qsizetype maxSize);
qsizetype readBrotli(char *data, qsizetype maxSize);
qsizetype readZstandard(char *data, qsizetype maxSize);
QByteDataBuffer compressedDataBuffer;
bool decoderHasData = false;

5
tests/auto/network/access/http2/tst_http2.cpp
View File

@ -801,6 +801,11 @@ void tst_Http2::contentEncoding_data()
"hello world");
#endif
#if QT_CONFIG(zstd)
contentEncodingData.emplace_back(
"zstd", QByteArray::fromBase64("KLUv/QRYWQAAaGVsbG8gd29ybGRoaR6y"), "hello world");
#endif
// Loop through and add the data...
for (const auto &data : contentEncodingData) {
const char *name = data.contentEncoding.data();

1
tests/auto/network/access/qdecompresshelper/CMakeLists.txt
View File

@ -9,6 +9,7 @@ qt_add_test(tst_qdecompresshelper
gzip.rcc.cpp
inflate.rcc.cpp
tst_qdecompresshelper.cpp
zstandard.rcc.cpp
DEFINES
SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR} # special case
PUBLIC_LIBRARIES

1
tests/auto/network/access/qdecompresshelper/qdecompresshelper.pro
View File

@ -7,5 +7,6 @@ SOURCES += \
tst_qdecompresshelper.cpp \
gzip.rcc.cpp \
inflate.rcc.cpp \
zstandard.rcc.cpp \
DEFINES += SRC_DIR="$$PWD"

21
tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp
View File

@ -73,9 +73,15 @@ void tst_QDecompressHelper::initTestCase()
{
Q_INIT_RESOURCE(gzip);
Q_INIT_RESOURCE(inflate);
#if QT_CONFIG(zstd)
Q_INIT_RESOURCE(zstandard);
#endif
}
void tst_QDecompressHelper::cleanupTestCase()
{
#if QT_CONFIG(zstd)
Q_CLEANUP_RESOURCE(zstandard);
#endif
Q_CLEANUP_RESOURCE(inflate);
Q_CLEANUP_RESOURCE(gzip);
}
@ -98,6 +104,11 @@ void tst_QDecompressHelper::encodingSupported()
++expected;
#endif
#if QT_CONFIG(zstd)
QVERIFY(QDecompressHelper::isSupportedEncoding("zstd"));
QVERIFY(accepted.contains("zstd"));
++expected;
#endif
QCOMPARE(expected, accepted.size());
}
@ -129,6 +140,12 @@ void tst_QDecompressHelper::sharedDecompress_data()
<< QByteArray("br") << QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD")
<< QByteArray("hello world");
#endif
#if QT_CONFIG(zstd)
QTest::newRow("zstandard-hello-world")
<< QByteArray("zstd") << QByteArray::fromBase64("KLUv/QRYWQAAaGVsbG8gd29ybGRoaR6y")
<< QByteArray("hello world");
#endif
}
void tst_QDecompressHelper::decompress_data()
@ -340,6 +357,10 @@ void tst_QDecompressHelper::decompressBigData_data()
#if QT_CONFIG(brotli)
QTest::newRow("brotli-4G") << QByteArray("br") << (srcDir + "/4G.br") << fourGiB;
#endif
#if QT_CONFIG(zstd)
QTest::newRow("zstandard-4G") << QByteArray("zstd") << (":/4G.zst") << fourGiB;
#endif
}
void tst_QDecompressHelper::decompressBigData()

116
tests/auto/network/access/qdecompresshelper/zstandard.rcc.cpp Normal file
View File

@ -0,0 +1,116 @@
/****************************************************************************
** Resource object code
**
** Created by: The Resource Compiler for Qt version 6.0.0
**
** WARNING! All changes made in this file will be lost!
*****************************************************************************/
static const unsigned char qt_resource_data[] = {
// D:/projects/qt/dev/src/qtbase/tests/auto/network/access/decompresshelper/4G.zst
0x0,0x0,0x1,0x75,
0x0,
0x2,0x3,0x93,0x78,0xda,0xed,0xd4,0x21,0xe,0x83,0x40,0x10,0x86,0xd1,0x29,0x98,
0x26,0x98,0x3d,0x46,0x1d,0x1a,0x8f,0xec,0x29,0x50,0xdc,0x84,0x13,0xe1,0x2b,0x90,
0x1c,0x89,0xcd,0x32,0xe9,0x25,0x2a,0xfa,0x26,0x79,0xc9,0xe8,0x5f,0x7c,0xaf,0x7d,
0xac,0xc7,0x1a,0x79,0x8f,0xf4,0x8e,0x78,0xe6,0x73,0xb5,0xa9,0x74,0x5d,0x94,0x0,
0xfe,0xcf,0xfc,0xed,0x41,0x6d,0xe7,0x50,0xcc,0x1,0x32,0x60,0xe,0x90,0x1,0x73,
0x80,0xc,0x98,0x4,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,
0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,
0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,
0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,
0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,
0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,
0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,
0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,
0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,
0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,
0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,
0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,
0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,
0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,
0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,
0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,
0x64,0x0,0x90,0x1,0x40,0x6,0x0,0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x0,
0x19,0x0,0x64,0x0,0x90,0x1,0x40,0x6,0x80,0x5f,0xe8,0xd3,0xf2,0x69,0xdb,0xd,
0xcd,0x15,0x90,0xe9,
};
static const unsigned char qt_resource_name[] = {
// 4G.zst
0x0,0x6,
0x3,0x8a,0x61,0xa4,
0x0,0x34,
0x0,0x47,0x0,0x2e,0x0,0x7a,0x0,0x73,0x0,0x74,
};
static const unsigned char qt_resource_struct[] = {
// :
0x0,0x0,0x0,0x0,0x0,0x2,0x0,0x0,0x0,0x1,0x0,0x0,0x0,0x1,
0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,
// :/4G.zst
0x0,0x0,0x0,0x0,0x0,0x1,0x0,0x0,0x0,0x1,0x0,0x0,0x0,0x0,
0x0,0x0,0x1,0x72,0x1c,0x8d,0x7,0xac,
};
#ifdef QT_NAMESPACE
# define QT_RCC_PREPEND_NAMESPACE(name) ::QT_NAMESPACE::name
# define QT_RCC_MANGLE_NAMESPACE0(x) x
# define QT_RCC_MANGLE_NAMESPACE1(a, b) a##_##b
# define QT_RCC_MANGLE_NAMESPACE2(a, b) QT_RCC_MANGLE_NAMESPACE1(a,b)
# define QT_RCC_MANGLE_NAMESPACE(name) QT_RCC_MANGLE_NAMESPACE2( \
QT_RCC_MANGLE_NAMESPACE0(name), QT_RCC_MANGLE_NAMESPACE0(QT_NAMESPACE))
#else
# define QT_RCC_PREPEND_NAMESPACE(name) name
# define QT_RCC_MANGLE_NAMESPACE(name) name
#endif
#ifdef QT_NAMESPACE
namespace QT_NAMESPACE {
#endif
bool qRegisterResourceData(int, const unsigned char *, const unsigned char *, const unsigned char *);
bool qUnregisterResourceData(int, const unsigned char *, const unsigned char *, const unsigned char *);
#if defined(__ELF__) || defined(__APPLE__)
static inline unsigned char qResourceFeatureZlib()
{
extern const unsigned char qt_resourceFeatureZlib;
return qt_resourceFeatureZlib;
}
#else
unsigned char qResourceFeatureZlib();
#endif
#ifdef QT_NAMESPACE
}
#endif
int QT_RCC_MANGLE_NAMESPACE(qInitResources_zstandard)();
int QT_RCC_MANGLE_NAMESPACE(qInitResources_zstandard)()
{
int version = 3;
QT_RCC_PREPEND_NAMESPACE(qRegisterResourceData)
(version, qt_resource_struct, qt_resource_name, qt_resource_data);
return 1;
}
int QT_RCC_MANGLE_NAMESPACE(qCleanupResources_zstandard)();
int QT_RCC_MANGLE_NAMESPACE(qCleanupResources_zstandard)()
{
int version = 3;
version += QT_RCC_PREPEND_NAMESPACE(qResourceFeatureZlib());
QT_RCC_PREPEND_NAMESPACE(qUnregisterResourceData)
(version, qt_resource_struct, qt_resource_name, qt_resource_data);
return 1;
}
namespace {
struct initializer {
initializer() { QT_RCC_MANGLE_NAMESPACE(qInitResources_zstandard)(); }
~initializer() { QT_RCC_MANGLE_NAMESPACE(qCleanupResources_zstandard)(); }
} dummy;
}

8
tests/auto/network/access/qnetworkreply/tst_qnetworkreply.cpp
View File

@ -9270,6 +9270,14 @@ void tst_QNetworkReply::contentEncoding_data()
<< QByteArray("br") << QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD")
<< QByteArray("hello world");
#endif
#if defined(QT_BUILD_INTERNAL) && QT_CONFIG(zstd)
QTest::newRow("zstandard-hello-world")
<< QByteArray("zstd") << QByteArray::fromBase64("KLUv/QRYWQAAaGVsbG8gd29ybGRoaR6y")
<< QByteArray("hello world");
#else
qDebug("Note: ZStandard testdata is only available for developer builds.");
#endif
}
void tst_QNetworkReply::contentEncoding()

BIN
tests/benchmarks/network/access/qdecompresshelper/50mb.txt.zst Normal file
View File

Binary file not shown.

4
tests/benchmarks/network/access/qdecompresshelper/main.cpp
View File

@ -56,6 +56,10 @@ void tst_QDecompressHelper::decompress_data()
#if QT_CONFIG(brotli)
QTest::addRow("brotli") << QByteArray("br") << srcDir + QString("50mb.txt.br");
dataAdded = true;
#endif
#if QT_CONFIG(zstd)
QTest::addRow("zstandard") << QByteArray("zstd") << srcDir + QString("50mb.txt.zst");
dataAdded = true;
#endif
if (!dataAdded)
QSKIP("There's no decompression support");