1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QResource: add support for resources compressed with Zstandard

Browse Source 
See next commit for details on why this is a good idea.

[ChangeLog][Important Behavior Changes] The Qt resource system now
supports compressing content using the Zstandard (https://zstd.net)
algorithm. Compared to zlib, it compresses better for the same CPU time,
so this algorithm is the default. QResource::isCompressed() returns true
for either compression algorithm. Use QResource::compressionAlgorithm()
to find out which algorithm to decompress. QFile will automatically
decompress using the correct algorithm.

Change-Id: I343f2beed55440a7ac0bfffd1562e9a8f94933a7
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@qt.io>
Reviewed-by: Samuel Gaist <samuel.gaist@idiap.ch>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>
This commit is contained in:
Thiago Macieira 2018-10-31 22:42:14 -07:00
parent 163b5c0278
commit f25bc30d8d
6 changed files with 140 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
configure.json
View File

@ -136,7 +136,8 @@
"Werror": { "type": "boolean", "name": "warnings_are_errors" },
"widgets": "boolean",
"xplatform": "string",
"zlib": { "type": "enum", "name": "system-zlib", "values": { "system": "yes", "qt": "no" } }
"zlib": { "type": "enum", "name": "system-zlib", "values": { "system": "yes", "qt": "no" } },
"zstd": "boolean"
},
"prefix": {
"D": "defines",
@ -167,6 +168,21 @@
{ "libs": "-s USE_ZLIB=1", "condition": "config.wasm" }
]
},
"zstd": {
"label": "Zstandard",
"test": {
"include": "zstd.h",
"main": [
"(void) ZSTD_compress(NULL, 0, NULL, 0, 1);",
"unsigned long long n = ZSTD_getFrameContentSize(NULL, 0);",
"(void) ZSTD_decompress(NULL, 0, NULL, n);"
]
},
"sources": [
{ "type": "pkgConfig", "args": "libzstd >= 1.3" },
"-lzstd"
]
},
"dbus": {
"label": "D-Bus >= 1.2",
"test": {
@ -1126,6 +1142,11 @@
"condition": "libs.zlib",
"output": [ "privateFeature" ]
},
"zstd": {
"label": "Zstandard support",
"condition": "libs.zstd",
"output": [ "privateFeature" ]
},
"thread": {
"label": "Thread support",
"purpose": "Provides QThread and related classes.",
@ -1452,7 +1473,8 @@ Configure with '-qreal float' to create a build that is binary-compatible with 5
"entries": [
"pkg-config",
"libudev",
"system-zlib"
"system-zlib",
"zstd"
]
}
]

1
src/corelib/global/qconfig-bootstrapped.h
View File

@ -121,6 +121,7 @@
#define QT_FEATURE_topleveldomain -1
#define QT_NO_TRANSLATION
#define QT_FEATURE_translation -1
#define QT_FEATURE_zstd -1
#ifdef QT_BUILD_QMAKE
#define QT_FEATURE_commandlineparser -1

2
src/corelib/io/io.pri
View File

@ -79,6 +79,8 @@ SOURCES += \
io/qloggingcategory.cpp \
io/qloggingregistry.cpp
qtConfig(zstd): QMAKE_USE_PRIVATE += zstd
qtConfig(filesystemwatcher) {
HEADERS += \
io/qfilesystemwatcher.h \

123
src/corelib/io/qresource.cpp
View File

@ -1,6 +1,7 @@
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Copyright (C) 2018 The Qt Company Ltd.
** Copyright (C) 2018 Intel Corporation.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtCore module of the Qt Toolkit.
@ -56,8 +57,13 @@
#include "private/qabstractfileengine_p.h"
#include "private/qnumeric_p.h"
#include "private/qsimd_p.h"
#include "private/qtools_p.h"
#include "private/qsystemerror_p.h"
#if QT_CONFIG(zstd)
# include <zstd.h>
#endif
#ifdef Q_OS_UNIX
# include "private/qcore_unix_p.h"
#endif
@ -100,8 +106,10 @@ class QResourceRoot
{
enum Flags
{
// must match rcc.h
Compressed = 0x01,
Directory = 0x02
Directory = 0x02,
CompressedZstd = 0x04
};
const uchar *tree, *names, *payloads;
int version;
@ -117,7 +125,15 @@ public:
virtual ~QResourceRoot() { }
int findNode(const QString &path, const QLocale &locale=QLocale()) const;
inline bool isContainer(int node) const { return flags(node) & Directory; }
inline bool isCompressed(int node) const { return flags(node) & Compressed; }
QResource::Compression compressionAlgo(int node)
{
uint compressionFlags = flags(node) & (Compressed | CompressedZstd);
if (compressionFlags == Compressed)
return QResource::ZlibCompression;
if (compressionFlags == CompressedZstd)
return QResource::ZstdCompression;
return QResource::NoCompression;
}
const uchar *data(int node, qint64 *size) const;
quint64 lastModified(int node) const;
QStringList children(int node) const;
@ -229,6 +245,23 @@ static inline QStringList *resourceSearchPaths()
\sa {The Qt Resource System}, QFile, QDir, QFileInfo
*/
/*!
\enum QResource::Compression
\since 5.13
This enum is used by compressionAlgorithm() to indicate which algorithm the
RCC tool used to compress the payload.
\value NoCompression Contents are not compressed (isCompressed() is false).
\value ZlibCompression Contents are compressed using \l{zlib}{https://zlib.net} and can
be decompressed using the qUncompress() function.
\value ZstdCompression Contents are compressed using \l{zstd}{https://zstd.net}. To
decompress, use the \c{ZSTD_decompress} function from the zstd
library.
\sa compressionAlgorithm(), isCopressed()
*/
class QResourcePrivate {
public:
inline QResourcePrivate(QResource *_q) : q_ptr(_q) { clear(); }
@ -243,12 +276,13 @@ public:
QLocale locale;
QString fileName, absoluteFilePath;
QList<QResourceRoot*> related;
uint container : 1;
mutable uint compressed : 1;
mutable qint64 size;
mutable quint64 lastModified;
mutable const uchar *data;
mutable QStringList children;
mutable quint64 lastModified;
mutable quint8 compressionAlgo;
bool container;
/* 2 or 6 padding bytes */
QResource *q_ptr;
Q_DECLARE_PUBLIC(QResource)
@ -258,7 +292,7 @@ void
QResourcePrivate::clear()
{
absoluteFilePath.clear();
compressed = 0;
compressionAlgo = QResource::NoCompression;
data = 0;
size = 0;
children.clear();
@ -287,11 +321,11 @@ QResourcePrivate::load(const QString &file)
container = res->isContainer(node);
if(!container) {
data = res->data(node, &size);
compressed = res->isCompressed(node);
compressionAlgo = res->compressionAlgo(node);
} else {
data = 0;
data = nullptr;
size = 0;
compressed = 0;
compressionAlgo = QResource::NoCompression;
}
lastModified = res->lastModified(node);
} else if(res->isContainer(node) != container) {
@ -301,9 +335,9 @@ QResourcePrivate::load(const QString &file)
related.append(res);
} else if(res->mappingRootSubdir(file)) {
container = true;
data = 0;
data = nullptr;
size = 0;
compressed = 0;
compressionAlgo = QResource::NoCompression;
lastModified = 0;
res->ref.ref();
related.append(res);
@ -493,16 +527,41 @@ bool QResource::isValid() const
/*!
Returns \c true if the resource represents a file and the data backing it
is in a compressed format, false otherwise.
is in a compressed format, false otherwise. If the data is compressed,
check compressionAlgorithm() to verify what algorithm to use to decompress
the data.
\sa data(), isFile()
\sa data(), compressionType(), isFile()
*/
bool QResource::isCompressed() const
{
return compressionAlgorithm() != NoCompression;
}
/*!
\since 5.13
Returns the compression type that this resource is compressed with, if any.
If it is not compressed, this function returns QResource::NoCompression.
If this function returns QResource::ZlibCompression, you may decompress the
data using the qUncompress() function. Up until Qt 5.13, this was the only
possible compression algorithm.
If this function returns QResource::ZstdCompression, you need to use the
Zstandard library functios (\c{<zstd.h> header). Qt does not provide a
wrapper.
See \l{http://facebook.github.io/zstd/zstd_manual.html}{Zstandard manual}.
\sa isCompressed(), data(), isFile()
*/
QResource::Compression QResource::compressionAlgorithm() const
{
Q_D(const QResource);
d->ensureInitialized();
return d->compressed;
return Compression(d->compressionAlgo);
}
/*!
@ -1531,12 +1590,40 @@ bool QResourceFileEnginePrivate::unmap(uchar *ptr)
void QResourceFileEnginePrivate::uncompress() const
{
if (resource.isCompressed() && uncompressed.isEmpty() && resource.size()) {
if (uncompressed.isEmpty() && resource.size()) {
quint64 size;
switch (resource.compressionAlgorithm()) {
case QResource::NoCompression:
return; // nothing to do
case QResource::ZlibCompression:
#ifndef QT_NO_COMPRESS
uncompressed = qUncompress(resource.data(), resource.size());
uncompressed = qUncompress(resource.data(), resource.size());
#else
Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for compression");
Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zlib compression");
#endif
break;
case QResource::ZstdCompression:
#if QT_CONFIG(zstd)
size = ZSTD_getFrameContentSize(resource.data(), resource.size());
if (!ZSTD_isError(size)) {
if (size >= MaxAllocSize) {
qWarning("QResourceFileEngine::open: content bigger than memory (size %lld)", size);
} else {
uncompressed = QByteArray(size, Qt::Uninitialized);
size = ZSTD_decompress(const_cast<char *>(uncompressed.data()), size,
resource.data(), resource.size());
}
}
if (ZSTD_isError(size))
qWarning("QResourceFileEngine::open: error decoding: %s", ZSTD_getErrorName(size));
#else
Q_UNUSED(size);
Q_ASSERT(!"QResourceFileEngine::open: Qt built without support for Zstd compression");
#endif
break;
}
}
}

7
src/corelib/io/qresource.h
View File

@ -54,6 +54,12 @@ class QResourcePrivate;
class Q_CORE_EXPORT QResource
{
public:
enum Compression {
NoCompression,
ZlibCompression,
ZstdCompression
};
QResource(const QString &file=QString(), const QLocale &locale=QLocale());
~QResource();
@ -67,6 +73,7 @@ public:
bool isValid() const;
bool isCompressed() const;
Compression compressionAlgorithm() const;
qint64 size() const;
const uchar *data() const;
QDateTime lastModified() const;

1
src/tools/rcc/rcc.cpp
View File

@ -94,6 +94,7 @@ class RCCFileInfo
public:
enum Flags
{
// must match qresource.cpp
NoFlags = 0x00,
Compressed = 0x01,
Directory = 0x02