Update baseline testing framework
Merge in various minor changes and fixes that have been done to the branched copy in qtquick3d, and clean out some outdated code. Mostly just coding style fixes and cleanups, but also: - adds -keeprunning command line parameter, intended for tests that by default exits early if it seems the platform is too unstable (e.g. crashing) for a meaningful testrun. - Changes behaviour for fuzzy matches, from SKIP to PASS. The (mis)use of QSKIP was done to force log output; now the output is just printed by qInfo() instead. Pick-to: 6.2 Change-Id: I46e77a94cc5b1980ac420086c2ae88dc9b84ef12 Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
This commit is contained in:
Eirik Aavitsland
parent
5750a9728e
commit
8a883dea1c
@ -36,6 +36,7 @@
|
||||
#endif
|
||||
#include <QFileInfo>
|
||||
#include <QDir>
|
||||
#include <QThread>
|
||||
#include <QTime>
|
||||
#include <QPointer>
|
||||
#include <QRegularExpression>
|
||||
@ -49,29 +50,7 @@ const QString PI_OSVersion(QLS("OSVersion"));
|
||||
const QString PI_QtVersion(QLS("QtVersion"));
|
||||
const QString PI_QtBuildMode(QLS("QtBuildMode"));
|
||||
const QString PI_GitCommit(QLS("GitCommit"));
|
||||
const QString PI_QMakeSpec(QLS("QMakeSpec"));
|
||||
const QString PI_PulseGitBranch(QLS("PulseGitBranch"));
|
||||
const QString PI_PulseTestrBranch(QLS("PulseTestrBranch"));
|
||||
|
||||
#ifndef QMAKESPEC
|
||||
#define QMAKESPEC "Unknown"
|
||||
#endif
|
||||
|
||||
#if defined(Q_OS_WIN)
|
||||
#include <QtCore/qt_windows.h>
|
||||
#endif
|
||||
#if defined(Q_OS_UNIX)
|
||||
#include <time.h>
|
||||
#endif
|
||||
void BaselineProtocol::sysSleep(int ms)
|
||||
{
|
||||
#if defined(Q_OS_WIN)
|
||||
Sleep(DWORD(ms));
|
||||
#else
|
||||
struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 };
|
||||
nanosleep(&ts, NULL);
|
||||
#endif
|
||||
}
|
||||
const QString PI_GitBranch(QLS("GitBranch"));
|
||||
|
||||
PlatformInfo::PlatformInfo()
|
||||
: QMap<QString, QString>(), adHoc(true)
|
||||
@ -83,7 +62,6 @@ PlatformInfo PlatformInfo::localHostInfo()
|
||||
PlatformInfo pi;
|
||||
pi.insert(PI_HostName, QHostInfo::localHostName());
|
||||
pi.insert(PI_QtVersion, QLS(qVersion()));
|
||||
pi.insert(PI_QMakeSpec, QString(QLS(QMAKESPEC)).remove(QRegularExpression(QLS("^.*mkspecs/"))));
|
||||
pi.insert(PI_QtBuildMode, QLibraryInfo::isDebugBuild() ? QLS("QtDebug") : QLS("QtRelease"));
|
||||
#if defined(Q_OS_LINUX) && QT_CONFIG(process)
|
||||
pi.insert(PI_OSName, QLS("Linux"));
|
||||
@ -113,27 +91,15 @@ PlatformInfo PlatformInfo::localHostInfo()
|
||||
pi.insert(PI_GitCommit, QString::fromLocal8Bit(git.readAllStandardOutput().constData()).simplified());
|
||||
else
|
||||
pi.insert(PI_GitCommit, QLS("Unknown"));
|
||||
|
||||
QByteArray gb = qgetenv("PULSE_GIT_BRANCH");
|
||||
if (!gb.isEmpty()) {
|
||||
pi.insert(PI_PulseGitBranch, QString::fromLatin1(gb));
|
||||
pi.setAdHocRun(false);
|
||||
}
|
||||
QByteArray tb = qgetenv("PULSE_TESTR_BRANCH");
|
||||
if (!tb.isEmpty()) {
|
||||
pi.insert(PI_PulseTestrBranch, QString::fromLatin1(tb));
|
||||
pi.setAdHocRun(false);
|
||||
}
|
||||
if (!qgetenv("JENKINS_HOME").isEmpty()) {
|
||||
pi.setAdHocRun(false);
|
||||
gb = qgetenv("GIT_BRANCH");
|
||||
if (!gb.isEmpty()) {
|
||||
// FIXME: the string "Pulse" should be eliminated, since that is not the used tool.
|
||||
pi.insert(PI_PulseGitBranch, QString::fromLatin1(gb));
|
||||
}
|
||||
}
|
||||
#endif // QT_CONFIG(process)
|
||||
|
||||
if (qEnvironmentVariableIsSet("JENKINS_HOME"))
|
||||
pi.setAdHocRun(false);
|
||||
|
||||
QString gb = qEnvironmentVariable("GIT_BRANCH");
|
||||
if (!gb.isEmpty())
|
||||
pi.insert(PI_GitBranch, gb);
|
||||
|
||||
return pi;
|
||||
}
|
||||
|
||||
@ -286,8 +252,7 @@ void ImageItem::writeImageToStream(QDataStream &out) const
|
||||
out << quint8('Q') << quint8(image.format());
|
||||
out << quint8(QSysInfo::ByteOrder) << quint8(0); // pad to multiple of 4 bytes
|
||||
out << quint32(image.width()) << quint32(image.height()) << quint32(image.bytesPerLine());
|
||||
out << qCompress(reinterpret_cast<const uchar *>(image.constBits()),
|
||||
int(image.sizeInBytes()));
|
||||
out << qCompress(reinterpret_cast<const uchar *>(image.constBits()), image.sizeInBytes());
|
||||
//# can be followed by colormap for formats that use it
|
||||
}
|
||||
|
||||
@ -360,7 +325,7 @@ bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const Plat
|
||||
|
||||
socket.connectToHost(serverName, ServerPort);
|
||||
if (!socket.waitForConnected(Timeout)) {
|
||||
sysSleep(3000); // Wait a bit and try again, the server might just be restarting
|
||||
QThread::msleep(3000); // Wait a bit and try again, the server might just be restarting
|
||||
if (!socket.waitForConnected(Timeout)) {
|
||||
errMsg += QLS("TCP connectToHost failed. Host:") + QLS(serverName) + QLS(" port:") + QString::number(ServerPort);
|
||||
return false;
|
||||
@ -425,7 +390,7 @@ bool BaselineProtocol::requestBaselineChecksums(const QString &testFunction, Ima
|
||||
if (!itemList)
|
||||
return false;
|
||||
|
||||
for(ImageItemList::iterator it = itemList->begin(); it != itemList->end(); it++)
|
||||
for (ImageItemList::iterator it = itemList->begin(); it != itemList->end(); it++)
|
||||
it->testFunction = testFunction;
|
||||
|
||||
QByteArray block;
|
||||
@ -448,7 +413,7 @@ bool BaselineProtocol::submitMatch(const ImageItem &item, QByteArray *serverMsg)
|
||||
{
|
||||
Command cmd;
|
||||
ImageItem smallItem = item;
|
||||
smallItem.image = QImage(); // No need to waste bandwith sending image (identical to baseline) to server
|
||||
smallItem.image = QImage(); // No need to waste bandwidth sending image (identical to baseline) to server
|
||||
return (sendItem(AcceptMatch, smallItem) && receiveBlock(&cmd, serverMsg) && cmd == Ack);
|
||||
}
|
||||
|
||||
|
||||
@ -51,9 +51,7 @@ extern const QString PI_OSVersion;
|
||||
extern const QString PI_QtVersion;
|
||||
extern const QString PI_QtBuildMode;
|
||||
extern const QString PI_GitCommit;
|
||||
extern const QString PI_QMakeSpec;
|
||||
extern const QString PI_PulseGitBranch;
|
||||
extern const QString PI_PulseTestrBranch;
|
||||
extern const QString PI_GitBranch;
|
||||
|
||||
class PlatformInfo : public QMap<QString, QString>
|
||||
{
|
||||
@ -174,7 +172,6 @@ private:
|
||||
|
||||
bool sendBlock(Command cmd, const QByteArray &block);
|
||||
bool receiveBlock(Command *cmd, QByteArray *block);
|
||||
void sysSleep(int ms);
|
||||
|
||||
QString errMsg;
|
||||
QTcpSocket socket;
|
||||
|
||||
@ -204,7 +204,7 @@ quint32 initval) /* the previous hash, or an arbitrary value */
|
||||
}
|
||||
|
||||
/*------------------------------------------- handle the last 3 quint32's */
|
||||
switch(length) /* all the case statements fall through */
|
||||
switch (length) /* all the case statements fall through */
|
||||
{
|
||||
case 3 : c+=k[2];
|
||||
Q_FALLTHROUGH();
|
||||
@ -253,7 +253,7 @@ quint32 *pb) /* IN: more seed OUT: secondary hash value */
|
||||
}
|
||||
|
||||
/*------------------------------------------- handle the last 3 quint32's */
|
||||
switch(length) /* all the case statements fall through */
|
||||
switch (length) /* all the case statements fall through */
|
||||
{
|
||||
case 3 : c+=k[2];
|
||||
Q_FALLTHROUGH();
|
||||
@ -332,7 +332,7 @@ quint32 hashlittle( const void *key, size_t length, quint32 initval)
|
||||
*/
|
||||
#ifndef VALGRIND
|
||||
|
||||
switch(length)
|
||||
switch (length)
|
||||
{
|
||||
case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
|
||||
case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break;
|
||||
@ -352,7 +352,7 @@ quint32 hashlittle( const void *key, size_t length, quint32 initval)
|
||||
#else /* make valgrind happy */
|
||||
|
||||
const quint8 *k8 = (const quint8 *)k;
|
||||
switch(length)
|
||||
switch (length)
|
||||
{
|
||||
case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
|
||||
case 11: c+=((quint32)k8[10])<<16;
|
||||
@ -396,7 +396,7 @@ quint32 hashlittle( const void *key, size_t length, quint32 initval)
|
||||
|
||||
/*----------------------------- handle the last (probably partial) block */
|
||||
k8 = (const quint8 *)k;
|
||||
switch(length)
|
||||
switch (length)
|
||||
{
|
||||
case 12: c+=k[4]+(((quint32)k[5])<<16);
|
||||
b+=k[2]+(((quint32)k[3])<<16);
|
||||
@ -455,7 +455,7 @@ quint32 hashlittle( const void *key, size_t length, quint32 initval)
|
||||
}
|
||||
|
||||
/*-------------------------------- last block: affect all 32 bits of (c) */
|
||||
switch(length) /* all the case statements fall through */
|
||||
switch (length) /* all the case statements fall through */
|
||||
{
|
||||
case 12: c+=((quint32)k[11])<<24;
|
||||
Q_FALLTHROUGH();
|
||||
@ -540,7 +540,7 @@ void hashlittle2(
|
||||
*/
|
||||
#ifndef VALGRIND
|
||||
|
||||
switch(length)
|
||||
switch (length)
|
||||
{
|
||||
case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
|
||||
case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break;
|
||||
@ -560,7 +560,7 @@ void hashlittle2(
|
||||
#else /* make valgrind happy */
|
||||
|
||||
const quint8 *k8 = (const quint8 *)k;
|
||||
switch(length)
|
||||
switch (length)
|
||||
{
|
||||
case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
|
||||
case 11: c+=((quint32)k8[10])<<16;
|
||||
@ -604,7 +604,7 @@ void hashlittle2(
|
||||
|
||||
/*----------------------------- handle the last (probably partial) block */
|
||||
k8 = (const quint8 *)k;
|
||||
switch(length)
|
||||
switch (length)
|
||||
{
|
||||
case 12: c+=k[4]+(((quint32)k[5])<<16);
|
||||
b+=k[2]+(((quint32)k[3])<<16);
|
||||
@ -663,7 +663,7 @@ void hashlittle2(
|
||||
}
|
||||
|
||||
/*-------------------------------- last block: affect all 32 bits of (c) */
|
||||
switch(length) /* all the case statements fall through */
|
||||
switch (length) /* all the case statements fall through */
|
||||
{
|
||||
case 12: c+=((quint32)k[11])<<24;
|
||||
Q_FALLTHROUGH();
|
||||
@ -740,7 +740,7 @@ quint32 hashbig( const void *key, size_t length, quint32 initval)
|
||||
*/
|
||||
#ifndef VALGRIND
|
||||
|
||||
switch(length)
|
||||
switch (length)
|
||||
{
|
||||
case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
|
||||
case 11: c+=k[2]&0xffffff00; b+=k[1]; a+=k[0]; break;
|
||||
@ -760,7 +760,7 @@ quint32 hashbig( const void *key, size_t length, quint32 initval)
|
||||
#else /* make valgrind happy */
|
||||
|
||||
const quint8 *k8 = (const quint8 *)k;
|
||||
switch(length) /* all the case statements fall through */
|
||||
switch (length) /* all the case statements fall through */
|
||||
{
|
||||
case 12: c+=k[2]; b+=k[1]; a+=k[0]; break;
|
||||
case 11: c+=((quint32)k8[10])<<8;
|
||||
@ -811,7 +811,7 @@ quint32 hashbig( const void *key, size_t length, quint32 initval)
|
||||
}
|
||||
|
||||
/*-------------------------------- last block: affect all 32 bits of (c) */
|
||||
switch(length) /* all the case statements fall through */
|
||||
switch (length) /* all the case statements fall through */
|
||||
{
|
||||
case 12: c+=k[11];
|
||||
Q_FALLTHROUGH();
|
||||
|
||||
@ -45,6 +45,7 @@ static bool connected = false;
|
||||
static bool triedConnecting = false;
|
||||
static bool dryRunMode = false;
|
||||
static enum { UploadMissing, UploadAll, UploadNone } baselinePolicy = UploadMissing;
|
||||
static bool abortIfUnstable = true;
|
||||
|
||||
static QByteArray curFunction;
|
||||
static ImageItemList itemList;
|
||||
@ -66,7 +67,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
|
||||
|
||||
for (int i = 0; i < numArgs; i++) {
|
||||
QByteArray arg = (*argvp)[i];
|
||||
QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : 0;
|
||||
QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : nullptr;
|
||||
|
||||
if (arg == "-simfail") {
|
||||
simfail = true;
|
||||
@ -88,6 +89,8 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
|
||||
customInfo.setAdHocRun(true);
|
||||
} else if (arg == "-setbaselines") {
|
||||
baselinePolicy = UploadAll;
|
||||
} else if (arg == "-keeprunning") {
|
||||
abortIfUnstable = false;
|
||||
} else if (arg == "-nosetbaselines") {
|
||||
baselinePolicy = UploadNone;
|
||||
} else if (arg == "-compareto") {
|
||||
@ -122,6 +125,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
|
||||
out << " -fuzzlevel <int> : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n";
|
||||
out << " -auto : Inform server that this run is done by a daemon, CI system or similar.\n";
|
||||
out << " -adhoc (default) : The inverse of -auto; this run is done by human, e.g. for testing.\n";
|
||||
out << " -keeprunning : Run all tests even if the system is unstable \n";
|
||||
out << " -setbaselines : Store ALL rendered images as new baselines. Forces replacement of previous baselines.\n";
|
||||
out << " -nosetbaselines : Do not store rendered images as new baselines when previous baselines are missing.\n";
|
||||
out << " -compareto KEY=VAL : Force comparison to baselines from a different client,\n";
|
||||
@ -131,6 +135,10 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
|
||||
}
|
||||
}
|
||||
|
||||
bool shouldAbortIfUnstable()
|
||||
{
|
||||
return abortIfUnstable;
|
||||
}
|
||||
|
||||
void addClientProperty(const QString& key, const QString& value)
|
||||
{
|
||||
@ -184,7 +192,8 @@ bool connect(QByteArray *msg, bool *error)
|
||||
// Merge the platform info set by the program with the protocols default info
|
||||
PlatformInfo clientInfo = customInfo;
|
||||
PlatformInfo defaultInfo = PlatformInfo::localHostInfo();
|
||||
foreach (QString key, defaultInfo.keys()) {
|
||||
const auto &defaultInfoKeys = defaultInfo.keys();
|
||||
for (const QString &key : defaultInfoKeys) {
|
||||
if (!clientInfo.contains(key))
|
||||
clientInfo.insert(key, defaultInfo.value(key));
|
||||
}
|
||||
@ -319,8 +328,7 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
|
||||
bool fuzzyMatch = false;
|
||||
bool res = proto.submitMismatch(item, &srvMsg, &fuzzyMatch);
|
||||
if (res && fuzzyMatch) {
|
||||
*error = true; // To force a QSKIP/debug output; somewhat kludgy
|
||||
*msg += srvMsg;
|
||||
qInfo() << "Baseline server reports:" << srvMsg;
|
||||
return true; // The server decides: a fuzzy match means no mismatch
|
||||
}
|
||||
*msg += "Mismatch. See report:\n " + srvMsg;
|
||||
|
||||
@ -30,6 +30,7 @@
|
||||
#define BASELINETEST_H
|
||||
|
||||
#include <QTest>
|
||||
#include <QString>
|
||||
|
||||
namespace QBaselineTest {
|
||||
void setAutoMode(bool mode);
|
||||
@ -41,6 +42,7 @@ bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArra
|
||||
bool testImage(const QImage& img, QByteArray *msg, bool *error);
|
||||
QTestData &newRow(const char *dataTag, quint16 checksum = 0);
|
||||
bool disconnectFromBaselineServer();
|
||||
bool shouldAbortIfUnstable();
|
||||
}
|
||||
|
||||
#define QBASELINE_CHECK_SUM(image, name, checksum)\
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user