1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add a few improvements to the baseline testing framework

Browse Source 
1) Add a QBASELINETEST_MAIN macro as a replacement for QTEST_MAIN,
relieving the clients of reproducing the kludgy workaround

2) Add a -server command line option to baseline tests, as an
alternative to specifying the server in an environment variable

3) Fix command line parsing so that it exits on syntax errors.

Change-Id: I36f38267143a308e971e2e7b2fdbe4be44370043
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
(cherry picked from commit 1036e9d4f1cbeb7c7e43b65eb1678b76ac4c2f2a)
This commit is contained in:
Eirik Aavitsland 2024-06-09 18:27:09 +02:00
parent 3cdc9084ba
commit a721428478
8 changed files with 42 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tests/baseline/painting/tst_baseline_painting.cpp
View File

@ -473,17 +473,6 @@ void tst_Lancelot::paint(QPaintDevice *device, GraphicsEngine engine, QImage::Fo
p.end();
}
#define main _realmain
QTEST_MAIN(tst_Lancelot)
#undef main
int main(int argc, char *argv[])
{
// Avoid rendering variations caused by QHash randomization
QHashSeed::setDeterministicGlobalSeed();
QBaselineTest::handleCmdLineArgs(&argc, &argv);
return _realmain(argc, argv);
}
QBASELINETEST_MAIN(tst_Lancelot);
#include "tst_baseline_painting.moc"

13
tests/baseline/shared/baselineprotocol.cpp
View File

@ -261,18 +261,21 @@ bool BaselineProtocol::disconnect()
}
bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo& clientInfo)
bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo &clientInfo, const QString &server)
{
errMsg.clear();
QByteArray serverName(qgetenv("QT_LANCELOT_SERVER"));
if (serverName.isNull())
serverName = "lancelot.test.qt-project.org";
QString serverName = server;
if (serverName.isEmpty()) {
serverName = qEnvironmentVariable("QT_LANCELOT_SERVER");
if (serverName.isEmpty())
serverName = QStringLiteral("lancelot.test.qt-project.org");
}
socket.connectToHost(serverName, ServerPort);
if (!socket.waitForConnected(Timeout)) {
QThread::sleep(std::chrono::seconds{3}); // Wait a bit and try again, the server might just be restarting
if (!socket.waitForConnected(Timeout)) {
errMsg += QLS("TCP connectToHost failed. Host:") + QLS(serverName) + QLS(" port:") + QString::number(ServerPort);
errMsg += QLS("TCP connectToHost failed. Host:") + serverName + QLS(" port:") + QString::number(ServerPort);
return false;
}
}

3
tests/baseline/shared/baselineprotocol.h
View File

@ -115,7 +115,8 @@ public:
// For client:
// For advanced client:
bool connect(const QString &testCase, bool *dryrun = nullptr, const PlatformInfo& clientInfo = PlatformInfo());
bool connect(const QString &testCase, bool *dryrun = nullptr,
const PlatformInfo &clientInfo = PlatformInfo(), const QString &server = QString());
bool disconnect();
bool requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList);
bool submitMatch(const ImageItem &item, QByteArray *serverMsg);

27
tests/baseline/shared/qbaselinetest.cpp
View File

@ -11,6 +11,7 @@
namespace QBaselineTest {
static char *fargv[MAXCMDLINEARGS];
static QString server;
static bool simfail = false;
static PlatformInfo customInfo;
static bool customAutoModeSet = false;
@ -33,6 +34,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
return;
bool showHelp = false;
bool abortOnHelp = true;
int fargc = 0;
int numArgs = *argcp;
@ -41,7 +43,16 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
QByteArray arg = (*argvp)[i];
QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : nullptr;
if (arg == "-simfail") {
if (arg == "-server") {
i++;
if (!nextArg.isEmpty()) {
server = QString::fromLocal8Bit(nextArg);
} else {
qWarning() << "-server requires parameter";
showHelp = true;
break;
}
} else if (arg == "-simfail") {
simfail = true;
} else if (arg == "-fuzzlevel") {
i++;
@ -77,10 +88,13 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
}
customInfo.addOverride(key, value);
} else {
if ( (arg == "-help") || (arg == "--help") )
if ( (arg == "-help") || (arg == "--help") ) {
showHelp = true;
abortOnHelp = false;
}
if (fargc >= MAXCMDLINEARGS) {
qWarning() << "Too many command line arguments!";
showHelp = true;
break;
}
fargv[fargc++] = (*argvp)[i];
@ -93,7 +107,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
// TBD: arrange for this to be printed *after* QTest's help
QTextStream out(stdout);
out << "\n Baseline testing (lancelot) options:\n";
out << " -simfail : Force an image comparison mismatch. For testing purposes.\n";
out << " -server <host> : Set the network baseline server to connect to.\n";
out << " The default is taken from the environment variable QT_LANCELOT_SERVER.\n";
out << " -simfail : Force an image comparison mismatch. For development purposes.\n";
out << " -fuzzlevel <int> : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n";
out << " -auto : Inform server that this run is done by a daemon, CI system or similar.\n";
out << " -adhoc (default) : The inverse of -auto; this run is done by human, e.g. for testing.\n";
@ -104,6 +120,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
out << " for example: -compareto QtVersion=4.8.0\n";
out << " Multiple -compareto client specifications may be given.\n";
out << "\n";
out.flush();
if (abortOnHelp)
std::exit(1);
}
}
@ -182,7 +201,7 @@ bool connect(QByteArray *msg, bool *error)
return false;
}
if (!proto.connect(testCase, &dryRunMode, clientInfo)) {
if (!proto.connect(testCase, &dryRunMode, clientInfo, server)) {
*msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1();
*error = true;
return false;

5
tests/baseline/shared/qbaselinetest.h
View File

@ -66,4 +66,9 @@ do {\
QSKIP("Blacklisted on baseline server.");\
} while (0)
#define QBASELINETEST_MAIN(TestObject) QTEST_MAIN_WRAPPER(TestObject, \
QHashSeed::setDeterministicGlobalSeed(); \
QBaselineTest::handleCmdLineArgs(&argc, &argv); \
QTEST_MAIN_SETUP())
#endif // BASELINETEST_H

13
tests/baseline/stylesheet/tst_baseline_stylesheet.cpp
View File

@ -224,17 +224,6 @@ void tst_Stylesheet::tst_QHeaderView()
QBASELINE_TEST(takeSnapshot());
}
#define main _realmain
QTEST_MAIN(tst_Stylesheet)
#undef main
int main(int argc, char *argv[])
{
// Avoid rendering variations caused by QHash randomization
QHashSeed::setDeterministicGlobalSeed();
QBaselineTest::handleCmdLineArgs(&argc, &argv);
return _realmain(argc, argv);
}
QBASELINETEST_MAIN(tst_Stylesheet)
#include "tst_baseline_stylesheet.moc"

13
tests/baseline/text/tst_baseline_text.cpp
View File

@ -103,17 +103,6 @@ void tst_Text::tst_differentScriptsBackgrounds()
}
#define main _realmain
QTEST_MAIN(tst_Text)
#undef main
int main(int argc, char *argv[])
{
// Avoid rendering variations caused by QHash randomization
QHashSeed::setDeterministicGlobalSeed();
QBaselineTest::handleCmdLineArgs(&argc, &argv);
return _realmain(argc, argv);
}
QBASELINETEST_MAIN(tst_Text)
#include "tst_baseline_text.moc"

13
tests/baseline/widgets/tst_baseline_widgets.cpp
View File

@ -1275,17 +1275,6 @@ void tst_Widgets::tst_QLCDNumber()
QBASELINE_CHECK_DEFERRED(takeSnapshot(), "lcdnumber");
}
#define main _realmain
QTEST_MAIN(tst_Widgets)
#undef main
int main(int argc, char *argv[])
{
// Avoid rendering variations caused by QHash randomization
QHashSeed::setDeterministicGlobalSeed();
QBaselineTest::handleCmdLineArgs(&argc, &argv);
return _realmain(argc, argv);
}
QBASELINETEST_MAIN(tst_Widgets)
#include "tst_baseline_widgets.moc"