Add a few improvements to the baseline testing framework

1) Add a QBASELINETEST_MAIN macro as a replacement for QTEST_MAIN,
relieving the clients of reproducing the kludgy workaround

2) Add a -server command line option to baseline tests, as an
alternative to specifying the server in an environment variable

3) Fix command line parsing so that it exits on syntax errors.

Change-Id: I36f38267143a308e971e2e7b2fdbe4be44370043
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>
(cherry picked from commit 1036e9d4f1cbeb7c7e43b65eb1678b76ac4c2f2a)

tests/baseline/painting/tst_baseline_painting.cpp

@@ -473,17 +473,6 @@ void tst_Lancelot::paint(QPaintDevice *device, GraphicsEngine engine, QImage::Fo
     p.end();
 }
 
-#define main _realmain
+QBASELINETEST_MAIN(tst_Lancelot);
-QTEST_MAIN(tst_Lancelot)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
 
 #include "tst_baseline_painting.moc"

tests/baseline/shared/baselineprotocol.cpp

@@ -261,18 +261,21 @@ bool BaselineProtocol::disconnect()
 }
 
 
-bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo& clientInfo)
+bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo &clientInfo, const QString &server)
 {
     errMsg.clear();
-    QByteArray serverName(qgetenv("QT_LANCELOT_SERVER"));
+    QString serverName = server;
-    if (serverName.isNull())
+    if (serverName.isEmpty()) {
-        serverName = "lancelot.test.qt-project.org";
+        serverName = qEnvironmentVariable("QT_LANCELOT_SERVER");
+        if (serverName.isEmpty())
+            serverName = QStringLiteral("lancelot.test.qt-project.org");
+    }
 
     socket.connectToHost(serverName, ServerPort);
     if (!socket.waitForConnected(Timeout)) {
         QThread::sleep(std::chrono::seconds{3});  // Wait a bit and try again, the server might just be restarting
         if (!socket.waitForConnected(Timeout)) {
-            errMsg += QLS("TCP connectToHost failed. Host:") + QLS(serverName) + QLS(" port:") + QString::number(ServerPort);
+            errMsg += QLS("TCP connectToHost failed. Host:") + serverName + QLS(" port:") + QString::number(ServerPort);
             return false;
         }
     }

tests/baseline/shared/baselineprotocol.h

@@ -115,7 +115,8 @@ public:
     // For client:
 
     // For advanced client:
-    bool connect(const QString &testCase, bool *dryrun = nullptr, const PlatformInfo& clientInfo = PlatformInfo());
+    bool connect(const QString &testCase, bool *dryrun = nullptr,
+                 const PlatformInfo &clientInfo = PlatformInfo(), const QString &server = QString());
     bool disconnect();
     bool requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList);
     bool submitMatch(const ImageItem &item, QByteArray *serverMsg);

tests/baseline/shared/qbaselinetest.cpp

@@ -11,6 +11,7 @@
 namespace QBaselineTest {
 
 static char *fargv[MAXCMDLINEARGS];
+static QString server;
 static bool simfail = false;
 static PlatformInfo customInfo;
 static bool customAutoModeSet = false;
@@ -33,6 +34,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         return;
 
     bool showHelp = false;
+    bool abortOnHelp = true;
 
     int fargc = 0;
     int numArgs = *argcp;
@@ -41,7 +43,16 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         QByteArray arg = (*argvp)[i];
         QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : nullptr;
 
-        if (arg == "-simfail") {
+        if (arg == "-server") {
+            i++;
+            if (!nextArg.isEmpty()) {
+                server = QString::fromLocal8Bit(nextArg);
+            } else {
+                qWarning() << "-server requires parameter";
+                showHelp = true;
+                break;
+            }
+        } else if (arg == "-simfail") {
             simfail = true;
         } else if (arg == "-fuzzlevel") {
             i++;
@@ -77,10 +88,13 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
             }
             customInfo.addOverride(key, value);
         } else {
-            if ( (arg == "-help") || (arg == "--help") )
+            if ( (arg == "-help") || (arg == "--help") ) {
                 showHelp = true;
+                abortOnHelp = false;
+            }
             if (fargc >= MAXCMDLINEARGS) {
                 qWarning() << "Too many command line arguments!";
+                showHelp = true;
                 break;
             }
             fargv[fargc++] = (*argvp)[i];
@@ -93,7 +107,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         // TBD: arrange for this to be printed *after* QTest's help
         QTextStream out(stdout);
         out << "\n Baseline testing (lancelot) options:\n";
-        out << " -simfail            : Force an image comparison mismatch. For testing purposes.\n";
+        out << " -server <host>      : Set the network baseline server to connect to.\n";
+        out << "                       The default is taken from the environment variable QT_LANCELOT_SERVER.\n";
+        out << " -simfail            : Force an image comparison mismatch. For development purposes.\n";
         out << " -fuzzlevel <int>    : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n";
         out << " -auto               : Inform server that this run is done by a daemon, CI system or similar.\n";
         out << " -adhoc (default)    : The inverse of -auto; this run is done by human, e.g. for testing.\n";
@@ -104,6 +120,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         out << "                       for example: -compareto QtVersion=4.8.0\n";
         out << "                       Multiple -compareto client specifications may be given.\n";
         out << "\n";
+        out.flush();
+        if (abortOnHelp)
+            std::exit(1);
     }
 }
 
@@ -182,7 +201,7 @@ bool connect(QByteArray *msg, bool *error)
         return false;
     }
 
-    if (!proto.connect(testCase, &dryRunMode, clientInfo)) {
+    if (!proto.connect(testCase, &dryRunMode, clientInfo, server)) {
         *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1();
         *error = true;
         return false;

tests/baseline/shared/qbaselinetest.h

@@ -66,4 +66,9 @@ do {\
         QSKIP("Blacklisted on baseline server.");\
 } while (0)
 
+#define QBASELINETEST_MAIN(TestObject) QTEST_MAIN_WRAPPER(TestObject, \
+    QHashSeed::setDeterministicGlobalSeed(); \
+    QBaselineTest::handleCmdLineArgs(&argc, &argv); \
+    QTEST_MAIN_SETUP())
+
 #endif // BASELINETEST_H

tests/baseline/stylesheet/tst_baseline_stylesheet.cpp

@@ -224,17 +224,6 @@ void tst_Stylesheet::tst_QHeaderView()
     QBASELINE_TEST(takeSnapshot());
 }
 
-#define main _realmain
+QBASELINETEST_MAIN(tst_Stylesheet)
-QTEST_MAIN(tst_Stylesheet)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
 
 #include "tst_baseline_stylesheet.moc"

tests/baseline/text/tst_baseline_text.cpp

@@ -103,17 +103,6 @@ void tst_Text::tst_differentScriptsBackgrounds()
 }
 
 
-#define main _realmain
+QBASELINETEST_MAIN(tst_Text)
-QTEST_MAIN(tst_Text)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
 
 #include "tst_baseline_text.moc"

tests/baseline/widgets/tst_baseline_widgets.cpp

@@ -1275,17 +1275,6 @@ void tst_Widgets::tst_QLCDNumber()
     QBASELINE_CHECK_DEFERRED(takeSnapshot(), "lcdnumber");
 }
 
-#define main _realmain
+QBASELINETEST_MAIN(tst_Widgets)
-QTEST_MAIN(tst_Widgets)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
 
 #include "tst_baseline_widgets.moc"