Add GPU_BLACKLIST support to QTestLib

In addition to BLACKLIST, Qt will now look for GPU_BLACKLIST too.
Test cases that are specified as disabled in the GPU blacklist
will be skipped. This is particularly relevant when running tests
on Embedded Linux devices.

For example, the following JSON would configure the test case
glxContextWrap to be skipped on drivers where GL_VENDOR contains
UnstableDriverVendor:

{
    "entries": [ {
            "gl_vendor": "UnstableDriverVendor",
            "features": [ "disable_glxContextWrap" ]
    } ]
}

In contrast to the regular blacklist, GPU-blacklisted test cases are
not run at all. This is because driver problems and instabilities
often lead to crashes.

Change-Id: I340cf5c0261a206109b78409774408981bba5c68
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@theqtcompany.com>
Reviewed-by: Simon Hausmann <simon.hausmann@theqtcompany.com>

src/gui/opengl/qopengl.cpp

@@ -47,6 +47,8 @@
 #include <QtCore/QFile>
 #include <QtCore/QDir>
 
+#include <set>
+
 QT_BEGIN_NAMESPACE
 
 #if defined(QT_OPENGL_3)
@@ -474,4 +476,13 @@ QOpenGLConfig::Gpu QOpenGLConfig::Gpu::fromContext()
     return gpu;
 }
 
+Q_GUI_EXPORT std::set<QByteArray> *qgpu_features(const QString &filename)
+{
+    const QSet<QString> features = QOpenGLConfig::gpuFeatures(QOpenGLConfig::Gpu::fromContext(), filename);
+    std::set<QByteArray> *result = new std::set<QByteArray>;
+    foreach (const QString &feature, features)
+        result->insert(feature.toUtf8());
+    return result;
+}
+
 QT_END_NAMESPACE

src/testlib/qtest.h

@@ -289,6 +289,18 @@ int main(int argc, char *argv[]) \
 }
 
 #include <QtTest/qtestsystem.h>
+#include <set>
+
+#ifndef QT_NO_OPENGL
+#  define QTEST_ADD_GPU_BLACKLIST_SUPPORT_DEFS \
+    extern Q_TESTLIB_EXPORT std::set<QByteArray> *(*qgpu_features_ptr)(const QString &); \
+    extern Q_GUI_EXPORT std::set<QByteArray> *qgpu_features(const QString &);
+#  define QTEST_ADD_GPU_BLACKLIST_SUPPORT \
+    qgpu_features_ptr = qgpu_features;
+#else
+#  define QTEST_ADD_GPU_BLACKLIST_SUPPORT_DEFS
+#  define QTEST_ADD_GPU_BLACKLIST_SUPPORT
+#endif
 
 #if defined(QT_WIDGETS_LIB)
 
@@ -301,11 +313,15 @@ int main(int argc, char *argv[]) \
 #endif
 
 #define QTEST_MAIN(TestObject) \
+QT_BEGIN_NAMESPACE \
+QTEST_ADD_GPU_BLACKLIST_SUPPORT_DEFS \
+QT_END_NAMESPACE \
 int main(int argc, char *argv[]) \
 { \
     QApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
     QTEST_DISABLE_KEYPAD_NAVIGATION \
+    QTEST_ADD_GPU_BLACKLIST_SUPPORT \
     TestObject tc; \
     QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \
@@ -316,10 +332,14 @@ int main(int argc, char *argv[]) \
 #include <QtTest/qtest_gui.h>
 
 #define QTEST_MAIN(TestObject) \
+QT_BEGIN_NAMESPACE \
+QTEST_ADD_GPU_BLACKLIST_SUPPORT_DEFS \
+QT_END_NAMESPACE \
 int main(int argc, char *argv[]) \
 { \
     QGuiApplication app(argc, argv); \
     app.setAttribute(Qt::AA_Use96Dpi, true); \
+    QTEST_ADD_GPU_BLACKLIST_SUPPORT \
     TestObject tc; \
     QTEST_SET_MAIN_SOURCE_PATH \
     return QTest::qExec(&tc, argc, argv); \

src/testlib/qtestblacklist.cpp

@@ -34,6 +34,7 @@
 #include "qtestresult_p.h"
 
 #include <QtTest/qtestcase.h>
+#include <QtTest/qtest.h>
 #include <QtCore/qbytearray.h>
 #include <QtCore/qfile.h>
 #include <QtCore/QSysInfo>
@@ -154,6 +155,9 @@ static bool checkCondition(const QByteArray &condition)
 
 static bool ignoreAll = false;
 static std::set<QByteArray> *ignoredTests = 0;
+static std::set<QByteArray> *gpuFeatures = 0;
+
+Q_TESTLIB_EXPORT std::set<QByteArray> *(*qgpu_features_ptr)(const QString &) = 0;
 
 namespace QTestPrivate {
 
@@ -189,7 +193,18 @@ void parseBlackList()
     }
 }
 
-void checkBlackList(const char *slot, const char *data)
+void parseGpuBlackList()
+{
+    if (!qgpu_features_ptr)
+        return;
+    QString filename = QTest::qFindTestData(QStringLiteral("GPU_BLACKLIST"));
+    if (filename.isEmpty())
+        return;
+    if (!gpuFeatures)
+        gpuFeatures = qgpu_features_ptr(filename);
+}
+
+void checkBlackLists(const char *slot, const char *data)
 {
     bool ignore = ignoreAll;
 
@@ -204,6 +219,16 @@ void checkBlackList(const char *slot, const char *data)
     }
 
     QTestResult::setBlacklistCurrentTest(ignore);
+
+    // Tests blacklisted in GPU_BLACKLIST are to be skipped. Just ignoring the result is
+    // not sufficient since these are expected to crash or behave in undefined ways.
+    if (!ignore && gpuFeatures) {
+        const QByteArray disableKey = QByteArrayLiteral("disable_") + QByteArray(slot);
+        if (gpuFeatures->find(disableKey) != gpuFeatures->end()) {
+            const QByteArray msg = QByteArrayLiteral("Skipped due to GPU blacklist: ") + disableKey;
+            QTest::qSkip(msg.constData(), __FILE__, __LINE__);
+        }
+    }
 }
 
 }

src/testlib/qtestblacklist_p.h

@@ -51,7 +51,8 @@ QT_BEGIN_NAMESPACE
 
 namespace QTestPrivate {
     void parseBlackList();
-    void checkBlackList(const char *slot, const char *data);
+    void parseGpuBlackList();
+    void checkBlackLists(const char *slot, const char *data);
 }
 
 QT_END_NAMESPACE

src/testlib/qtestcase.cpp

@@ -2074,7 +2074,7 @@ static bool qInvokeTestMethod(const char *slotName, const char *data=0)
                 if (!data || !qstrcmp(data, table.testData(curDataIndex)->dataTag())) {
                     foundFunction = true;
 
-                    QTestPrivate::checkBlackList(slot, dataCount ? table.testData(curDataIndex)->dataTag() : 0);
+                    QTestPrivate::checkBlackLists(slot, dataCount ? table.testData(curDataIndex)->dataTag() : 0);
 
                     QTestDataSetter s(curDataIndex >= dataCount ? static_cast<QTestData *>(0)
                                                       : table.testData(curDataIndex));
@@ -2583,6 +2583,7 @@ int QTest::qExec(QObject *testObject, int argc, char **argv)
 #endif
 
     QTestPrivate::parseBlackList();
+    QTestPrivate::parseGpuBlackList();
 
     QTestResult::reset();