vulkan: Add debug message filtering

[ChangeLog][QtGui] Added support for filtering Vulkan debug messages in
QVulkanInstance. This is especially useful for processing or suppressing
messages from the validation layers.

Change-Id: Idf0d7889085948daf5b1a53d2a9b11081e967609
Reviewed-by: Paul Olav Tvete <paul.tvete@qt.io>

src/gui/vulkan/qplatformvulkaninstance.cpp

@@ -85,4 +85,9 @@ void QPlatformVulkanInstance::presentQueued(QWindow *window)
     Q_UNUSED(window);
 }
 
+void QPlatformVulkanInstance::setDebugFilters(const QVector<QVulkanInstance::DebugFilter> &filters)
+{
+    Q_UNUSED(filters);
+}
+
 QT_END_NAMESPACE

src/gui/vulkan/qplatformvulkaninstance.h

@@ -78,6 +78,7 @@ public:
     virtual PFN_vkVoidFunction getInstanceProcAddr(const char *name) = 0;
     virtual bool supportsPresent(VkPhysicalDevice physicalDevice, uint32_t queueFamilyIndex, QWindow *window) = 0;
     virtual void presentQueued(QWindow *window);
+    virtual void setDebugFilters(const QVector<QVulkanInstance::DebugFilter> &filters);
 
 private:
     QScopedPointer<QPlatformVulkanInstancePrivate> d_ptr;

src/gui/vulkan/qvulkaninstance.cpp

@@ -269,6 +269,7 @@ public:
     VkResult errorCode;
     QScopedPointer<QVulkanFunctions> funcs;
     QHash<VkDevice, QVulkanDeviceFunctions *> deviceFuncs;
+    QVector<QVulkanInstance::DebugFilter> debugFilters;
 };
 
 bool QVulkanInstancePrivate::ensureVulkan()
@@ -570,6 +571,7 @@ bool QVulkanInstance::create()
         d_ptr->extensions = d_ptr->platformInst->enabledExtensions();
         d_ptr->errorCode = VK_SUCCESS;
         d_ptr->funcs.reset(new QVulkanFunctions(this));
+        d_ptr->platformInst->setDebugFilters(d_ptr->debugFilters);
         return true;
     }
 
@@ -785,6 +787,50 @@ void QVulkanInstance::presentQueued(QWindow *window)
     d_ptr->platformInst->presentQueued(window);
 }
 
+/*!
+    \typedef QVulkanInstance::DebugFilter
+
+    Typedef for debug filtering callback functions.
+
+    \sa installDebugOutputFilter(), removeDebugOutputFilter()
+ */
+
+/*!
+    Installs a \a filter function that is called for every Vulkan debug
+    message. When the callback returns \c true, the message is stopped (filtered
+    out) and will not appear on the debug output.
+
+    \note Filtering is only effective when NoDebugOutputRedirect is not
+    \l{setFlags()}{set}. Installing filters has no effect otherwise.
+
+    \note This function can be called before create().
+
+    \sa removeDebugOutputFilter()
+ */
+void QVulkanInstance::installDebugOutputFilter(DebugFilter filter)
+{
+    if (!d_ptr->debugFilters.contains(filter)) {
+        d_ptr->debugFilters.append(filter);
+        if (d_ptr->platformInst)
+            d_ptr->platformInst->setDebugFilters(d_ptr->debugFilters);
+    }
+}
+
+/*!
+    Removes a \a filter function previously installed by
+    installDebugOutputFilter().
+
+    \note This function can be called before create().
+
+    \sa installDebugOutputFilter()
+ */
+void QVulkanInstance::removeDebugOutputFilter(DebugFilter filter)
+{
+    d_ptr->debugFilters.removeOne(filter);
+    if (d_ptr->platformInst)
+        d_ptr->platformInst->setDebugFilters(d_ptr->debugFilters);
+}
+
 #ifndef QT_NO_DEBUG_STREAM
 QDebug operator<<(QDebug dbg, const QVulkanLayer &layer)
 {

src/gui/vulkan/qvulkaninstance.h

@@ -188,6 +188,11 @@ public:
 
     void presentQueued(QWindow *window);
 
+    typedef bool (*DebugFilter)(VkDebugReportFlagsEXT flags, VkDebugReportObjectTypeEXT objectType, uint64_t object,
+                                size_t location, int32_t messageCode, const char *pLayerPrefix, const char *pMessage);
+    void installDebugOutputFilter(DebugFilter filter);
+    void removeDebugOutputFilter(DebugFilter filter);
+
 private:
     QScopedPointer<QVulkanInstancePrivate> d_ptr;
     Q_DISABLE_COPY(QVulkanInstance)

src/platformsupport/vkconvenience/qbasicvulkanplatforminstance.cpp

@@ -330,6 +330,11 @@ bool QBasicPlatformVulkanInstance::supportsPresent(VkPhysicalDevice physicalDevi
     return supported;
 }
 
+void QBasicPlatformVulkanInstance::setDebugFilters(const QVector<QVulkanInstance::DebugFilter> &filters)
+{
+    m_debugFilters = filters;
+}
+
 void QBasicPlatformVulkanInstance::destroySurface(VkSurfaceKHR surface) const
 {
     if (m_destroySurface && surface)
@@ -345,11 +350,11 @@ static VKAPI_ATTR VkBool32 VKAPI_CALL defaultDebugCallbackFunc(VkDebugReportFlag
                                                                const char *pMessage,
                                                                void *pUserData)
 {
-    Q_UNUSED(flags);
-    Q_UNUSED(objectType);
-    Q_UNUSED(object);
-    Q_UNUSED(location);
-    Q_UNUSED(pUserData);
+    QBasicPlatformVulkanInstance *self = static_cast<QBasicPlatformVulkanInstance *>(pUserData);
+    for (QVulkanInstance::DebugFilter filter : *self->debugFilters()) {
+        if (filter(flags, objectType, object, location, messageCode, pLayerPrefix, pMessage))
+            return VK_FALSE;
+    }
 
     // not categorized, just route to plain old qDebug
     qDebug("vkDebug: %s: %d: %s", pLayerPrefix, messageCode, pMessage);
@@ -374,6 +379,7 @@ void QBasicPlatformVulkanInstance::setupDebugOutput()
             | VK_DEBUG_REPORT_WARNING_BIT_EXT
             | VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT;
     dbgCallbackInfo.pfnCallback = defaultDebugCallbackFunc;
+    dbgCallbackInfo.pUserData = this;
 
     VkResult err = createDebugReportCallback(m_vkInst, &dbgCallbackInfo, nullptr, &m_debugCallback);
     if (err != VK_SUCCESS)

src/platformsupport/vkconvenience/qbasicvulkanplatforminstance_p.h

@@ -73,7 +73,10 @@ public:
     QByteArrayList enabledExtensions() const override;
     PFN_vkVoidFunction getInstanceProcAddr(const char *name) override;
     bool supportsPresent(VkPhysicalDevice physicalDevice, uint32_t queueFamilyIndex, QWindow *window) override;
+    void setDebugFilters(const QVector<QVulkanInstance::DebugFilter> &filters) override;
+
     void destroySurface(VkSurfaceKHR surface) const;
+    const QVector<QVulkanInstance::DebugFilter> *debugFilters() const { return &m_debugFilters; }
 
 protected:
     void loadVulkanLibrary(const QString &defaultLibraryName);
@@ -105,6 +108,7 @@ private:
 
     VkDebugReportCallbackEXT m_debugCallback;
     PFN_vkDestroyDebugReportCallbackEXT m_vkDestroyDebugReportCallbackEXT;
+    QVector<QVulkanInstance::DebugFilter> m_debugFilters;
 };
 
 QT_END_NAMESPACE