From 13a885d044688dd1a0cf904526e5e1550abb0eed Mon Sep 17 00:00:00 2001
From: Laszlo Agocs <laszlo.agocs@qt.io>
Date: Wed, 25 Jan 2023 12:30:27 +0100
Subject: [PATCH] Modernize the VK_EXT_debug_utils callback
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

...as suggested by review comments.

Switching to std::function implies that we cannot easily
do the install-remove pattern anymore as there is no way to
compare an std::function to something other than null. Instead
of making it more complicated by returning a key or something
like that, change the remove function to a clear (that now clears
both the legacy VK_EXT_debug_report and the new VK_EXT_debug_utils
callback lists).

Also add a missing call that registers the new-style callbacks that
are installed before create().

Change-Id: I66c1dd8e8dcc8eee0f5eb9671f94c2c80319dcaf
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Kristoffer Skau <kristoffer.skau@qt.io>
Reviewed-by: Christian Strømme <christian.stromme@qt.io>
(cherry picked from commit 0b20f243f7efbd612d41bab8c2000e6c9fcb13e9)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
---
 src/gui/vulkan/qvulkaninstance.cpp | 31 ++++++++++++++++--------------
 src/gui/vulkan/qvulkaninstance.h   |  4 ++--
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/src/gui/vulkan/qvulkaninstance.cpp b/src/gui/vulkan/qvulkaninstance.cpp
index a1d709fbbdf..946a183d89b 100644
--- a/src/gui/vulkan/qvulkaninstance.cpp
+++ b/src/gui/vulkan/qvulkaninstance.cpp
@@ -567,6 +567,7 @@ bool QVulkanInstance::create()
         d_ptr->errorCode = VK_SUCCESS;
         d_ptr->funcs.reset(new QVulkanFunctions(this));
         d_ptr->platformInst->setDebugFilters(d_ptr->debugFilters);
+        d_ptr->platformInst->setDebugUtilsFilters(d_ptr->debugUtilsFilters);
         return true;
     }
 
@@ -886,12 +887,13 @@ void QVulkanInstance::removeDebugOutputFilter(DebugFilter filter)
     Typedef for debug filtering callback functions, with the following signature:
 
     \code
-    bool myDebugUtilsFilter(DebugMessageSeverityFlags severity, DebugMessageTypeFlags type, const void *callbackData);
+    std::function<bool(DebugMessageSeverityFlags severity, DebugMessageTypeFlags type, const void *message)>;
     \endcode
 
-    The \c callbackData
-    argument is a pointer to the VkDebugUtilsMessengerCallbackDataEXT
-    structure.
+    The \c message argument is a pointer to the
+    VkDebugUtilsMessengerCallbackDataEXT structure. Refer to the documentation
+    of \c{VK_EXT_debug_utils} for details. The Qt headers do not use the real
+    type in order to avoid introducing a dependency on post-1.0 Vulkan headers.
 
     Returning \c true suppresses the printing of the message.
 
@@ -928,20 +930,18 @@ void QVulkanInstance::removeDebugOutputFilter(DebugFilter filter)
 
     \note This function can be called before create().
 
-    \sa removeDebugOutputFilter()
+    \sa clearDebugOutputFilters()
     \since 6.5
  */
 void QVulkanInstance::installDebugOutputFilter(DebugUtilsFilter filter)
 {
-    if (!d_ptr->debugUtilsFilters.contains(filter)) {
-        d_ptr->debugUtilsFilters.append(filter);
-        if (d_ptr->platformInst)
-            d_ptr->platformInst->setDebugUtilsFilters(d_ptr->debugUtilsFilters);
-    }
+    d_ptr->debugUtilsFilters.append(filter);
+    if (d_ptr->platformInst)
+        d_ptr->platformInst->setDebugUtilsFilters(d_ptr->debugUtilsFilters);
 }
 
 /*!
-    Removes a \a filter function previously installed by
+    Removes all filter functions installed previously by
     installDebugOutputFilter().
 
     \note This function can be called before create().
@@ -949,11 +949,14 @@ void QVulkanInstance::installDebugOutputFilter(DebugUtilsFilter filter)
     \sa installDebugOutputFilter()
     \since 6.5
  */
-void QVulkanInstance::removeDebugOutputFilter(DebugUtilsFilter filter)
+void QVulkanInstance::clearDebugOutputFilters()
 {
-    d_ptr->debugUtilsFilters.removeOne(filter);
-    if (d_ptr->platformInst)
+    d_ptr->debugFilters.clear();
+    d_ptr->debugUtilsFilters.clear();
+    if (d_ptr->platformInst) {
+        d_ptr->platformInst->setDebugFilters(d_ptr->debugFilters);
         d_ptr->platformInst->setDebugUtilsFilters(d_ptr->debugUtilsFilters);
+    }
 }
 
 #ifndef QT_NO_DEBUG_STREAM
diff --git a/src/gui/vulkan/qvulkaninstance.h b/src/gui/vulkan/qvulkaninstance.h
index 74ff0f62cdc..221f605fa2a 100644
--- a/src/gui/vulkan/qvulkaninstance.h
+++ b/src/gui/vulkan/qvulkaninstance.h
@@ -203,9 +203,9 @@ public:
     };
     Q_DECLARE_FLAGS(DebugMessageTypeFlags, DebugMessageTypeFlag)
 
-    typedef bool (*DebugUtilsFilter)(DebugMessageSeverityFlags severity, DebugMessageTypeFlags type, const void *callbackData);
+    using DebugUtilsFilter = std::function<bool(DebugMessageSeverityFlags severity, DebugMessageTypeFlags type, const void *message)>;
     void installDebugOutputFilter(DebugUtilsFilter filter);
-    void removeDebugOutputFilter(DebugUtilsFilter filter);
+    void clearDebugOutputFilters();
 
 private:
     friend class QVulkanInstancePrivate;