From 9c9bb25c723f56d1d7ded10318c4e2bc74411fb9 Mon Sep 17 00:00:00 2001
From: Laszlo Agocs <laszlo.agocs@qt.io>
Date: Mon, 20 Nov 2023 14:57:14 +0100
Subject: [PATCH] rhi: d3d12: Try without debug layer when failed
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This matches what the D3D11 backend does: if the debug device (D3D11) /
debug factory (D3D12) is not available, we retry without the flag and
if succeeded, continue without the debug layer while printing a log
message. This way create() succeeds even when the debug layer is
requested but is not available at runtime (because the necessary SDK
or Visual Studio components are not installed, which can happen
on an end-user PC not set up for development)

Task-number: QTBUG-119240
Change-Id: I02e3bf45728e74b8fe196e880372f584de7aa5d2
Reviewed-by: Kristoffer Skau <kristoffer.skau@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Christian Strømme <christian.stromme@qt.io>
(cherry picked from commit 1248979968435c9d35db4c5ca53282a31f40bb51)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>
---
 src/gui/rhi/qrhid3d12.cpp | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/gui/rhi/qrhid3d12.cpp b/src/gui/rhi/qrhid3d12.cpp
index 38e4fbd2d63..4a5e52bfc61 100644
--- a/src/gui/rhi/qrhid3d12.cpp
+++ b/src/gui/rhi/qrhid3d12.cpp
@@ -176,9 +176,20 @@ bool QRhiD3D12::create(QRhi::Flags flags)
         factoryFlags |= DXGI_CREATE_FACTORY_DEBUG;
     HRESULT hr = CreateDXGIFactory2(factoryFlags, __uuidof(IDXGIFactory2), reinterpret_cast<void **>(&dxgiFactory));
     if (FAILED(hr)) {
-        qWarning("CreateDXGIFactory2() failed to create DXGI factory: %s",
-                 qPrintable(QSystemError::windowsComString(hr)));
-        return false;
+        // retry without debug, if it was requested (to match D3D11 backend behavior)
+        if (debugLayer) {
+            qCDebug(QRHI_LOG_INFO, "Debug layer was requested but is not available. "
+                                   "Attempting to create DXGIFactory2 without it.");
+            factoryFlags &= ~DXGI_CREATE_FACTORY_DEBUG;
+            hr = CreateDXGIFactory2(factoryFlags, __uuidof(IDXGIFactory2), reinterpret_cast<void **>(&dxgiFactory));
+        }
+        if (SUCCEEDED(hr)) {
+            debugLayer = false;
+        } else {
+            qWarning("CreateDXGIFactory2() failed to create DXGI factory: %s",
+                     qPrintable(QSystemError::windowsComString(hr)));
+            return false;
+        }
     }
 
     supportsAllowTearing = false;