rhi: d3d: Switch to DirectComposition for semi-transparent windows

A window with alpha > 0 is no longer forced to using a legacy,
non-flip swapchain. Instead, the Win8+ DirectComposition API is used
in that case to get a properly composited semi-transparent window.

The remainings of the non-flip model swapchain support, that was left
after purging Windows 7 in order to support semi-transparent windows,
is now gone completely. Setting QT_D3D_NO_FLIP has no effect from now
on. DXGIFactory2 and flip model swapchains are now always required and
in use.

There is one downside when it comes to the swap effect: for
semi-transparent windows we will be stuck with SCALING_STRETCH as
SCALING_NONE is not supported on this path. Using NONE over STRETCH
was preferable for typical Qt applications because user interfaces
typically look "nicer" during resize without stretching when ending up
with a size-mismatched Present. For normal (non-translucent) windows
this will continue to be the case, but enabling alpha will now move
back to SCALING_STRETCH implicitly.

In addition, it looks like AlphaMode is ignored when using
CreateSwapChainForComposition. Setting DXGI_ALPHA_MODE_STRAIGHT fails
always whereas other values lead to an identical looking (with
transparency) result. Therefore we just set it to
DXGI_ALPHA_MODE_PREMULTIPLIED regaredless of which QRhiSwapChain alpha
flag is set.

Fixes: QTBUG-105271
Change-Id: Ie7a584e1656ccfdb506489ea21c41706f57c4018
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

src/gui/CMakeLists.txt

@@ -424,6 +424,7 @@ qt_internal_extend_target(Gui CONDITION WIN32
         d3d11
         dxgi
         dxguid
+        dcomp
 )
 
 #### Keys ignored in scope 7:.:.:gui.pro:WIN32:

src/gui/rhi/qrhid3d11.cpp

@@ -149,17 +149,6 @@ static IDXGIFactory1 *createDXGIFactory2()
     return result;
 }
 
-static IDXGIFactory1 *createDXGIFactory1()
-{
-    IDXGIFactory1 *result = nullptr;
-    const HRESULT hr = CreateDXGIFactory1(__uuidof(IDXGIFactory1), reinterpret_cast<void **>(&result));
-    if (FAILED(hr)) {
-        qWarning("CreateDXGIFactory1() failed to create DXGI factory: %s", qPrintable(comErrorMessage(hr)));
-        result = nullptr;
-    }
-    return result;
-}
-
 bool QRhiD3D11::create(QRhi::Flags flags)
 {
     rhiFlags = flags;
@@ -169,22 +158,13 @@ bool QRhiD3D11::create(QRhi::Flags flags)
         devFlags |= D3D11_CREATE_DEVICE_DEBUG;
 
     dxgiFactory = createDXGIFactory2();
-    if (dxgiFactory != nullptr) {
+    if (!dxgiFactory)
-        supportsFlipSwapchain = !qEnvironmentVariableIntValue("QT_D3D_NO_FLIP");
-    } else {
-        dxgiFactory = createDXGIFactory1();
-        supportsFlipSwapchain = false;
-    }
-
-    if (dxgiFactory == nullptr)
         return false;
 
-    supportsAllowTearing = false;
-    forceFlipDiscard = false;
-    if (supportsFlipSwapchain) {
     // For a FLIP_* swapchain Present(0, 0) is not necessarily
     // sufficient to get non-blocking behavior, try using ALLOW_TEARING
     // when available.
+    supportsAllowTearing = false;
     IDXGIFactory5 *factory5 = nullptr;
     if (SUCCEEDED(dxgiFactory->QueryInterface(__uuidof(IDXGIFactory5), reinterpret_cast<void **>(&factory5)))) {
         BOOL allowTearing = false;
@@ -192,17 +172,18 @@ bool QRhiD3D11::create(QRhi::Flags flags)
             supportsAllowTearing = allowTearing;
         factory5->Release();
     }
+
     // if we default to FLIP_SEQUENTIAL, have a way to request FLIP_DISCARD
     forceFlipDiscard = qEnvironmentVariableIntValue("QT_D3D_FLIP_DISCARD");
-    }
 
-    qCDebug(QRHI_LOG_INFO, "FLIP_* swapchain supported = %s, ALLOW_TEARING supported = %s",
+    if (qEnvironmentVariableIntValue("QT_D3D_NO_FLIP"))
-            supportsFlipSwapchain ? "true" : "false",
+        qWarning("Non-FLIP swapchains are no longer supported, QT_D3D_NO_FLIP is now ignored");
+
+    qCDebug(QRHI_LOG_INFO, "FLIP_* swapchain supported = true, ALLOW_TEARING supported = %s",
             supportsAllowTearing ? "true" : "false");
 
     qCDebug(QRHI_LOG_INFO, "Default swap effect: %s",
-            supportsFlipSwapchain ? (forceFlipDiscard ? "FLIP_DISCARD" : "FLIP_SEQUENTIAL")
+            forceFlipDiscard ? "FLIP_DISCARD" : "FLIP_SEQUENTIAL");
-                                  : "DISCARD");
 
     if (!importedDeviceAndContext) {
         IDXGIAdapter1 *adapter;
@@ -381,6 +362,11 @@ void QRhiD3D11::destroy()
         }
     }
 
+    if (dcompDevice) {
+        dcompDevice->Release();
+        dcompDevice = nullptr;
+    }
+
     if (activeAdapter) {
         activeAdapter->Release();
         activeAdapter = nullptr;
@@ -1327,6 +1313,9 @@ QRhi::FrameOpResult QRhiD3D11::endFrame(QRhiSwapChain *swapChain, QRhi::EndFrame
             return QRhi::FrameOpError;
         }
 
+        if (dcompDevice && swapChainD->dcompTarget && swapChainD->dcompVisual)
+            dcompDevice->Commit();
+
         // move on to the next buffer
         swapChainD->currentFrameSlot = (swapChainD->currentFrameSlot + 1) % QD3D11SwapChain::BUFFER_COUNT;
     } else {
@@ -4603,6 +4592,16 @@ void QD3D11SwapChain::destroy()
     swapChain->Release();
     swapChain = nullptr;
 
+    if (dcompVisual) {
+        dcompVisual->Release();
+        dcompVisual = nullptr;
+    }
+
+    if (dcompTarget) {
+        dcompTarget->Release();
+        dcompTarget = nullptr;
+    }
+
     QRHI_RES_RHI(QRhiD3D11);
     if (rhiD)
         rhiD->unregisterResource(this);
@@ -4740,6 +4739,22 @@ bool QD3D11SwapChain::newColorBuffer(const QSize &size, DXGI_FORMAT format, DXGI
 static const DXGI_FORMAT DEFAULT_FORMAT = DXGI_FORMAT_R8G8B8A8_UNORM;
 static const DXGI_FORMAT DEFAULT_SRGB_FORMAT = DXGI_FORMAT_R8G8B8A8_UNORM_SRGB;
 
+bool QRhiD3D11::ensureDirectCompositionDevice()
+{
+    if (dcompDevice)
+        return true;
+
+    qCDebug(QRHI_LOG_INFO, "Creating Direct Composition device (needed for semi-transparent windows)");
+
+    HRESULT hr = DCompositionCreateDevice(nullptr, __uuidof(IDCompositionDevice), reinterpret_cast<void **>(&dcompDevice));
+    if (FAILED(hr)) {
+        qWarning("Failed to Direct Composition device: %s", qPrintable(comErrorMessage(hr)));
+        return false;
+    }
+
+    return true;
+}
+
 bool QD3D11SwapChain::createOrResize()
 {
     // Can be called multiple times due to window resizes - that is not the
@@ -4759,16 +4774,29 @@ bool QD3D11SwapChain::createOrResize()
     if (pixelSize.isEmpty())
         return false;
 
-    QRHI_RES_RHI(QRhiD3D11);
+    HWND hwnd = reinterpret_cast<HWND>(window->winId());
-    bool useFlipModel = rhiD->supportsFlipSwapchain;
+    HRESULT hr;
+
+    QRHI_RES_RHI(QRhiD3D11);
 
-    // Take a shortcut for alpha: whatever the platform plugin does to enable
-    // transparency for our QWindow will be sufficient on the legacy (DISCARD)
-    // path. For FLIP_* we'd need to use DirectComposition (create a
-    // IDCompositionDevice/Target/Visual), avoid that for now. (this though
-    // means HDR and semi-transparent windows cannot be combined)
     if (m_flags.testFlag(SurfaceHasPreMulAlpha) || m_flags.testFlag(SurfaceHasNonPreMulAlpha)) {
-        useFlipModel = false;
+        if (rhiD->ensureDirectCompositionDevice()) {
+            if (!dcompTarget) {
+                hr = rhiD->dcompDevice->CreateTargetForHwnd(hwnd, true, &dcompTarget);
+                if (FAILED(hr)) {
+                    qWarning("Failed to create Direct Compsition target for the window: %s",
+                             qPrintable(comErrorMessage(hr)));
+                }
+            }
+            if (dcompTarget && !dcompVisual) {
+                hr = rhiD->dcompDevice->CreateVisual(&dcompVisual);
+                if (FAILED(hr)) {
+                    qWarning("Failed to create DirectComposition visual: %s",
+                             qPrintable(comErrorMessage(hr)));
+                }
+            }
+        }
+        // simple consistency check
         if (window->requestedFormat().alphaBufferSize() <= 0)
             qWarning("Swapchain says surface has alpha but the window has no alphaBufferSize set. "
                      "This may lead to problems.");
@@ -4781,17 +4809,14 @@ bool QD3D11SwapChain::createOrResize()
     // ALLOW_TEARING, and ALLOW_TEARING is not compatible with it at all so the
     // flag must not be set then. Whereas for flip we should use it, if
     // supported, to get better results for 'unthrottled' presentation.
-    if (swapInterval == 0 && useFlipModel && rhiD->supportsAllowTearing)
+    if (swapInterval == 0 && rhiD->supportsAllowTearing)
         swapChainFlags |= DXGI_SWAP_CHAIN_FLAG_ALLOW_TEARING;
 
     if (!swapChain) {
-        HWND hwnd = reinterpret_cast<HWND>(window->winId());
         sampleDesc = rhiD->effectiveSampleCount(m_sampleCount);
         colorFormat = DEFAULT_FORMAT;
         srgbAdjustedColorFormat = m_flags.testFlag(sRGB) ? DEFAULT_SRGB_FORMAT : DEFAULT_FORMAT;
 
-        HRESULT hr;
-        if (useFlipModel) {
         DXGI_COLOR_SPACE_TYPE hdrColorSpace = DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709; // SDR
         DXGI_OUTPUT_DESC1 hdrOutputDesc;
         if (outputDesc1ForWindow(m_window, rhiD->activeAdapter, &hdrOutputDesc) && m_format != SDR) {
@@ -4834,10 +4859,11 @@ bool QD3D11SwapChain::createOrResize()
         desc.SampleDesc.Count = 1;
         desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
         desc.BufferCount = BUFFER_COUNT;
+        desc.Flags = swapChainFlags;
 
         // Normally we'd want FLIP_DISCARD, but that comes with the default
         // SCALING_STRETCH, as SCALING_NONE is documented to be only
-            // available for FLIP_SEQUENTIAl. The problem with stretch is that
+        // available for FLIP_SEQUENTIAL. The problem with stretch is that
         // Qt Quick and similar apps typically running in resizable windows
         // will not like how that looks in practice: the content will
         // appear to be "jumping" around during a window resize. So choose
@@ -4850,13 +4876,24 @@ bool QD3D11SwapChain::createOrResize()
             desc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL;
         }
 
-            // Do not bother with AlphaMode, if won't work unless we go through
+        if (dcompVisual) {
-            // DirectComposition. Instead, we just take the other (DISCARD)
+            // With DirectComposition setting AlphaMode to STRAIGHT fails the
-            // path for now when alpha is requested.
+            // swapchain creation, whereas the result seems to be identical
-            desc.Flags = swapChainFlags;
+            // with any of the other values, including IGNORE. (?)
+            desc.AlphaMode = DXGI_ALPHA_MODE_PREMULTIPLIED;
+
+            // DirectComposition has its own limitations, cannot use
+            // SCALING_NONE. So with semi-transparency requested we are forced
+            // to SCALING_STRETCH.
+            desc.Scaling = DXGI_SCALING_STRETCH;
+        }
 
         IDXGIFactory2 *fac = static_cast<IDXGIFactory2 *>(rhiD->dxgiFactory);
         IDXGISwapChain1 *sc1;
+
+        if (dcompVisual)
+            hr = fac->CreateSwapChainForComposition(rhiD->dev, &desc, nullptr, &sc1);
+        else
             hr = fac->CreateSwapChainForHwnd(rhiD->dev, hwnd, &desc, nullptr, nullptr, &sc1);
 
         // If failed and we tried a HDR format, then try with SDR. This
@@ -4865,6 +4902,9 @@ bool QD3D11SwapChain::createOrResize()
         if (FAILED(hr) && m_format != SDR) {
             colorFormat = DEFAULT_FORMAT;
             desc.Format = DEFAULT_FORMAT;
+            if (dcompVisual)
+                hr = fac->CreateSwapChainForComposition(rhiD->dev, &desc, nullptr, &sc1);
+            else
                 hr = fac->CreateSwapChainForHwnd(rhiD->dev, hwnd, &desc, nullptr, nullptr, &sc1);
         }
 
@@ -4881,28 +4921,19 @@ bool QD3D11SwapChain::createOrResize()
                     qWarning("IDXGISwapChain3 not available, HDR swapchain will not work as expected");
                 }
             }
+            if (dcompVisual) {
+                hr = dcompVisual->SetContent(sc1);
+                if (SUCCEEDED(hr)) {
+                    hr = dcompTarget->SetRoot(dcompVisual);
+                    if (FAILED(hr)) {
+                        qWarning("Failed to associate Direct Composition visual with the target: %s",
+                                 qPrintable(comErrorMessage(hr)));
                     }
                 } else {
-            // Fallback: use DISCARD mode. Regardless, keep on using our manual
+                    qWarning("Failed to set content for Direct Composition visual: %s",
-            // resolve for symmetry with the FLIP_* code path when MSAA is
+                             qPrintable(comErrorMessage(hr)));
-            // requested. This has no HDR support.
+                }
-
+            }
-            DXGI_SWAP_CHAIN_DESC desc;
-            memset(&desc, 0, sizeof(desc));
-            desc.BufferDesc.Width = UINT(pixelSize.width());
-            desc.BufferDesc.Height = UINT(pixelSize.height());
-            desc.BufferDesc.RefreshRate.Numerator = 60;
-            desc.BufferDesc.RefreshRate.Denominator = 1;
-            desc.BufferDesc.Format = colorFormat;
-            desc.SampleDesc.Count = 1;
-            desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
-            desc.BufferCount = 1;
-            desc.OutputWindow = hwnd;
-            desc.Windowed = true;
-            desc.SwapEffect = DXGI_SWAP_EFFECT_DISCARD;
-            desc.Flags = swapChainFlags;
-
-            hr = rhiD->dxgiFactory->CreateSwapChain(rhiD->dev, &desc, &swapChain);
         }
         if (FAILED(hr)) {
             qWarning("Failed to create D3D11 swapchain: %s", qPrintable(comErrorMessage(hr)));
@@ -4911,8 +4942,8 @@ bool QD3D11SwapChain::createOrResize()
         rhiD->dxgiFactory->MakeWindowAssociation(hwnd, DXGI_MWA_NO_WINDOW_CHANGES);
     } else {
         releaseBuffers();
-        const UINT count = useFlipModel ? BUFFER_COUNT : 1;
+        // flip model -> buffer count is the real buffer count, not 1 like with the legacy modes
-        HRESULT hr = swapChain->ResizeBuffers(count, UINT(pixelSize.width()), UINT(pixelSize.height()),
+        hr = swapChain->ResizeBuffers(UINT(BUFFER_COUNT), UINT(pixelSize.width()), UINT(pixelSize.height()),
                                       colorFormat, swapChainFlags);
         if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET) {
             qWarning("Device loss detected in ResizeBuffers()");
@@ -4938,7 +4969,7 @@ bool QD3D11SwapChain::createOrResize()
     // swapchain."
 
     // So just query index 0 once (per resize) and be done with it.
-    HRESULT hr = swapChain->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast<void **>(&backBufferTex));
+    hr = swapChain->GetBuffer(0, __uuidof(ID3D11Texture2D), reinterpret_cast<void **>(&backBufferTex));
     if (FAILED(hr)) {
         qWarning("Failed to query swapchain backbuffer: %s", qPrintable(comErrorMessage(hr)));
         return false;
@@ -4997,7 +5028,7 @@ bool QD3D11SwapChain::createOrResize()
         for (int i = 0; i < BUFFER_COUNT; ++i) {
             if (!timestampDisjointQuery[i]) {
                 queryDesc.Query = D3D11_QUERY_TIMESTAMP_DISJOINT;
-                HRESULT hr = rhiD->dev->CreateQuery(&queryDesc, &timestampDisjointQuery[i]);
+                hr = rhiD->dev->CreateQuery(&queryDesc, &timestampDisjointQuery[i]);
                 if (FAILED(hr)) {
                     qWarning("Failed to create timestamp disjoint query: %s", qPrintable(comErrorMessage(hr)));
                     break;
@@ -5007,7 +5038,7 @@ bool QD3D11SwapChain::createOrResize()
             for (int j = 0; j < 2; ++j) {
                 const int idx = BUFFER_COUNT * i + j; // one pair per buffer (frame)
                 if (!timestampQuery[idx]) {
-                    HRESULT hr = rhiD->dev->CreateQuery(&queryDesc, &timestampQuery[idx]);
+                    hr = rhiD->dev->CreateQuery(&queryDesc, &timestampQuery[idx]);
                     if (FAILED(hr)) {
                         qWarning("Failed to create timestamp query: %s", qPrintable(comErrorMessage(hr)));
                         break;

src/gui/rhi/qrhid3d11_p_p.h

@@ -22,6 +22,7 @@
 
 #include <d3d11_1.h>
 #include <dxgi1_6.h>
+#include <dcomp.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -551,6 +552,8 @@ struct QD3D11SwapChain : public QRhiSwapChain
     ID3D11Query *timestampDisjointQuery[BUFFER_COUNT];
     ID3D11Query *timestampQuery[BUFFER_COUNT * 2];
     UINT swapInterval = 1;
+    IDCompositionTarget *dcompTarget = nullptr;
+    IDCompositionVisual *dcompVisual = nullptr;
 };
 
 class QRhiD3D11 : public QRhiImplementation
@@ -681,6 +684,7 @@ public:
     void clearShaderCache();
     QByteArray compileHlslShaderSource(const QShader &shader, QShader::Variant shaderVariant, uint flags,
                                        QString *error, QShaderKey *usedShaderKey);
+    bool ensureDirectCompositionDevice();
 
     QRhi::Flags rhiFlags;
     bool debugLayer = false;
@@ -692,7 +696,7 @@ public:
     ID3DUserDefinedAnnotation *annotations = nullptr;
     IDXGIAdapter1 *activeAdapter = nullptr;
     IDXGIFactory1 *dxgiFactory = nullptr;
-    bool supportsFlipSwapchain = false;
+    IDCompositionDevice *dcompDevice = nullptr;
     bool supportsAllowTearing = false;
     bool forceFlipDiscard = false;
     bool deviceLost = false;