macOS: Use dedicated content CALayer for Metal/Raster content

By making the content layer a sublayer of the view's root layer (our
container layer) we open up the possibility of manually managing the
z-order between the content and other sublayers of the view's layer,
for example sublayers added as a result of adding NSVisualEffectView
child views to our view.

[ChangeLog][macOS] Metal and Raster windows no longer render
their content directly to the root CALayer of the window's
NSView, but to a sublayer of the root layer. This is an implementation
detail that should not be relied on, but may affect client code
that pokes into the NSView of the QWindow in unsupported ways.
To opt out of the new mode, set QT_MAC_NO_CONTAINER_LAYER=1.

Change-Id: I7053d7530b6966ed7dd4d1a4d1b7e94754767c57
Reviewed-by: Richard Moe Gustavsen <richard.gustavsen@qt.io>

src/gui/kernel/qplatformwindow_p.h

@@ -28,6 +28,10 @@
 struct wl_surface;
 #endif
 
+#if defined(Q_OS_MACOS)
+Q_FORWARD_DECLARE_OBJC_CLASS(CALayer);
+#endif
+
 QT_BEGIN_NAMESPACE
 
 class QMargins;
@@ -55,9 +59,10 @@ struct Q_GUI_EXPORT QWasmWindow
 #if defined(Q_OS_MACOS) || defined(Q_QDOC)
 struct Q_GUI_EXPORT QCocoaWindow
 {
-    QT_DECLARE_NATIVE_INTERFACE(QCocoaWindow, 1, QWindow)
+    QT_DECLARE_NATIVE_INTERFACE(QCocoaWindow, 2, QWindow)
     virtual void setContentBorderEnabled(bool enable) = 0;
     virtual QPoint bottomLeftClippedByNSWindowOffset() const = 0;
+    virtual CALayer *contentLayer() const = 0;
 };
 #endif
 

src/gui/platform/darwin/qmetallayer.mm

@@ -73,5 +73,4 @@ QT_USE_NAMESPACE
     return [super nextDrawable];
 }
 
-
 @end

src/gui/rhi/qrhimetal.mm

@@ -14,6 +14,7 @@
 
 #include <QtCore/private/qcore_mac_p.h>
 #include <QtGui/private/qmetallayer_p.h>
+#include <QtGui/qpa/qplatformwindow_p.h>
 
 #ifdef Q_OS_MACOS
 #include <AppKit/AppKit.h>
@@ -6321,13 +6322,15 @@ QRhiRenderTarget *QMetalSwapChain::currentFrameRenderTarget()
 static inline CAMetalLayer *layerForWindow(QWindow *window)
 {
     Q_ASSERT(window);
+    CALayer *layer = nullptr;
 #ifdef Q_OS_MACOS
-    NSView *view = reinterpret_cast<NSView *>(window->winId());
+    if (auto *cocoaWindow = window->nativeInterface<QNativeInterface::Private::QCocoaWindow>())
+        layer = cocoaWindow->contentLayer();
 #else
-    UIView *view = reinterpret_cast<UIView *>(window->winId());
+    layer = reinterpret_cast<UIView *>(window->winId()).layer;
 #endif
-    Q_ASSERT(view);
-    return static_cast<CAMetalLayer *>(view.layer);
+    Q_ASSERT(layer);
+    return static_cast<CAMetalLayer *>(layer);
 }
 
 // If someone calls this, it is hopefully from the main thread, and they will

src/plugins/platforms/cocoa/qcocoabackingstore.mm

@@ -350,6 +350,7 @@ void QCALayerBackingStore::flush(QWindow *flushedWindow, const QRegion &region,
     QMacAutoReleasePool pool;
 
     NSView *flushedView = static_cast<QCocoaWindow *>(flushedWindow->handle())->view();
+    CALayer *layer = static_cast<QCocoaWindow *>(flushedWindow->handle())->contentLayer();
 
     // If the backingstore is just flushed, without being painted to first, then we may
     // end in a situation where the backingstore is flushed to a layer with a different
@@ -360,11 +361,11 @@ void QCALayerBackingStore::flush(QWindow *flushedWindow, const QRegion &region,
     // we at least cover the whole layer. This is necessary since we set the view's
     // contents placement policy to NSViewLayerContentsPlacementTopLeft, which means
     // AppKit will not do any scaling on our behalf.
-    if (m_buffers.back()->devicePixelRatio() != flushedView.layer.contentsScale) {
+    if (m_buffers.back()->devicePixelRatio() != layer.contentsScale) {
         qCWarning(lcQpaBackingStore) << "Back buffer dpr of" << m_buffers.back()->devicePixelRatio()
-            << "doesn't match" << flushedView.layer << "contents scale of" << flushedView.layer.contentsScale
+            << "doesn't match" << layer << "contents scale of" << layer.contentsScale
             << "- updating layer to match.";
-        flushedView.layer.contentsScale = m_buffers.back()->devicePixelRatio();
+        layer.contentsScale = m_buffers.back()->devicePixelRatio();
     }
 
     const bool isSingleBuffered = window()->format().swapBehavior() == QSurfaceFormat::SingleBuffer;
@@ -380,13 +381,13 @@ void QCALayerBackingStore::flush(QWindow *flushedWindow, const QRegion &region,
     if (isSingleBuffered) {
         // The private API [CALayer reloadValueForKeyPath:@"contents"] would be preferable,
         // but barring any side effects or performance issues we opt for the hammer for now.
-        flushedView.layer.contents = nil;
+        layer.contents = nil;
     }
 
     qCInfo(lcQpaBackingStore) << "Flushing" << backBufferSurface
-        << "to" << flushedView.layer << "of" << flushedView;
+        << "to" << layer << "of" << flushedView;
 
-    flushedView.layer.contents = backBufferSurface;
+    layer.contents = backBufferSurface;
 
     if (!isSingleBuffered) {
         // Mark the surface as in use, so that we don't end up rendering
@@ -428,7 +429,8 @@ void QCALayerBackingStore::flushSubWindow(QWindow *subWindow)
     NSView *backingStoreView = static_cast<QCocoaWindow *>(window()->handle())->view();
     NSView *flushedView = static_cast<QCocoaWindow *>(subWindow->handle())->view();
     auto subviewRect = [flushedView convertRect:flushedView.bounds toView:backingStoreView];
-    auto scale = flushedView.layer.contentsScale;
+    CALayer *layer = static_cast<QCocoaWindow *>(subWindow->handle())->contentLayer();
+    auto scale = layer.contentsScale;
     subviewRect = CGRectApplyAffineTransform(subviewRect, CGAffineTransformMakeScale(scale, scale));
 
     m_buffers.back()->lock(QPlatformGraphicsBuffer::SWReadAccess);

src/plugins/platforms/cocoa/qcocoawindow.h

@@ -225,6 +225,8 @@ public: // for QNSView
     static void setupPopupMonitor();
     static void removePopupMonitor();
 
+    CALayer *contentLayer() const override;
+
     NSView *m_view = nil;
     QCocoaNSWindow *m_nsWindow = nil;
 

src/plugins/platforms/cocoa/qcocoawindow.mm

@@ -1732,7 +1732,7 @@ void QCocoaWindow::deliverUpdateRequest()
 {
     qCDebug(lcQpaDrawing) << "Delivering update request to" << window();
 
-    if (auto *qtMetalLayer = qt_objc_cast<QMetalLayer*>(m_view.layer)) {
+    if (auto *qtMetalLayer = qt_objc_cast<QMetalLayer*>(contentLayer())) {
         // We attempt a read lock here, so that the animation/render thread is
         // prioritized lower than the main thread's displayLayer processing.
         // Without this the two threads might fight over the next drawable,
@@ -2216,6 +2216,14 @@ void QCocoaWindow::setFrameStrutEventsEnabled(bool enabled)
     m_frameStrutEventsEnabled = enabled;
 }
 
+CALayer *QCocoaWindow::contentLayer() const
+{
+    auto *layer = m_view.layer;
+    if (auto *containerLayer = qt_objc_cast<QContainerLayer*>(layer))
+        layer = containerLayer.contentLayer;
+    return layer;
+}
+
 #ifndef QT_NO_DEBUG_STREAM
 QDebug operator<<(QDebug debug, const QCocoaWindow *window)
 {

src/plugins/platforms/cocoa/qnsview.h

@@ -5,6 +5,7 @@
 #define QNSVIEW_H
 
 #include <AppKit/NSView.h>
+#include <QuartzCore/CALayer.h>
 
 #include <QtCore/private/qcore_mac_p.h>
 
@@ -41,7 +42,12 @@ Q_FORWARD_DECLARE_OBJC_CLASS(NSColorSpace);
 @interface QNSView (QtExtras)
 @property (nonatomic, readonly) QCocoaWindow *platformWindow;
 @end
+
+QT_DECLARE_NAMESPACED_OBJC_INTERFACE(QContainerLayer, CALayer
+- (instancetype)initWithContentLayer:(CALayer *)contentLayer;
+@property (readonly) CALayer *contentLayer;
+)
+
 #endif // __OBJC__
 
-
 #endif //QNSVIEW_H

src/plugins/platforms/cocoa/qnsview_drawing.mm

@@ -3,6 +3,36 @@
 
 // This file is included from qnsview.mm, and only used to organize the code
 
+@implementation QContainerLayer {
+    CALayer *m_contentLayer;
+}
+- (instancetype)initWithContentLayer:(CALayer *)contentLayer
+{
+    if ((self = [super init])) {
+        m_contentLayer = contentLayer;
+        [self addSublayer:contentLayer];
+        contentLayer.autoresizingMask = kCALayerWidthSizable | kCALayerHeightSizable;
+    }
+    return self;
+}
+
+- (CALayer*)contentLayer
+{
+    return m_contentLayer;
+}
+
+- (void)setNeedsDisplay
+{
+    [self setNeedsDisplayInRect:CGRectInfinite];
+}
+
+- (void)setNeedsDisplayInRect:(CGRect)rect
+{
+    [super setNeedsDisplayInRect:rect];
+    [self.contentLayer setNeedsDisplayInRect:rect];
+}
+@end
+
 @implementation QNSView (Drawing)
 
 - (void)initDrawing
@@ -107,6 +137,17 @@
         layer.delegate = self;
     }
 
+    layer.name = @"Qt content layer";
+
+    static const bool containerLayerOptOut = qEnvironmentVariableIsSet("QT_MAC_NO_CONTAINER_LAYER");
+    if (m_platformWindow->window()->surfaceType() != QSurface::OpenGLSurface && !containerLayerOptOut) {
+        qCDebug(lcQpaDrawing) << "Wrapping content layer" << layer << "in container layer";
+        auto *containerLayer = [[QContainerLayer alloc] initWithContentLayer:layer];
+        containerLayer.name = @"Qt container layer";
+        containerLayer.delegate = self;
+        layer = containerLayer;
+    }
+
     [super setLayer:layer];
 
     [self propagateBackingProperties];
@@ -117,7 +158,6 @@
         // where it doesn't.
         layer.backgroundColor = NSColor.magentaColor.CGColor;
     }
-
 }
 
 // ----------------------- Layer updates -----------------------
@@ -172,11 +212,12 @@
     // to NO. In this case the window will have a backingScaleFactor of 2,
     // but the QWindow will have a devicePixelRatio of 1.
     auto devicePixelRatio = m_platformWindow->devicePixelRatio();
-    qCDebug(lcQpaDrawing) << "Updating" << self.layer << "content scale to" << devicePixelRatio;
-    self.layer.contentsScale = devicePixelRatio;
+    auto *contentLayer = m_platformWindow->contentLayer();
+    qCDebug(lcQpaDrawing) << "Updating" << contentLayer << "content scale to" << devicePixelRatio;
+    contentLayer.contentsScale = devicePixelRatio;
 
-    if ([self.layer isKindOfClass:CAMetalLayer.class]) {
-        CAMetalLayer *metalLayer = static_cast<CAMetalLayer *>(self.layer);
+    if ([contentLayer isKindOfClass:CAMetalLayer.class]) {
+        CAMetalLayer *metalLayer = static_cast<CAMetalLayer *>(contentLayer);
         metalLayer.colorspace = self.colorSpace.CGColorSpace;
         qCDebug(lcQpaDrawing) << "Set" << metalLayer << "color space to" << metalLayer.colorspace;
     }
@@ -205,8 +246,11 @@
 */
 - (void)displayLayer:(CALayer *)layer
 {
-    Q_ASSERT_X(self.layer && layer == self.layer, "QNSView",
-        "The displayLayer code path should only be hit for our own layer");
+    if (auto *containerLayer = qt_objc_cast<QContainerLayer*>(layer)) {
+        qCDebug(lcQpaDrawing) << "Skipping display of" << containerLayer
+            << "as display is handled by content layer" << containerLayer.contentLayer;
+        return;
+    }
 
     if (!m_platformWindow)
         return;
@@ -226,7 +270,7 @@
         m_platformWindow->handleExposeEvent(bounds);
     };
 
-    if (auto *qtMetalLayer = qt_objc_cast<QMetalLayer*>(self.layer)) {
+    if (auto *qtMetalLayer = qt_objc_cast<QMetalLayer*>(layer)) {
         const bool presentedWithTransaction = qtMetalLayer.presentsWithTransaction;
         qtMetalLayer.presentsWithTransaction = YES;