macOS: Modernize masking of windows

Instead of masking window blitting via a CGImage mask, we use the window's
mask directly to intersect the region that we blit during flushing of the
QCocoaBackingStore. This approach also enables masking of child windows.

We now also support setting a mask for layer-backed views, by setting a
CAShapeLayer as the layer's mask.

The window shadow invalidation has been moved out of QNSView, as the view
should not be involved in that process. For layer-backed views, the shadow
is not invalidated as expected after the initial mask has been set, but
this bug has been left as a fix for a later stage as it requires more
research.

Change-Id: Ie0127d8df49d95b2d6144816b19559f3d3c95d13
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

src/plugins/platforms/cocoa/cocoa.pro

@@ -75,7 +75,7 @@ qtConfig(opengl.*) {
 
 RESOURCES += qcocoaresources.qrc
 
-LIBS += -framework AppKit -framework Carbon -framework IOKit -lcups
+LIBS += -framework AppKit -framework Carbon -framework IOKit -framework QuartzCore -lcups
 
 QT += \
     core-private gui-private \

src/plugins/platforms/cocoa/qcocoabackingstore.mm

@@ -190,14 +190,15 @@ void QCocoaBackingStore::flush(QWindow *window, const QRegion &region, const QPo
     // Create temporary image to use for blitting, without copying image data
     NSImage *backingStoreImage = [[[NSImage alloc] initWithCGImage:m_cgImage size:NSZeroSize] autorelease];
 
-    if ([topLevelView hasMask]) {
-        // FIXME: Implement via NSBezierPath and addClip
-        CGRect boundingRect = region.boundingRect().toCGRect();
-        QCFType<CGImageRef> subMask = CGImageCreateWithImageInRect([topLevelView maskImage], boundingRect);
-        CGContextClipToMask(graphicsContext.CGContext, boundingRect, subMask);
+    QRegion clippedRegion = region;
+    for (QWindow *w = window; w; w = w->parent()) {
+        if (!w->mask().isEmpty()) {
+            clippedRegion &= w == window ? w->mask()
+                : w->mask().translated(window->mapFromGlobal(w->mapToGlobal(QPoint(0, 0))));
+        }
     }
 
-    for (const QRect &viewLocalRect : region) {
+    for (const QRect &viewLocalRect : clippedRegion) {
         QPoint backingStoreOffset = viewLocalRect.topLeft() + offset;
         QRect backingStoreRect(backingStoreOffset * devicePixelRatio, viewLocalRect.size() * devicePixelRatio);
         if (graphicsContext.flipped) // Flip backingStoreRect to match graphics context
@@ -225,6 +226,12 @@ void QCocoaBackingStore::flush(QWindow *window, const QRegion &region, const QPo
 #endif
     }
 
+    QCocoaWindow *topLevelCocoaWindow = static_cast<QCocoaWindow *>(topLevelWindow->handle());
+    if (Q_UNLIKELY(topLevelCocoaWindow->m_needsInvalidateShadow)) {
+        [topLevelView.window invalidateShadow];
+        topLevelCocoaWindow->m_needsInvalidateShadow = false;
+    }
+
     // -------------------------------------------------------------------------
 
     if (shouldHandleViewLockManually)
@@ -234,9 +241,6 @@ void QCocoaBackingStore::flush(QWindow *window, const QRegion &region, const QPo
         redrawRoundedBottomCorners([view convertRect:region.boundingRect().toCGRect() toView:nil]);
         [view.window flushWindow];
     }
-
-    // FIXME: Tie to changing window flags and/or mask instead
-    [view invalidateWindowShadowIfNeeded];
 }
 
 /*

src/plugins/platforms/cocoa/qcocoawindow.h

@@ -260,6 +260,8 @@ public: // for QNSView
     QCocoaMenuBar *m_menubar;
     NSCursor *m_windowCursor;
 
+    bool m_needsInvalidateShadow;
+
     bool m_hasModalSession;
     bool m_frameStrutEventsEnabled;
     bool m_isExposed;

src/plugins/platforms/cocoa/qcocoawindow.mm

@@ -57,6 +57,7 @@
 #include <QtGui/private/qhighdpiscaling_p.h>
 
 #include <AppKit/AppKit.h>
+#include <QuartzCore/QuartzCore.h>
 
 #include <QDebug>
 
@@ -150,6 +151,7 @@ QCocoaWindow::QCocoaWindow(QWindow *win, WId nativeHandle)
 #endif
     , m_menubar(0)
     , m_windowCursor(0)
+    , m_needsInvalidateShadow(false)
     , m_hasModalSession(false)
     , m_frameStrutEventsEnabled(false)
     , m_isExposed(false)
@@ -699,7 +701,7 @@ bool QCocoaWindow::isOpaque() const
 
     bool translucent = window()->format().alphaBufferSize() > 0
                         || window()->opacity() < 1
-                        || [qnsview_cast(m_view) hasMask]
+                        || !window()->mask().isEmpty()
                         || (surface()->supportsOpenGL() && openglSourfaceOrder == -1);
     return !translucent;
 }
@@ -755,7 +757,34 @@ void QCocoaWindow::setMask(const QRegion &region)
 {
     qCDebug(lcQpaCocoaWindow) << "QCocoaWindow::setMask" << window() << region;
 
-    [qnsview_cast(m_view) setMaskRegion:&region];
+    if (m_view.layer) {
+        if (!region.isEmpty()) {
+            QCFType<CGMutablePathRef> maskPath = CGPathCreateMutable();
+            for (const QRect &r : region)
+                CGPathAddRect(maskPath, nullptr, r.toCGRect());
+            CAShapeLayer *maskLayer = [CAShapeLayer layer];
+            maskLayer.path = maskPath;
+            m_view.layer.mask = maskLayer;
+        } else {
+            m_view.layer.mask = nil;
+        }
+    }
+
+    if (isContentView()) {
+        // Setting the mask requires invalidating the NSWindow shadow, but that needs
+        // to happen after the backingstore has been redrawn, so that AppKit can pick
+        // up the new window shape based on the backingstore content. Doing a display
+        // directly here is not an option, as the window might not be exposed at this
+        // time, and so would not result in an updated backingstore.
+        m_needsInvalidateShadow = true;
+        [m_view setNeedsDisplay:YES];
+
+        // FIXME: [NSWindow invalidateShadow] has no effect when in layer-backed mode,
+        // so if the mask is changed after the initial mask is applied, it will not
+        // result in any visual change to the shadow. This is an Apple bug, and there
+        // may be ways to work around it, such as calling setFrame on the window to
+        // trigger some internal invalidation, but that needs more research.
+    }
 }
 
 bool QCocoaWindow::setKeyboardGrabEnabled(bool grab)

src/plugins/platforms/cocoa/qnsview.h

@@ -57,9 +57,6 @@ QT_END_NAMESPACE
 Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 
 @interface QT_MANGLE_NAMESPACE(QNSView) : NSView <NSTextInputClient> {
-    QRegion m_maskRegion;
-    CGImageRef m_maskImage;
-    bool m_shouldInvalidateWindowShadow;
     QPointer<QCocoaWindow> m_platformWindow;
     NSTrackingArea *m_trackingArea;
     Qt::MouseButtons m_buttons;
@@ -90,9 +87,6 @@ Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 #ifndef QT_NO_OPENGL
 - (void)setQCocoaGLContext:(QCocoaGLContext *)context;
 #endif
-- (void)setMaskRegion:(const QRegion *)region;
-- (CGImageRef)maskImage;
-- (void)invalidateWindowShadowIfNeeded;
 - (void)drawRect:(NSRect)dirtyRect;
 - (void)textInputContextKeyboardSelectionDidChangeNotification : (NSNotification *) textInputContextKeyboardSelectionDidChangeNotification;
 - (void)viewDidHide;
@@ -101,7 +95,6 @@ Q_FORWARD_DECLARE_OBJC_CLASS(QT_MANGLE_NAMESPACE(QNSViewMouseMoveHelper));
 - (BOOL)isFlipped;
 - (BOOL)acceptsFirstResponder;
 - (BOOL)becomeFirstResponder;
-- (BOOL)hasMask;
 - (BOOL)isOpaque;
 
 - (void)convertFromScreen:(NSPoint)mouseLocation toWindowPoint:(QPointF *)qtWindowPoint andScreenPoint:(QPointF *)qtScreenPoint;

src/plugins/platforms/cocoa/qnsview.mm

@@ -128,8 +128,6 @@ static QTouchDevice *touchDevice = 0;
 - (id) init
 {
     if (self = [super initWithFrame:NSZeroRect]) {
-        m_maskImage = 0;
-        m_shouldInvalidateWindowShadow = false;
         m_buttons = Qt::NoButton;
         m_acceptedMouseDowns = Qt::NoButton;
         m_frameStrutButtons = Qt::NoButton;
@@ -163,12 +161,10 @@ static QTouchDevice *touchDevice = 0;
 
 - (void)dealloc
 {
-    CGImageRelease(m_maskImage);
     if (m_trackingArea) {
         [self removeTrackingArea:m_trackingArea];
         [m_trackingArea release];
     }
-    m_maskImage = 0;
     [m_inputSource release];
     [[NSNotificationCenter defaultCenter] removeObserver:self];
     [m_mouseMoveHelper release];
@@ -304,11 +300,6 @@ static QTouchDevice *touchDevice = 0;
     [super removeFromSuperview];
 }
 
-- (BOOL) hasMask
-{
-    return !m_maskRegion.isEmpty();
-}
-
 - (BOOL) isOpaque
 {
     if (!m_platformWindow)
@@ -316,48 +307,6 @@ static QTouchDevice *touchDevice = 0;
     return m_platformWindow->isOpaque();
 }
 
-- (void) setMaskRegion:(const QRegion *)region
-{
-    m_shouldInvalidateWindowShadow = true;
-    m_maskRegion = *region;
-    if (m_maskImage)
-        CGImageRelease(m_maskImage);
-    if (region->isEmpty()) {
-        m_maskImage = 0;
-        return;
-    }
-
-    const QRect &rect = region->boundingRect();
-    QImage tmp(rect.size(), QImage::Format_RGB32);
-    tmp.fill(Qt::white);
-    QPainter p(&tmp);
-    p.setClipRegion(*region);
-    p.fillRect(rect, Qt::black);
-    p.end();
-    QImage maskImage = QImage(rect.size(), QImage::Format_Indexed8);
-    for (int y=0; y<rect.height(); ++y) {
-        const uint *src = (const uint *) tmp.constScanLine(y);
-        uchar *dst = maskImage.scanLine(y);
-        for (int x=0; x<rect.width(); ++x) {
-            dst[x] = src[x] & 0xff;
-        }
-    }
-    m_maskImage = qt_mac_toCGImageMask(maskImage);
-}
-
-- (CGImageRef)maskImage
-{
-    return m_maskImage;
-}
-
-- (void)invalidateWindowShadowIfNeeded
-{
-    if (m_shouldInvalidateWindowShadow && m_platformWindow->isContentView()) {
-        [m_platformWindow->nativeWindow() invalidateShadow];
-        m_shouldInvalidateWindowShadow = false;
-    }
-}
-
 - (void)drawRect:(NSRect)dirtyRect
 {
     Q_UNUSED(dirtyRect);
@@ -612,7 +561,8 @@ static QTouchDevice *touchDevice = 0;
     Q_UNUSED(qtScreenPoint);
 
     // Maintain masked state for the button for use by MouseDragged and MouseUp.
-    const bool masked = [self hasMask] && !m_maskRegion.contains(qtWindowPoint.toPoint());
+    QRegion mask = m_platformWindow->window()->mask();
+    const bool masked = !mask.isEmpty() && !mask.contains(qtWindowPoint.toPoint());
     if (masked)
         m_acceptedMouseDowns &= ~button;
     else
@@ -710,8 +660,8 @@ static QTouchDevice *touchDevice = 0;
     [self convertFromScreen:[self screenMousePoint:theEvent] toWindowPoint:&qtWindowPoint andScreenPoint:&qtScreenPoint];
     Q_UNUSED(qtScreenPoint);
 
-    const bool masked = [self hasMask] && !m_maskRegion.contains(qtWindowPoint.toPoint());
-
+    QRegion mask = m_platformWindow->window()->mask();
+    const bool masked = !mask.isEmpty() && !mask.contains(qtWindowPoint.toPoint());
     // Maintain masked state for the button for use by MouseDragged and Up.
     if (masked)
         m_acceptedMouseDowns &= ~Qt::LeftButton;