macOS: Invalidate window shadow after QNSWindowBackingStore resize

The window shadow rendered by AppKit is based on the shape/content of the
NSWindow surface. If the backingstore is partially transparent, we need
to invalidate the window shadow after each resize (and subsequent flush)
of the backingstore.

Change-Id: I451370af5a8c0c25faea26beb3faa2483a33a5cf
Fixes: QTBUG-74560
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

src/plugins/platforms/cocoa/qcocoabackingstore.h

@@ -55,6 +55,7 @@ public:
     QNSWindowBackingStore(QWindow *window);
     ~QNSWindowBackingStore();
 
+    void resize(const QSize &size, const QRegion &staticContents) override;
     void flush(QWindow *, const QRegion &, const QPoint &) override;
 
 private:

src/plugins/platforms/cocoa/qcocoabackingstore.mm

@@ -71,6 +71,24 @@ QImage::Format QNSWindowBackingStore::format() const
     return QRasterBackingStore::format();
 }
 
+void QNSWindowBackingStore::resize(const QSize &size, const QRegion &staticContents)
+{
+    qCDebug(lcQpaBackingStore) << "Resize requested to" << size;
+    QRasterBackingStore::resize(size, staticContents);
+
+    // The window shadow rendered by AppKit is based on the shape/content of the
+    // NSWindow surface. Technically any flush of the backingstore can result in
+    // a potentially new shape of the window, and would need a shadow invalidation,
+    // but this is likely too expensive to do at every flush for the few cases where
+    // clients change the shape dynamically. One case where we do know that the shadow
+    // likely needs invalidation, if the window has partially transparent content,
+    // is after a resize, where AppKit's default shadow may be based on the previous
+    // window content.
+    QCocoaWindow *cocoaWindow = static_cast<QCocoaWindow *>(window()->handle());
+    if (cocoaWindow->isContentView() && !cocoaWindow->isOpaque())
+        cocoaWindow->m_needsInvalidateShadow = true;
+}
+
 /*!
     Flushes the given \a region from the specified \a window onto the
     screen.
@@ -217,6 +235,7 @@ void QNSWindowBackingStore::flush(QWindow *window, const QRegion &region, const
 
     QCocoaWindow *topLevelCocoaWindow = static_cast<QCocoaWindow *>(topLevelWindow->handle());
     if (Q_UNLIKELY(topLevelCocoaWindow->m_needsInvalidateShadow)) {
+        qCDebug(lcQpaBackingStore) << "Invalidating window shadow for" << topLevelCocoaWindow;
         [topLevelView.window invalidateShadow];
         topLevelCocoaWindow->m_needsInvalidateShadow = false;
     }

src/plugins/platforms/cocoa/qcocoawindow.mm

@@ -1010,16 +1010,16 @@ void QCocoaWindow::setMask(const QRegion &region)
         } else {
             m_view.layer.mask = nil;
         }
-    }
+    } else {
-
+        if (isContentView()) {
-    if (isContentView()) {
+            // Setting the mask requires invalidating the NSWindow shadow, but that needs
-        // Setting the mask requires invalidating the NSWindow shadow, but that needs
+            // to happen after the backingstore has been redrawn, so that AppKit can pick
-        // to happen after the backingstore has been redrawn, so that AppKit can pick
+            // up the new window shape based on the backingstore content. Doing a display
-        // up the new window shape based on the backingstore content. Doing a display
+            // directly here is not an option, as the window might not be exposed at this
-        // directly here is not an option, as the window might not be exposed at this
+            // time, and so would not result in an updated backingstore.
-        // time, and so would not result in an updated backingstore.
+            m_needsInvalidateShadow = true;
-        m_needsInvalidateShadow = true;
+            [m_view setNeedsDisplay:YES];
-        [m_view setNeedsDisplay:YES];
+        }
     }
 }