wayland: Reset surface on QWindow type change

Changing the role between toplevel and popups is illegal on wayland even
if the window is not visible. We need to reset to the wl_surface.

This meant we cannot call setWindowFlags in the constructor.

QWindow::flags / QWindow::windowType changes after
QPlatformWindow::setFlags has finished, so we need to use our own flags
rather in XdgShell rather than pulling them from the QWindow.

Task-number: QTBUG-136110
Change-Id: I8b54b7ea8a768a539178395e53cc63a64fd80232
Reviewed-by: David Edmundson <davidedmundson@kde.org>

src/plugins/platforms/wayland/plugins/shellintegration/xdg-shell/qwaylandxdgshell.cpp

@@ -311,7 +311,7 @@ QWaylandXdgSurface::QWaylandXdgSurface(QWaylandXdgShell *shell, ::xdg_surface *s
     , m_window(window)
 {
     QWaylandDisplay *display = window->display();
-    Qt::WindowType type = window->window()->type();
+    Qt::WindowType type =  static_cast<Qt::WindowType>(int(window->windowFlags() & Qt::WindowType_Mask));
     auto *transientParent = window->transientParent();
 
     if (type == Qt::ToolTip) {

src/plugins/platforms/wayland/qwaylandwindow.cpp

@@ -62,6 +62,7 @@ QWaylandWindow::QWaylandWindow(QWindow *window, QWaylandDisplay *display)
     }
 
     initializeWlSurface();
+    mFlags = window->flags();
 
     setWindowIcon(window->icon());
 
@@ -180,6 +181,9 @@ void QWaylandWindow::initWindow()
         }
     }
 
+    createDecoration();
+    updateInputRegion();
+
     // Enable high-dpi rendering. Scale() returns the screen scale factor and will
     // typically be integer 1 (normal-dpi) or 2 (high-dpi). Call set_buffer_scale()
     // to inform the compositor that high-resolution buffers will be provided.
@@ -188,7 +192,6 @@ void QWaylandWindow::initWindow()
     else if (mSurface->version() >= 3)
         mSurface->set_buffer_scale(std::ceil(scale()));
 
-    setWindowFlags(window()->flags());
     QRect geometry = windowGeometry();
     QRect defaultGeometry = this->defaultGeometry();
     if (geometry.width() <= 0)
@@ -198,10 +201,11 @@ void QWaylandWindow::initWindow()
 
     setGeometry_helper(geometry);
     setMask(window()->mask());
-    if (mShellSurface)
+    if (mShellSurface) {
         mShellSurface->requestWindowStates(window()->windowStates());
+        mShellSurface->setWindowFlags(mFlags);
+    }
     handleContentOrientationChange(window()->contentOrientation());
-    mFlags = window()->flags();
 
     if (mShellSurface && mShellSurface->commitSurfaceRole())
         mSurface->commit();
@@ -1046,16 +1050,33 @@ void QWaylandWindow::setWindowState(Qt::WindowStates states)
 
 void QWaylandWindow::setWindowFlags(Qt::WindowFlags flags)
 {
-    if (mShellSurface)
+    const bool wasPopup = mFlags.testFlag(Qt::Popup);
-        mShellSurface->setWindowFlags(flags);
+    const bool isPopup = flags.testFlag(Qt::Popup);
 
     mFlags = flags;
+    // changing role is not allowed on XdgShell on the same wl_surface
+    if (wasPopup != isPopup) {
+        reset();
+        initializeWlSurface();
+        if (window()->isVisible()) {
+            initWindow();
+        }
+    } else {
+        if (mShellSurface)
+            mShellSurface->setWindowFlags(flags);
+    }
+
     createDecoration();
 
     QReadLocker locker(&mSurfaceLock);
     updateInputRegion();
 }
 
+Qt::WindowFlags QWaylandWindow::windowFlags() const
+{
+    return mFlags;
+}
+
 bool QWaylandWindow::createDecoration()
 {
     Q_ASSERT_X(QThread::currentThreadId() == QThreadData::get2(thread())->threadId.loadRelaxed(),

src/plugins/platforms/wayland/qwaylandwindow_p.h

@@ -156,6 +156,7 @@ public:
     Qt::WindowStates windowStates() const;
     void setWindowState(Qt::WindowStates states) override;
     void setWindowFlags(Qt::WindowFlags flags) override;
+    Qt::WindowFlags windowFlags() const;
     void handleWindowStatesChanged(Qt::WindowStates states);
 
     void raise() override;

tests/auto/wayland/xdgshell/tst_xdgshell.cpp

@@ -24,6 +24,7 @@ private slots:
     void popup();
     void tooltipOnPopup();
     void tooltipAndSiblingPopup();
+    void windowTypeChanges();
     void switchPopups();
     void hidePopupParent();
     void popupsWithoutParent();
@@ -475,6 +476,43 @@ void tst_xdgshell::tooltipAndSiblingPopup()
     QCOMPOSITOR_TRY_COMPARE(xdgPopup(0), nullptr);
 }
 
+void tst_xdgshell::windowTypeChanges()
+{
+    QRasterWindow parentWindow;
+    parentWindow.resize(200, 200);
+    parentWindow.show();
+
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel());
+    exec([&] { xdgToplevel()->sendCompleteConfigure(); });
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel()->m_xdgSurface->m_committedConfigureSerial);
+
+    // show a toplevel
+    QRasterWindow window;
+    window.resize(100, 100);
+    window.setTransientParent(&parentWindow);
+    window.show();
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel(1));
+    exec([&] { xdgToplevel(1)->sendCompleteConfigure(); });
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel(1)->m_xdgSurface->m_committedConfigureSerial);
+
+    // now change it to a popup
+    window.setFlag(Qt::ToolTip, true);
+    QCOMPOSITOR_TRY_VERIFY(!xdgToplevel(1));
+    QCOMPOSITOR_TRY_VERIFY(xdgPopup());
+    exec([&] { xdgPopup()->sendCompleteConfigure(QRect(100, 100, 100, 100)); });
+    QCOMPOSITOR_TRY_VERIFY(xdgPopup()->m_xdgSurface->m_committedConfigureSerial);
+
+    window.hide();
+    QCOMPOSITOR_TRY_VERIFY(!xdgPopup());
+
+    // change to a toplevel again this time whilst hidden
+    window.setFlag(Qt::ToolTip, false);
+    window.show();
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel(1));
+    exec([&] { xdgToplevel(1)->sendCompleteConfigure(); });
+    QCOMPOSITOR_TRY_VERIFY(xdgToplevel(1)->m_xdgSurface->m_committedConfigureSerial);
+}
+
 // QTBUG-65680
 void tst_xdgshell::switchPopups()
 {