diff --git a/src/gui/kernel/qevent.cpp b/src/gui/kernel/qevent.cpp
index b5418347b45..adc10297541 100644
--- a/src/gui/kernel/qevent.cpp
+++ b/src/gui/kernel/qevent.cpp
@@ -400,7 +400,7 @@ QList<QPointer<QObject> > QPointerEvent::passiveGrabbers(const QEventPoint &poin
         qWarning() << "point is not in activePoints" << point;
         return {};
     }
-    return persistentPoint->passiveGrabbers.keys();
+    return persistentPoint->passiveGrabbers;
 }
 
 /*!
diff --git a/src/gui/kernel/qpointingdevice.cpp b/src/gui/kernel/qpointingdevice.cpp
index 58f318f2f94..6ef032098f2 100644
--- a/src/gui/kernel/qpointingdevice.cpp
+++ b/src/gui/kernel/qpointingdevice.cpp
@@ -504,8 +504,6 @@ void QPointingDevicePrivate::setExclusiveGrabber(const QPointerEvent *event, con
     QMutableEventPoint::from(persistentPoint->eventPoint).setGlobalGrabPosition(point.globalPosition());
     if (exclusiveGrabber)
         emit q->grabChanged(exclusiveGrabber, QPointingDevice::GrabExclusive, event, point);
-    else
-        persistentPoint->exclusiveGrabberContext.clear();
 }
 
 /*!
@@ -538,7 +536,7 @@ bool QPointingDevicePrivate::addPassiveGrabber(const QPointerEvent *event, const
         qCDebug(lcPointerGrab) << name << "point" << point.id() << point.state()
                                << ": grab (passive)" << grabber;
     }
-    persistentPoint->passiveGrabbers.insert(grabber, {});
+    persistentPoint->passiveGrabbers << grabber;
     emit q->grabChanged(grabber, QPointingDevice::GrabPassive, event, point);
     return true;
 }
@@ -551,14 +549,14 @@ bool QPointingDevicePrivate::removePassiveGrabber(const QPointerEvent *event, co
         qWarning() << "point is not in activePoints" << point;
         return false;
     }
-    auto pgit = persistentPoint->passiveGrabbers.find(grabber);
-    if (pgit != persistentPoint->passiveGrabbers.end()) {
+    int i = persistentPoint->passiveGrabbers.indexOf(grabber);
+    if (i >= 0) {
         if (Q_UNLIKELY(lcPointerGrab().isDebugEnabled())) {
             qCDebug(lcPointerGrab) << name << "point" << point.id() << point.state()
                                    << ": removing passive grabber" << grabber;
         }
         emit q->grabChanged(grabber, QPointingDevice::UngrabPassive, event, point);
-        persistentPoint->passiveGrabbers.erase(pgit);
+        persistentPoint->passiveGrabbers.removeAt(i);
         return true;
     }
     return false;
@@ -576,9 +574,9 @@ void QPointingDevicePrivate::clearPassiveGrabbers(const QPointerEvent *event, co
         return;
     if (Q_UNLIKELY(lcPointerGrab().isDebugEnabled())) {
         qCDebug(lcPointerGrab) << name << "point" << point.id() << point.state()
-                               << ": clearing" << persistentPoint->passiveGrabbers.keys();
+                               << ": clearing" << persistentPoint->passiveGrabbers;
     }
-    for (auto g : persistentPoint->passiveGrabbers.keys())
+    for (auto g : persistentPoint->passiveGrabbers)
         emit q->grabChanged(g, QPointingDevice::UngrabPassive, event, point);
     persistentPoint->passiveGrabbers.clear();
 }
@@ -604,16 +602,15 @@ void QPointingDevicePrivate::removeGrabber(QObject *grabber, bool cancel)
                                    << "@" << epd.eventPoint.scenePosition()
                                    << ": grab" << grabber << "-> nullptr";
             epd.exclusiveGrabber.clear();
-            epd.exclusiveGrabberContext.clear();
             emit q->grabChanged(grabber,
                                 cancel ? QPointingDevice::CancelGrabExclusive : QPointingDevice::UngrabExclusive,
                                 nullptr, epd.eventPoint);
         }
-        auto pgit = epd.passiveGrabbers.find(grabber);
-        if (pgit != epd.passiveGrabbers.end()) {
+        int pi = epd.passiveGrabbers.indexOf(grabber);
+        if (pi >= 0) {
             qCDebug(lcPointerGrab) << name << "point" << epd.eventPoint.id() << epd.eventPoint.state()
                                    << ": removing passive grabber" << grabber;
-            epd.passiveGrabbers.erase(pgit);
+            epd.passiveGrabbers.removeAt(pi);
             emit q->grabChanged(grabber,
                                 cancel ? QPointingDevice::CancelGrabPassive : QPointingDevice::UngrabPassive,
                                 nullptr, epd.eventPoint);
diff --git a/src/gui/kernel/qpointingdevice_p.h b/src/gui/kernel/qpointingdevice_p.h
index 0198b0d1426..645c82673b8 100644
--- a/src/gui/kernel/qpointingdevice_p.h
+++ b/src/gui/kernel/qpointingdevice_p.h
@@ -85,12 +85,10 @@ public:
     /*! \internal
         This struct (stored in activePoints) holds persistent state between event deliveries.
     */
-    using PassiveGrabbersMap = QFlatMap<QPointer<QObject>, QPointer<QObject>>;
     struct EventPointData {
         QEventPoint eventPoint;
         QPointer<QObject> exclusiveGrabber;
-        QPointer<QObject> exclusiveGrabberContext;  // extra info about where the grab happened
-        PassiveGrabbersMap passiveGrabbers;         // key: passive grabber; value: context (where the grab happened)
+        QList<QPointer <QObject> > passiveGrabbers;
     };
     EventPointData *queryPointById(int id) const;
     EventPointData *pointById(int id) const;
diff --git a/tests/auto/gui/kernel/qmouseevent/tst_qmouseevent.cpp b/tests/auto/gui/kernel/qmouseevent/tst_qmouseevent.cpp
index cdf11ec54ac..d5174e09421 100644
--- a/tests/auto/gui/kernel/qmouseevent/tst_qmouseevent.cpp
+++ b/tests/auto/gui/kernel/qmouseevent/tst_qmouseevent.cpp
@@ -290,7 +290,7 @@ void tst_QMouseEvent::grabbers()
     QCOMPARE(firstEPD->exclusiveGrabber, grabExclusive ? testMouseWidget : nullptr);
     QCOMPARE(firstEPD->passiveGrabbers.count(), grabPassive ? 1 : 0);
     if (grabPassive)
-        QCOMPARE(firstEPD->passiveGrabbers.keys().first(), testMouseWidget);
+        QCOMPARE(firstEPD->passiveGrabbers.first(), testMouseWidget);
 
     // Ensure that grabbers are forgotten after release delivery
     QTest::mouseRelease(testMouseWidget, Qt::LeftButton, Qt::KeyboardModifiers(), {10, 10});
diff --git a/tests/auto/gui/kernel/qtouchevent/tst_qtouchevent.cpp b/tests/auto/gui/kernel/qtouchevent/tst_qtouchevent.cpp
index fa3f495c236..28c33e82265 100644
--- a/tests/auto/gui/kernel/qtouchevent/tst_qtouchevent.cpp
+++ b/tests/auto/gui/kernel/qtouchevent/tst_qtouchevent.cpp
@@ -1949,7 +1949,7 @@ void tst_QTouchEvent::grabbers()
     QCOMPARE(devPriv->pointById(0)->exclusiveGrabber, grabExclusive ? &w : nullptr);
     QCOMPARE(devPriv->pointById(0)->passiveGrabbers.count(), grabPassive ? 1 : 0);
     if (grabPassive)
-        QCOMPARE(devPriv->pointById(0)->passiveGrabbers.keys().first(), &w);
+        QCOMPARE(devPriv->pointById(0)->passiveGrabbers.first(), &w);
 
     // Ensure that eventpoints are forgotten after release delivery
     points.first().state = QEventPoint::State::Released;