From f65d11ecbf4d5417087f434c8974e4ea9a339b98 Mon Sep 17 00:00:00 2001 From: Andy Nichols Date: Fri, 24 May 2024 15:11:20 +0200 Subject: [PATCH] visionOS: Handle Spatial Events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch enables the handling of spatialEvents from SwiftUI. Since these events do not have ObjectiveC structs, for now we serialize them to JSON and pass them to Qt for usage later in Spatial applications. These events just cover the built-in gestures like pinch "clicking" etc. Change-Id: I8368683259eb64277083cf345ca3a5ed9af32ecf Reviewed-by: Tor Arne Vestbø --- src/gui/kernel/qguiapplication_platform.h | 1 + .../platforms/ios/qiosapplication.swift | 116 ++++++++++++++++++ src/plugins/platforms/ios/qiosintegration.h | 1 + src/plugins/platforms/ios/qiosintegration.mm | 13 ++ 4 files changed, 131 insertions(+) diff --git a/src/gui/kernel/qguiapplication_platform.h b/src/gui/kernel/qguiapplication_platform.h index d9ff01bf14c..98e19427ae7 100644 --- a/src/gui/kernel/qguiapplication_platform.h +++ b/src/gui/kernel/qguiapplication_platform.h @@ -84,6 +84,7 @@ struct Q_GUI_EXPORT QVisionOSApplication struct ImmersiveSpaceCompositorLayer { virtual void configure(cp_layer_renderer_capabilities_t, cp_layer_renderer_configuration_t) const {} virtual void render(cp_layer_renderer_t) = 0; + virtual void handleSpatialEvents(const QJsonObject &) {}; }; virtual void setImmersiveSpaceCompositorLayer(ImmersiveSpaceCompositorLayer *layer) = 0; virtual void openImmersiveSpace() = 0; diff --git a/src/plugins/platforms/ios/qiosapplication.swift b/src/plugins/platforms/ios/qiosapplication.swift index 6f75ebd0b5e..9eb172a8964 100644 --- a/src/plugins/platforms/ios/qiosapplication.swift +++ b/src/plugins/platforms/ios/qiosapplication.swift @@ -17,6 +17,11 @@ struct QIOSSwiftApplication: App { ImmersiveSpace(id: "QIOSImmersiveSpace") { CompositorLayer(configuration: QIOSLayerConfiguration()) { layerRenderer in QIOSIntegration.instance().renderCompositorLayer(layerRenderer) + + // Handle any events in the scene. + layerRenderer.onSpatialEvent = { eventCollection in + QIOSIntegration.instance().handleSpatialEvents(jsonStringFromEventCollection(eventCollection)) + } } } // CompositorLayer immersive spaces are always full, and should not need @@ -80,3 +85,114 @@ public class ImmersiveSpaceManager : NSObject { ImmersiveState.shared.showImmersiveSpace = false } } + +extension SpatialEventCollection.Event.Kind: Encodable { + enum CodingKeys: String, CodingKey { + case touch + case directPinch + case indirectPinch + case pointer + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .touch: + try container.encode("touch") + case .directPinch: + try container.encode("directPinch") + case .indirectPinch: + try container.encode("indirectPinch") + case .pointer: + try container.encode("pointer") + @unknown default: + try container.encode("unknown") + } + } +} +extension SpatialEventCollection.Event.Phase: Encodable { + enum CodingKeys: String, CodingKey { + case active + case ending + case cancled + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch self { + case .active: + try container.encode("active") + case .ended: + try container.encode("ended") + case .cancelled: + try container.encode("canceled") + @unknown default: + try container.encode("unknown") + } + } +} +extension SpatialEventCollection.Event.InputDevicePose: Encodable { + enum CodingKeys: String, CodingKey { + case altitude + case azimuth + case pose3D + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(altitude.radians, forKey: .altitude) + try container.encode(azimuth.radians, forKey: .azimuth) + try container.encode(pose3D, forKey: .pose3D) + } +} + +extension SpatialEventCollection.Event: Encodable { + enum CodingKeys: String, CodingKey { + case id + case timestamp + case kind + case location + case phase + case modifierKeys + case inputDevicePose + case location3D + case selectionRay + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(id.hashValue, forKey: .id) + try container.encode(timestamp, forKey: .timestamp) + try container.encode(kind, forKey: .kind) + try container.encode(location, forKey: .location) + try container.encode(phase, forKey: .phase) + try container.encode(modifierKeys.rawValue, forKey: .modifierKeys) + try container.encode(inputDevicePose, forKey: .inputDevicePose) + try container.encode(location3D, forKey: .location3D) + try container.encode(selectionRay, forKey: .selectionRay) + } +} + +extension SpatialEventCollection: Encodable { + enum CodingKeys: String, CodingKey { + case events + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(Array(self), forKey: .events) + } +} + +func jsonStringFromEventCollection(_ eventCollection: SpatialEventCollection) -> String { + let encoder = JSONEncoder() + encoder.dateEncodingStrategy = .iso8601 + + do { + let jsonData = try encoder.encode(eventCollection) + return String(data: jsonData, encoding: .utf8) ?? "{}" + } catch { + print("Failed to encode event collection: \(error)") + return "{}" + } +} diff --git a/src/plugins/platforms/ios/qiosintegration.h b/src/plugins/platforms/ios/qiosintegration.h index 53f64c1748e..6c2014d048f 100644 --- a/src/plugins/platforms/ios/qiosintegration.h +++ b/src/plugins/platforms/ios/qiosintegration.h @@ -99,6 +99,7 @@ public: void configureCompositorLayer(cp_layer_renderer_capabilities_t, cp_layer_renderer_configuration_t); void renderCompositorLayer(cp_layer_renderer_t); + void handleSpatialEvents(const char *jsonString); #endif private: diff --git a/src/plugins/platforms/ios/qiosintegration.mm b/src/plugins/platforms/ios/qiosintegration.mm index 2c32957c037..76173ce830a 100644 --- a/src/plugins/platforms/ios/qiosintegration.mm +++ b/src/plugins/platforms/ios/qiosintegration.mm @@ -329,6 +329,19 @@ void QIOSIntegration::renderCompositorLayer(cp_layer_renderer_t renderer) m_immersiveSpaceCompositorLayer->render(renderer); } +void QIOSIntegration::handleSpatialEvents(const char *jsonString) +{ + if (m_immersiveSpaceCompositorLayer) { + QJsonParseError error; + QJsonDocument doc = QJsonDocument::fromJson(QByteArray(jsonString), &error); + if (error.error != QJsonParseError::NoError) { + qWarning() << "Error parsing JSON: " << error.errorString(); + return; + } + m_immersiveSpaceCompositorLayer->handleSpatialEvents(doc.object()); + } +} + #endif // ---------------------------------------------------------