From 66a76a5def46d0e4a330f7130ad440c639b87cf7 Mon Sep 17 00:00:00 2001 From: Lorn Potter Date: Mon, 17 Jan 2022 07:25:26 +1000 Subject: [PATCH] wasm: enable mobile native keyboarding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This works on iOS and Android, and Windows with touchscreen. On Android, we need to listen to the input event of a hidden text element and synthesize Qt keyboard events from that in order to get input events into Qt. On Windows, we need to be more creative about bringing the native virtual keyboard up. Because the entire canvas is contenteditable, we need to specify the inputmode is set to 'none', otherwise the v keyboard pops up when user clicks anywhere on the canvas. Therefore we set a hidden element as contenteditable, which pops up keyboard when Qt needs it for editable widgets. On Android, this is the same element that is used to proxy the keyboard input. [ChangeLog][wasm] Add support for native mobile keyboard Done-with: Morten Johan Sørvig Fixes: QTBUG-83064 Fixes: QTBUG-88803 Change-Id: I769fe344fc10c17971bd1c0a603501040fe82653 Reviewed-by: David Skoland Reviewed-by: Morten Johan Sørvig --- cmake/QtWasmHelpers.cmake | 2 + src/plugins/platforms/wasm/CMakeLists.txt | 1 + .../platforms/wasm/qwasmcompositor.cpp | 20 +- src/plugins/platforms/wasm/qwasmcompositor.h | 2 + .../platforms/wasm/qwasmeventtranslator.h | 2 +- .../platforms/wasm/qwasminputcontext.cpp | 176 ++++++++++++++++++ .../platforms/wasm/qwasminputcontext.h | 71 +++++++ .../platforms/wasm/qwasmintegration.cpp | 23 +++ src/plugins/platforms/wasm/qwasmintegration.h | 18 +- src/plugins/platforms/wasm/qwasmscreen.cpp | 3 + 10 files changed, 303 insertions(+), 15 deletions(-) create mode 100644 src/plugins/platforms/wasm/qwasminputcontext.cpp create mode 100644 src/plugins/platforms/wasm/qwasminputcontext.h diff --git a/cmake/QtWasmHelpers.cmake b/cmake/QtWasmHelpers.cmake index 47af621861b..f458f9a560f 100644 --- a/cmake/QtWasmHelpers.cmake +++ b/cmake/QtWasmHelpers.cmake @@ -65,6 +65,8 @@ function (qt_internal_setup_wasm_target_properties wasmTarget) "SHELL:-s DEMANGLE_SUPPORT=1" "SHELL:-s GL_DEBUG=1" "SHELL:-s ASSERTIONS=2" + "SHELL:-s SAFE_HEAP=1" + "SHELL:-s SAFE_HEAP_LOG=1" --profiling-funcs>) # target_link_options("${wasmTarget}" INTERFACE "SHELL:-s LIBRARY_DEBUG=1") # print out library calls, verbose diff --git a/src/plugins/platforms/wasm/CMakeLists.txt b/src/plugins/platforms/wasm/CMakeLists.txt index 130af2dec69..3fc1f6e5956 100644 --- a/src/plugins/platforms/wasm/CMakeLists.txt +++ b/src/plugins/platforms/wasm/CMakeLists.txt @@ -26,6 +26,7 @@ qt_internal_add_plugin(QWasmIntegrationPlugin qwasmstylepixmaps_p.h qwasmtheme.cpp qwasmtheme.h qwasmwindow.cpp qwasmwindow.h + qwasminputcontext.cpp qwasminputcontext.h DEFINES QT_EGL_NO_X11 QT_NO_FOREACH diff --git a/src/plugins/platforms/wasm/qwasmcompositor.cpp b/src/plugins/platforms/wasm/qwasmcompositor.cpp index 51890b32109..fe0ea81692a 100644 --- a/src/plugins/platforms/wasm/qwasmcompositor.cpp +++ b/src/plugins/platforms/wasm/qwasmcompositor.cpp @@ -160,17 +160,9 @@ void QWasmCompositor::initEventHandlers() { QByteArray canvasSelector = "#" + screen()->canvasId().toUtf8(); - // The Platform Detect: expand coverage and move as needed - enum Platform { - GenericPlatform, - MacOSPlatform - }; - Platform platform = Platform(emscripten::val::global("navigator")["platform"] - .call("includes", emscripten::val("Mac"))); - - eventTranslator->setIsMac(platform == MacOSPlatform); - - if (platform == MacOSPlatform) { + eventTranslator->g_usePlatformMacSpecifics + = (QWasmIntegration::get()->platform == QWasmIntegration::MacOSPlatform); + if (QWasmIntegration::get()->platform == QWasmIntegration::MacOSPlatform) { g_useNaturalScrolling = false; // make this !default on macOS if (!emscripten::val::global("window")["safari"].isUndefined()) { @@ -1299,11 +1291,13 @@ int QWasmCompositor::handleTouch(int eventType, const EmscriptenTouchEvent *touc QFlags keyModifier = eventTranslator->translateTouchEventModifier(touchEvent); - bool accepted = QWindowSystemInterface::handleTouchEvent( - window2, QWasmIntegration::getTimestamp(), touchDevice, touchPointList, keyModifier); + bool accepted = false; if (eventType == EMSCRIPTEN_EVENT_TOUCHCANCEL) accepted = QWindowSystemInterface::handleTouchCancelEvent(window2, QWasmIntegration::getTimestamp(), touchDevice, keyModifier); + else + accepted = QWindowSystemInterface::handleTouchEvent( + window2, QWasmIntegration::getTimestamp(), touchDevice, touchPointList, keyModifier); return static_cast(accepted); } diff --git a/src/plugins/platforms/wasm/qwasmcompositor.h b/src/plugins/platforms/wasm/qwasmcompositor.h index 1feda048702..d405f8035b8 100644 --- a/src/plugins/platforms/wasm/qwasmcompositor.h +++ b/src/plugins/platforms/wasm/qwasmcompositor.h @@ -42,6 +42,8 @@ #include #include +#include +#include QT_BEGIN_NAMESPACE diff --git a/src/plugins/platforms/wasm/qwasmeventtranslator.h b/src/plugins/platforms/wasm/qwasmeventtranslator.h index 341971d79fa..d7a1fa331a8 100644 --- a/src/plugins/platforms/wasm/qwasmeventtranslator.h +++ b/src/plugins/platforms/wasm/qwasmeventtranslator.h @@ -67,11 +67,11 @@ public: void setStickyDeadKey(const EmscriptenKeyboardEvent *keyEvent); void setIsMac(bool is_mac) {g_usePlatformMacSpecifics = is_mac;}; + bool g_usePlatformMacSpecifics = false; Q_SIGNALS: void getWindowAt(const QPoint &point, QWindow **window); private: - bool g_usePlatformMacSpecifics = false; static Qt::Key translateDeadKey(Qt::Key deadKey, Qt::Key accentBaseKey, bool is_mac = false); private: diff --git a/src/plugins/platforms/wasm/qwasminputcontext.cpp b/src/plugins/platforms/wasm/qwasminputcontext.cpp new file mode 100644 index 00000000000..f50395c059c --- /dev/null +++ b/src/plugins/platforms/wasm/qwasminputcontext.cpp @@ -0,0 +1,176 @@ +/**************************************************************************** +** +** Copyright (C) 2019 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the plugins of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 or (at your option) any later version +** approved by the KDE Free Qt Foundation. The licenses are as published by +** the Free Software Foundation and appearing in the file LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include + +#include "qwasminputcontext.h" +#include "qwasmintegration.h" +#include +#include +#include "qwasmeventtranslator.h" +#include "qwasmscreen.h" +#include +#include +#include +#include +using namespace qstdweb; + +static void inputCallback(emscripten::val event) +{ + QString str = QString::fromStdString(event["target"]["value"].as()); + QWasmInputContext *wasmInput = + reinterpret_cast(event["target"]["data-context"].as()); + wasmInput->inputStringChanged(str, wasmInput); + + // this stops suggestions + // but allows us to send only one character like a normal keyboard + event["target"].set("value", ""); +} + +EMSCRIPTEN_BINDINGS(clipboard_module) { + function("qt_InputContextCallback", &inputCallback); +} + +QWasmInputContext::QWasmInputContext() +{ + emscripten::val document = emscripten::val::global("document"); + m_inputElement = document.call("createElement", std::string("input")); + m_inputElement.set("type", "text"); + m_inputElement.set("style", "position:absolute;left:-1000px;top:-1000px"); // offscreen + m_inputElement.set("contentaediable","true"); + + if (QWasmIntegration::get()->platform == QWasmIntegration::AndroidPlatform) { + emscripten::val body = document["body"]; + body.call("appendChild", m_inputElement); + + m_inputElement.call("addEventListener", std::string("input"), + emscripten::val::module_property("qt_InputContextCallback"), + emscripten::val(false)); + m_inputElement.set("data-context", + emscripten::val(quintptr(reinterpret_cast(this)))); + + // android sends Enter through target window, let's just handle this here + emscripten_set_keydown_callback(EMSCRIPTEN_EVENT_TARGET_WINDOW, (void *)this, 1, + &androidKeyboardCallback); + + } + if (QWasmIntegration::get()->platform == QWasmIntegration::MacOSPlatform) + { + auto callback = [=](emscripten::val) { + m_inputElement["parentElement"].call("removeChild", m_inputElement); }; + m_blurEventHandler.reset(new EventCallback(m_inputElement, "blur", callback)); + inputPanelIsOpen = false; + } + + QObject::connect(qGuiApp, &QGuiApplication::focusWindowChanged, this, + &QWasmInputContext::focusWindowChanged); +} + +QWasmInputContext::~QWasmInputContext() +{ + if (QWasmIntegration::get()->platform == QWasmIntegration::AndroidPlatform) + emscripten_set_keydown_callback(EMSCRIPTEN_EVENT_TARGET_WINDOW, 0, 0, NULL); +} + +void QWasmInputContext::focusWindowChanged(QWindow *focusWindow) +{ + m_focusWindow = focusWindow; +} + +emscripten::val QWasmInputContext::focusCanvas() +{ + if (!m_focusWindow) + return emscripten::val::undefined(); + QScreen *screen = m_focusWindow->screen(); + if (!screen) + return emscripten::val::undefined(); + return QWasmScreen::get(screen)->canvas(); +} + +void QWasmInputContext::update(Qt::InputMethodQueries queries) +{ + QPlatformInputContext::update(queries); +} + +void QWasmInputContext::showInputPanel() +{ + if (QWasmIntegration::get()->platform == QWasmIntegration::WindowsPlatform + && inputPanelIsOpen) // call this only once for win32 + return; + // this is called each time the keyboard is touched + + // Add the input element as a child of the canvas for the + // currently focused window and give it focus. The browser + // will not display the input element, but mobile browsers + // should display the virtual keyboard. Key events will be + // captured by the keyboard event handler installed on the + // canvas. + + if (QWasmIntegration::get()->platform == QWasmIntegration::MacOSPlatform + || QWasmIntegration::get()->platform == QWasmIntegration::WindowsPlatform) { + emscripten::val canvas = focusCanvas(); + if (canvas == emscripten::val::undefined()) + return; + canvas.call("appendChild", m_inputElement); + } + + m_inputElement.call("focus"); + inputPanelIsOpen = true; +} + +void QWasmInputContext::hideInputPanel() +{ + if (QWasmIntegration::get()->touchPoints < 1) + return; + m_inputElement.call("blur"); + inputPanelIsOpen = false; +} + +void QWasmInputContext::inputStringChanged(QString &inputString, QWasmInputContext *context) +{ + Q_UNUSED(context) + QKeySequence keys = QKeySequence::fromString(inputString); + // synthesize this keyevent as android is not normal + QWindowSystemInterface::handleKeyEvent( + 0, QEvent::KeyPress,keys[0].key(), keys[0].keyboardModifiers(), inputString); +} + +int QWasmInputContext::androidKeyboardCallback(int eventType, + const EmscriptenKeyboardEvent *keyEvent, + void *userData) +{ + Q_UNUSED(eventType) + QString strKey(keyEvent->key); + if (strKey == "Unidentified") + return false; + QWasmInputContext *wasmInput = reinterpret_cast(userData); + wasmInput->inputStringChanged(strKey, wasmInput); + + return true; +} diff --git a/src/plugins/platforms/wasm/qwasminputcontext.h b/src/plugins/platforms/wasm/qwasminputcontext.h new file mode 100644 index 00000000000..22e556d9bf0 --- /dev/null +++ b/src/plugins/platforms/wasm/qwasminputcontext.h @@ -0,0 +1,71 @@ +/**************************************************************************** +** +** Copyright (C) 2019 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the plugins of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:GPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3 or (at your option) any later version +** approved by the KDE Free Qt Foundation. The licenses are as published by +** the Free Software Foundation and appearing in the file LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QWASMINPUTCONTEXT_H +#define QWASMINPUTCONTEXT_H + + +#include +#include +#include +#include +#include +#include + +class QWasmInputContext : public QPlatformInputContext +{ + Q_DISABLE_COPY(QWasmInputContext) + Q_OBJECT +public: + explicit QWasmInputContext(); + ~QWasmInputContext() override; + + void update(Qt::InputMethodQueries) override; + + void showInputPanel() override; + void hideInputPanel() override; + bool isValid() const override { return true; } + + void focusWindowChanged(QWindow *focusWindow); + emscripten::val focusCanvas(); + void inputStringChanged(QString &, QWasmInputContext *context); + +private: + bool m_inputPanelVisible = false; + + QPointer m_focusWindow; + emscripten::val m_inputElement = emscripten::val::null(); + std::unique_ptr m_blurEventHandler; + std::unique_ptr m_inputEventHandler; + static int androidKeyboardCallback(int eventType, + const EmscriptenKeyboardEvent *keyEvent, void *userData); + bool inputPanelIsOpen = false; +}; + +#endif // QWASMINPUTCONTEXT_H diff --git a/src/plugins/platforms/wasm/qwasmintegration.cpp b/src/plugins/platforms/wasm/qwasmintegration.cpp index 031128563ee..01fb9f1cd05 100644 --- a/src/plugins/platforms/wasm/qwasmintegration.cpp +++ b/src/plugins/platforms/wasm/qwasmintegration.cpp @@ -106,6 +106,22 @@ QWasmIntegration::QWasmIntegration() { s_instance = this; + touchPoints = emscripten::val::global("navigator")["maxTouchPoints"].as(); + // The Platform Detect: expand coverage as needed + platform = GenericPlatform; + emscripten::val rawPlatform = emscripten::val::global("navigator")["platform"]; + + if (rawPlatform.call("includes", emscripten::val("Mac"))) + platform = MacOSPlatform; + if (rawPlatform.call("includes", emscripten::val("Win32"))) + platform = WindowsPlatform; + if (rawPlatform.call("includes", emscripten::val("Linux"))) { + platform = LinuxPlatform; + emscripten::val uAgent = emscripten::val::global("navigator")["userAgent"]; + if (uAgent.call("includes", emscripten::val("Android"))) + platform = AndroidPlatform; + } + // We expect that qtloader.js has populated Module.qtCanvasElements with one or more canvases. emscripten::val qtCanvaseElements = val::module_property("qtCanvasElements"); emscripten::val canvas = val::module_property("canvas"); // TODO: remove for Qt 6.0 @@ -156,6 +172,8 @@ QWasmIntegration::~QWasmIntegration() delete m_fontDb; delete m_desktopServices; + if (m_platformInputContext) + delete m_platformInputContext; for (const auto &canvasAndScreen : m_screens) QWindowSystemInterface::handleScreenRemoved(canvasAndScreen.second); @@ -210,9 +228,14 @@ QPlatformOpenGLContext *QWasmIntegration::createPlatformOpenGLContext(QOpenGLCon void QWasmIntegration::initialize() { + if (touchPoints < 1) // only touchscreen need inputcontexts + return; + QString icStr = QPlatformInputContextFactory::requested(); if (!icStr.isNull()) m_inputContext.reset(QPlatformInputContextFactory::create(icStr)); + else + m_inputContext.reset(new QWasmInputContext()); } QPlatformInputContext *QWasmIntegration::inputContext() const diff --git a/src/plugins/platforms/wasm/qwasmintegration.h b/src/plugins/platforms/wasm/qwasmintegration.h index 46fab8e8183..316f6eef40e 100644 --- a/src/plugins/platforms/wasm/qwasmintegration.h +++ b/src/plugins/platforms/wasm/qwasmintegration.h @@ -42,6 +42,9 @@ #include #include +#include "qwasminputcontext.h" +#include + QT_BEGIN_NAMESPACE class QWasmEventTranslator; @@ -58,6 +61,14 @@ class QWasmIntegration : public QObject, public QPlatformIntegration { Q_OBJECT public: + enum Platform { + GenericPlatform, + MacOSPlatform, + WindowsPlatform, + LinuxPlatform, + AndroidPlatform + }; + QWasmIntegration(); ~QWasmIntegration(); @@ -80,7 +91,7 @@ public: QPlatformInputContext *inputContext() const override; QWasmClipboard *getWasmClipboard() { return m_clipboard; } - + QWasmInputContext *getWasmInputContext() { return m_platformInputContext; } static QWasmIntegration *get() { return s_instance; } void addScreen(const emscripten::val &canvas); @@ -91,6 +102,9 @@ public: void removeBackingStore(QWindow* window); static quint64 getTimestamp(); + Platform platform; + int touchPoints; + private: mutable QWasmFontDatabase *m_fontDb; mutable QWasmServices *m_desktopServices; @@ -100,6 +114,8 @@ private: qreal m_fontDpi = -1; mutable QScopedPointer m_inputContext; static QWasmIntegration *s_instance; + + mutable QWasmInputContext *m_platformInputContext = nullptr; }; QT_END_NAMESPACE diff --git a/src/plugins/platforms/wasm/qwasmscreen.cpp b/src/plugins/platforms/wasm/qwasmscreen.cpp index 51020c45bf5..2ee56bc5aec 100644 --- a/src/plugins/platforms/wasm/qwasmscreen.cpp +++ b/src/plugins/platforms/wasm/qwasmscreen.cpp @@ -65,6 +65,9 @@ QWasmScreen::QWasmScreen(const emscripten::val &canvas) // Set contenteditable so that the canvas gets clipboard events, // then hide the resulting focus frame, and reset the cursor. m_canvas.set("contentEditable", std::string("true")); + // set inputmode to none to stop mobile keyboard opening + // when user clicks anywhere on the canvas. + m_canvas.set("inputmode", std::string("none")); style.set("outline", std::string("0px solid transparent")); style.set("caret-color", std::string("transparent")); style.set("cursor", std::string("default"));