wasm: enable mobile native keyboarding

This works on iOS and Android, and Windows with touchscreen.

On Android, we need to listen to the input event
of a hidden text element and synthesize Qt keyboard
events from that in order to get input events into Qt.

On Windows, we need to be more creative about bringing the native
virtual keyboard up.

Because the entire canvas is contenteditable, we need to specify the
inputmode is set to 'none', otherwise the v keyboard pops up
when user clicks anywhere on the canvas. Therefore we set a hidden
element as contenteditable, which pops up keyboard when Qt
needs it for editable widgets. On Android, this is the same
element that is used to proxy the keyboard input.

[ChangeLog][wasm] Add support for native mobile keyboard

Done-with: Morten Johan Sørvig <morten.sorvig@qt.io>
Fixes: QTBUG-83064
Fixes: QTBUG-88803
Change-Id: I769fe344fc10c17971bd1c0a603501040fe82653
Reviewed-by: David Skoland <david.skoland@qt.io>
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>

cmake/QtWasmHelpers.cmake

@@ -65,6 +65,8 @@ function (qt_internal_setup_wasm_target_properties wasmTarget)
         "SHELL:-s DEMANGLE_SUPPORT=1"
         "SHELL:-s GL_DEBUG=1"
         "SHELL:-s ASSERTIONS=2"
+        "SHELL:-s SAFE_HEAP=1"
+        "SHELL:-s SAFE_HEAP_LOG=1"
         --profiling-funcs>)
 
     # target_link_options("${wasmTarget}" INTERFACE "SHELL:-s LIBRARY_DEBUG=1") # print out library calls, verbose

src/plugins/platforms/wasm/CMakeLists.txt

@@ -26,6 +26,7 @@ qt_internal_add_plugin(QWasmIntegrationPlugin
         qwasmstylepixmaps_p.h
         qwasmtheme.cpp qwasmtheme.h
         qwasmwindow.cpp qwasmwindow.h
+        qwasminputcontext.cpp qwasminputcontext.h
     DEFINES
         QT_EGL_NO_X11
         QT_NO_FOREACH

src/plugins/platforms/wasm/qwasmcompositor.cpp

@@ -160,17 +160,9 @@ void QWasmCompositor::initEventHandlers()
 {
     QByteArray canvasSelector = "#" + screen()->canvasId().toUtf8();
 
-    // The Platform Detect: expand coverage and move as needed
-    enum Platform {
-        GenericPlatform,
-        MacOSPlatform
-    };
-    Platform platform = Platform(emscripten::val::global("navigator")["platform"]
-                                         .call<bool>("includes", emscripten::val("Mac")));
-
-    eventTranslator->setIsMac(platform == MacOSPlatform);
-
-    if (platform == MacOSPlatform) {
+    eventTranslator->g_usePlatformMacSpecifics
+    = (QWasmIntegration::get()->platform == QWasmIntegration::MacOSPlatform);
+    if (QWasmIntegration::get()->platform == QWasmIntegration::MacOSPlatform) {
         g_useNaturalScrolling = false; // make this !default on macOS
 
         if (!emscripten::val::global("window")["safari"].isUndefined()) {
@@ -1299,11 +1291,13 @@ int QWasmCompositor::handleTouch(int eventType, const EmscriptenTouchEvent *touc
 
     QFlags<Qt::KeyboardModifier> keyModifier = eventTranslator->translateTouchEventModifier(touchEvent);
 
-    bool accepted = QWindowSystemInterface::handleTouchEvent<QWindowSystemInterface::SynchronousDelivery>(
-            window2, QWasmIntegration::getTimestamp(), touchDevice, touchPointList, keyModifier);
+    bool accepted = false;
 
     if (eventType == EMSCRIPTEN_EVENT_TOUCHCANCEL)
         accepted = QWindowSystemInterface::handleTouchCancelEvent(window2, QWasmIntegration::getTimestamp(), touchDevice, keyModifier);
+    else
+        accepted = QWindowSystemInterface::handleTouchEvent<QWindowSystemInterface::SynchronousDelivery>(
+                window2, QWasmIntegration::getTimestamp(), touchDevice, touchPointList, keyModifier);
 
     return static_cast<int>(accepted);
 }

src/plugins/platforms/wasm/qwasmcompositor.h

@@ -42,6 +42,8 @@
 #include <QPointingDevice>
 
 #include <emscripten/html5.h>
+#include <emscripten/emscripten.h>
+#include <emscripten/bind.h>
 
 QT_BEGIN_NAMESPACE
 

src/plugins/platforms/wasm/qwasmeventtranslator.h

@@ -67,11 +67,11 @@ public:
     void setStickyDeadKey(const EmscriptenKeyboardEvent *keyEvent);
 
     void setIsMac(bool is_mac) {g_usePlatformMacSpecifics = is_mac;};
+    bool g_usePlatformMacSpecifics = false;
 
 Q_SIGNALS:
     void getWindowAt(const QPoint &point, QWindow **window);
 private:
-    bool g_usePlatformMacSpecifics = false;
     static Qt::Key translateDeadKey(Qt::Key deadKey, Qt::Key accentBaseKey, bool is_mac = false);
 
 private:

src/plugins/platforms/wasm/qwasminputcontext.cpp

@@ -0,0 +1,176 @@
+/****************************************************************************
+**
+** Copyright (C) 2019 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the plugins of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 or (at your option) any later version
+** approved by the KDE Free Qt Foundation. The licenses are as published by
+** the Free Software Foundation and appearing in the file LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <emscripten/bind.h>
+
+#include "qwasminputcontext.h"
+#include "qwasmintegration.h"
+#include <QRectF>
+#include <qpa/qplatforminputcontext.h>
+#include "qwasmeventtranslator.h"
+#include "qwasmscreen.h"
+#include <qguiapplication.h>
+#include <qwindow.h>
+#include <QKeySequence>
+#include <qpa/qwindowsysteminterface.h>
+using namespace qstdweb;
+
+static void inputCallback(emscripten::val event)
+{
+    QString str = QString::fromStdString(event["target"]["value"].as<std::string>());
+    QWasmInputContext *wasmInput =
+            reinterpret_cast<QWasmInputContext*>(event["target"]["data-context"].as<quintptr>());
+    wasmInput->inputStringChanged(str, wasmInput);
+
+    // this stops suggestions
+    // but allows us to send only one character like a normal keyboard
+    event["target"].set("value", "");
+}
+
+EMSCRIPTEN_BINDINGS(clipboard_module) {
+    function("qt_InputContextCallback", &inputCallback);
+}
+
+QWasmInputContext::QWasmInputContext()
+{
+    emscripten::val document = emscripten::val::global("document");
+    m_inputElement = document.call<emscripten::val>("createElement", std::string("input"));
+    m_inputElement.set("type", "text");
+    m_inputElement.set("style", "position:absolute;left:-1000px;top:-1000px"); // offscreen
+    m_inputElement.set("contentaediable","true");
+
+    if (QWasmIntegration::get()->platform == QWasmIntegration::AndroidPlatform) {
+        emscripten::val body = document["body"];
+        body.call<void>("appendChild", m_inputElement);
+
+        m_inputElement.call<void>("addEventListener", std::string("input"),
+                                  emscripten::val::module_property("qt_InputContextCallback"),
+                                  emscripten::val(false));
+        m_inputElement.set("data-context",
+                           emscripten::val(quintptr(reinterpret_cast<void *>(this))));
+
+        // android sends Enter through target window, let's just handle this here
+        emscripten_set_keydown_callback(EMSCRIPTEN_EVENT_TARGET_WINDOW, (void *)this, 1,
+                                        &androidKeyboardCallback);
+
+    }
+    if (QWasmIntegration::get()->platform == QWasmIntegration::MacOSPlatform)
+     {
+        auto callback = [=](emscripten::val) {
+            m_inputElement["parentElement"].call<void>("removeChild", m_inputElement); };
+        m_blurEventHandler.reset(new EventCallback(m_inputElement, "blur", callback));
+        inputPanelIsOpen = false;
+    }
+
+    QObject::connect(qGuiApp, &QGuiApplication::focusWindowChanged, this,
+                     &QWasmInputContext::focusWindowChanged);
+}
+
+QWasmInputContext::~QWasmInputContext()
+{
+    if (QWasmIntegration::get()->platform == QWasmIntegration::AndroidPlatform)
+        emscripten_set_keydown_callback(EMSCRIPTEN_EVENT_TARGET_WINDOW, 0, 0, NULL);
+}
+
+void QWasmInputContext::focusWindowChanged(QWindow *focusWindow)
+{
+    m_focusWindow = focusWindow;
+}
+
+emscripten::val QWasmInputContext::focusCanvas()
+{
+    if (!m_focusWindow)
+        return emscripten::val::undefined();
+    QScreen *screen = m_focusWindow->screen();
+    if (!screen)
+        return emscripten::val::undefined();
+    return QWasmScreen::get(screen)->canvas();
+}
+
+void QWasmInputContext::update(Qt::InputMethodQueries queries)
+{
+    QPlatformInputContext::update(queries);
+}
+
+void QWasmInputContext::showInputPanel()
+{
+    if (QWasmIntegration::get()->platform == QWasmIntegration::WindowsPlatform
+        && inputPanelIsOpen) // call this only once for win32
+        return;
+    // this is called each time the keyboard is touched
+
+    // Add the input element as a child of the canvas for the
+    // currently focused window and give it focus. The browser
+    // will not display the input element, but mobile browsers
+    // should display the virtual keyboard. Key events will be
+    // captured by the keyboard event handler installed on the
+    // canvas.
+
+    if (QWasmIntegration::get()->platform == QWasmIntegration::MacOSPlatform
+     || QWasmIntegration::get()->platform == QWasmIntegration::WindowsPlatform) {
+        emscripten::val canvas = focusCanvas();
+        if (canvas == emscripten::val::undefined())
+            return;
+        canvas.call<void>("appendChild", m_inputElement);
+    }
+
+    m_inputElement.call<void>("focus");
+    inputPanelIsOpen = true;
+}
+
+void QWasmInputContext::hideInputPanel()
+{
+    if (QWasmIntegration::get()->touchPoints < 1)
+        return;
+    m_inputElement.call<void>("blur");
+    inputPanelIsOpen = false;
+}
+
+void QWasmInputContext::inputStringChanged(QString &inputString, QWasmInputContext *context)
+{
+    Q_UNUSED(context)
+    QKeySequence keys = QKeySequence::fromString(inputString);
+    // synthesize this keyevent as android is not normal
+    QWindowSystemInterface::handleKeyEvent<QWindowSystemInterface::SynchronousDelivery>(
+                0, QEvent::KeyPress,keys[0].key(), keys[0].keyboardModifiers(), inputString);
+}
+
+int QWasmInputContext::androidKeyboardCallback(int eventType,
+                                               const EmscriptenKeyboardEvent *keyEvent,
+                                               void *userData)
+{
+    Q_UNUSED(eventType)
+    QString strKey(keyEvent->key);
+    if (strKey == "Unidentified")
+        return false;
+    QWasmInputContext *wasmInput = reinterpret_cast<QWasmInputContext*>(userData);
+    wasmInput->inputStringChanged(strKey, wasmInput);
+
+    return true;
+}

src/plugins/platforms/wasm/qwasminputcontext.h

@@ -0,0 +1,71 @@
+/****************************************************************************
+**
+** Copyright (C) 2019 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the plugins of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 or (at your option) any later version
+** approved by the KDE Free Qt Foundation. The licenses are as published by
+** the Free Software Foundation and appearing in the file LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QWASMINPUTCONTEXT_H
+#define QWASMINPUTCONTEXT_H
+
+
+#include <qpa/qplatforminputcontext.h>
+#include <QtCore/qpointer.h>
+#include <private/qstdweb_p.h>
+#include <emscripten/bind.h>
+#include <emscripten/html5.h>
+#include <emscripten/emscripten.h>
+
+class QWasmInputContext : public QPlatformInputContext
+{
+    Q_DISABLE_COPY(QWasmInputContext)
+    Q_OBJECT
+public:
+    explicit QWasmInputContext();
+    ~QWasmInputContext() override;
+
+    void update(Qt::InputMethodQueries) override;
+
+    void showInputPanel() override;
+    void hideInputPanel() override;
+    bool isValid() const override { return true; }
+
+    void focusWindowChanged(QWindow *focusWindow);
+    emscripten::val focusCanvas();
+    void inputStringChanged(QString &, QWasmInputContext *context);
+
+private:
+    bool m_inputPanelVisible = false;
+
+    QPointer<QWindow> m_focusWindow;
+    emscripten::val m_inputElement = emscripten::val::null();
+    std::unique_ptr<qstdweb::EventCallback> m_blurEventHandler;
+    std::unique_ptr<qstdweb::EventCallback> m_inputEventHandler;
+    static int androidKeyboardCallback(int eventType,
+                                       const EmscriptenKeyboardEvent *keyEvent, void *userData);
+    bool inputPanelIsOpen = false;
+};
+
+#endif // QWASMINPUTCONTEXT_H

src/plugins/platforms/wasm/qwasmintegration.cpp

@@ -106,6 +106,22 @@ QWasmIntegration::QWasmIntegration()
 {
     s_instance = this;
 
+   touchPoints = emscripten::val::global("navigator")["maxTouchPoints"].as<int>();
+    // The Platform Detect: expand coverage as needed
+    platform = GenericPlatform;
+    emscripten::val rawPlatform = emscripten::val::global("navigator")["platform"];
+
+    if (rawPlatform.call<bool>("includes", emscripten::val("Mac")))
+        platform = MacOSPlatform;
+    if (rawPlatform.call<bool>("includes", emscripten::val("Win32")))
+        platform = WindowsPlatform;
+    if (rawPlatform.call<bool>("includes", emscripten::val("Linux"))) {
+        platform = LinuxPlatform;
+        emscripten::val uAgent = emscripten::val::global("navigator")["userAgent"];
+        if (uAgent.call<bool>("includes", emscripten::val("Android")))
+            platform = AndroidPlatform;
+    }
+
     // We expect that qtloader.js has populated Module.qtCanvasElements with one or more canvases.
     emscripten::val qtCanvaseElements = val::module_property("qtCanvasElements");
     emscripten::val canvas = val::module_property("canvas"); // TODO: remove for Qt 6.0
@@ -156,6 +172,8 @@ QWasmIntegration::~QWasmIntegration()
 
     delete m_fontDb;
     delete m_desktopServices;
+    if (m_platformInputContext)
+        delete m_platformInputContext;
 
     for (const auto &canvasAndScreen : m_screens)
         QWindowSystemInterface::handleScreenRemoved(canvasAndScreen.second);
@@ -210,9 +228,14 @@ QPlatformOpenGLContext *QWasmIntegration::createPlatformOpenGLContext(QOpenGLCon
 
 void QWasmIntegration::initialize()
 {
+    if (touchPoints < 1) // only touchscreen need inputcontexts
+        return;
+
     QString icStr = QPlatformInputContextFactory::requested();
     if (!icStr.isNull())
         m_inputContext.reset(QPlatformInputContextFactory::create(icStr));
+    else
+        m_inputContext.reset(new QWasmInputContext());
 }
 
 QPlatformInputContext *QWasmIntegration::inputContext() const

src/plugins/platforms/wasm/qwasmintegration.h

@@ -42,6 +42,9 @@
 #include <emscripten/html5.h>
 #include <emscripten/val.h>
 
+#include "qwasminputcontext.h"
+#include <private/qstdweb_p.h>
+
 QT_BEGIN_NAMESPACE
 
 class QWasmEventTranslator;
@@ -58,6 +61,14 @@ class QWasmIntegration : public QObject, public QPlatformIntegration
 {
     Q_OBJECT
 public:
+    enum Platform {
+        GenericPlatform,
+        MacOSPlatform,
+        WindowsPlatform,
+        LinuxPlatform,
+        AndroidPlatform
+    };
+
     QWasmIntegration();
     ~QWasmIntegration();
 
@@ -80,7 +91,7 @@ public:
     QPlatformInputContext *inputContext() const override;
 
     QWasmClipboard *getWasmClipboard() { return m_clipboard; }
-
+    QWasmInputContext *getWasmInputContext() { return m_platformInputContext; }
     static QWasmIntegration *get() { return s_instance; }
 
     void addScreen(const emscripten::val &canvas);
@@ -91,6 +102,9 @@ public:
     void removeBackingStore(QWindow* window);
     static quint64 getTimestamp();
 
+    Platform platform;
+    int touchPoints;
+
 private:
     mutable QWasmFontDatabase *m_fontDb;
     mutable QWasmServices *m_desktopServices;
@@ -100,6 +114,8 @@ private:
     qreal m_fontDpi = -1;
     mutable QScopedPointer<QPlatformInputContext> m_inputContext;
     static QWasmIntegration *s_instance;
+
+    mutable QWasmInputContext *m_platformInputContext = nullptr;
 };
 
 QT_END_NAMESPACE

src/plugins/platforms/wasm/qwasmscreen.cpp

@@ -65,6 +65,9 @@ QWasmScreen::QWasmScreen(const emscripten::val &canvas)
     // Set contenteditable so that the canvas gets clipboard events,
     // then hide the resulting focus frame, and reset the cursor.
     m_canvas.set("contentEditable", std::string("true"));
+    // set inputmode to none to stop mobile keyboard opening
+    // when user clicks  anywhere on the canvas.
+    m_canvas.set("inputmode", std::string("none"));
     style.set("outline", std::string("0px solid transparent"));
     style.set("caret-color", std::string("transparent"));
     style.set("cursor", std::string("default"));