wasm: add QIODevices for accessing JS data

BlobIoDevice: Supports reading data from a JS Blob,
which can be a File (on disk) or some other object
which can provide data. The native access functions
are async and using this class requires that asyncify
is available.

Uint8ArrayIODevice: Supports reading and writing to
a Uint8Array / ArrayBuffer. Similar to the existing
QByteArray::fromEcmaUint8Array() API, except that it
supports incremental accesss.

Change-Id: Ic5de3534ff75eb6c745287b73b15ccd92d74ac2c
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>

src/corelib/platform/wasm/qstdweb.cpp

@@ -503,6 +503,11 @@ uint32_t ArrayBuffer::byteLength() const
     return m_arrayBuffer["byteLength"].as<uint32_t>();
 }
 
+ArrayBuffer ArrayBuffer::slice(uint32_t begin, uint32_t end) const
+{
+    return ArrayBuffer(m_arrayBuffer.call<emscripten::val>("slice", begin, end));
+}
+
 emscripten::val ArrayBuffer::val() const
 {
     return m_arrayBuffer;
@@ -514,6 +519,13 @@ Blob::Blob(const emscripten::val &blob)
 
 }
 
+Blob Blob::fromArrayBuffer(const ArrayBuffer &arrayBuffer)
+{
+    auto array = emscripten::val::array();
+    array.call<void>("push", arrayBuffer.val());
+    return Blob(emscripten::val::global("Blob").new_(array));
+}
+
 uint32_t Blob::size() const
 {
     return m_blob["size"].as<uint32_t>();
@@ -536,6 +548,25 @@ Blob Blob::copyFrom(const char *buffer, uint32_t size)
     return copyFrom(buffer, size, "application/octet-stream");
 }
 
+Blob Blob::slice(uint32_t begin, uint32_t end) const
+{
+    return Blob(m_blob.call<emscripten::val>("slice", begin, end));
+}
+
+ArrayBuffer Blob::arrayBuffer_sync() const
+{
+    QEventLoop loop;
+    emscripten::val buffer;
+    qstdweb::Promise::make(m_blob, "arrayBuffer", {
+        .thenFunc = [&loop, &buffer](emscripten::val arrayBuffer) {
+            buffer = arrayBuffer;
+            loop.quit();
+        }
+    });
+    loop.exec();
+    return ArrayBuffer(buffer);
+}
+
 emscripten::val Blob::val() const
 {
     return m_blob;
@@ -706,6 +737,13 @@ void Uint8Array::set(const Uint8Array &source)
     m_uint8Array.call<void>("set", source.m_uint8Array); // copies source content
 }
 
+Uint8Array Uint8Array::subarray(uint32_t begin, uint32_t end)
+{
+    // Note: using uint64_t here errors with "Cannot convert a BigInt value to a number"
+    // (see JS BigInt and Number types). Use uint32_t for now.
+    return Uint8Array(m_uint8Array.call<emscripten::val>("subarray", begin, end));
+}
+
 // Copies the Uint8Array content to a destination on the heap
 void Uint8Array::copyTo(char *destination) const
 {
@@ -890,6 +928,101 @@ readDataTransfer(emscripten::val webDataTransfer, std::function<QVariant(QByteAr
     return DataTransferReader::read(webDataTransfer, std::move(imageReader), std::move(onDone));
 }
 
+BlobIODevice::BlobIODevice(Blob blob)
+    : m_blob(blob)
+{
+
+}
+
+bool BlobIODevice::open(QIODevice::OpenMode mode)
+{
+    if (mode.testFlag(QIODevice::WriteOnly))
+        return false;
+    return QIODevice::open(mode);
+}
+
+bool BlobIODevice::isSequential() const
+{
+    return false;
+}
+
+qint64 BlobIODevice::size() const
+{
+    return m_blob.size();
+}
+
+bool BlobIODevice::seek(qint64 pos)
+{
+    if (pos >= size())
+        return false;
+    return QIODevice::seek(pos);
+}
+
+qint64 BlobIODevice::readData(char *data, qint64 maxSize)
+{
+    uint64_t begin = QIODevice::pos();
+    uint64_t end = std::min<uint64_t>(begin + maxSize, size());
+    uint64_t size = end - begin;
+    if (size > 0) {
+        qstdweb::ArrayBuffer buffer = m_blob.slice(begin, end).arrayBuffer_sync();
+        qstdweb::Uint8Array(buffer).copyTo(data);
+    }
+    return size;
+}
+
+qint64 BlobIODevice::writeData(const char *, qint64)
+{
+    Q_UNREACHABLE();
+}
+
+Uint8ArrayIODevice::Uint8ArrayIODevice(Uint8Array array)
+    : m_array(array)
+{
+
+}
+
+bool Uint8ArrayIODevice::open(QIODevice::OpenMode mode)
+{
+    return QIODevice::open(mode);
+}
+
+bool Uint8ArrayIODevice::isSequential() const
+{
+    return false;
+}
+
+qint64 Uint8ArrayIODevice::size() const
+{
+    return m_array.length();
+}
+
+bool Uint8ArrayIODevice::seek(qint64 pos)
+{
+    if (pos >= size())
+        return false;
+    return QIODevice::seek(pos);
+}
+
+qint64 Uint8ArrayIODevice::readData(char *data, qint64 maxSize)
+{
+    uint64_t begin = QIODevice::pos();
+    uint64_t end = std::min<uint64_t>(begin + maxSize, size());
+    uint64_t size = end - begin;
+    if (size > 0)
+        m_array.subarray(begin, end).copyTo(data);
+    return size;
+}
+
+qint64 Uint8ArrayIODevice::writeData(const char *data, qint64 maxSize)
+{
+    uint64_t begin = QIODevice::pos();
+    uint64_t end = std::min<uint64_t>(begin + maxSize, size());
+    uint64_t size = end - begin;
+    if (size > 0)
+        m_array.subarray(begin, end).set(Uint8Array(data, size));
+    return size;
+}
+
 } // namespace qstdweb
 
 QT_END_NAMESPACE

src/corelib/platform/wasm/qstdweb_p.h

@@ -58,6 +58,7 @@ namespace qstdweb {
         explicit ArrayBuffer(uint32_t size);
         explicit ArrayBuffer(const emscripten::val &arrayBuffer);
         uint32_t byteLength() const;
+        ArrayBuffer slice(uint32_t begin, uint32_t end) const;
         emscripten::val val() const;
 
     private:
@@ -68,9 +69,12 @@ namespace qstdweb {
     class Q_CORE_EXPORT Blob {
     public:
         explicit Blob(const emscripten::val &blob);
+        static Blob fromArrayBuffer(const ArrayBuffer &arrayBuffer);
         uint32_t size() const;
         static Blob copyFrom(const char *buffer, uint32_t size, std::string mimeType);
         static Blob copyFrom(const char *buffer, uint32_t size);
+        Blob slice(uint32_t begin, uint32_t end) const;
+        ArrayBuffer arrayBuffer_sync() const;
         emscripten::val val() const;
         std::string type() const;
 
@@ -140,6 +144,7 @@ namespace qstdweb {
         ArrayBuffer buffer() const;
         uint32_t length() const;
         void set(const Uint8Array &source);
+        Uint8Array subarray(uint32_t begin, uint32_t end);
 
         void copyTo(char *destination) const;
         QByteArray copyToQByteArray() const;
@@ -207,6 +212,40 @@ namespace qstdweb {
         return wrappedCallback;
     }
 
+    class Q_CORE_EXPORT BlobIODevice: public QIODevice
+    {
+    public:
+        BlobIODevice(Blob blob);
+        bool open(QIODeviceBase::OpenMode mode) override;
+        bool isSequential() const override;
+        qint64 size() const override;
+        bool seek(qint64 pos) override;
+
+    protected:
+        qint64 readData(char *data, qint64 maxSize) override;
+        qint64 writeData(const char *, qint64) override;
+
+    private:
+        Blob m_blob;
+    };
+
+    class Uint8ArrayIODevice: public QIODevice
+    {
+    public:
+        Uint8ArrayIODevice(Uint8Array array);
+        bool open(QIODevice::OpenMode mode) override;
+        bool isSequential() const override;
+        qint64 size() const override;
+        bool seek(qint64 pos) override;
+
+    protected:
+        qint64 readData(char *data, qint64 maxSize) override;
+        qint64 writeData(const char *data, qint64 size) override;
+
+    private:
+        Uint8Array m_array;
+    };
+
     inline emscripten::val window()
     {
         static emscripten::val savedWindow = emscripten::val::global("window");
@@ -225,6 +264,7 @@ namespace qstdweb {
     readDataTransfer(emscripten::val webObject, std::function<QVariant(QByteArray)> imageReader,
                      std::function<void(std::unique_ptr<QMimeData>)> onDone);
 
+
 #if QT_CONFIG(thread)
     template<class T>
     T proxyCall(std::function<T()> task, emscripten::ProxyingQueue *queue)
@@ -261,6 +301,7 @@ namespace qstdweb {
         return task();
     }
 #endif // QT_CONFIG(thread)
+
 }
 
 QT_END_NAMESPACE

tests/manual/wasm/qstdweb/CMakeLists.txt

@@ -70,3 +70,28 @@ add_custom_command(
             ${CMAKE_CURRENT_BINARY_DIR}/qtwasmtestlib.js)
 
 target_link_options(qwasmcompositor_auto PRIVATE -sASYNCIFY -Os)
+
+qt_internal_add_manual_test(iodevices_auto
+    SOURCES
+        iodevices_main.cpp
+        ../qtwasmtestlib/qtwasmtestlib.cpp
+    LIBRARIES
+        Qt::Core
+        Qt::CorePrivate
+        Qt::GuiPrivate
+)
+
+add_custom_command(
+    TARGET iodevices_auto POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy
+            ${CMAKE_CURRENT_SOURCE_DIR}/iodevices_auto.html
+            ${CMAKE_CURRENT_BINARY_DIR}/iodevices_auto.html)
+
+add_custom_command(
+    TARGET iodevices_auto POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy
+            ${CMAKE_CURRENT_SOURCE_DIR}/../qtwasmtestlib/qtwasmtestlib.js
+            ${CMAKE_CURRENT_BINARY_DIR}/qtwasmtestlib.js)
+
+target_link_options(iodevices_auto PRIVATE -sASYNCIFY -Os)
+

tests/manual/wasm/qstdweb/iodevices_auto.html

@@ -0,0 +1,10 @@
+<!doctype html>
+<script type="text/javascript" src="qtwasmtestlib.js"></script>
+<script type="text/javascript" src="iodevices_auto.js"></script>
+<script>
+    window.onload = () => {
+        runTestCase(iodevices_auto_entry, document.getElementById("log"));
+    };
+</script>
+<p>Running qstdweb iodevices auto test.</p>
+<div id="log"></div>

tests/manual/wasm/qstdweb/iodevices_main.cpp

@@ -0,0 +1,103 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QtCore/QtCore>
+#include <QtCore/private/qstdweb_p.h>
+
+#include <qtwasmtestlib.h>
+
+#include "emscripten.h"
+
+using qstdweb::ArrayBuffer;
+using qstdweb::Uint8Array;
+using qstdweb::Blob;
+using qstdweb::BlobIODevice;
+using qstdweb::Uint8ArrayIODevice;
+
+class WasmIoDevicesTest: public QObject
+{
+    Q_OBJECT
+
+private slots:
+    void blobIODevice();
+    void uint8ArrayIODevice();
+};
+
+// Creates a test arraybuffer with byte values [0..size] % 256 * 2
+char testByteValue(int i) { return (i % 256) * 2; }
+ArrayBuffer createTestArrayBuffer(int size)
+{
+    ArrayBuffer buffer(size);
+    Uint8Array array(buffer);
+    for (int i = 0; i < size; ++i)
+        array.val().set(i, testByteValue(i));
+    return buffer;
+}
+
+void WasmIoDevicesTest::blobIODevice()
+{
+    if (!qstdweb::canBlockCallingThread()) {
+        QtWasmTest::completeTestFunction(QtWasmTest::TestResult::Skip, "requires asyncify");
+        return;
+    }
+
+    // Create test buffer and BlobIODevice
+    const int bufferSize = 16;
+    BlobIODevice blobDevice(Blob::fromArrayBuffer(createTestArrayBuffer(bufferSize)));
+
+    // Read back byte for byte from the device
+    QWASMVERIFY(blobDevice.open(QIODevice::ReadOnly));
+    for (int i = 0; i < bufferSize; ++i) {
+        char byte;
+        blobDevice.seek(i);
+        blobDevice.read(&byte, 1);
+        QWASMCOMPARE(byte, testByteValue(i));
+    }
+
+    blobDevice.close();
+    QWASMVERIFY(!blobDevice.open(QIODevice::WriteOnly));
+    QWASMSUCCESS();
+}
+
+void WasmIoDevicesTest::uint8ArrayIODevice()
+{
+    // Create test buffer and Uint8ArrayIODevice
+    const int bufferSize = 1024;
+    Uint8Array array(createTestArrayBuffer(bufferSize));
+    Uint8ArrayIODevice arrayDevice(array);
+
+    // Read back byte for byte from the device
+    QWASMVERIFY(arrayDevice.open(QIODevice::ReadWrite));
+    for (int i = 0; i < bufferSize; ++i) {
+        char byte;
+        arrayDevice.seek(i);
+        arrayDevice.read(&byte, 1);
+        QWASMCOMPARE(byte, testByteValue(i));
+    }
+
+    // Write a different set of bytes
+    QWASMCOMPARE(arrayDevice.seek(0), true);
+    for (int i = 0; i < bufferSize; ++i) {
+        char byte = testByteValue(i + 1);
+        arrayDevice.seek(i);
+        QWASMCOMPARE(arrayDevice.write(&byte, 1), 1);
+    }
+
+    // Verify that the original array was updated
+    QByteArray copy = QByteArray::fromEcmaUint8Array(array.val());
+    for (int i = 0; i < bufferSize; ++i)
+        QWASMCOMPARE(copy.at(i), testByteValue(i + 1));
+
+    arrayDevice.close();
+    QWASMSUCCESS();
+}
+
+int main(int argc, char **argv)
+{
+    auto testObject = std::make_shared<WasmIoDevicesTest>();
+    QtWasmTest::initTestCase<QCoreApplication>(argc, argv, testObject);
+    return 0;
+}
+
+#include "iodevices_main.moc"
+