wasm: add "preload" qtloader config property

Add support for downloading files from the web server
to the in-memory file system at application load time.

See included documentation for usage.

This preload functionality is different from Emscripten's
--preload-file and --embed-file in that the files are
not packed to a single data file or embedded in the
JavaScript runtime. Instead, the files are downloaded
individually from the web server, which means that they
can be cached individually, and also updated individually
without rebuilding the application.

Any file type can be preloaded. The primary use case
(at the moment) is preloading Qt plugins and QML imports.

Pick-to: 6.6
Task-number: QTBUG-63925
Change-Id: I2b71b0d6a2c12ecd3ec58e319c679cd3f6b16631
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>

src/plugins/platforms/wasm/qtloader.js

@@ -29,6 +29,23 @@
  *      The path is set to 'qt' by default, and is relative to the path of the web page's html file.
  *      This property is not in use when static linking is used, since this build mode includes all
  *      libraries and plugins in the wasm file.
+ * - preload: [string]: Array of file paths to json-encoded files which specifying which files to preload.
+ *      The preloaded files will be downloaded at application startup and copied to the in-memory file
+ *      system provided by Emscripten.
+ *
+ *      Each json file must contain an array of source, destination objects:
+ *      [
+ *           {
+ *               "source": "path/to/source",
+ *               "destination": "/path/to/destination"
+ *           },
+ *           ...
+ *      ]
+ *      The source path is relative to the html file path. The destination path must be
+ *      an absolute path.
+ *
+ *      $QTDIR may be used as a placeholder for the "qtdir" configuration property (see @qtdir), for instance:
+ *          "source": "$QTDIR/plugins/imageformats/libqjpeg.so"
  *
  * @return Promise<instance: EmscriptenModule>
  *      The promise is resolved when the module has been instantiated and its main function has been
@@ -49,6 +66,16 @@ async function qtLoad(config)
             throw new Error('ENV must be exported if environment variables are passed');
     };
 
+    const throwIfFsUsedButNotExported = (instance, config) =>
+    {
+        const environment = config.environment;
+        if (!environment || Object.keys(environment).length === 0)
+            return;
+        const isFsExported = typeof instance.FS === 'object';
+        if (!isFsExported)
+            throw new Error('FS must be exported if preload is used');
+    };
+
     if (typeof config !== 'object')
         throw new Error('config is required, expected an object');
     if (typeof config.qt !== 'object')
@@ -57,6 +84,7 @@ async function qtLoad(config)
         throw new Error('config.qt.entryFunction is required, expected a function');
 
     config.qt.qtdir ??= 'qt';
+    config.qt.preload ??= [];
 
     config.qtContainerElements = config.qt.containerElements;
     delete config.qt.containerElements;
@@ -91,6 +119,30 @@ async function qtLoad(config)
         throwIfEnvUsedButNotExported(instance, config);
         for (const [name, value] of Object.entries(config.qt.environment ?? {}))
             instance.ENV[name] = value;
+
+        const makeDirs = (FS, filePath) => {
+            const parts = filePath.split("/");
+            let path = "/";
+            for (let i = 0; i < parts.length - 1; ++i) {
+                const part = parts[i];
+                if (part == "")
+                    continue;
+                path += part + "/";
+                try {
+                    FS.mkdir(path);
+                } catch (error) {
+                    const EEXIST = 20;
+                    if (error.errno != EEXIST)
+                        throw error;
+                }
+            }
+        }
+
+        throwIfFsUsedButNotExported(instance, config);
+        for ({destination, data} of self.preloadData) {
+            makeDirs(instance.FS, destination);
+            instance.FS.writeFile(destination, new Uint8Array(data));
+        }
     };
 
     config.onRuntimeInitialized = () => config.qt.onLoaded?.();
@@ -137,6 +189,22 @@ async function qtLoad(config)
         });
     };
 
+    const fetchPreloadFiles = async () => {
+        const fetchJson = async path => (await fetch(path)).json();
+        const fetchArrayBuffer = async path => (await fetch(path)).arrayBuffer();
+        const loadFiles = async (paths) => {
+            const source = paths['source'].replace('$QTDIR', config.qt.qtdir);
+            return {
+                destination: paths['destination'],
+                data: await fetchArrayBuffer(source)
+            };
+        }
+        const fileList = (await Promise.all(config.qt.preload.map(fetchJson))).flat();
+        self.preloadData = (await Promise.all(fileList.map(loadFiles))).flat();
+    }
+
+    await fetchPreloadFiles();
+
     // Call app/emscripten module entry function. It may either come from the emscripten
     // runtime script or be customized as needed.
     const instance = await Promise.race(

tests/manual/wasm/qtloader_integration/CMakeLists.txt

@@ -37,3 +37,9 @@ add_custom_command(
     COMMAND ${CMAKE_COMMAND} -E copy
             ${CMAKE_CURRENT_SOURCE_DIR}/test_body.js
             ${CMAKE_CURRENT_BINARY_DIR}/test_body.js)
+
+add_custom_command(
+    TARGET tst_qtloader_integration POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy
+            ${CMAKE_CURRENT_SOURCE_DIR}/preload.json
+            ${CMAKE_CURRENT_BINARY_DIR}/preload.json)

tests/manual/wasm/qtloader_integration/main.cpp

@@ -49,6 +49,21 @@ std::string logicalDpi()
     return out.str();
 }
 
+std::string preloadedFiles()
+{
+    QStringList files = QDir("/preload").entryList(QDir::Files);
+    std::ostringstream out;
+    out << "[";
+    const char *separator = "";
+    for (const auto &file : files) {
+        out << separator;
+        out << file.toStdString();
+        separator = ",";
+    }
+    out << "]";
+    return out.str();
+}
+
 void crash()
 {
     std::abort();
@@ -145,6 +160,7 @@ EMSCRIPTEN_BINDINGS(qtLoaderIntegrationTest)
 
     emscripten::function("screenInformation", &screenInformation);
     emscripten::function("logicalDpi", &logicalDpi);
+    emscripten::function("preloadedFiles", &preloadedFiles);
     emscripten::function("crash", &crash);
     emscripten::function("exitApp", &exitApp);
     emscripten::function("produceOutput", &produceOutput);

tests/manual/wasm/qtloader_integration/preload.json

@@ -0,0 +1,10 @@
+[
+    {
+        "source": "qtloader.js",
+        "destination": "/preload/qtloader.js"
+    },
+    {
+        "source": "$QTDIR/qtlogo.svg",
+        "destination": "/preload/qtlogo.svg"
+    }
+]

tests/manual/wasm/qtloader_integration/test_body.js

@@ -439,6 +439,20 @@ export class QtLoaderIntegrationTests
         assert.equal('ExitStatus', exception.name);
     }
 
+    async preloadFiles()
+    {
+        const instance = await qtLoad({
+            arguments: ["--no-gui"],
+            qt: {
+                preload: ['preload.json'],
+                qtdir: '.',
+            }
+        });
+        const preloadedFiles = instance.preloadedFiles();
+        // Verify that preloaded file list matches files specified in preload.json
+        assert.equal("[qtloader.js,qtlogo.svg]", preloadedFiles);
+    }
+
     #callTestInstanceApi(instance, apiName)
     {
         return eval(instance[apiName]());