From 64007c749703090ebf7f9b1b49b4267bb9993b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morten=20S=C3=B8rvig?= Date: Thu, 1 Jun 2023 16:24:05 +0200 Subject: [PATCH] wasm: add "preload" qtloader config property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for downloading files from the web server to the in-memory file system at application load time. See included documentation for usage. This preload functionality is different from Emscripten's --preload-file and --embed-file in that the files are not packed to a single data file or embedded in the JavaScript runtime. Instead, the files are downloaded individually from the web server, which means that they can be cached individually, and also updated individually without rebuilding the application. Any file type can be preloaded. The primary use case (at the moment) is preloading Qt plugins and QML imports. Pick-to: 6.6 Task-number: QTBUG-63925 Change-Id: I2b71b0d6a2c12ecd3ec58e319c679cd3f6b16631 Reviewed-by: Morten Johan Sørvig --- src/plugins/platforms/wasm/qtloader.js | 68 +++++++++++++++++++ .../wasm/qtloader_integration/CMakeLists.txt | 6 ++ .../manual/wasm/qtloader_integration/main.cpp | 16 +++++ .../wasm/qtloader_integration/preload.json | 10 +++ .../wasm/qtloader_integration/test_body.js | 14 ++++ 5 files changed, 114 insertions(+) create mode 100644 tests/manual/wasm/qtloader_integration/preload.json diff --git a/src/plugins/platforms/wasm/qtloader.js b/src/plugins/platforms/wasm/qtloader.js index 4ce27de6a54..5f3f2d1a91c 100644 --- a/src/plugins/platforms/wasm/qtloader.js +++ b/src/plugins/platforms/wasm/qtloader.js @@ -29,6 +29,23 @@ * The path is set to 'qt' by default, and is relative to the path of the web page's html file. * This property is not in use when static linking is used, since this build mode includes all * libraries and plugins in the wasm file. + * - preload: [string]: Array of file paths to json-encoded files which specifying which files to preload. + * The preloaded files will be downloaded at application startup and copied to the in-memory file + * system provided by Emscripten. + * + * Each json file must contain an array of source, destination objects: + * [ + * { + * "source": "path/to/source", + * "destination": "/path/to/destination" + * }, + * ... + * ] + * The source path is relative to the html file path. The destination path must be + * an absolute path. + * + * $QTDIR may be used as a placeholder for the "qtdir" configuration property (see @qtdir), for instance: + * "source": "$QTDIR/plugins/imageformats/libqjpeg.so" * * @return Promise * The promise is resolved when the module has been instantiated and its main function has been @@ -49,6 +66,16 @@ async function qtLoad(config) throw new Error('ENV must be exported if environment variables are passed'); }; + const throwIfFsUsedButNotExported = (instance, config) => + { + const environment = config.environment; + if (!environment || Object.keys(environment).length === 0) + return; + const isFsExported = typeof instance.FS === 'object'; + if (!isFsExported) + throw new Error('FS must be exported if preload is used'); + }; + if (typeof config !== 'object') throw new Error('config is required, expected an object'); if (typeof config.qt !== 'object') @@ -57,6 +84,7 @@ async function qtLoad(config) throw new Error('config.qt.entryFunction is required, expected a function'); config.qt.qtdir ??= 'qt'; + config.qt.preload ??= []; config.qtContainerElements = config.qt.containerElements; delete config.qt.containerElements; @@ -91,6 +119,30 @@ async function qtLoad(config) throwIfEnvUsedButNotExported(instance, config); for (const [name, value] of Object.entries(config.qt.environment ?? {})) instance.ENV[name] = value; + + const makeDirs = (FS, filePath) => { + const parts = filePath.split("/"); + let path = "/"; + for (let i = 0; i < parts.length - 1; ++i) { + const part = parts[i]; + if (part == "") + continue; + path += part + "/"; + try { + FS.mkdir(path); + } catch (error) { + const EEXIST = 20; + if (error.errno != EEXIST) + throw error; + } + } + } + + throwIfFsUsedButNotExported(instance, config); + for ({destination, data} of self.preloadData) { + makeDirs(instance.FS, destination); + instance.FS.writeFile(destination, new Uint8Array(data)); + } }; config.onRuntimeInitialized = () => config.qt.onLoaded?.(); @@ -137,6 +189,22 @@ async function qtLoad(config) }); }; + const fetchPreloadFiles = async () => { + const fetchJson = async path => (await fetch(path)).json(); + const fetchArrayBuffer = async path => (await fetch(path)).arrayBuffer(); + const loadFiles = async (paths) => { + const source = paths['source'].replace('$QTDIR', config.qt.qtdir); + return { + destination: paths['destination'], + data: await fetchArrayBuffer(source) + }; + } + const fileList = (await Promise.all(config.qt.preload.map(fetchJson))).flat(); + self.preloadData = (await Promise.all(fileList.map(loadFiles))).flat(); + } + + await fetchPreloadFiles(); + // Call app/emscripten module entry function. It may either come from the emscripten // runtime script or be customized as needed. const instance = await Promise.race( diff --git a/tests/manual/wasm/qtloader_integration/CMakeLists.txt b/tests/manual/wasm/qtloader_integration/CMakeLists.txt index 24c5607cd88..2603a05135c 100644 --- a/tests/manual/wasm/qtloader_integration/CMakeLists.txt +++ b/tests/manual/wasm/qtloader_integration/CMakeLists.txt @@ -37,3 +37,9 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/test_body.js ${CMAKE_CURRENT_BINARY_DIR}/test_body.js) + +add_custom_command( + TARGET tst_qtloader_integration POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_SOURCE_DIR}/preload.json + ${CMAKE_CURRENT_BINARY_DIR}/preload.json) diff --git a/tests/manual/wasm/qtloader_integration/main.cpp b/tests/manual/wasm/qtloader_integration/main.cpp index ed8a57bcc6e..b9bed0d49b0 100644 --- a/tests/manual/wasm/qtloader_integration/main.cpp +++ b/tests/manual/wasm/qtloader_integration/main.cpp @@ -49,6 +49,21 @@ std::string logicalDpi() return out.str(); } +std::string preloadedFiles() +{ + QStringList files = QDir("/preload").entryList(QDir::Files); + std::ostringstream out; + out << "["; + const char *separator = ""; + for (const auto &file : files) { + out << separator; + out << file.toStdString(); + separator = ","; + } + out << "]"; + return out.str(); +} + void crash() { std::abort(); @@ -145,6 +160,7 @@ EMSCRIPTEN_BINDINGS(qtLoaderIntegrationTest) emscripten::function("screenInformation", &screenInformation); emscripten::function("logicalDpi", &logicalDpi); + emscripten::function("preloadedFiles", &preloadedFiles); emscripten::function("crash", &crash); emscripten::function("exitApp", &exitApp); emscripten::function("produceOutput", &produceOutput); diff --git a/tests/manual/wasm/qtloader_integration/preload.json b/tests/manual/wasm/qtloader_integration/preload.json new file mode 100644 index 00000000000..d7e09911ff8 --- /dev/null +++ b/tests/manual/wasm/qtloader_integration/preload.json @@ -0,0 +1,10 @@ +[ + { + "source": "qtloader.js", + "destination": "/preload/qtloader.js" + }, + { + "source": "$QTDIR/qtlogo.svg", + "destination": "/preload/qtlogo.svg" + } +] diff --git a/tests/manual/wasm/qtloader_integration/test_body.js b/tests/manual/wasm/qtloader_integration/test_body.js index 385a3816d3f..233c64f2f60 100644 --- a/tests/manual/wasm/qtloader_integration/test_body.js +++ b/tests/manual/wasm/qtloader_integration/test_body.js @@ -439,6 +439,20 @@ export class QtLoaderIntegrationTests assert.equal('ExitStatus', exception.name); } + async preloadFiles() + { + const instance = await qtLoad({ + arguments: ["--no-gui"], + qt: { + preload: ['preload.json'], + qtdir: '.', + } + }); + const preloadedFiles = instance.preloadedFiles(); + // Verify that preloaded file list matches files specified in preload.json + assert.equal("[qtloader.js,qtlogo.svg]", preloadedFiles); + } + #callTestInstanceApi(instance, apiName) { return eval(instance[apiName]());