From c28e68c729a3a2639315928b486d95b82db42acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Morten=20S=C3=B8rvig?= Date: Mon, 11 Dec 2023 23:48:12 +0100 Subject: [PATCH] wasm: implement async drag-and-drop Make toMimeDataWithFile() handle writing files to the in-memory file system correctly: this is an async operation and we need to collect the files as the writes complete and then invoke the callback once all files have been saved. There's then no need for a global static QMimeData. Use toMimeDataWithFile() for both the paste and drop handling, however QPlatformClipboard::setMimeData() takes ownership of the passed in QMimeData and the callback API must be designed accordingly. An open question is when we should delete the files. Deleting them right away (after calling the app event handler) is predictable, however it looks like QPlatformClipboard::setMimeData() retains the current QMimeData until a new one is set, so maybe we should follow that. Change-Id: Ia9b825eaef1134ff9a554e51ee7e41d1c2ee779a Reviewed-by: Lorn Potter (cherry picked from commit 0ffe8050bd5b55d64da37f5177a7e20dd9d14232) Reviewed-by: Qt Cherry-pick Bot --- src/plugins/platforms/wasm/qwasmclipboard.cpp | 16 +- src/plugins/platforms/wasm/qwasmdom.cpp | 140 +++++++++++------- src/plugins/platforms/wasm/qwasmdom.h | 4 +- src/plugins/platforms/wasm/qwasmdrag.cpp | 40 ++--- 4 files changed, 119 insertions(+), 81 deletions(-) diff --git a/src/plugins/platforms/wasm/qwasmclipboard.cpp b/src/plugins/platforms/wasm/qwasmclipboard.cpp index 786267277af..1aa3ffa5b36 100644 --- a/src/plugins/platforms/wasm/qwasmclipboard.cpp +++ b/src/plugins/platforms/wasm/qwasmclipboard.cpp @@ -285,20 +285,20 @@ void QWasmClipboard::writeToClipboard() void QWasmClipboard::sendClipboardData(emscripten::val event) { - dom::DataTransfer transfer(event["clipboardData"]); - QMimeData *mData; - const auto pointerCallback = std::function([&](QMimeData &data) { - mData = &data; + qDebug() << "sendClipboardData"; + + dom::DataTransfer *transfer = new dom::DataTransfer(event["clipboardData"]); + const auto mimeCallback = std::function([transfer](QMimeData *data) { + // Persist clipboard data so that the app can read it when handling the CTRL+V - QWasmIntegration::get()->clipboard()->QPlatformClipboard::setMimeData(mData, - QClipboard::Clipboard); + QWasmIntegration::get()->clipboard()->QPlatformClipboard::setMimeData(data, QClipboard::Clipboard); QWindowSystemInterface::handleKeyEvent(0, QEvent::KeyPress, Qt::Key_V, Qt::ControlModifier, "V"); QWindowSystemInterface::handleKeyEvent(0, QEvent::KeyRelease, Qt::Key_V, Qt::ControlModifier, "V"); + delete transfer; }); - transfer.toMimeDataWithFile(pointerCallback); // mimedata - + transfer->toMimeDataWithFile(mimeCallback); } QT_END_NAMESPACE diff --git a/src/plugins/platforms/wasm/qwasmdom.cpp b/src/plugins/platforms/wasm/qwasmdom.cpp index a2a9c129771..0045f43745c 100644 --- a/src/plugins/platforms/wasm/qwasmdom.cpp +++ b/src/plugins/platforms/wasm/qwasmdom.cpp @@ -14,9 +14,6 @@ QT_BEGIN_NAMESPACE -// this needs to live outside the life of DataTransfer -Q_GLOBAL_STATIC(QMimeData, resultMimeData); - namespace dom { namespace { std::string dropActionToDropEffect(Qt::DropAction action) @@ -82,17 +79,56 @@ void DataTransfer::setDataFromMimeData(const QMimeData &mimeData) } } -QMimeData *DataTransfer::toMimeDataWithFile(std::function callback) +// Converts a DataTransfer instance to a QMimeData instance. Invokes the +// given callback when the conversion is complete. The callback takes ownership +// of the QMimeData. +void DataTransfer::toMimeDataWithFile(std::function callback) { - resultMimeData->clear(); enum class ItemKind { File, String, }; + class MimeContext { + + public: + MimeContext(int itemCount, std::function callback) + :m_remainingItemCount(itemCount), m_callback(callback) + { + + } + + void deref() { + if (--m_remainingItemCount > 0) + return; + + mimeData->setUrls(fileUrls); + + m_callback(mimeData); + + // Delete files; we expect that the user callback reads/copies + // file content before returning. + // Fixme: tie file lifetime to lifetime of the QMimeData? + for (QUrl fileUrl: fileUrls) + QFile(fileUrl.toLocalFile()).remove(); + + delete this; + } + + QMimeData *mimeData = new QMimeData(); + QList fileUrls; + + private: + int m_remainingItemCount; + std::function m_callback; + }; + const auto items = webDataTransfer["items"]; - QList uriList; - for (int i = 0; i < items["length"].as(); ++i) { + const int itemCount = items["length"].as(); + const int fileCount = webDataTransfer["files"]["length"].as(); + MimeContext *mimeContext = new MimeContext(itemCount, callback); + + for (int i = 0; i < itemCount; ++i) { const auto item = items[i]; const auto itemKind = item["kind"].as() == "string" ? ItemKind::String : ItemKind::File; @@ -100,50 +136,58 @@ QMimeData *DataTransfer::toMimeDataWithFile(std::function cal switch (itemKind) { case ItemKind::File: { - m_webFile = item.call("getAsFile"); - qstdweb::File webfile(m_webFile); - if (webfile.size() == 0 - || webfile.size() > 1e+9) { // limit file size to 1 GB - qWarning() << "Something happened, File size is" << webfile.size(); + qstdweb::File webfile(item.call("getAsFile")); + + if (webfile.size() > 1e+9) { // limit file size to 1 GB + qWarning() << "File is too large (> 1GB) and will be skipped. File size is" << webfile.size(); + mimeContext->deref(); continue; } - QByteArray fileContent(webfile.size(), Qt::Uninitialized); - QString mimeFormat = QString::fromStdString(webfile.type()); + + QString mimeFormat = QString::fromStdString(webfile.type()); QString fileName = QString::fromStdString(webfile.name()); // there's a file, now read it + QByteArray fileContent(webfile.size(), Qt::Uninitialized); webfile.stream(fileContent.data(), [=]() { - QList fileUriList; - if (!fileContent.isEmpty()) { - if (mimeFormat.contains("image/")) { - QImage image; - image.loadFromData(fileContent, nullptr); - resultMimeData->setImageData(image); - } else { - QUrl fileUrl(QStringLiteral("file:///") + - QString::fromStdString(webfile.name())); - fileUriList.append(fileUrl); - QFile file(fileName); - if (!file.open(QFile::WriteOnly)) { - qWarning() << "File was not opened"; - return; - } - - if (file.write(fileContent) < 0) - qWarning() << "Write failed"; - - file.close(); - resultMimeData->setUrls(fileUriList); - } - callback(*resultMimeData); + + // If we get a single file, and that file is an image, then + // try to decode the image data. This handles the case where + // image data (i.e. not an image file) is pasted. The browsers + // will then create a fake "image.png" file which has the image + // data. As a side effect Qt will also decode the image for + // single-image-file drops, since there is no way to differentiate + // the fake "image.png" from a real one. + if (fileCount == 1 && mimeFormat.contains("image/")) { + QImage image; + if (image.loadFromData(fileContent)) + mimeContext->mimeData->setImageData(image); } - }); + QDir qtTmpDir("/qt/tmp/"); // "tmp": indicate that these files won't stay around + qtTmpDir.mkpath(qtTmpDir.path()); + + QUrl fileUrl = QUrl::fromLocalFile(qtTmpDir.filePath(QString::fromStdString(webfile.name()))); + mimeContext->fileUrls.append(fileUrl); + + QFile file(fileName); + if (!file.open(QFile::WriteOnly)) { + qWarning() << "File was not opened"; + mimeContext->deref(); + return; + } + if (file.write(fileContent) < 0) { + qWarning() << "Write failed"; + file.close(); + } + mimeContext->deref(); + }); break; } case ItemKind::String: if (itemMimeType.contains("STRING", Qt::CaseSensitive) || itemMimeType.contains("TEXT", Qt::CaseSensitive)) { + mimeContext->deref(); break; } QString a; @@ -152,33 +196,25 @@ QMimeData *DataTransfer::toMimeDataWithFile(std::function cal if (!data.isEmpty()) { if (itemMimeType == "text/html") - resultMimeData->setHtml(data); + mimeContext->mimeData->setHtml(data); else if (itemMimeType.isEmpty() || itemMimeType == "text/plain") - resultMimeData->setText(data); // the type can be empty + mimeContext->mimeData->setText(data); // the type can be empty else { // TODO improve encoding if (data.startsWith("QB64")) { data.remove(0, 4); - resultMimeData->setData(itemMimeType, + mimeContext->mimeData->setData(itemMimeType, QByteArray::fromBase64(QByteArray::fromStdString( data.toStdString()))); } else { - resultMimeData->setData(itemMimeType, data.toLocal8Bit()); + mimeContext->mimeData->setData(itemMimeType, data.toLocal8Bit()); } } } + mimeContext->deref(); break; } - } // end items - - if (!uriList.isEmpty()) { - resultMimeData->setUrls(uriList); - } - - if (resultMimeData->formats().length() > 0) - callback(*resultMimeData); - - return resultMimeData; + } // for items } QMimeData *DataTransfer::toMimeDataPreview() diff --git a/src/plugins/platforms/wasm/qwasmdom.h b/src/plugins/platforms/wasm/qwasmdom.h index 37c7bc97f13..0a520815a3c 100644 --- a/src/plugins/platforms/wasm/qwasmdom.h +++ b/src/plugins/platforms/wasm/qwasmdom.h @@ -36,7 +36,7 @@ struct DataTransfer DataTransfer &operator=(const DataTransfer &other); DataTransfer &operator=(DataTransfer &&other); - QMimeData *toMimeDataWithFile(std::function callback); + void toMimeDataWithFile(std::function callback); QMimeData *toMimeDataPreview(); void setDragImage(emscripten::val element, const QPoint &hotspot); void setData(std::string format, std::string data); @@ -44,8 +44,6 @@ struct DataTransfer void setDataFromMimeData(const QMimeData &mimeData); emscripten::val webDataTransfer; - emscripten::val m_webFile = emscripten::val::undefined(); - qstdweb::File m_file; }; inline emscripten::val document() diff --git a/src/plugins/platforms/wasm/qwasmdrag.cpp b/src/plugins/platforms/wasm/qwasmdrag.cpp index b765fe8268d..f088ecb2f6f 100644 --- a/src/plugins/platforms/wasm/qwasmdrag.cpp +++ b/src/plugins/platforms/wasm/qwasmdrag.cpp @@ -146,11 +146,11 @@ void QWasmDrag::onNativeDrop(DragEvent *event) { QWasmWindow *wasmWindow = QWasmWindow::fromWindow(event->targetWindow); - const auto localScreenElementPoint = dom::mapPoint( + const auto screenElementPos = dom::mapPoint( event->target(), wasmWindow->platformScreen()->element(), event->localPoint); - const auto pointInQtScreen = - wasmWindow->platformScreen()->mapFromLocal(localScreenElementPoint); - const QPointF pointInTargetWindowCoords = event->targetWindow->mapFromGlobal(pointInQtScreen); + const auto screenPos = + wasmWindow->platformScreen()->mapFromLocal(screenElementPos); + const QPoint targetWindowPos = event->targetWindow->mapFromGlobal(screenPos).toPoint(); const Qt::DropActions actions = m_dragState ? m_dragState->drag->supportedActions() @@ -159,25 +159,29 @@ void QWasmDrag::onNativeDrop(DragEvent *event) Qt::MouseButton mouseButton = event->mouseButton; QFlags modifiers = event->modifiers; - const auto pointerCallback = std::function([&, wasmWindow, pointInTargetWindowCoords, - actions, mouseButton, modifiers](QMimeData &data) { - if (data.formats().count() == 0) - return; - auto dropResponse = std::make_shared(true, Qt::DropAction::CopyAction); + // Accept the native drop event: We are going to async read any dropped + // files, but the + event->acceptDrop(); - *dropResponse = QWindowSystemInterface::handleDrop(wasmWindow->window(), &data, - pointInTargetWindowCoords.toPoint(), actions, + qDebug() << "QWasmDrag::onNativeDrop" << event; + + const auto dropCallback = [&m_dragState = m_dragState, wasmWindow, targetWindowPos, + actions, mouseButton, modifiers](QMimeData *mimeData) { + + qDebug() << "QWasmDrag::onNativeDrop callback"; + + auto dropResponse = std::make_shared(true, Qt::DropAction::CopyAction); + *dropResponse = QWindowSystemInterface::handleDrop(wasmWindow->window(), mimeData, + targetWindowPos, actions, mouseButton, modifiers); - if (dropResponse->isAccepted()) { -// event->acceptDrop(); // boom -// event->dataTransfer.setDropAction(dropResponse->acceptedAction()); - + if (dropResponse->isAccepted()) m_dragState->dropAction = dropResponse->acceptedAction(); - } - }); - event->dataTransfer.toMimeDataWithFile(pointerCallback); + delete mimeData; + }; + + event->dataTransfer.toMimeDataWithFile(dropCallback); } void QWasmDrag::onNativeDragFinished(DragEvent *event)