From 42852168608f5da12105de1e94c9014e475bcabb Mon Sep 17 00:00:00 2001 From: Jarek Kobus Date: Tue, 4 Jun 2024 19:21:07 +0200 Subject: [PATCH] AssetDownloader: Implement the downloader using TaskTree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task-number: QTBUG-122550 Change-Id: I990d0db1c5f0246aab0d796f438b26976650dc2c Reviewed-by: Ali Kianian Reviewed-by: Kai Köhne (cherry picked from commit 6dff842b2a55cc941e7868a12c725e0a8c6afdb1) Reviewed-by: Qt Cherry-pick Bot --- src/assets/downloader/CMakeLists.txt | 9 +- src/assets/downloader/assetdownloader.cpp | 1100 ++++++++------------- src/assets/downloader/assetdownloader.h | 76 +- 3 files changed, 426 insertions(+), 759 deletions(-) diff --git a/src/assets/downloader/CMakeLists.txt b/src/assets/downloader/CMakeLists.txt index e2dc5b2f44a..872932ad2a6 100644 --- a/src/assets/downloader/CMakeLists.txt +++ b/src/assets/downloader/CMakeLists.txt @@ -4,7 +4,6 @@ qt_internal_add_module(ExamplesAssetDownloaderPrivate CONFIG_MODULE_NAME examples_asset_downloader STATIC - EXCEPTIONS INTERNAL_MODULE SOURCES assetdownloader.cpp assetdownloader.h @@ -15,11 +14,13 @@ qt_internal_add_module(ExamplesAssetDownloaderPrivate tasking/tasking_global.h tasking/tasktree.cpp tasking/tasktree.h tasking/tasktreerunner.cpp tasking/tasktreerunner.h - LIBRARIES + DEFINES + QT_NO_CAST_FROM_ASCII + PUBLIC_LIBRARIES Qt6::Concurrent + Qt6::Core Qt6::CorePrivate Qt6::Network - PUBLIC_LIBRARIES - Qt6::Core + NO_GENERATE_CPP_EXPORTS ) diff --git a/src/assets/downloader/assetdownloader.cpp b/src/assets/downloader/assetdownloader.cpp index 85699cfe733..47caf58bf74 100644 --- a/src/assets/downloader/assetdownloader.cpp +++ b/src/assets/downloader/assetdownloader.cpp @@ -3,26 +3,26 @@ #include "assetdownloader.h" -#include -#include +#include "tasking/concurrentcall.h" +#include "tasking/networkquery.h" +#include "tasking/tasktreerunner.h" + #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include -using QtExamples::AssetDownloader; -using QtExamples::AssetDownloaderPrivate; +using namespace Tasking; -namespace { +QT_BEGIN_NAMESPACE -enum class UnzipRule { DeleteArchive, KeepArchive }; +namespace Assets::Downloader { struct DownloadableAssets { @@ -30,87 +30,61 @@ struct DownloadableAssets QList files; }; -class Exception : public QException +class AssetDownloaderPrivate { public: - Exception(const QString &errorString) - : m_errorString(errorString) - {} + AssetDownloaderPrivate(AssetDownloader *q) : m_q(q) {} + AssetDownloader *m_q = nullptr; - QString message() const { return m_errorString; } + std::unique_ptr m_manager; + std::unique_ptr m_temporaryDir; + TaskTreeRunner m_taskTreeRunner; + QString m_lastProgressText; + QDir m_localDownloadDir; -private: - QString m_errorString; + QString m_jsonFileName; + QString m_zipFileName; + QDir m_preferredLocalDownloadDir = + QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation); + QUrl m_offlineAssetsFilePath; + QUrl m_downloadBase; + + void setLocalDownloadDir(const QDir &dir) + { + if (m_localDownloadDir != dir) { + m_localDownloadDir = dir; + emit m_q->localDownloadDirChanged(QUrl::fromLocalFile(m_localDownloadDir.absolutePath())); + } + } + void setProgress(int progressValue, int progressMaximum, const QString &progressText) + { + m_lastProgressText = progressText; + emit m_q->progressChanged(progressValue, progressMaximum, progressText); + } + void updateProgress(int progressValue, int progressMaximum) + { + setProgress(progressValue, progressMaximum, m_lastProgressText); + } + void clearProgress(const QString &progressText) + { + setProgress(0, 0, progressText); + } + + void setupDownload(NetworkQuery *query, const QString &progressText) + { + query->setNetworkAccessManager(m_manager.get()); + clearProgress(progressText); + QObject::connect(query, &NetworkQuery::started, query, [this, query] { + QNetworkReply *reply = query->reply(); + QObject::connect(reply, &QNetworkReply::downloadProgress, + query, [this](qint64 bytesReceived, qint64 totalBytes) { + updateProgress((totalBytes > 0) ? 100.0 * bytesReceived / totalBytes : 0, 100); + }); + }); + } }; -class NetworkException : public Exception -{ -public: - NetworkException(QNetworkReply *reply) - : Exception(reply->errorString()) - {} - - void raise() const override { throw *this; } - NetworkException *clone() const override { return new NetworkException(*this); } -}; - -class FileException : public Exception -{ -public: - FileException(QFile *file) - : Exception(QString::fromUtf8("%1 \"%2\"").arg(file->errorString(), file->fileName())) - {} - - void raise() const override { throw *this; } - FileException *clone() const override { return new FileException(*this); } -}; - -class FileRemoveException : public Exception -{ -public: - FileRemoveException(QFile *file) - : Exception(QString::fromLatin1("Unable to remove file \"%1\"") - .arg(QFileInfo(file->fileName()).absoluteFilePath())) - {} - - void raise() const override { throw *this; } - FileRemoveException *clone() const override { return new FileRemoveException(*this); } -}; - -class FileCopyException : public Exception -{ -public: - FileCopyException(const QString &src, const QString &dst) - : Exception(QString::fromLatin1("Unable to copy file \"%1\" to \"%2\"") - .arg(QFileInfo(src).absoluteFilePath(), QFileInfo(dst).absoluteFilePath())) - {} - - void raise() const override { throw *this; } - FileCopyException *clone() const override { return new FileCopyException(*this); } -}; - -class DirException : public Exception -{ -public: - DirException(const QDir &dir) - : Exception(QString::fromLatin1("Cannot create directory %1").arg(dir.absolutePath())) - {} - - void raise() const override { throw *this; } - DirException *clone() const override { return new DirException(*this); } -}; - -class AssetFileException : public Exception -{ -public: - AssetFileException() - : Exception(QString::fromUtf8("Asset file is broken.")) - {} - void raise() const override { throw *this; } - AssetFileException *clone() const override { return new AssetFileException(*this); } -}; - -bool isWritableDir(const QDir &dir) +static bool isWritableDir(const QDir &dir) { if (dir.exists()) { QTemporaryFile file(dir.filePath(QString::fromLatin1("tmp"))); @@ -119,7 +93,7 @@ bool isWritableDir(const QDir &dir) return false; } -bool sameFileContent(const QFileInfo &first, const QFileInfo &second) +static bool sameFileContent(const QFileInfo &first, const QFileInfo &second) { if (first.exists() ^ second.exists()) return false; @@ -149,7 +123,7 @@ bool sameFileContent(const QFileInfo &first, const QFileInfo &second) return false; } -bool createDirectory(const QDir &dir) +static bool createDirectory(const QDir &dir) { if (dir.exists()) return true; @@ -160,694 +134,424 @@ bool createDirectory(const QDir &dir) return dir.mkpath(QString::fromUtf8(".")); } -bool canBeALocalBaseDir(QDir &dir) +static bool canBeALocalBaseDir(const QDir &dir) { - if (dir.exists()) { - if (dir.isEmpty()) - return isWritableDir(dir); - return true; - } else { - return createDirectory(dir) && isWritableDir(dir); - } - return false; + if (dir.exists()) + return !dir.isEmpty() || isWritableDir(dir); + return createDirectory(dir) && isWritableDir(dir); } -bool unzip(const QString &fileName, const QDir &directory, UnzipRule rule) +static QDir baseLocalDir(const QDir &preferredLocalDir) { - QZipReader zipFile(fileName); - const bool zipExtracted = zipFile.extractAll(directory.absolutePath()); - zipFile.close(); - if (zipExtracted && (rule == UnzipRule::DeleteArchive)) - QFile::remove(fileName); - return zipExtracted; + if (canBeALocalBaseDir(preferredLocalDir)) + return preferredLocalDir; + + qWarning().noquote() << "AssetDownloader: Cannot set \"" << preferredLocalDir + << "\" as a local download directory!"; + return QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation); } -void ensureDirOrThrow(const QDir &dir) -{ - if (createDirectory(dir)) - return; - - throw DirException(dir); -} - -void ensureRemoveFileOrThrow(QFile &file) -{ - if (!file.exists() || file.remove()) - return; - - throw FileRemoveException(&file); -} - -bool shouldDownloadEverything(const QList &assetFiles, const QDir &expectedDir) -{ - return std::any_of(assetFiles.begin(), assetFiles.end(), [&](const QUrl &imgPath) -> bool { - const QString localPath = expectedDir.absoluteFilePath(imgPath.toString()); - return !QFileInfo::exists(localPath); - }); -} - -QString pathFromUrl(const QUrl &url) +static QString pathFromUrl(const QUrl &url) { return url.isLocalFile() ? url.toLocalFile() : url.toString(); } -DownloadableAssets readAssetsFile(const QByteArray &assetFileContent) -{ - DownloadableAssets result; - QJsonObject json = QJsonDocument::fromJson(assetFileContent).object(); - - const QJsonArray assetsArray = json[u"assets"].toArray(); - for (const QJsonValue &asset : assetsArray) - result.files.append(asset.toString()); - - result.remoteUrl = json[u"url"].toString(); - - if (result.files.isEmpty() || result.remoteUrl.isEmpty()) - throw AssetFileException{}; - return result; -} - -QList filterDownloadableAssets(const QList &assetFiles, const QDir &expectedDir) +static QList filterDownloadableAssets(const QList &assetFiles, const QDir &expectedDir) { QList downloadList; - std::copy_if(assetFiles.begin(), - assetFiles.end(), - std::back_inserter(downloadList), - [&](const QUrl &imgPath) { - const QString tempFilePath = expectedDir.absoluteFilePath(imgPath.toString()); - return !QFileInfo::exists(tempFilePath); - }); + std::copy_if(assetFiles.begin(), assetFiles.end(), std::back_inserter(downloadList), + [&](const QUrl &assetPath) { + return !QFileInfo::exists(expectedDir.absoluteFilePath(assetPath.toString())); + }); return downloadList; } -} // namespace - -class QtExamples::AssetDownloaderPrivate +static bool allAssetsPresent(const QList &assetFiles, const QDir &expectedDir) { - AssetDownloader *q_ptr = nullptr; - Q_DECLARE_PUBLIC(AssetDownloader) - -public: - using State = AssetDownloader::State; - - AssetDownloaderPrivate(AssetDownloader *parent); - ~AssetDownloaderPrivate(); - - void setupBase(); - void setBaseLocalDir(const QDir &dir); - void moveAllAssets(); - void setAllDownloadsCount(int count); - void setCompletedDownloadsCount(int count); - void finalizeDownload(); - void setState(State state); - - QFuture download(const QUrl &url); - QFuture downloadToFile(const QUrl &url, const QUrl &destPath); - QFuture downloadOrRead(const QUrl &url, const QUrl &localFile); - QFuture downloadAssets(const DownloadableAssets &assets); - QFuture maybeReadZipFile(const DownloadableAssets &assets); - - State state = State::NotDownloadedState; - int allDownloadsCount = 0; - std::unique_ptr providedTempDir; - QDir baseLocal; - QDir tempDir; - int completedDownloadsCount = 0; - double progress = 0; - bool downloadStarted = false; - DownloadableAssets downloadableAssets; - QNetworkAccessManager *networkManager = nullptr; - QFutureWatcher *downloadWatcher = nullptr; - QFutureWatcher *fileDownloadWatcher = nullptr; - - QString jsonFileName; - QString zipFileName; - - QDir preferredLocalDownloadDir; - QUrl offlineAssetsFilePath; - - QUrl downloadBase = {QString::fromLatin1("https://download.qt.io/learning/examples/")}; -}; - -AssetDownloaderPrivate::AssetDownloaderPrivate(AssetDownloader *parent) - : q_ptr(parent) - , networkManager(new QNetworkAccessManager(parent)) - , downloadWatcher(new QFutureWatcher(parent)) - , fileDownloadWatcher(new QFutureWatcher(parent)) - , preferredLocalDownloadDir( - QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation)) -{ - QObject::connect(downloadWatcher, - &QFutureWatcherBase::progressValueChanged, - parent, - &AssetDownloader::setProgressPercent); - - QObject::connect(fileDownloadWatcher, - &QFutureWatcherBase::progressValueChanged, - parent, - &AssetDownloader::setProgressPercent); -} - -AssetDownloaderPrivate::~AssetDownloaderPrivate() -{ - downloadWatcher->cancel(); - fileDownloadWatcher->cancel(); - downloadWatcher->waitForFinished(); - fileDownloadWatcher->waitForFinished(); -} - -void AssetDownloaderPrivate::setupBase() -{ - providedTempDir.reset(new QTemporaryDir{}); - Q_ASSERT_X(providedTempDir->isValid(), - "AssetDownloader::setupBase()", - "Cannot create a temporary directory!"); - - if (canBeALocalBaseDir(preferredLocalDownloadDir)) { - setBaseLocalDir(preferredLocalDownloadDir); - } else { - qWarning().noquote() << "AssetDownloader: Cannot set \"" << preferredLocalDownloadDir - << "\" as a local download directory!"; - setBaseLocalDir(QStandardPaths::writableLocation(QStandardPaths::AppLocalDataLocation)); - } - - tempDir = providedTempDir->path(); -} - -void AssetDownloaderPrivate::setBaseLocalDir(const QDir &dir) -{ - Q_Q(AssetDownloader); - if (baseLocal != dir) { - baseLocal = dir; - emit q->localDownloadDirChanged(QUrl::fromLocalFile(baseLocal.absolutePath())); - } -} - -void AssetDownloaderPrivate::moveAllAssets() -{ - if (tempDir != baseLocal && !tempDir.isEmpty()) { - try { - setState(State::MovingFilesState); - QtConcurrent::blockingMap(downloadableAssets.files, [this](const QUrl &asset) { - QFile srcFile(tempDir.absoluteFilePath(asset.toString())); - QFile dstFile(baseLocal.absoluteFilePath(asset.toString())); - QFileInfo srcFileInfo(srcFile.fileName()); - QFileInfo dstFileInfo(dstFile.fileName()); - - ensureRemoveFileOrThrow(dstFile); - ensureDirOrThrow(dstFileInfo.absolutePath()); - - if (!srcFile.copy(dstFile.fileName())) { - if (!sameFileContent(srcFileInfo, dstFileInfo)) - throw FileCopyException(srcFile.fileName(), dstFile.fileName()); - } - }); - } catch (const Exception &exception) { - qWarning() << exception.message(); - setBaseLocalDir(tempDir); - } catch (const std::exception &exception) { - qWarning() << exception.what(); - setBaseLocalDir(tempDir); - } - } -} - -void AssetDownloaderPrivate::setAllDownloadsCount(int count) -{ - Q_Q(AssetDownloader); - if (allDownloadsCount != count) { - allDownloadsCount = count; - emit q->allDownloadsCountChanged(allDownloadsCount); - } -} - -void AssetDownloaderPrivate::setCompletedDownloadsCount(int count) -{ - Q_Q(AssetDownloader); - if (completedDownloadsCount != count) { - completedDownloadsCount = count; - emit q->completedDownloadsCountChanged(completedDownloadsCount); - } -} - -void AssetDownloaderPrivate::finalizeDownload() -{ - Q_Q(AssetDownloader); - if (downloadStarted) { - downloadStarted = false; - emit q->downloadFinished(); - } -} - -void AssetDownloaderPrivate::setState(State state) -{ - Q_Q(AssetDownloader); - if (this->state != state) { - this->state = state; - emit q->stateChanged(this->state); - } -} - -QFuture AssetDownloaderPrivate::download(const QUrl &url) -{ - Q_Q(AssetDownloader); - Q_ASSERT_X(QThread::currentThread() != q->thread(), - "AssetDownloader::download()", - "Method called from wrong thread"); - - QSharedPointer> promise(new QPromise); - QFuture result = promise->future(); - promise->start(); - - QNetworkReply *reply = networkManager->get(QNetworkRequest(url)); - QObject::connect(reply, &QNetworkReply::finished, reply, [reply, promise]() { - if (reply->error() == QNetworkReply::NoError) - promise->addResult(reply->readAll()); - else - promise->setException(NetworkException(reply)); - promise->finish(); - reply->deleteLater(); + return std::all_of(assetFiles.begin(), assetFiles.end(), [&](const QUrl &assetPath) { + return QFileInfo::exists(expectedDir.absoluteFilePath(assetPath.toString())); }); - - QObject::connect(reply, - &QNetworkReply::downloadProgress, - reply, - [promise, reply](qint64 bytesReceived, qint64 totalBytes) { - if (promise->isCanceled()) { - reply->abort(); - return; - } - int progress = (totalBytes > 0) ? 100.0 * bytesReceived / totalBytes : 0; - promise->setProgressRange(0, 100); - promise->setProgressValue(progress); - }); - - downloadWatcher->setFuture(result); - emit q->downloadingFileChanged(url); - - return result; -} - -QFuture AssetDownloaderPrivate::downloadToFile(const QUrl &url, const QUrl &destPath) -{ - Q_Q(AssetDownloader); - Q_ASSERT_X(QThread::currentThread() != q->thread(), - "AssetDownloader::download()", - "Method called from wrong thread"); - - QSharedPointer> promise(new QPromise); - QFuture result = promise->future(); - promise->start(); - - QFileInfo fileInfo(pathFromUrl(destPath)); - createDirectory(fileInfo.absolutePath()); - - QSharedPointer file(new QFile(fileInfo.absoluteFilePath())); - - if (!file->open(QFile::WriteOnly)) { - promise->setException(FileException(file.data())); - promise->finish(); - return result; - } - - QNetworkReply *reply = networkManager->get(QNetworkRequest(url), file.data()); - QObject::connect(reply, &QNetworkReply::finished, reply, [reply, promise, destPath, file]() { - if (reply->error() == QNetworkReply::NoError) - promise->addResult(destPath); - else - promise->setException(NetworkException(reply)); - - const qint64 bytesAvailable = reply->bytesAvailable(); - if (bytesAvailable) - file->write(reply->read(bytesAvailable)); - file->close(); - promise->finish(); - reply->deleteLater(); - }); - - QObject::connect(reply, &QNetworkReply::readyRead, reply, [promise, reply, file]() { - if (promise->isCanceled()) { - reply->abort(); - return; - } - const qint64 available = reply->bytesAvailable(); - file->write(reply->read(available)); - }); - - QObject::connect(reply, - &QNetworkReply::downloadProgress, - reply, - [promise, reply](qint64 bytesReceived, qint64 totalBytes) { - if (promise->isCanceled()) { - reply->abort(); - return; - } - int progress = (totalBytes > 0) ? 100.0 * bytesReceived / totalBytes : 0; - promise->setProgressRange(0, 100); - promise->setProgressValue(progress); - }); - - fileDownloadWatcher->setFuture(result); - emit q->downloadingFileChanged(url); - - return result; -} - -QFuture AssetDownloaderPrivate::downloadOrRead(const QUrl &url, const QUrl &localFile) -{ - Q_Q(AssetDownloader); - Q_ASSERT_X(QThread::currentThread() != q->thread(), - "AssetDownloader::downloadOrRead()", - "Method called from wrong thread"); - - QSharedPointer> promise(new QPromise); - QFuture futureResult = promise->future(); - promise->start(); - - // Precheck local file - if (!localFile.isEmpty()) { - QFile file(pathFromUrl(localFile)); - if (!file.open(QIODevice::ReadOnly)) - qWarning() << "Cannot open local file" << localFile; - } - - download(url) - .then([promise](const QByteArray &content) { - if (promise->isCanceled()) - return; - - promise->addResult(content); - promise->finish(); - }) - .onFailed([promise, url, localFile](const QException &downloadException) { - QFile file(pathFromUrl(localFile)); - qWarning() << "Cannot download" << url; - - if (promise->isCanceled()) - return; - - if (localFile.isEmpty()) { - qWarning() << "Also there is no local file as a replacement"; - promise->setException(downloadException); - } else if (!file.open(QIODevice::ReadOnly)) { - qWarning() << "Also failed to open" << localFile; - promise->setException(FileException(&file)); - } else { - promise->addResult(file.readAll()); - } - promise->finish(); - }); - - return futureResult; -} - -/*! - * \internal - * Schedules \a assets for downloading. - * Returns a future representing the files which are not downloaded. - * \note call this method on the AssetDownloader thread - */ -QFuture AssetDownloaderPrivate::downloadAssets(const DownloadableAssets &assets) -{ - Q_Q(AssetDownloader); - - Q_ASSERT_X(QThread::currentThread() != q->thread(), - "AssetDownloader::downloadAssets()", - "Method called from wrong thread"); - - QSharedPointer> promise(new QPromise); - QFuture future = promise->future(); - - promise->start(); - - if (assets.files.isEmpty()) { - promise->addResult(assets); - promise->finish(); - return future; - } - - setCompletedDownloadsCount(0); - setAllDownloadsCount(assets.files.size()); - setState(State::DownloadingFilesState); - - QFuture schedule = QtConcurrent::run([] {}); - - for (const QUrl &assetFile : assets.files) { - const QUrl url = assets.remoteUrl.resolved(assetFile); - const QUrl dstFilePath = tempDir.absoluteFilePath(assetFile.toString()); - schedule = schedule - .then(q, - [this, url, dstFilePath] { return downloadToFile(url, dstFilePath); }) - .unwrap() - .then(q, - [this, promise]([[maybe_unused]] const QUrl &url) { - if (promise->isCanceled()) - return; - setCompletedDownloadsCount(completedDownloadsCount + 1); - }) - .onFailed([](const Exception &exception) { - qWarning() << "Download failed" << exception.message(); - }); - ; - } - schedule = schedule - .then([assets, promise, tempDir = tempDir]() { - if (promise->isCanceled()) - return; - - DownloadableAssets result = assets; - result.files = filterDownloadableAssets(result.files, tempDir); - promise->addResult(result); - promise->finish(); - }) - .onFailed([promise](const QException &exception) { - promise->setException(exception); - promise->finish(); - }); - return future; -} - -QFuture AssetDownloaderPrivate::maybeReadZipFile(const DownloadableAssets &assets) -{ - Q_Q(AssetDownloader); - Q_ASSERT_X(QThread::currentThread() != q->thread(), - "AssetDownloader::maybeReadZipFile()", - "Method called from wrong thread"); - - QSharedPointer> promise(new QPromise); - QFuture future = promise->future(); - - promise->start(); - - if (assets.files.isEmpty() || zipFileName.isEmpty()) { - promise->addResult(assets); - promise->finish(); - return future; - } - - setState(State::DownloadingZipState); - - const QUrl zipUrl = downloadBase.resolved(zipFileName); - const QUrl dstUrl = tempDir.filePath(zipFileName); - downloadToFile(zipUrl, dstUrl) - .then(q, - [this, promise](const QUrl &downloadedFile) -> QUrl { - if (promise->isCanceled()) - return {}; - - setState(State::ExtractingZipState); - return downloadedFile; - }) - .then(QtFuture::Launch::Async, - [promise, zipFileName = zipFileName, tempDir = tempDir](const QUrl &downloadedFile) { - if (promise->isCanceled()) - return; - - unzip(pathFromUrl(downloadedFile), tempDir, UnzipRule::KeepArchive); - }) - .then([promise, assets, tempDir = tempDir]() { - if (promise->isCanceled()) - return; - - DownloadableAssets result = assets; - result.files = filterDownloadableAssets(result.files, tempDir); - promise->addResult(result); - promise->finish(); - }) - .onFailed([promise, assets](const Exception &exception) { - qWarning() << "ZipFile failed" << exception.message(); - promise->addResult(assets); - promise->finish(); - }); - - return future; } AssetDownloader::AssetDownloader(QObject *parent) : QObject(parent) - , d_ptr(new AssetDownloaderPrivate(this)) + , d(new AssetDownloaderPrivate(this)) {} AssetDownloader::~AssetDownloader() = default; QUrl AssetDownloader::downloadBase() const { - Q_D(const AssetDownloader); - return d->downloadBase; + return d->m_downloadBase; } void AssetDownloader::setDownloadBase(const QUrl &downloadBase) { - Q_D(AssetDownloader); - if (d->downloadBase != downloadBase) { - d->downloadBase = downloadBase; - emit downloadBaseChanged(d->downloadBase); + if (d->m_downloadBase != downloadBase) { + d->m_downloadBase = downloadBase; + emit downloadBaseChanged(d->m_downloadBase); } } -int AssetDownloader::allDownloadsCount() const -{ - Q_D(const AssetDownloader); - return d->allDownloadsCount; -} - -int AssetDownloader::completedDownloadsCount() const -{ - Q_D(const AssetDownloader); - return d->completedDownloadsCount; -} - -QUrl AssetDownloader::localDownloadDir() const -{ - Q_D(const AssetDownloader); - return QUrl::fromLocalFile(d->baseLocal.absolutePath()); -} - -double AssetDownloader::progress() const -{ - Q_D(const AssetDownloader); - return d->progress; -} - QUrl AssetDownloader::preferredLocalDownloadDir() const { - Q_D(const AssetDownloader); - return QUrl::fromLocalFile(d->preferredLocalDownloadDir.absolutePath()); + return QUrl::fromLocalFile(d->m_preferredLocalDownloadDir.absolutePath()); } void AssetDownloader::setPreferredLocalDownloadDir(const QUrl &localDir) { - Q_D(AssetDownloader); - if (!localDir.isLocalFile()) qWarning() << "preferredLocalDownloadDir Should be a local directory"; const QString path = pathFromUrl(localDir); - if (d->preferredLocalDownloadDir != path) { - d->preferredLocalDownloadDir.setPath(path); + if (d->m_preferredLocalDownloadDir != path) { + d->m_preferredLocalDownloadDir.setPath(path); emit preferredLocalDownloadDirChanged(preferredLocalDownloadDir()); } } QUrl AssetDownloader::offlineAssetsFilePath() const { - Q_D(const AssetDownloader); - return d->offlineAssetsFilePath; + return d->m_offlineAssetsFilePath; } void AssetDownloader::setOfflineAssetsFilePath(const QUrl &offlineAssetsFilePath) { - Q_D(AssetDownloader); - if (d->offlineAssetsFilePath != offlineAssetsFilePath) { - d->offlineAssetsFilePath = offlineAssetsFilePath; - emit offlineAssetsFilePathChanged(d->offlineAssetsFilePath); + if (d->m_offlineAssetsFilePath != offlineAssetsFilePath) { + d->m_offlineAssetsFilePath = offlineAssetsFilePath; + emit offlineAssetsFilePathChanged(d->m_offlineAssetsFilePath); } } QString AssetDownloader::jsonFileName() const { - Q_D(const AssetDownloader); - return d->jsonFileName; + return d->m_jsonFileName; } void AssetDownloader::setJsonFileName(const QString &jsonFileName) { - Q_D(AssetDownloader); - if (d->jsonFileName != jsonFileName) { - d->jsonFileName = jsonFileName; - emit jsonFileNameChanged(d->jsonFileName); + if (d->m_jsonFileName != jsonFileName) { + d->m_jsonFileName = jsonFileName; + emit jsonFileNameChanged(d->m_jsonFileName); } } QString AssetDownloader::zipFileName() const { - Q_D(const AssetDownloader); - return d->zipFileName; + return d->m_zipFileName; } void AssetDownloader::setZipFileName(const QString &zipFileName) { - Q_D(AssetDownloader); - if (d->zipFileName != zipFileName) { - d->zipFileName = zipFileName; - emit zipFileNameChanged(d->zipFileName); + if (d->m_zipFileName != zipFileName) { + d->m_zipFileName = zipFileName; + emit zipFileNameChanged(d->m_zipFileName); } } -AssetDownloader::State AssetDownloader::state() const +QUrl AssetDownloader::localDownloadDir() const { - Q_D(const AssetDownloader); - return d->state; + return QUrl::fromLocalFile(d->m_localDownloadDir.absolutePath()); +} + +static void precheckLocalFile(const QUrl &url) +{ + if (url.isEmpty()) + return; + QFile file(pathFromUrl(url)); + if (!file.open(QIODevice::ReadOnly)) + qWarning() << "Cannot open local file" << url; +} + +static void readAssetsFileContent(QPromise &promise, const QByteArray &content) +{ + const QJsonObject json = QJsonDocument::fromJson(content).object(); + const QJsonArray assetsArray = json[u"assets"].toArray(); + DownloadableAssets result; + result.remoteUrl = json[u"url"].toString(); + for (const QJsonValue &asset : assetsArray) { + if (promise.isCanceled()) + return; + result.files.append(asset.toString()); + } + + if (result.files.isEmpty() || result.remoteUrl.isEmpty()) + promise.future().cancel(); + else + promise.addResult(result); +} + +static void unzip(QPromise &promise, const QByteArray &content, const QDir &directory, + const QString &fileName) +{ + const QString zipFilePath = directory.absoluteFilePath(fileName); + QFile zipFile(zipFilePath); + if (!zipFile.open(QIODevice::WriteOnly)) { + promise.future().cancel(); + return; + } + zipFile.write(content); + zipFile.close(); + + if (promise.isCanceled()) + return; + + QZipReader reader(zipFilePath); + const bool extracted = reader.extractAll(directory.absolutePath()); + reader.close(); + if (extracted) + QFile::remove(zipFilePath); + else + promise.future().cancel(); +} + +static void writeAsset(QPromise &promise, const QByteArray &content, const QString &filePath) +{ + const QFileInfo fileInfo(filePath); + QFile file(fileInfo.absoluteFilePath()); + if (!createDirectory(fileInfo.dir()) || !file.open(QFile::WriteOnly)) { + promise.future().cancel(); + return; + } + + if (promise.isCanceled()) + return; + + file.write(content); + file.close(); +} + +static void copyAndCheck(QPromise &promise, const QString &sourcePath, const QString &destPath) +{ + QFile sourceFile(sourcePath); + QFile destFile(destPath); + const QFileInfo sourceFileInfo(sourceFile.fileName()); + const QFileInfo destFileInfo(destFile.fileName()); + + if (destFile.exists() && !destFile.remove()) { + qWarning().noquote() << QString::fromLatin1("Unable to remove file \"%1\".") + .arg(QFileInfo(destFile.fileName()).absoluteFilePath()); + promise.future().cancel(); + return; + } + + if (!createDirectory(destFileInfo.absolutePath())) { + qWarning().noquote() << QString::fromLatin1("Cannot create directory \"%1\".") + .arg(destFileInfo.absolutePath()); + promise.future().cancel(); + return; + } + + if (promise.isCanceled()) + return; + + if (!sourceFile.copy(destFile.fileName()) && !sameFileContent(sourceFileInfo, destFileInfo)) + promise.future().cancel(); } void AssetDownloader::start() { - Q_D(AssetDownloader); - if (d->downloadStarted) + if (d->m_taskTreeRunner.isRunning()) return; - d->setState(State::NotDownloadedState); - d->setupBase(); + struct StorageData + { + QDir tempDir; + QByteArray jsonContent; + DownloadableAssets assets; + QList assetsToDownload; + QByteArray zipContent; + int doneCount = 0; + }; - d->downloadOrRead(d->downloadBase.resolved(d->jsonFileName), d->offlineAssetsFilePath) - .then(QtFuture::Launch::Async, &readAssetsFile) - .then(this, - [this, d](const DownloadableAssets &assets) { - d->downloadStarted = true; - d->downloadableAssets = assets; - emit downloadStarted(); - return assets; - }) - .then(QtFuture::Launch::Async, - [localDir = d->baseLocal](DownloadableAssets assets) -> DownloadableAssets { - if (shouldDownloadEverything(assets.files, localDir)) - return assets; - return {}; - }) - .then(this, [d](const DownloadableAssets &assets) { return d->maybeReadZipFile(assets); }) - .unwrap() - .then(this, [d](const DownloadableAssets &assets) { return d->downloadAssets(assets); }) - .unwrap() - .then(this, [d](const DownloadableAssets &) { return d->moveAllAssets(); }) - .then(this, - [d]() { - d->setState(State::DownloadedState); - d->finalizeDownload(); - }) - .onFailed(this, [d](const Exception &exception) { - qWarning() << "Exception: " << exception.message(); - d->setState(State::ErrorState); - d->finalizeDownload(); - }); - ; + const Storage storage; + + const auto onSetup = [this, storage] { + if (!d->m_manager) + d->m_manager = std::make_unique(); + if (!d->m_temporaryDir) + d->m_temporaryDir = std::make_unique(); + if (!d->m_temporaryDir->isValid()) { + qWarning() << "Cannot create a temporary directory."; + return SetupResult::StopWithError; + } + storage->tempDir = d->m_temporaryDir->path(); + d->setLocalDownloadDir(baseLocalDir(d->m_preferredLocalDownloadDir)); + precheckLocalFile(d->m_offlineAssetsFilePath); + return SetupResult::Continue; + }; + + const auto onJsonDownloadSetup = [this](NetworkQuery &query) { + query.setRequest(QNetworkRequest(d->m_downloadBase.resolved(d->m_jsonFileName))); + d->setupDownload(&query, tr("Downloading JSON file...")); + }; + const auto onJsonDownloadDone = [this, storage](const NetworkQuery &query, DoneWith result) { + if (result == DoneWith::Success) { + storage->jsonContent = query.reply()->readAll(); + return DoneResult::Success; + } + qWarning() << "Cannot download" << d->m_downloadBase.resolved(d->m_jsonFileName) + << query.reply()->errorString(); + if (d->m_offlineAssetsFilePath.isEmpty()) { + qWarning() << "Also there is no local file as a replacement"; + return DoneResult::Error; + } + + QFile file(pathFromUrl(d->m_offlineAssetsFilePath)); + if (!file.open(QIODevice::ReadOnly)) { + qWarning() << "Also failed to open" << d->m_offlineAssetsFilePath; + return DoneResult::Error; + } + + storage->jsonContent = file.readAll(); + return DoneResult::Success; + }; + + const auto onReadAssetsFileSetup = [storage](ConcurrentCall &async) { + async.setConcurrentCallData(readAssetsFileContent, storage->jsonContent); + }; + const auto onReadAssetsFileDone = [storage](const ConcurrentCall &async) { + storage->assets = async.result(); + storage->assetsToDownload = storage->assets.files; + }; + + const auto onSkipIfAllAssetsPresent = [this, storage] { + return allAssetsPresent(storage->assets.files, d->m_localDownloadDir) + ? SetupResult::StopWithSuccess : SetupResult::Continue; + }; + + const auto onZipDownloadSetup = [this, storage](NetworkQuery &query) { + if (d->m_zipFileName.isEmpty()) + return SetupResult::StopWithSuccess; + + query.setRequest(QNetworkRequest(d->m_downloadBase.resolved(d->m_zipFileName))); + d->setupDownload(&query, tr("Downloading zip file...")); + return SetupResult::Continue; + }; + const auto onZipDownloadDone = [storage](const NetworkQuery &query, DoneWith result) { + if (result == DoneWith::Success) + storage->zipContent = query.reply()->readAll(); + return DoneResult::Success; // Ignore zip download failure + }; + + const auto onUnzipSetup = [this, storage](ConcurrentCall &async) { + if (storage->zipContent.isEmpty()) + return SetupResult::StopWithSuccess; + + async.setConcurrentCallData(unzip, storage->zipContent, storage->tempDir, d->m_zipFileName); + d->clearProgress(tr("Unzipping...")); + return SetupResult::Continue; + }; + const auto onUnzipDone = [storage](DoneWith result) { + if (result == DoneWith::Success) { + // Avoid downloading assets that are present in unzipped tree + StorageData &storageData = *storage; + storageData.assetsToDownload = + filterDownloadableAssets(storageData.assets.files, storageData.tempDir); + } else { + qWarning() << "ZipFile failed"; + } + return DoneResult::Success; // Ignore unzip failure + }; + + const LoopUntil downloadIterator([storage](int iteration) { + return iteration < storage->assetsToDownload.count(); + }); + + const Storage assetStorage; + + const auto onAssetsDownloadGroupSetup = [this, storage] { + d->setProgress(0, storage->assetsToDownload.size(), tr("Downloading assets...")); + }; + + const auto onAssetDownloadSetup = [this, storage, downloadIterator](NetworkQuery &query) { + query.setNetworkAccessManager(d->m_manager.get()); + query.setRequest(QNetworkRequest(storage->assets.remoteUrl.resolved( + storage->assetsToDownload.at(downloadIterator.iteration())))); + }; + const auto onAssetDownloadDone = [assetStorage](const NetworkQuery &query, DoneWith result) { + if (result == DoneWith::Success) + *assetStorage = query.reply()->readAll(); + }; + + const auto onAssetWriteSetup = [storage, downloadIterator, assetStorage]( + ConcurrentCall &async) { + const QString filePath = storage->tempDir.absoluteFilePath( + storage->assetsToDownload.at(downloadIterator.iteration()).toString()); + async.setConcurrentCallData(writeAsset, *assetStorage, filePath); + }; + const auto onAssetWriteDone = [this, storage](DoneWith result) { + if (result != DoneWith::Success) { + qWarning() << "Asset write failed"; + return; + } + StorageData &storageData = *storage; + ++storageData.doneCount; + d->updateProgress(storageData.doneCount, storageData.assetsToDownload.size()); + }; + + const LoopUntil copyIterator([storage](int iteration) { + return iteration < storage->assets.files.count(); + }); + + const auto onAssetsCopyGroupSetup = [this, storage] { + storage->doneCount = 0; + d->setProgress(0, storage->assets.files.size(), tr("Copying assets...")); + }; + + const auto onAssetCopySetup = [this, storage, copyIterator](ConcurrentCall &async) { + const QString fileName = storage->assets.files.at(copyIterator.iteration()).toString(); + const QString sourcePath = storage->tempDir.absoluteFilePath(fileName); + const QString destPath = d->m_localDownloadDir.absoluteFilePath(fileName); + async.setConcurrentCallData(copyAndCheck, sourcePath, destPath); + }; + const auto onAssetCopyDone = [this, storage] { + StorageData &storageData = *storage; + ++storageData.doneCount; + d->updateProgress(storageData.doneCount, storageData.assets.files.size()); + }; + + const auto onAssetsCopyGroupDone = [this, storage](DoneWith result) { + if (result != DoneWith::Success) { + d->setLocalDownloadDir(storage->tempDir); + qWarning() << "Asset copy failed"; + return; + } + d->m_temporaryDir.reset(); + }; + + const Group recipe { + storage, + onGroupSetup(onSetup), + NetworkQueryTask(onJsonDownloadSetup, onJsonDownloadDone), + ConcurrentCallTask(onReadAssetsFileSetup, onReadAssetsFileDone, CallDoneIf::Success), + Group { + onGroupSetup(onSkipIfAllAssetsPresent), + NetworkQueryTask(onZipDownloadSetup, onZipDownloadDone), + ConcurrentCallTask(onUnzipSetup, onUnzipDone), + Group { + parallelIdealThreadCountLimit, + downloadIterator, + onGroupSetup(onAssetsDownloadGroupSetup), + Group { + assetStorage, + NetworkQueryTask(onAssetDownloadSetup, onAssetDownloadDone), + ConcurrentCallTask(onAssetWriteSetup, onAssetWriteDone) + } + }, + Group { + parallelIdealThreadCountLimit, + copyIterator, + onGroupSetup(onAssetsCopyGroupSetup), + ConcurrentCallTask(onAssetCopySetup, onAssetCopyDone, CallDoneIf::Success), + onGroupDone(onAssetsCopyGroupDone) + } + } + }; + d->m_taskTreeRunner.start(recipe, [this](TaskTree *) { emit started(); }, + [this](DoneWith result) { emit finished(result == DoneWith::Success); }); } -void AssetDownloader::setProgressPercent(int progress) -{ - Q_D(AssetDownloader); - const double progressInRange = 0.01 * progress; - if (d->progress != progressInRange) { - d->progress = progressInRange; - emit progressChanged(d->progress); - } -} +} // namespace Assets::Downloader + +QT_END_NAMESPACE diff --git a/src/assets/downloader/assetdownloader.h b/src/assets/downloader/assetdownloader.h index 70b710ff198..e000122b41c 100644 --- a/src/assets/downloader/assetdownloader.h +++ b/src/assets/downloader/assetdownloader.h @@ -8,22 +8,24 @@ // W A R N I N G // ------------- // -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to // version without notice, or even be removed. // // We mean it. // #include -#include #include +#include + QT_BEGIN_NAMESPACE -namespace QtExamples { +namespace Assets::Downloader { class AssetDownloaderPrivate; + class AssetDownloader : public QObject { Q_OBJECT @@ -34,32 +36,12 @@ class AssetDownloader : public QObject WRITE setDownloadBase NOTIFY downloadBaseChanged) - Q_PROPERTY( - int completedDownloadsCount - READ completedDownloadsCount - NOTIFY completedDownloadsCountChanged) - - Q_PROPERTY( - int allDownloadsCount - READ allDownloadsCount - NOTIFY allDownloadsCountChanged) - - Q_PROPERTY( - double progress - READ progress - NOTIFY progressChanged) - Q_PROPERTY( QUrl preferredLocalDownloadDir READ preferredLocalDownloadDir WRITE setPreferredLocalDownloadDir NOTIFY preferredLocalDownloadDirChanged) - Q_PROPERTY( - QUrl localDownloadDir - READ localDownloadDir - NOTIFY localDownloadDirChanged) - Q_PROPERTY( QUrl offlineAssetsFilePath READ offlineAssetsFilePath @@ -78,31 +60,18 @@ class AssetDownloader : public QObject WRITE setZipFileName NOTIFY zipFileNameChanged) - Q_PROPERTY(State state READ state NOTIFY stateChanged) + Q_PROPERTY( + QUrl localDownloadDir + READ localDownloadDir + NOTIFY localDownloadDirChanged) public: - enum class State { - NotDownloadedState, - DownloadingZipState, - ExtractingZipState, - DownloadingFilesState, - MovingFilesState, - DownloadedState, - ErrorState - }; - Q_ENUM(State) - AssetDownloader(QObject *parent = nullptr); ~AssetDownloader(); QUrl downloadBase() const; void setDownloadBase(const QUrl &downloadBase); - int allDownloadsCount() const; - int completedDownloadsCount() const; - QUrl localDownloadDir() const; - double progress() const; - QUrl preferredLocalDownloadDir() const; void setPreferredLocalDownloadDir(const QUrl &localDir); @@ -115,35 +84,28 @@ public: QString zipFileName() const; void setZipFileName(const QString &zipFileName); - State state() const; + QUrl localDownloadDir() const; public Q_SLOTS: void start(); -private Q_SLOTS: - void setProgressPercent(int); - Q_SIGNALS: - void stateChanged(State); - void downloadBaseChanged(const QUrl &); - void allDownloadsCountChanged(int count); - void downloadStarted(); - void completedDownloadsCountChanged(int count); - void downloadFinished(); - void downloadingFileChanged(const QUrl &url); - void preferredLocalDownloadDirChanged(const QUrl &url); + void started(); + void finished(bool success); + void progressChanged(int progressValue, int progressMaximum, const QString &progressText); void localDownloadDirChanged(const QUrl &url); - void progressChanged(double progress); + + void downloadBaseChanged(const QUrl &); + void preferredLocalDownloadDirChanged(const QUrl &url); void offlineAssetsFilePathChanged(const QUrl &); void jsonFileNameChanged(const QString &); void zipFileNameChanged(const QString &); private: - Q_DECLARE_PRIVATE(AssetDownloader) - QScopedPointer d_ptr; + std::unique_ptr d; }; -} // namespace QtExamples +} // namespace Assets::Downloader QT_END_NAMESPACE