From 856da3ee19a6c0f6c88ecc77b69ad41b4d4e6ffa Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Tue, 5 Apr 2011 16:24:01 +0200 Subject: [PATCH 01/26] QNAM: Improve internal authenticationRequired() Make it independant from the backend architecture to improve refactorability. Reviewed-by: Peter Hartmann --- src/network/access/qnetworkaccessbackend.cpp | 2 +- src/network/access/qnetworkaccessmanager.cpp | 22 ++++++++++---------- src/network/access/qnetworkaccessmanager_p.h | 6 +++++- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/network/access/qnetworkaccessbackend.cpp b/src/network/access/qnetworkaccessbackend.cpp index 6220abed024..2847cd5a502 100644 --- a/src/network/access/qnetworkaccessbackend.cpp +++ b/src/network/access/qnetworkaccessbackend.cpp @@ -321,7 +321,7 @@ void QNetworkAccessBackend::proxyAuthenticationRequired(const QNetworkProxy &pro void QNetworkAccessBackend::authenticationRequired(QAuthenticator *authenticator) { - manager->authenticationRequired(this, authenticator); + manager->authenticationRequired(authenticator, reply->q_func(), synchronous, reply->url, &reply->urlForLastAuthentication); } void QNetworkAccessBackend::metaDataChanged() diff --git a/src/network/access/qnetworkaccessmanager.cpp b/src/network/access/qnetworkaccessmanager.cpp index 5a7521e33e7..a8095835505 100644 --- a/src/network/access/qnetworkaccessmanager.cpp +++ b/src/network/access/qnetworkaccessmanager.cpp @@ -1055,35 +1055,35 @@ void QNetworkAccessManagerPrivate::createCookieJar() const } } -void QNetworkAccessManagerPrivate::authenticationRequired(QNetworkAccessBackend *backend, - QAuthenticator *authenticator) +void QNetworkAccessManagerPrivate::authenticationRequired(QAuthenticator *authenticator, + QNetworkReply *reply, + bool synchronous, + QUrl &url, + QUrl *urlForLastAuthentication) { Q_Q(QNetworkAccessManager); - // FIXME: Add support for domains (i.e., the leading path) - QUrl url = backend->reply->url; - // don't try the cache for the same URL twice in a row // being called twice for the same URL means the authentication failed // also called when last URL is empty, e.g. on first call - if (backend->reply->urlForLastAuthentication.isEmpty() - || url != backend->reply->urlForLastAuthentication) { + if (urlForLastAuthentication->isEmpty() + || url != *urlForLastAuthentication) { QNetworkAuthenticationCredential cred = authenticationManager->fetchCachedCredentials(url, authenticator); if (!cred.isNull()) { authenticator->setUser(cred.user); authenticator->setPassword(cred.password); - backend->reply->urlForLastAuthentication = url; + *urlForLastAuthentication = url; return; } } // if we emit a signal here in synchronous mode, the user might spin // an event loop, which might recurse and lead to problems - if (backend->isSynchronous()) + if (synchronous) return; - backend->reply->urlForLastAuthentication = url; - emit q->authenticationRequired(backend->reply->q_func(), authenticator); + *urlForLastAuthentication = url; + emit q->authenticationRequired(reply, authenticator); authenticationManager->cacheCredentials(url, authenticator); } diff --git a/src/network/access/qnetworkaccessmanager_p.h b/src/network/access/qnetworkaccessmanager_p.h index f64cc4dc79c..a8237b30d2b 100644 --- a/src/network/access/qnetworkaccessmanager_p.h +++ b/src/network/access/qnetworkaccessmanager_p.h @@ -94,7 +94,11 @@ public: QNetworkReply *postProcess(QNetworkReply *reply); void createCookieJar() const; - void authenticationRequired(QNetworkAccessBackend *backend, QAuthenticator *authenticator); + void authenticationRequired(QAuthenticator *authenticator, + QNetworkReply *reply, + bool synchronous, + QUrl &url, + QUrl *urlForLastAuthentication); void cacheCredentials(const QUrl &url, const QAuthenticator *auth); QNetworkAuthenticationCredential *fetchCachedCredentials(const QUrl &url, const QAuthenticator *auth = 0); From be4d62d3b70fabbf730566a9040ac731f61539da Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Tue, 5 Apr 2011 16:42:33 +0200 Subject: [PATCH 02/26] QNAM: Improve internal proxyAuthenticationRequired() Make it independent from the backend architecture to improve refactorability. Reviewed-by: Peter Hartmann --- src/network/access/qnetworkaccessbackend.cpp | 2 +- src/network/access/qnetworkaccessmanager.cpp | 13 +++++++------ src/network/access/qnetworkaccessmanager_p.h | 6 ++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/network/access/qnetworkaccessbackend.cpp b/src/network/access/qnetworkaccessbackend.cpp index 2847cd5a502..8a53d2dfa63 100644 --- a/src/network/access/qnetworkaccessbackend.cpp +++ b/src/network/access/qnetworkaccessbackend.cpp @@ -315,7 +315,7 @@ void QNetworkAccessBackend::error(QNetworkReply::NetworkError code, const QStrin void QNetworkAccessBackend::proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *authenticator) { - manager->proxyAuthenticationRequired(this, proxy, authenticator); + manager->proxyAuthenticationRequired(proxy, synchronous, authenticator, &reply->lastProxyAuthentication); } #endif diff --git a/src/network/access/qnetworkaccessmanager.cpp b/src/network/access/qnetworkaccessmanager.cpp index a8095835505..34ac17037be 100644 --- a/src/network/access/qnetworkaccessmanager.cpp +++ b/src/network/access/qnetworkaccessmanager.cpp @@ -1088,9 +1088,10 @@ void QNetworkAccessManagerPrivate::authenticationRequired(QAuthenticator *authen } #ifndef QT_NO_NETWORKPROXY -void QNetworkAccessManagerPrivate::proxyAuthenticationRequired(QNetworkAccessBackend *backend, - const QNetworkProxy &proxy, - QAuthenticator *authenticator) +void QNetworkAccessManagerPrivate::proxyAuthenticationRequired(const QNetworkProxy &proxy, + bool synchronous, + QAuthenticator *authenticator, + QNetworkProxy *lastProxyAuthentication) { Q_Q(QNetworkAccessManager); // ### FIXME Tracking of successful authentications @@ -1100,7 +1101,7 @@ void QNetworkAccessManagerPrivate::proxyAuthenticationRequired(QNetworkAccessBac // proxyAuthenticationRequired gets emitted again // possible solution: some tracking inside the authenticator // or a new function proxyAuthenticationSucceeded(true|false) - if (proxy != backend->reply->lastProxyAuthentication) { + if (proxy != *lastProxyAuthentication) { QNetworkAuthenticationCredential cred = authenticationManager->fetchCachedProxyCredentials(proxy); if (!cred.isNull()) { authenticator->setUser(cred.user); @@ -1111,10 +1112,10 @@ void QNetworkAccessManagerPrivate::proxyAuthenticationRequired(QNetworkAccessBac // if we emit a signal here in synchronous mode, the user might spin // an event loop, which might recurse and lead to problems - if (backend->isSynchronous()) + if (synchronous) return; - backend->reply->lastProxyAuthentication = proxy; + *lastProxyAuthentication = proxy; emit q->proxyAuthenticationRequired(proxy, authenticator); authenticationManager->cacheProxyCredentials(proxy, authenticator); } diff --git a/src/network/access/qnetworkaccessmanager_p.h b/src/network/access/qnetworkaccessmanager_p.h index a8237b30d2b..f5e13e3a097 100644 --- a/src/network/access/qnetworkaccessmanager_p.h +++ b/src/network/access/qnetworkaccessmanager_p.h @@ -104,8 +104,10 @@ public: const QAuthenticator *auth = 0); #ifndef QT_NO_NETWORKPROXY - void proxyAuthenticationRequired(QNetworkAccessBackend *backend, const QNetworkProxy &proxy, - QAuthenticator *authenticator); + void proxyAuthenticationRequired(const QNetworkProxy &proxy, + bool synchronous, + QAuthenticator *authenticator, + QNetworkProxy *lastProxyAuthentication); void cacheProxyCredentials(const QNetworkProxy &proxy, const QAuthenticator *auth); QNetworkAuthenticationCredential *fetchCachedProxyCredentials(const QNetworkProxy &proxy, const QAuthenticator *auth = 0); From f78640e5c2274ecd4ccda1aba7607003c5bf00bf Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Mon, 28 Mar 2011 14:14:57 +0200 Subject: [PATCH 03/26] QNAM: Copy files for new architecture --- src/network/access/qnetworkreplyhttpimpl.cpp | 1190 ++++++++++++++++++ src/network/access/qnetworkreplyhttpimpl_p.h | 169 +++ 2 files changed, 1359 insertions(+) create mode 100644 src/network/access/qnetworkreplyhttpimpl.cpp create mode 100644 src/network/access/qnetworkreplyhttpimpl_p.h diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp new file mode 100644 index 00000000000..e75347f78a6 --- /dev/null +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -0,0 +1,1190 @@ +/**************************************************************************** +** +** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the QtNetwork module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +//#define QNETWORKACCESSHTTPBACKEND_DEBUG + +#include "qnetworkaccesshttpbackend_p.h" +#include "qnetworkaccessmanager_p.h" +#include "qnetworkaccesscache_p.h" +#include "qabstractnetworkcache.h" +#include "qnetworkrequest.h" +#include "qnetworkreply.h" +#include "qnetworkrequest_p.h" +#include "qnetworkcookie_p.h" +#include "QtCore/qdatetime.h" +#include "QtCore/qelapsedtimer.h" +#include "QtNetwork/qsslconfiguration.h" +#include "qhttpthreaddelegate_p.h" +#include "qthread.h" + +#ifndef QT_NO_HTTP + +#include // for strchr + +Q_DECLARE_METATYPE(QSharedPointer) + +QT_BEGIN_NAMESPACE + +class QNetworkProxy; + +static inline bool isSeparator(register char c) +{ + static const char separators[] = "()<>@,;:\\\"/[]?={}"; + return isLWS(c) || strchr(separators, c) != 0; +} + +// ### merge with nextField in cookiejar.cpp +static QHash parseHttpOptionHeader(const QByteArray &header) +{ + // The HTTP header is of the form: + // header = #1(directives) + // directives = token | value-directive + // value-directive = token "=" (token | quoted-string) + QHash result; + + int pos = 0; + while (true) { + // skip spaces + pos = nextNonWhitespace(header, pos); + if (pos == header.length()) + return result; // end of parsing + + // pos points to a non-whitespace + int comma = header.indexOf(',', pos); + int equal = header.indexOf('=', pos); + if (comma == pos || equal == pos) + // huh? Broken header. + return result; + + // The key name is delimited by either a comma, an equal sign or the end + // of the header, whichever comes first + int end = comma; + if (end == -1) + end = header.length(); + if (equal != -1 && end > equal) + end = equal; // equal sign comes before comma/end + QByteArray key = QByteArray(header.constData() + pos, end - pos).trimmed().toLower(); + pos = end + 1; + + if (uint(equal) < uint(comma)) { + // case: token "=" (token | quoted-string) + // skip spaces + pos = nextNonWhitespace(header, pos); + if (pos == header.length()) + // huh? Broken header + return result; + + QByteArray value; + value.reserve(header.length() - pos); + if (header.at(pos) == '"') { + // case: quoted-string + // quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) + // qdtext = > + // quoted-pair = "\" CHAR + ++pos; + while (pos < header.length()) { + register char c = header.at(pos); + if (c == '"') { + // end of quoted text + break; + } else if (c == '\\') { + ++pos; + if (pos >= header.length()) + // broken header + return result; + c = header.at(pos); + } + + value += c; + ++pos; + } + } else { + // case: token + while (pos < header.length()) { + register char c = header.at(pos); + if (isSeparator(c)) + break; + value += c; + ++pos; + } + } + + result.insert(key, value); + + // find the comma now: + comma = header.indexOf(',', pos); + if (comma == -1) + return result; // end of parsing + pos = comma + 1; + } else { + // case: token + // key is already set + result.insert(key, QByteArray()); + } + } +} + +QNetworkAccessBackend * +QNetworkAccessHttpBackendFactory::create(QNetworkAccessManager::Operation op, + const QNetworkRequest &request) const +{ + // check the operation + switch (op) { + case QNetworkAccessManager::GetOperation: + case QNetworkAccessManager::PostOperation: + case QNetworkAccessManager::HeadOperation: + case QNetworkAccessManager::PutOperation: + case QNetworkAccessManager::DeleteOperation: + case QNetworkAccessManager::CustomOperation: + break; + + default: + // no, we can't handle this request + return 0; + } + + QUrl url = request.url(); + QString scheme = url.scheme().toLower(); + if (scheme == QLatin1String("http") || scheme == QLatin1String("https")) + return new QNetworkAccessHttpBackend; + + return 0; +} + +QNetworkAccessHttpBackend::QNetworkAccessHttpBackend() + : QNetworkAccessBackend() + , statusCode(0) + , pendingDownloadDataEmissions(new QAtomicInt()) + , pendingDownloadProgressEmissions(new QAtomicInt()) + , loadingFromCache(false) + , usingZerocopyDownloadBuffer(false) +#ifndef QT_NO_OPENSSL + , pendingSslConfiguration(0), pendingIgnoreAllSslErrors(false) +#endif + , resumeOffset(0) +{ +} + +QNetworkAccessHttpBackend::~QNetworkAccessHttpBackend() +{ + // This will do nothing if the request was already finished or aborted + emit abortHttpRequest(); + +#ifndef QT_NO_OPENSSL + delete pendingSslConfiguration; +#endif +} + +/* + For a given httpRequest + 1) If AlwaysNetwork, return + 2) If we have a cache entry for this url populate headers so the server can return 304 + 3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true + */ +void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache) +{ + QNetworkRequest::CacheLoadControl CacheLoadControlAttribute = + (QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt(); + if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) { + // If the request does not already specify preferred cache-control + // force reload from the network and tell any caching proxy servers to reload too + if (!request().rawHeaderList().contains("Cache-Control")) { + httpRequest.setHeaderField("Cache-Control", "no-cache"); + httpRequest.setHeaderField("Pragma", "no-cache"); + } + return; + } + + // The disk cache API does not currently support partial content retrieval. + // That is why we don't use the disk cache for any such requests. + if (request().hasRawHeader("Range")) + return; + + QAbstractNetworkCache *nc = networkCache(); + if (!nc) + return; // no local cache + + QNetworkCacheMetaData metaData = nc->metaData(url()); + if (!metaData.isValid()) + return; // not in cache + + if (!metaData.saveToDisk()) + return; + + QNetworkHeadersPrivate cacheHeaders; + QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; + cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); + + it = cacheHeaders.findRawHeader("etag"); + if (it != cacheHeaders.rawHeaders.constEnd()) + httpRequest.setHeaderField("If-None-Match", it->second); + + QDateTime lastModified = metaData.lastModified(); + if (lastModified.isValid()) + httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified)); + + if (CacheLoadControlAttribute == QNetworkRequest::PreferNetwork) { + it = cacheHeaders.findRawHeader("Cache-Control"); + if (it != cacheHeaders.rawHeaders.constEnd()) { + QHash cacheControl = parseHttpOptionHeader(it->second); + if (cacheControl.contains("must-revalidate")) + return; + } + } + + QDateTime currentDateTime = QDateTime::currentDateTime(); + QDateTime expirationDate = metaData.expirationDate(); + +#if 0 + /* + * age_value + * is the value of Age: header received by the cache with + * this response. + * date_value + * is the value of the origin server's Date: header + * request_time + * is the (local) time when the cache made the request + * that resulted in this cached response + * response_time + * is the (local) time when the cache received the + * response + * now + * is the current (local) time + */ + int age_value = 0; + it = cacheHeaders.findRawHeader("age"); + if (it != cacheHeaders.rawHeaders.constEnd()) + age_value = it->second.toInt(); + + QDateTime dateHeader; + int date_value = 0; + it = cacheHeaders.findRawHeader("date"); + if (it != cacheHeaders.rawHeaders.constEnd()) { + dateHeader = QNetworkHeadersPrivate::fromHttpDate(it->second); + date_value = dateHeader.toTime_t(); + } + + int now = currentDateTime.toUTC().toTime_t(); + int request_time = now; + int response_time = now; + + // Algorithm from RFC 2616 section 13.2.3 + int apparent_age = qMax(0, response_time - date_value); + int corrected_received_age = qMax(apparent_age, age_value); + int response_delay = response_time - request_time; + int corrected_initial_age = corrected_received_age + response_delay; + int resident_time = now - response_time; + int current_age = corrected_initial_age + resident_time; + + // RFC 2616 13.2.4 Expiration Calculations + if (!expirationDate.isValid()) { + if (lastModified.isValid()) { + int diff = currentDateTime.secsTo(lastModified); + expirationDate = lastModified; + expirationDate.addSecs(diff / 10); + if (httpRequest.headerField("Warning").isEmpty()) { + QDateTime dt; + dt.setTime_t(current_age); + if (dt.daysTo(currentDateTime) > 1) + httpRequest.setHeaderField("Warning", "113"); + } + } + } + + // the cache-saving code below sets the expirationDate with date+max_age + // if "max-age" is present, or to Expires otherwise + int freshness_lifetime = dateHeader.secsTo(expirationDate); + bool response_is_fresh = (freshness_lifetime > current_age); +#else + bool response_is_fresh = currentDateTime.secsTo(expirationDate) >= 0; +#endif + + if (!response_is_fresh) + return; + + loadedFromCache = true; +#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) + qDebug() << "response_is_fresh" << CacheLoadControlAttribute; +#endif + if (!sendCacheContents(metaData)) + loadedFromCache = false; +} + +static QHttpNetworkRequest::Priority convert(const QNetworkRequest::Priority& prio) +{ + switch (prio) { + case QNetworkRequest::LowPriority: + return QHttpNetworkRequest::LowPriority; + case QNetworkRequest::HighPriority: + return QHttpNetworkRequest::HighPriority; + case QNetworkRequest::NormalPriority: + default: + return QHttpNetworkRequest::NormalPriority; + } +} + +void QNetworkAccessHttpBackend::postRequest() +{ + QThread *thread = 0; + if (isSynchronous()) { + // A synchronous HTTP request uses its own thread + thread = new QThread(); + QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); + thread->start(); + } else if (!manager->httpThread) { + // We use the manager-global thread. + // At some point we could switch to having multiple threads if it makes sense. + manager->httpThread = new QThread(); + QObject::connect(manager->httpThread, SIGNAL(finished()), manager->httpThread, SLOT(deleteLater())); + manager->httpThread->start(); +#ifndef QT_NO_NETWORKPROXY + qRegisterMetaType("QNetworkProxy"); +#endif +#ifndef QT_NO_OPENSSL + qRegisterMetaType >("QList"); + qRegisterMetaType("QSslConfiguration"); +#endif + qRegisterMetaType > >("QList >"); + qRegisterMetaType("QHttpNetworkRequest"); + qRegisterMetaType("QNetworkReply::NetworkError"); + qRegisterMetaType >("QSharedPointer"); + + thread = manager->httpThread; + } else { + // Asynchronous request, thread already exists + thread = manager->httpThread; + } + + QUrl url = request().url(); + httpRequest.setUrl(url); + + bool ssl = url.scheme().toLower() == QLatin1String("https"); + setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl); + httpRequest.setSsl(ssl); + + +#ifndef QT_NO_NETWORKPROXY + QNetworkProxy transparentProxy, cacheProxy; + + foreach (const QNetworkProxy &p, proxyList()) { + // use the first proxy that works + // for non-encrypted connections, any transparent or HTTP proxy + // for encrypted, only transparent proxies + if (!ssl + && (p.capabilities() & QNetworkProxy::CachingCapability) + && (p.type() == QNetworkProxy::HttpProxy || + p.type() == QNetworkProxy::HttpCachingProxy)) { + cacheProxy = p; + transparentProxy = QNetworkProxy::NoProxy; + break; + } + if (p.isTransparentProxy()) { + transparentProxy = p; + cacheProxy = QNetworkProxy::NoProxy; + break; + } + } + + // check if at least one of the proxies + if (transparentProxy.type() == QNetworkProxy::DefaultProxy && + cacheProxy.type() == QNetworkProxy::DefaultProxy) { + // unsuitable proxies + QMetaObject::invokeMethod(this, "error", isSynchronous() ? Qt::DirectConnection : Qt::QueuedConnection, + Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProxyNotFoundError), + Q_ARG(QString, tr("No suitable proxy found"))); + QMetaObject::invokeMethod(this, "finished", isSynchronous() ? Qt::DirectConnection : Qt::QueuedConnection); + return; + } +#endif + + + bool loadedFromCache = false; + httpRequest.setPriority(convert(request().priority())); + + switch (operation()) { + case QNetworkAccessManager::GetOperation: + httpRequest.setOperation(QHttpNetworkRequest::Get); + validateCache(httpRequest, loadedFromCache); + break; + + case QNetworkAccessManager::HeadOperation: + httpRequest.setOperation(QHttpNetworkRequest::Head); + validateCache(httpRequest, loadedFromCache); + break; + + case QNetworkAccessManager::PostOperation: + invalidateCache(); + httpRequest.setOperation(QHttpNetworkRequest::Post); + createUploadByteDevice(); + break; + + case QNetworkAccessManager::PutOperation: + invalidateCache(); + httpRequest.setOperation(QHttpNetworkRequest::Put); + createUploadByteDevice(); + break; + + case QNetworkAccessManager::DeleteOperation: + invalidateCache(); + httpRequest.setOperation(QHttpNetworkRequest::Delete); + break; + + case QNetworkAccessManager::CustomOperation: + invalidateCache(); // for safety reasons, we don't know what the operation does + httpRequest.setOperation(QHttpNetworkRequest::Custom); + createUploadByteDevice(); + httpRequest.setCustomVerb(request().attribute( + QNetworkRequest::CustomVerbAttribute).toByteArray()); + break; + + default: + break; // can't happen + } + + QList headers = request().rawHeaderList(); + if (resumeOffset != 0) { + if (headers.contains("Range")) { + // Need to adjust resume offset for user specified range + + headers.removeOne("Range"); + + // We've already verified that requestRange starts with "bytes=", see canResume. + QByteArray requestRange = request().rawHeader("Range").mid(6); + + int index = requestRange.indexOf('-'); + + quint64 requestStartOffset = requestRange.left(index).toULongLong(); + quint64 requestEndOffset = requestRange.mid(index + 1).toULongLong(); + + requestRange = "bytes=" + QByteArray::number(resumeOffset + requestStartOffset) + + '-' + QByteArray::number(requestEndOffset); + + httpRequest.setHeaderField("Range", requestRange); + } else { + httpRequest.setHeaderField("Range", "bytes=" + QByteArray::number(resumeOffset) + '-'); + } + } + + foreach (const QByteArray &header, headers) + httpRequest.setHeaderField(header, request().rawHeader(header)); + + if (loadedFromCache) { + // commented this out since it will be called later anyway + // by copyFinished() + //QNetworkAccessBackend::finished(); + return; // no need to send the request! :) + } + + if (request().attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool() == true) + httpRequest.setPipeliningAllowed(true); + + if (static_cast + (request().attribute(QNetworkRequest::AuthenticationReuseAttribute, + QNetworkRequest::Automatic).toInt()) == QNetworkRequest::Manual) + httpRequest.setWithCredentials(false); + + + // Create the HTTP thread delegate + QHttpThreadDelegate *delegate = new QHttpThreadDelegate; + + // For the synchronous HTTP, this is the normal way the delegate gets deleted + // For the asynchronous HTTP this is a safety measure, the delegate deletes itself when HTTP is finished + connect(thread, SIGNAL(finished()), delegate, SLOT(deleteLater())); + + // Set the properties it needs + delegate->httpRequest = httpRequest; +#ifndef QT_NO_NETWORKPROXY + delegate->cacheProxy = cacheProxy; + delegate->transparentProxy = transparentProxy; +#endif + delegate->ssl = ssl; +#ifndef QT_NO_OPENSSL + if (ssl) + delegate->incomingSslConfiguration = request().sslConfiguration(); +#endif + + // Do we use synchronous HTTP? + delegate->synchronous = isSynchronous(); + + // The authentication manager is used to avoid the BlockingQueuedConnection communication + // from HTTP thread to user thread in some cases. + delegate->authenticationManager = manager->authenticationManager; + + if (!isSynchronous()) { + // Tell our zerocopy policy to the delegate + delegate->downloadBufferMaximumSize = + request().attribute(QNetworkRequest::MaximumDownloadBufferSizeAttribute).toLongLong(); + + // These atomic integers are used for signal compression + delegate->pendingDownloadData = pendingDownloadDataEmissions; + delegate->pendingDownloadProgress = pendingDownloadProgressEmissions; + + // Connect the signals of the delegate to us + connect(delegate, SIGNAL(downloadData(QByteArray)), + this, SLOT(replyDownloadData(QByteArray)), + Qt::QueuedConnection); + connect(delegate, SIGNAL(downloadFinished()), + this, SLOT(replyFinished()), + Qt::QueuedConnection); + connect(delegate, SIGNAL(downloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)), + this, SLOT(replyDownloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)), + Qt::QueuedConnection); + connect(delegate, SIGNAL(downloadProgress(qint64,qint64)), + this, SLOT(replyDownloadProgressSlot(qint64,qint64)), + Qt::QueuedConnection); + connect(delegate, SIGNAL(error(QNetworkReply::NetworkError,QString)), + this, SLOT(httpError(QNetworkReply::NetworkError, const QString)), + Qt::QueuedConnection); +#ifndef QT_NO_OPENSSL + connect(delegate, SIGNAL(sslConfigurationChanged(QSslConfiguration)), + this, SLOT(replySslConfigurationChanged(QSslConfiguration)), + Qt::QueuedConnection); +#endif + // Those need to report back, therefire BlockingQueuedConnection + connect(delegate, SIGNAL(authenticationRequired(QHttpNetworkRequest,QAuthenticator*)), + this, SLOT(httpAuthenticationRequired(QHttpNetworkRequest,QAuthenticator*)), + Qt::BlockingQueuedConnection); +#ifndef QT_NO_NETWORKPROXY + connect (delegate, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), + this, SLOT(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), + Qt::BlockingQueuedConnection); +#endif +#ifndef QT_NO_OPENSSL + connect(delegate, SIGNAL(sslErrors(QList,bool*,QList*)), + this, SLOT(replySslErrors(const QList &, bool *, QList *)), + Qt::BlockingQueuedConnection); +#endif + // This signal we will use to start the request. + connect(this, SIGNAL(startHttpRequest()), delegate, SLOT(startRequest())); + connect(this, SIGNAL(abortHttpRequest()), delegate, SLOT(abortRequest())); + + if (uploadByteDevice) { + QNonContiguousByteDeviceThreadForwardImpl *forwardUploadDevice = + new QNonContiguousByteDeviceThreadForwardImpl(uploadByteDevice->atEnd(), uploadByteDevice->size()); + if (uploadByteDevice->isResetDisabled()) + forwardUploadDevice->disableReset(); + forwardUploadDevice->setParent(delegate); // needed to make sure it is moved on moveToThread() + delegate->httpRequest.setUploadByteDevice(forwardUploadDevice); + + // From main thread to user thread: + QObject::connect(this, SIGNAL(haveUploadData(QByteArray, bool, qint64)), + forwardUploadDevice, SLOT(haveDataSlot(QByteArray, bool, qint64)), Qt::QueuedConnection); + QObject::connect(uploadByteDevice.data(), SIGNAL(readyRead()), + forwardUploadDevice, SIGNAL(readyRead()), + Qt::QueuedConnection); + + // From http thread to user thread: + QObject::connect(forwardUploadDevice, SIGNAL(wantData(qint64)), + this, SLOT(wantUploadDataSlot(qint64))); + QObject::connect(forwardUploadDevice, SIGNAL(processedData(qint64)), + this, SLOT(sentUploadDataSlot(qint64))); + connect(forwardUploadDevice, SIGNAL(resetData(bool*)), + this, SLOT(resetUploadDataSlot(bool*)), + Qt::BlockingQueuedConnection); // this is the only one with BlockingQueued! + } + } else if (isSynchronous()) { + connect(this, SIGNAL(startHttpRequestSynchronously()), delegate, SLOT(startRequestSynchronously()), Qt::BlockingQueuedConnection); + + if (uploadByteDevice) { + // For the synchronous HTTP use case the use thread (this one here) is blocked + // so we cannot use the asynchronous upload architecture. + // We therefore won't use the QNonContiguousByteDeviceThreadForwardImpl but directly + // use the uploadByteDevice provided to us by the QNetworkReplyImpl. + // The code that is in QNetworkReplyImplPrivate::setup() makes sure it is safe to use from a thread + // since it only wraps a QRingBuffer + delegate->httpRequest.setUploadByteDevice(uploadByteDevice.data()); + } + } + + + // Move the delegate to the http thread + delegate->moveToThread(thread); + // This call automatically moves the uploadDevice too for the asynchronous case. + + // Send an signal to the delegate so it starts working in the other thread + if (isSynchronous()) { + emit startHttpRequestSynchronously(); // This one is BlockingQueuedConnection, so it will return when all work is done + + if (delegate->incomingErrorCode != QNetworkReply::NoError) { + replyDownloadMetaData + (delegate->incomingHeaders, + delegate->incomingStatusCode, + delegate->incomingReasonPhrase, + delegate->isPipeliningUsed, + QSharedPointer(), + delegate->incomingContentLength); + httpError(delegate->incomingErrorCode, delegate->incomingErrorDetail); + } else { + replyDownloadMetaData + (delegate->incomingHeaders, + delegate->incomingStatusCode, + delegate->incomingReasonPhrase, + delegate->isPipeliningUsed, + QSharedPointer(), + delegate->incomingContentLength); + replyDownloadData(delegate->synchronousDownloadData); + } + + // End the thread. It will delete itself from the finished() signal + thread->quit(); + + finished(); + } else { + emit startHttpRequest(); // Signal to the HTTP thread and go back to user. + } +} + +void QNetworkAccessHttpBackend::invalidateCache() +{ + QAbstractNetworkCache *nc = networkCache(); + if (nc) + nc->remove(url()); +} + +void QNetworkAccessHttpBackend::open() +{ + postRequest(); +} + +void QNetworkAccessHttpBackend::closeDownstreamChannel() +{ + // FIXME Maybe we can get rid of this whole architecture part +} + +void QNetworkAccessHttpBackend::downstreamReadyWrite() +{ + // FIXME Maybe we can get rid of this whole architecture part +} + +void QNetworkAccessHttpBackend::setDownstreamLimited(bool b) +{ + Q_UNUSED(b); + // We know that readBuffer maximum size limiting is broken since quite a while. + // The task to fix this is QTBUG-15065 +} + +void QNetworkAccessHttpBackend::replyDownloadData(QByteArray d) +{ + int pendingSignals = (int)pendingDownloadDataEmissions->fetchAndAddAcquire(-1) - 1; + + if (pendingSignals > 0) { + // Some more signal emissions to this slot are pending. + // Instead of writing the downstream data, we wait + // and do it in the next call we get + // (signal comppression) + pendingDownloadData.append(d); + return; + } + + pendingDownloadData.append(d); + d.clear(); + // We need to usa a copy for calling writeDownstreamData as we could + // possibly recurse into this this function when we call + // appendDownstreamDataSignalEmissions because the user might call + // processEvents() or spin an event loop when this occur. + QByteDataBuffer pendingDownloadDataCopy = pendingDownloadData; + pendingDownloadData.clear(); + writeDownstreamData(pendingDownloadDataCopy); +} + +void QNetworkAccessHttpBackend::replyFinished() +{ + // We are already loading from cache, we still however + // got this signal because it was posted already + if (loadingFromCache) + return; + + finished(); +} + +void QNetworkAccessHttpBackend::checkForRedirect(const int statusCode) +{ + switch (statusCode) { + case 301: // Moved Permanently + case 302: // Found + case 303: // See Other + case 307: // Temporary Redirect + // What do we do about the caching of the HTML note? + // The response to a 303 MUST NOT be cached, while the response to + // all of the others is cacheable if the headers indicate it to be + QByteArray header = rawHeader("location"); + QUrl url = QUrl::fromEncoded(header); + if (!url.isValid()) + url = QUrl(QLatin1String(header)); + redirectionRequested(url); + } +} + +void QNetworkAccessHttpBackend::replyDownloadMetaData + (QList > hm, + int sc,QString rp,bool pu, + QSharedPointer db, + qint64 contentLength) +{ + statusCode = sc; + reasonPhrase = rp; + + // Download buffer + if (!db.isNull()) { + reply->setDownloadBuffer(db, contentLength); + usingZerocopyDownloadBuffer = true; + } else { + usingZerocopyDownloadBuffer = false; + } + + setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, pu); + + // reconstruct the HTTP header + QList > headerMap = hm; + QList >::ConstIterator it = headerMap.constBegin(), + end = headerMap.constEnd(); + QByteArray header; + + for (; it != end; ++it) { + QByteArray value = rawHeader(it->first); + if (!value.isEmpty()) { + if (qstricmp(it->first.constData(), "set-cookie") == 0) + value += '\n'; + else + value += ", "; + } + value += it->second; + setRawHeader(it->first, value); + } + + setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode); + setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); + + // is it a redirection? + checkForRedirect(statusCode); + + if (statusCode >= 500 && statusCode < 600) { + QAbstractNetworkCache *nc = networkCache(); + if (nc) { + QNetworkCacheMetaData metaData = nc->metaData(url()); + QNetworkHeadersPrivate cacheHeaders; + cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); + QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; + it = cacheHeaders.findRawHeader("Cache-Control"); + bool mustReValidate = false; + if (it != cacheHeaders.rawHeaders.constEnd()) { + QHash cacheControl = parseHttpOptionHeader(it->second); + if (cacheControl.contains("must-revalidate")) + mustReValidate = true; + } + if (!mustReValidate && sendCacheContents(metaData)) + return; + } + } + + if (statusCode == 304) { +#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) + qDebug() << "Received a 304 from" << url(); +#endif + QAbstractNetworkCache *nc = networkCache(); + if (nc) { + QNetworkCacheMetaData oldMetaData = nc->metaData(url()); + QNetworkCacheMetaData metaData = fetchCacheMetaData(oldMetaData); + if (oldMetaData != metaData) + nc->updateMetaData(metaData); + if (sendCacheContents(metaData)) + return; + } + } + + + if (statusCode != 304 && statusCode != 303) { + if (!isCachingEnabled()) + setCachingEnabled(true); + } + + metaDataChanged(); +} + +void QNetworkAccessHttpBackend::replyDownloadProgressSlot(qint64 received, qint64 total) +{ + // we can be sure here that there is a download buffer + + int pendingSignals = (int)pendingDownloadProgressEmissions->fetchAndAddAcquire(-1) - 1; + if (pendingSignals > 0) { + // Let's ignore this signal and look at the next one coming in + // (signal comppression) + return; + } + + // Now do the actual notification of new bytes + writeDownstreamDataDownloadBuffer(received, total); +} + +void QNetworkAccessHttpBackend::httpAuthenticationRequired(const QHttpNetworkRequest &, + QAuthenticator *auth) +{ + authenticationRequired(auth); +} + +void QNetworkAccessHttpBackend::httpError(QNetworkReply::NetworkError errorCode, + const QString &errorString) +{ +#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) + qDebug() << "http error!" << errorCode << errorString; +#endif + + error(errorCode, errorString); +} + +#ifndef QT_NO_OPENSSL +void QNetworkAccessHttpBackend::replySslErrors( + const QList &list, bool *ignoreAll, QList *toBeIgnored) +{ + // Go to generic backend + sslErrors(list); + // Check if the callback set any ignore and return this here to http thread + if (pendingIgnoreAllSslErrors) + *ignoreAll = true; + if (!pendingIgnoreSslErrorsList.isEmpty()) + *toBeIgnored = pendingIgnoreSslErrorsList; +} + +void QNetworkAccessHttpBackend::replySslConfigurationChanged(const QSslConfiguration &c) +{ + // Receiving the used SSL configuration from the HTTP thread + if (pendingSslConfiguration) + *pendingSslConfiguration = c; + else if (!c.isNull()) + pendingSslConfiguration = new QSslConfiguration(c); +} +#endif + +// Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread +void QNetworkAccessHttpBackend::resetUploadDataSlot(bool *r) +{ + *r = uploadByteDevice->reset(); +} + +// Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread +void QNetworkAccessHttpBackend::sentUploadDataSlot(qint64 amount) +{ + uploadByteDevice->advanceReadPointer(amount); +} + +// Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread +void QNetworkAccessHttpBackend::wantUploadDataSlot(qint64 maxSize) +{ + // call readPointer + qint64 currentUploadDataLength = 0; + char *data = const_cast(uploadByteDevice->readPointer(maxSize, currentUploadDataLength)); + // Let's make a copy of this data + QByteArray dataArray(data, currentUploadDataLength); + + // Communicate back to HTTP thread + emit haveUploadData(dataArray, uploadByteDevice->atEnd(), uploadByteDevice->size()); +} + +/* + A simple web page that can be used to test us: http://www.procata.com/cachetest/ + */ +bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &metaData) +{ + setCachingEnabled(false); + if (!metaData.isValid()) + return false; + + QAbstractNetworkCache *nc = networkCache(); + Q_ASSERT(nc); + QIODevice *contents = nc->data(url()); + if (!contents) { +#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) + qDebug() << "Can not send cache, the contents are 0" << url(); +#endif + return false; + } + contents->setParent(this); + + QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes(); + int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (status < 100) + status = 200; // fake it + + setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status); + setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); + setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true); + + QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders(); + QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(), + end = rawHeaders.constEnd(); + for ( ; it != end; ++it) + setRawHeader(it->first, it->second); + + checkForRedirect(status); + + // This needs to be emitted in the event loop because it can be reached at + // the direct code path of qnam.get(...) before the user has a chance + // to connect any signals. + QMetaObject::invokeMethod(this, "metaDataChanged", Qt::QueuedConnection); + qRegisterMetaType("QIODevice*"); + QMetaObject::invokeMethod(this, "writeDownstreamData", Qt::QueuedConnection, Q_ARG(QIODevice*, contents)); + + +#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) + qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes"; +#endif + + // Set the following flag so we can ignore some signals from HTTP thread + // that would still come + loadingFromCache = true; + return true; +} + +void QNetworkAccessHttpBackend::copyFinished(QIODevice *dev) +{ + delete dev; + finished(); +} + +#ifndef QT_NO_OPENSSL +void QNetworkAccessHttpBackend::ignoreSslErrors() +{ + pendingIgnoreAllSslErrors = true; +} + +void QNetworkAccessHttpBackend::ignoreSslErrors(const QList &errors) +{ + // the pending list is set if QNetworkReply::ignoreSslErrors(const QList &errors) + // is called before QNetworkAccessManager::get() (or post(), etc.) + pendingIgnoreSslErrorsList = errors; +} + +void QNetworkAccessHttpBackend::fetchSslConfiguration(QSslConfiguration &config) const +{ + if (pendingSslConfiguration) + config = *pendingSslConfiguration; + else + config = request().sslConfiguration(); +} + +void QNetworkAccessHttpBackend::setSslConfiguration(const QSslConfiguration &newconfig) +{ + // Setting a SSL configuration on a reply is not supported. The user needs to set + // her/his QSslConfiguration on the QNetworkRequest. + Q_UNUSED(newconfig); +} +#endif + +QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const +{ + QNetworkCacheMetaData metaData = oldMetaData; + + QNetworkHeadersPrivate cacheHeaders; + cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); + QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; + + QList newHeaders = rawHeaderList(); + foreach (QByteArray header, newHeaders) { + QByteArray originalHeader = header; + header = header.toLower(); + bool hop_by_hop = + (header == "connection" + || header == "keep-alive" + || header == "proxy-authenticate" + || header == "proxy-authorization" + || header == "te" + || header == "trailers" + || header == "transfer-encoding" + || header == "upgrade"); + if (hop_by_hop) + continue; + + // for 4.6.0, we were planning to not store the date header in the + // cached resource; through that we planned to reduce the number + // of writes to disk when using a QNetworkDiskCache (i.e. don't + // write to disk when only the date changes). + // However, without the date we cannot calculate the age of the page + // anymore. + //if (header == "date") + //continue; + + // Don't store Warning 1xx headers + if (header == "warning") { + QByteArray v = rawHeader(header); + if (v.length() == 3 + && v[0] == '1' + && v[1] >= '0' && v[1] <= '9' + && v[2] >= '0' && v[2] <= '9') + continue; + } + + it = cacheHeaders.findRawHeader(header); + if (it != cacheHeaders.rawHeaders.constEnd()) { + // Match the behavior of Firefox and assume Cache-Control: "no-transform" + if (header == "content-encoding" + || header == "content-range" + || header == "content-type") + continue; + + // For MS servers that send "Content-Length: 0" on 304 responses + // ignore this too + if (header == "content-length") + continue; + } + +#if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) + QByteArray n = rawHeader(header); + QByteArray o; + if (it != cacheHeaders.rawHeaders.constEnd()) + o = (*it).second; + if (n != o && header != "date") { + qDebug() << "replacing" << header; + qDebug() << "new" << n; + qDebug() << "old" << o; + } +#endif + cacheHeaders.setRawHeader(originalHeader, rawHeader(header)); + } + metaData.setRawHeaders(cacheHeaders.rawHeaders); + + bool checkExpired = true; + + QHash cacheControl; + it = cacheHeaders.findRawHeader("Cache-Control"); + if (it != cacheHeaders.rawHeaders.constEnd()) { + cacheControl = parseHttpOptionHeader(it->second); + QByteArray maxAge = cacheControl.value("max-age"); + if (!maxAge.isEmpty()) { + checkExpired = false; + QDateTime dt = QDateTime::currentDateTime(); + dt = dt.addSecs(maxAge.toInt()); + metaData.setExpirationDate(dt); + } + } + if (checkExpired) { + it = cacheHeaders.findRawHeader("expires"); + if (it != cacheHeaders.rawHeaders.constEnd()) { + QDateTime expiredDateTime = QNetworkHeadersPrivate::fromHttpDate(it->second); + metaData.setExpirationDate(expiredDateTime); + } + } + + it = cacheHeaders.findRawHeader("last-modified"); + if (it != cacheHeaders.rawHeaders.constEnd()) + metaData.setLastModified(QNetworkHeadersPrivate::fromHttpDate(it->second)); + + bool canDiskCache; + // only cache GET replies by default, all other replies (POST, PUT, DELETE) + // are not cacheable by default (according to RFC 2616 section 9) + if (httpRequest.operation() == QHttpNetworkRequest::Get) { + + canDiskCache = true; + // 14.32 + // HTTP/1.1 caches SHOULD treat "Pragma: no-cache" as if the client + // had sent "Cache-Control: no-cache". + it = cacheHeaders.findRawHeader("pragma"); + if (it != cacheHeaders.rawHeaders.constEnd() + && it->second == "no-cache") + canDiskCache = false; + + // HTTP/1.1. Check the Cache-Control header + if (cacheControl.contains("no-cache")) + canDiskCache = false; + else if (cacheControl.contains("no-store")) + canDiskCache = false; + + // responses to POST might be cacheable + } else if (httpRequest.operation() == QHttpNetworkRequest::Post) { + + canDiskCache = false; + // some pages contain "expires:" and "cache-control: no-cache" field, + // so we only might cache POST requests if we get "cache-control: max-age ..." + if (cacheControl.contains("max-age")) + canDiskCache = true; + + // responses to PUT and DELETE are not cacheable + } else { + canDiskCache = false; + } + + metaData.setSaveToDisk(canDiskCache); + QNetworkCacheMetaData::AttributesMap attributes; + if (statusCode != 304) { + // update the status code + attributes.insert(QNetworkRequest::HttpStatusCodeAttribute, statusCode); + attributes.insert(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); + } else { + // this is a redirection, keep the attributes intact + attributes = oldMetaData.attributes(); + } + metaData.setAttributes(attributes); + return metaData; +} + +bool QNetworkAccessHttpBackend::canResume() const +{ + // Only GET operation supports resuming. + if (operation() != QNetworkAccessManager::GetOperation) + return false; + + // Can only resume if server/resource supports Range header. + QByteArray acceptRangesheaderName("Accept-Ranges"); + if (!hasRawHeader(acceptRangesheaderName) || rawHeader(acceptRangesheaderName) == "none") + return false; + + // We only support resuming for byte ranges. + if (request().hasRawHeader("Range")) { + QByteArray range = request().rawHeader("Range"); + if (!range.startsWith("bytes=")) + return false; + } + + // If we're using a download buffer then we don't support resuming/migration + // right now. Too much trouble. + if (usingZerocopyDownloadBuffer) + return false; + + return true; +} + +void QNetworkAccessHttpBackend::setResumeOffset(quint64 offset) +{ + resumeOffset = offset; +} + +QT_END_NAMESPACE + +#endif // QT_NO_HTTP diff --git a/src/network/access/qnetworkreplyhttpimpl_p.h b/src/network/access/qnetworkreplyhttpimpl_p.h new file mode 100644 index 00000000000..712dd2ff642 --- /dev/null +++ b/src/network/access/qnetworkreplyhttpimpl_p.h @@ -0,0 +1,169 @@ +/**************************************************************************** +** +** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the QtNetwork module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QNETWORKACCESSHTTPBACKEND_P_H +#define QNETWORKACCESSHTTPBACKEND_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of the Network Access API. This header file may change from +// version to version without notice, or even be removed. +// +// We mean it. +// + +#include "qhttpnetworkconnection_p.h" +#include "qnetworkaccessbackend_p.h" +#include "qnetworkrequest.h" +#include "qnetworkreply.h" +#include "qabstractsocket.h" + +#include "QtCore/qpointer.h" +#include "QtCore/qdatetime.h" +#include "QtCore/qsharedpointer.h" +#include "qatomic.h" + +#ifndef QT_NO_HTTP + +QT_BEGIN_NAMESPACE + +class QNetworkAccessCachedHttpConnection; + +class QNetworkAccessHttpBackendIODevice; + +class QNetworkAccessHttpBackend: public QNetworkAccessBackend +{ + Q_OBJECT +public: + QNetworkAccessHttpBackend(); + virtual ~QNetworkAccessHttpBackend(); + + virtual void open(); + virtual void closeDownstreamChannel(); + + virtual void downstreamReadyWrite(); + virtual void setDownstreamLimited(bool b); + + virtual void copyFinished(QIODevice *); +#ifndef QT_NO_OPENSSL + virtual void ignoreSslErrors(); + virtual void ignoreSslErrors(const QList &errors); + + virtual void fetchSslConfiguration(QSslConfiguration &configuration) const; + virtual void setSslConfiguration(const QSslConfiguration &configuration); +#endif + QNetworkCacheMetaData fetchCacheMetaData(const QNetworkCacheMetaData &metaData) const; + + // we return true since HTTP needs to send PUT/POST data again after having authenticated + bool needsResetableUploadData() { return true; } + + bool canResume() const; + void setResumeOffset(quint64 offset); + +signals: + // To HTTP thread: + void startHttpRequest(); + void abortHttpRequest(); + + void startHttpRequestSynchronously(); + + void haveUploadData(QByteArray dataArray, bool dataAtEnd, qint64 dataSize); +private slots: + // From HTTP thread: + void replyDownloadData(QByteArray); + void replyFinished(); + void replyDownloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64); + void replyDownloadProgressSlot(qint64,qint64); + void httpAuthenticationRequired(const QHttpNetworkRequest &request, QAuthenticator *auth); + void httpError(QNetworkReply::NetworkError error, const QString &errorString); +#ifndef QT_NO_OPENSSL + void replySslErrors(const QList &, bool *, QList *); + void replySslConfigurationChanged(const QSslConfiguration&); +#endif + + // From QNonContiguousByteDeviceThreadForwardImpl in HTTP thread: + void resetUploadDataSlot(bool *r); + void wantUploadDataSlot(qint64); + void sentUploadDataSlot(qint64); + + bool sendCacheContents(const QNetworkCacheMetaData &metaData); + +private: + QHttpNetworkRequest httpRequest; // There is also a copy in the HTTP thread + int statusCode; + QString reasonPhrase; + // Will be increased by HTTP thread: + QSharedPointer pendingDownloadDataEmissions; + QSharedPointer pendingDownloadProgressEmissions; + bool loadingFromCache; + QByteDataBuffer pendingDownloadData; + bool usingZerocopyDownloadBuffer; + +#ifndef QT_NO_OPENSSL + QSslConfiguration *pendingSslConfiguration; + bool pendingIgnoreAllSslErrors; + QList pendingIgnoreSslErrorsList; +#endif + + quint64 resumeOffset; + + void validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache); + void invalidateCache(); + void postRequest(); + void readFromHttp(); + void checkForRedirect(const int statusCode); +}; + +class QNetworkAccessHttpBackendFactory : public QNetworkAccessBackendFactory +{ +public: + virtual QNetworkAccessBackend *create(QNetworkAccessManager::Operation op, + const QNetworkRequest &request) const; +}; + +QT_END_NAMESPACE + +#endif // QT_NO_HTTP + +#endif From a0beeac097db8ead6f2a56e21699f6007b9686d4 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Mon, 28 Mar 2011 14:56:26 +0200 Subject: [PATCH 04/26] QNAM: Start new HTTP backend architecture --- src/network/access/access.pri | 4 +- src/network/access/qnetworkaccessmanager.cpp | 32 +- src/network/access/qnetworkaccessmanager.h | 1 + src/network/access/qnetworkreplyhttpimpl.cpp | 1193 ++++++++++++++---- src/network/access/qnetworkreplyhttpimpl_p.h | 253 +++- src/network/access/qnetworkreplyimpl.cpp | 9 +- 6 files changed, 1207 insertions(+), 285 deletions(-) diff --git a/src/network/access/access.pri b/src/network/access/access.pri index 5ead3ad37f4..0f901b873d9 100644 --- a/src/network/access/access.pri +++ b/src/network/access/access.pri @@ -14,7 +14,6 @@ HEADERS += \ access/qnetworkaccesscache_p.h \ access/qnetworkaccessbackend_p.h \ access/qnetworkaccessdebugpipebackend_p.h \ - access/qnetworkaccesshttpbackend_p.h \ access/qnetworkaccessfilebackend_p.h \ access/qnetworkaccesscachebackend_p.h \ access/qnetworkaccessftpbackend_p.h \ @@ -29,6 +28,7 @@ HEADERS += \ access/qnetworkreply_p.h \ access/qnetworkreplyimpl_p.h \ access/qnetworkreplydataimpl_p.h \ + access/qnetworkreplyhttpimpl_p.h \ access/qnetworkreplyfileimpl_p.h \ access/qabstractnetworkcache_p.h \ access/qabstractnetworkcache.h \ @@ -54,13 +54,13 @@ SOURCES += \ access/qnetworkaccessfilebackend.cpp \ access/qnetworkaccesscachebackend.cpp \ access/qnetworkaccessftpbackend.cpp \ - access/qnetworkaccesshttpbackend.cpp \ access/qnetworkcookie.cpp \ access/qnetworkcookiejar.cpp \ access/qnetworkrequest.cpp \ access/qnetworkreply.cpp \ access/qnetworkreplyimpl.cpp \ access/qnetworkreplydataimpl.cpp \ + access/qnetworkreplyhttpimpl.cpp \ access/qnetworkreplyfileimpl.cpp \ access/qabstractnetworkcache.cpp \ access/qnetworkdiskcache.cpp \ diff --git a/src/network/access/qnetworkaccessmanager.cpp b/src/network/access/qnetworkaccessmanager.cpp index 34ac17037be..090a25c7f0f 100644 --- a/src/network/access/qnetworkaccessmanager.cpp +++ b/src/network/access/qnetworkaccessmanager.cpp @@ -67,13 +67,12 @@ #include "QtNetwork/qhttpmultipart.h" #include "qhttpmultipart_p.h" +#include "qnetworkreplyhttpimpl_p.h" + #include "qthread.h" QT_BEGIN_NAMESPACE -#ifndef QT_NO_HTTP -Q_GLOBAL_STATIC(QNetworkAccessHttpBackendFactory, httpBackend) -#endif // QT_NO_HTTP Q_GLOBAL_STATIC(QNetworkAccessFileBackendFactory, fileBackend) #ifndef QT_NO_FTP Q_GLOBAL_STATIC(QNetworkAccessFtpBackendFactory, ftpBackend) @@ -85,10 +84,6 @@ Q_GLOBAL_STATIC(QNetworkAccessDebugPipeBackendFactory, debugpipeBackend) static void ensureInitialized() { -#ifndef QT_NO_HTTP - (void) httpBackend(); -#endif // QT_NO_HTTP - #ifndef QT_NO_FTP (void) ftpBackend(); #endif @@ -356,6 +351,17 @@ QNetworkAccessManager::QNetworkAccessManager(QObject *parent) ensureInitialized(); qRegisterMetaType("QNetworkReply::NetworkError"); +#ifndef QT_NO_NETWORKPROXY + qRegisterMetaType("QNetworkProxy"); +#endif +#ifndef QT_NO_OPENSSL + qRegisterMetaType >("QList"); + qRegisterMetaType("QSslConfiguration"); +#endif + qRegisterMetaType > >("QList >"); + qRegisterMetaType("QHttpNetworkRequest"); + qRegisterMetaType("QNetworkReply::NetworkError"); + qRegisterMetaType >("QSharedPointer"); } /*! @@ -967,6 +973,18 @@ QNetworkReply *QNetworkAccessManager::createRequest(QNetworkAccessManager::Opera } } +#ifndef QT_NO_HTTP + // Since Qt 5 we use the new QNetworkReplyHttpImpl + if (scheme == QLatin1String("http") || scheme == QLatin1String("https") ) { + QNetworkReplyHttpImpl *reply = new QNetworkReplyHttpImpl(this, request, op, outgoingData); +#ifndef QT_NO_BEARERMANAGEMENT + connect(this, SIGNAL(networkSessionConnected()), + reply, SLOT(_q_networkSessionConnected())); +#endif + return reply; + } +#endif // QT_NO_HTTP + // first step: create the reply QUrl url = request.url(); QNetworkReplyImpl *reply = new QNetworkReplyImpl(this); diff --git a/src/network/access/qnetworkaccessmanager.h b/src/network/access/qnetworkaccessmanager.h index 47760b210ba..b22be91a965 100644 --- a/src/network/access/qnetworkaccessmanager.h +++ b/src/network/access/qnetworkaccessmanager.h @@ -160,6 +160,7 @@ protected: private: friend class QNetworkReplyImplPrivate; friend class QNetworkAccessHttpBackend; + friend class QNetworkReplyHttpImpl; Q_DECLARE_PRIVATE(QNetworkAccessManager) Q_PRIVATE_SLOT(d_func(), void _q_replyFinished()) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index e75347f78a6..d38850b450b 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -41,7 +41,7 @@ //#define QNETWORKACCESSHTTPBACKEND_DEBUG -#include "qnetworkaccesshttpbackend_p.h" +#include "qnetworkreplyhttpimpl_p.h" #include "qnetworkaccessmanager_p.h" #include "qnetworkaccesscache_p.h" #include "qabstractnetworkcache.h" @@ -55,6 +55,8 @@ #include "qhttpthreaddelegate_p.h" #include "qthread.h" +#include "qnetworkcookiejar.h" + #ifndef QT_NO_HTTP #include // for strchr @@ -162,55 +164,231 @@ static QHash parseHttpOptionHeader(const QByteArray &hea } } -QNetworkAccessBackend * -QNetworkAccessHttpBackendFactory::create(QNetworkAccessManager::Operation op, - const QNetworkRequest &request) const +QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manager, + const QNetworkRequest& request, + QNetworkAccessManager::Operation& operation, + QIODevice* outgoingData) + : QNetworkReply(*new QNetworkReplyHttpImplPrivate, manager) { - // check the operation - switch (op) { - case QNetworkAccessManager::GetOperation: - case QNetworkAccessManager::PostOperation: - case QNetworkAccessManager::HeadOperation: - case QNetworkAccessManager::PutOperation: - case QNetworkAccessManager::DeleteOperation: - case QNetworkAccessManager::CustomOperation: - break; + Q_D(QNetworkReplyHttpImpl); + d->manager = manager; + d->managerPrivate = manager->d_func(); + d->request = request; + d->operation = operation; + d->outgoingData = outgoingData; + d->url = request.url(); +#ifndef QT_NO_OPENSSL + d->sslConfiguration = request.sslConfiguration(); +#endif - default: - // no, we can't handle this request - return 0; + // FIXME Later maybe set to Unbuffered, especially if it is zerocopy or from cache? + QIODevice::open(QIODevice::ReadOnly); + + + // Internal code that does a HTTP reply for the synchronous Ajax + // in QtWebKit. + QVariant synchronousHttpAttribute = request.attribute( + static_cast(QNetworkRequest::SynchronousRequestAttribute)); + if (synchronousHttpAttribute.isValid()) { + d->synchronous = synchronousHttpAttribute.toBool(); + if (d->synchronous && outgoingData) { + // The synchronous HTTP is a corner case, we will put all upload data in one big QByteArray in the outgoingDataBuffer. + // Yes, this is not the most efficient thing to do, but on the other hand synchronous XHR needs to die anyway. + d->outgoingDataBuffer = QSharedPointer(new QRingBuffer()); + qint64 previousDataSize = 0; + do { + previousDataSize = d->outgoingDataBuffer->size(); + d->outgoingDataBuffer->append(d->outgoingData->readAll()); + } while (d->outgoingDataBuffer->size() != previousDataSize); + d->_q_startOperation(); + return; + } } - QUrl url = request.url(); - QString scheme = url.scheme().toLower(); - if (scheme == QLatin1String("http") || scheme == QLatin1String("https")) - return new QNetworkAccessHttpBackend; - return 0; + if (outgoingData) { + // there is data to be uploaded, e.g. HTTP POST. + + if (!d->outgoingData->isSequential()) { + // fixed size non-sequential (random-access) + // just start the operation + QMetaObject::invokeMethod(this, "_q_startOperation", Qt::QueuedConnection); + // FIXME make direct call? + } else { + bool bufferingDisallowed = + request.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute, + false).toBool(); + + if (bufferingDisallowed) { + // if a valid content-length header for the request was supplied, we can disable buffering + // if not, we will buffer anyway + if (request.header(QNetworkRequest::ContentLengthHeader).isValid()) { + QMetaObject::invokeMethod(this, "_q_startOperation", Qt::QueuedConnection); + // FIXME make direct call? + } else { + d->state = d->Buffering; + QMetaObject::invokeMethod(this, "_q_bufferOutgoingData", Qt::QueuedConnection); + } + } else { + // _q_startOperation will be called when the buffering has finished. + d->state = d->Buffering; + QMetaObject::invokeMethod(this, "_q_bufferOutgoingData", Qt::QueuedConnection); + } + } + } else { + // No outgoing data (POST, ..) + d->_q_startOperation(); + } } -QNetworkAccessHttpBackend::QNetworkAccessHttpBackend() - : QNetworkAccessBackend() +QNetworkReplyHttpImpl::~QNetworkReplyHttpImpl() +{ +} + +void QNetworkReplyHttpImpl::close() +{ + Q_D(QNetworkReplyHttpImpl); + QNetworkReply::close(); + // FIXME +} + +void QNetworkReplyHttpImpl::abort() +{ + Q_D(QNetworkReplyHttpImpl); + QNetworkReply::close(); + // FIXME +} + +qint64 QNetworkReplyHttpImpl::bytesAvailable() const +{ + Q_D(const QNetworkReplyHttpImpl); + qDebug() << "QNetworkReplyHttpImpl::bytesAvailable()"; + + // FIXME cache device + if (d->cacheLoadDevice) { + return QNetworkReply::bytesAvailable() + d->cacheLoadDevice->bytesAvailable() + d->downloadMultiBuffer.byteAmount(); + } + + // FIXME 0-copy buffer + if (d->downloadZerocopyBuffer) { + return QNetworkReply::bytesAvailable() + d->downloadBufferCurrentSize - d->downloadBufferReadPosition; + } + + // FIXME normal buffer + qDebug() << "QNetworkReplyHttpImpl::bytesAvailable() ==" << QNetworkReply::bytesAvailable() + d->downloadMultiBuffer.byteAmount(); + return QNetworkReply::bytesAvailable() + d->downloadMultiBuffer.byteAmount(); + + // FIXME +} + +bool QNetworkReplyHttpImpl::isSequential () const +{ + // FIXME Maybe not for cache or 0-copy buffer + return true; +} + +qint64 QNetworkReplyHttpImpl::size() const +{ + Q_D(const QNetworkReplyHttpImpl); + //return -1; + return QNetworkReply::size(); +} + +qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen) +{ + Q_D(QNetworkReplyHttpImpl); + qDebug() << "QNetworkReplyHttpImpl::readData()" << maxlen; + + // FIXME cacheload device + if (d->cacheLoadDevice) { + // FIXME bytesdownloaded, position etc? + + // There is something already in the buffer we buffered before because the user did not read() + // anything, so we read there first: + if (!d->downloadMultiBuffer.isEmpty()) { + return d->downloadMultiBuffer.read(data, maxlen); + } + + qint64 ret = d->cacheLoadDevice->read(data, maxlen); + return ret; + } + + // FIXME 0-copy buffer + if (d->downloadZerocopyBuffer) { + // bla + qint64 howMuch = qMin(maxlen, (d->downloadBufferCurrentSize - d->downloadBufferReadPosition)); + memcpy(data, d->downloadZerocopyBuffer + d->downloadBufferReadPosition, howMuch); + d->downloadBufferReadPosition += howMuch; + return howMuch; + + } + + // FIXME normal buffer + if (d->downloadMultiBuffer.isEmpty()) + return d->state == d->Finished ? -1 : 0; + // FIXME what about "Aborted" state? + + //d->backendNotify(QNetworkReplyImplPrivate::NotifyDownstreamReadyWrite); + if (maxlen == 1) { + // optimization for getChar() + *data = d->downloadMultiBuffer.getChar(); + return 1; + } + + maxlen = qMin(maxlen, d->downloadMultiBuffer.byteAmount()); + return d->downloadMultiBuffer.read(data, maxlen); +} + +void QNetworkReplyHttpImpl::setReadBufferSize(qint64 size) +{ + return; // FIXME, unsupported right now +} + +bool QNetworkReplyHttpImpl::canReadLine () const +{ + Q_D(const QNetworkReplyHttpImpl); + + if (QNetworkReply::canReadLine()) + return true; + + if (d->cacheLoadDevice) + return d->cacheLoadDevice->canReadLine() || d->downloadMultiBuffer.canReadLine(); + + return d->downloadMultiBuffer.canReadLine(); +} + +QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate() +// FIXME order etc + : QNetworkReplyPrivate() , statusCode(0) , pendingDownloadDataEmissions(new QAtomicInt()) , pendingDownloadProgressEmissions(new QAtomicInt()) , loadingFromCache(false) - , usingZerocopyDownloadBuffer(false) #ifndef QT_NO_OPENSSL - , pendingSslConfiguration(0), pendingIgnoreAllSslErrors(false) + , pendingIgnoreAllSslErrors(false) #endif , resumeOffset(0) + , outgoingData(0), + cacheLoadDevice(0), + cacheEnabled(false), cacheSaveDevice(0), + // notificationHandlingPaused(false), + bytesDownloaded(0), lastBytesDownloaded(-1), bytesUploaded(-1), preMigrationDownloaded(-1), + //httpStatusCode(0), + state(Idle) + , downloadBufferReadPosition(0) + , downloadBufferCurrentSize(0) + , downloadBufferMaximumSize(0) + , downloadZerocopyBuffer(0) + , synchronous(false) + { } -QNetworkAccessHttpBackend::~QNetworkAccessHttpBackend() +QNetworkReplyHttpImplPrivate::~QNetworkReplyHttpImplPrivate() { + Q_Q(QNetworkReplyHttpImpl); // This will do nothing if the request was already finished or aborted - emit abortHttpRequest(); - -#ifndef QT_NO_OPENSSL - delete pendingSslConfiguration; -#endif + emit q->abortHttpRequest(); } /* @@ -219,14 +397,14 @@ QNetworkAccessHttpBackend::~QNetworkAccessHttpBackend() 2) If we have a cache entry for this url populate headers so the server can return 304 3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true */ -void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache) +void QNetworkReplyHttpImplPrivate::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache) { QNetworkRequest::CacheLoadControl CacheLoadControlAttribute = - (QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt(); + (QNetworkRequest::CacheLoadControl)request.attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt(); if (CacheLoadControlAttribute == QNetworkRequest::AlwaysNetwork) { // If the request does not already specify preferred cache-control // force reload from the network and tell any caching proxy servers to reload too - if (!request().rawHeaderList().contains("Cache-Control")) { + if (!request.rawHeaderList().contains("Cache-Control")) { httpRequest.setHeaderField("Cache-Control", "no-cache"); httpRequest.setHeaderField("Pragma", "no-cache"); } @@ -235,14 +413,14 @@ void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, // The disk cache API does not currently support partial content retrieval. // That is why we don't use the disk cache for any such requests. - if (request().hasRawHeader("Range")) + if (request.hasRawHeader("Range")) return; - QAbstractNetworkCache *nc = networkCache(); + QAbstractNetworkCache *nc = managerPrivate->networkCache; if (!nc) return; // no local cache - QNetworkCacheMetaData metaData = nc->metaData(url()); + QNetworkCacheMetaData metaData = nc->metaData(request.url()); if (!metaData.isValid()) return; // not in cache @@ -348,7 +526,7 @@ void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, loadedFromCache = false; } -static QHttpNetworkRequest::Priority convert(const QNetworkRequest::Priority& prio) +QHttpNetworkRequest::Priority QNetworkReplyHttpImplPrivate::convert(const QNetworkRequest::Priority& prio) { switch (prio) { case QNetworkRequest::LowPriority: @@ -361,50 +539,44 @@ static QHttpNetworkRequest::Priority convert(const QNetworkRequest::Priority& pr } } -void QNetworkAccessHttpBackend::postRequest() +void QNetworkReplyHttpImplPrivate::postRequest() { + Q_Q(QNetworkReplyHttpImpl); + QThread *thread = 0; - if (isSynchronous()) { + if (synchronous) { // A synchronous HTTP request uses its own thread + qDebug() << "sync!"; thread = new QThread(); QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); thread->start(); - } else if (!manager->httpThread) { + } else if (!managerPrivate->httpThread) { // We use the manager-global thread. // At some point we could switch to having multiple threads if it makes sense. - manager->httpThread = new QThread(); - QObject::connect(manager->httpThread, SIGNAL(finished()), manager->httpThread, SLOT(deleteLater())); - manager->httpThread->start(); -#ifndef QT_NO_NETWORKPROXY - qRegisterMetaType("QNetworkProxy"); -#endif -#ifndef QT_NO_OPENSSL - qRegisterMetaType >("QList"); - qRegisterMetaType("QSslConfiguration"); -#endif - qRegisterMetaType > >("QList >"); - qRegisterMetaType("QHttpNetworkRequest"); - qRegisterMetaType("QNetworkReply::NetworkError"); - qRegisterMetaType >("QSharedPointer"); + managerPrivate->httpThread = new QThread(); + QObject::connect(managerPrivate->httpThread, SIGNAL(finished()), managerPrivate->httpThread, SLOT(deleteLater())); + managerPrivate->httpThread->start(); - thread = manager->httpThread; + thread = managerPrivate->httpThread; } else { // Asynchronous request, thread already exists - thread = manager->httpThread; + thread = managerPrivate->httpThread; } - QUrl url = request().url(); + QUrl url = request.url(); httpRequest.setUrl(url); bool ssl = url.scheme().toLower() == QLatin1String("https"); - setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl); + q->setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl); httpRequest.setSsl(ssl); #ifndef QT_NO_NETWORKPROXY QNetworkProxy transparentProxy, cacheProxy; - foreach (const QNetworkProxy &p, proxyList()) { + // FIXME the proxy stuff should be done in the HTTP thread + foreach (const QNetworkProxy &p, managerPrivate->queryProxy(QNetworkProxyQuery(request.url()))) { + //foreach (const QNetworkProxy &p, proxyList()) { // use the first proxy that works // for non-encrypted connections, any transparent or HTTP proxy // for encrypted, only transparent proxies @@ -427,19 +599,19 @@ void QNetworkAccessHttpBackend::postRequest() if (transparentProxy.type() == QNetworkProxy::DefaultProxy && cacheProxy.type() == QNetworkProxy::DefaultProxy) { // unsuitable proxies - QMetaObject::invokeMethod(this, "error", isSynchronous() ? Qt::DirectConnection : Qt::QueuedConnection, + QMetaObject::invokeMethod(q, "error", synchronous ? Qt::DirectConnection : Qt::QueuedConnection, Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProxyNotFoundError), - Q_ARG(QString, tr("No suitable proxy found"))); - QMetaObject::invokeMethod(this, "finished", isSynchronous() ? Qt::DirectConnection : Qt::QueuedConnection); + Q_ARG(QString, q->tr("No suitable proxy found"))); + QMetaObject::invokeMethod(q, "finished", synchronous ? Qt::DirectConnection : Qt::QueuedConnection); return; } #endif bool loadedFromCache = false; - httpRequest.setPriority(convert(request().priority())); + httpRequest.setPriority(convert(request.priority())); - switch (operation()) { + switch (operation) { case QNetworkAccessManager::GetOperation: httpRequest.setOperation(QHttpNetworkRequest::Get); validateCache(httpRequest, loadedFromCache); @@ -471,7 +643,7 @@ void QNetworkAccessHttpBackend::postRequest() invalidateCache(); // for safety reasons, we don't know what the operation does httpRequest.setOperation(QHttpNetworkRequest::Custom); createUploadByteDevice(); - httpRequest.setCustomVerb(request().attribute( + httpRequest.setCustomVerb(request.attribute( QNetworkRequest::CustomVerbAttribute).toByteArray()); break; @@ -479,7 +651,7 @@ void QNetworkAccessHttpBackend::postRequest() break; // can't happen } - QList headers = request().rawHeaderList(); + QList headers = request.rawHeaderList(); if (resumeOffset != 0) { if (headers.contains("Range")) { // Need to adjust resume offset for user specified range @@ -487,7 +659,7 @@ void QNetworkAccessHttpBackend::postRequest() headers.removeOne("Range"); // We've already verified that requestRange starts with "bytes=", see canResume. - QByteArray requestRange = request().rawHeader("Range").mid(6); + QByteArray requestRange = request.rawHeader("Range").mid(6); int index = requestRange.indexOf('-'); @@ -504,20 +676,17 @@ void QNetworkAccessHttpBackend::postRequest() } foreach (const QByteArray &header, headers) - httpRequest.setHeaderField(header, request().rawHeader(header)); + httpRequest.setHeaderField(header, request.rawHeader(header)); if (loadedFromCache) { - // commented this out since it will be called later anyway - // by copyFinished() - //QNetworkAccessBackend::finished(); return; // no need to send the request! :) } - if (request().attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool() == true) + if (request.attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool() == true) httpRequest.setPipeliningAllowed(true); if (static_cast - (request().attribute(QNetworkRequest::AuthenticationReuseAttribute, + (request.attribute(QNetworkRequest::AuthenticationReuseAttribute, QNetworkRequest::Automatic).toInt()) == QNetworkRequest::Manual) httpRequest.setWithCredentials(false); @@ -527,7 +696,7 @@ void QNetworkAccessHttpBackend::postRequest() // For the synchronous HTTP, this is the normal way the delegate gets deleted // For the asynchronous HTTP this is a safety measure, the delegate deletes itself when HTTP is finished - connect(thread, SIGNAL(finished()), delegate, SLOT(deleteLater())); + QObject::connect(thread, SIGNAL(finished()), delegate, SLOT(deleteLater())); // Set the properties it needs delegate->httpRequest = httpRequest; @@ -538,63 +707,63 @@ void QNetworkAccessHttpBackend::postRequest() delegate->ssl = ssl; #ifndef QT_NO_OPENSSL if (ssl) - delegate->incomingSslConfiguration = request().sslConfiguration(); + delegate->incomingSslConfiguration = request.sslConfiguration(); #endif // Do we use synchronous HTTP? - delegate->synchronous = isSynchronous(); + delegate->synchronous = synchronous; // The authentication manager is used to avoid the BlockingQueuedConnection communication // from HTTP thread to user thread in some cases. - delegate->authenticationManager = manager->authenticationManager; + delegate->authenticationManager = managerPrivate->authenticationManager; - if (!isSynchronous()) { + if (!synchronous) { // Tell our zerocopy policy to the delegate delegate->downloadBufferMaximumSize = - request().attribute(QNetworkRequest::MaximumDownloadBufferSizeAttribute).toLongLong(); + request.attribute(QNetworkRequest::MaximumDownloadBufferSizeAttribute).toLongLong(); // These atomic integers are used for signal compression delegate->pendingDownloadData = pendingDownloadDataEmissions; delegate->pendingDownloadProgress = pendingDownloadProgressEmissions; // Connect the signals of the delegate to us - connect(delegate, SIGNAL(downloadData(QByteArray)), - this, SLOT(replyDownloadData(QByteArray)), + QObject::connect(delegate, SIGNAL(downloadData(QByteArray)), + q, SLOT(replyDownloadData(QByteArray)), Qt::QueuedConnection); - connect(delegate, SIGNAL(downloadFinished()), - this, SLOT(replyFinished()), + QObject::connect(delegate, SIGNAL(downloadFinished()), + q, SLOT(replyFinished()), Qt::QueuedConnection); - connect(delegate, SIGNAL(downloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)), - this, SLOT(replyDownloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)), + QObject::connect(delegate, SIGNAL(downloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)), + q, SLOT(replyDownloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)), Qt::QueuedConnection); - connect(delegate, SIGNAL(downloadProgress(qint64,qint64)), - this, SLOT(replyDownloadProgressSlot(qint64,qint64)), + QObject::connect(delegate, SIGNAL(downloadProgress(qint64,qint64)), + q, SLOT(replyDownloadProgressSlot(qint64,qint64)), Qt::QueuedConnection); - connect(delegate, SIGNAL(error(QNetworkReply::NetworkError,QString)), - this, SLOT(httpError(QNetworkReply::NetworkError, const QString)), + QObject::connect(delegate, SIGNAL(error(QNetworkReply::NetworkError,QString)), + q, SLOT(httpError(QNetworkReply::NetworkError, const QString)), Qt::QueuedConnection); #ifndef QT_NO_OPENSSL - connect(delegate, SIGNAL(sslConfigurationChanged(QSslConfiguration)), - this, SLOT(replySslConfigurationChanged(QSslConfiguration)), + QObject::connect(delegate, SIGNAL(sslConfigurationChanged(QSslConfiguration)), + q, SLOT(replySslConfigurationChanged(QSslConfiguration)), Qt::QueuedConnection); #endif // Those need to report back, therefire BlockingQueuedConnection - connect(delegate, SIGNAL(authenticationRequired(QHttpNetworkRequest,QAuthenticator*)), - this, SLOT(httpAuthenticationRequired(QHttpNetworkRequest,QAuthenticator*)), + QObject::connect(delegate, SIGNAL(authenticationRequired(QHttpNetworkRequest,QAuthenticator*)), + q, SLOT(httpAuthenticationRequired(QHttpNetworkRequest,QAuthenticator*)), Qt::BlockingQueuedConnection); #ifndef QT_NO_NETWORKPROXY - connect (delegate, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), - this, SLOT(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), + QObject::connect(delegate, SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), + q, SLOT(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)), Qt::BlockingQueuedConnection); #endif #ifndef QT_NO_OPENSSL - connect(delegate, SIGNAL(sslErrors(QList,bool*,QList*)), - this, SLOT(replySslErrors(const QList &, bool *, QList *)), + QObject::connect(delegate, SIGNAL(sslErrors(QList,bool*,QList*)), + q, SLOT(replySslErrors(const QList &, bool *, QList *)), Qt::BlockingQueuedConnection); #endif // This signal we will use to start the request. - connect(this, SIGNAL(startHttpRequest()), delegate, SLOT(startRequest())); - connect(this, SIGNAL(abortHttpRequest()), delegate, SLOT(abortRequest())); + QObject::connect(q, SIGNAL(startHttpRequest()), delegate, SLOT(startRequest())); + QObject::connect(q, SIGNAL(abortHttpRequest()), delegate, SLOT(abortRequest())); if (uploadByteDevice) { QNonContiguousByteDeviceThreadForwardImpl *forwardUploadDevice = @@ -605,7 +774,7 @@ void QNetworkAccessHttpBackend::postRequest() delegate->httpRequest.setUploadByteDevice(forwardUploadDevice); // From main thread to user thread: - QObject::connect(this, SIGNAL(haveUploadData(QByteArray, bool, qint64)), + QObject::connect(q, SIGNAL(haveUploadData(QByteArray, bool, qint64)), forwardUploadDevice, SLOT(haveDataSlot(QByteArray, bool, qint64)), Qt::QueuedConnection); QObject::connect(uploadByteDevice.data(), SIGNAL(readyRead()), forwardUploadDevice, SIGNAL(readyRead()), @@ -613,22 +782,22 @@ void QNetworkAccessHttpBackend::postRequest() // From http thread to user thread: QObject::connect(forwardUploadDevice, SIGNAL(wantData(qint64)), - this, SLOT(wantUploadDataSlot(qint64))); + q, SLOT(wantUploadDataSlot(qint64))); QObject::connect(forwardUploadDevice, SIGNAL(processedData(qint64)), - this, SLOT(sentUploadDataSlot(qint64))); - connect(forwardUploadDevice, SIGNAL(resetData(bool*)), - this, SLOT(resetUploadDataSlot(bool*)), + q, SLOT(sentUploadDataSlot(qint64))); + QObject::connect(forwardUploadDevice, SIGNAL(resetData(bool*)), + q, SLOT(resetUploadDataSlot(bool*)), Qt::BlockingQueuedConnection); // this is the only one with BlockingQueued! } - } else if (isSynchronous()) { - connect(this, SIGNAL(startHttpRequestSynchronously()), delegate, SLOT(startRequestSynchronously()), Qt::BlockingQueuedConnection); + } else if (synchronous) { + QObject::connect(q, SIGNAL(startHttpRequestSynchronously()), delegate, SLOT(startRequestSynchronously()), Qt::BlockingQueuedConnection); if (uploadByteDevice) { // For the synchronous HTTP use case the use thread (this one here) is blocked // so we cannot use the asynchronous upload architecture. // We therefore won't use the QNonContiguousByteDeviceThreadForwardImpl but directly // use the uploadByteDevice provided to us by the QNetworkReplyImpl. - // The code that is in QNetworkReplyImplPrivate::setup() makes sure it is safe to use from a thread + // The code that is in start() makes sure it is safe to use from a thread // since it only wraps a QRingBuffer delegate->httpRequest.setUploadByteDevice(uploadByteDevice.data()); } @@ -640,8 +809,8 @@ void QNetworkAccessHttpBackend::postRequest() // This call automatically moves the uploadDevice too for the asynchronous case. // Send an signal to the delegate so it starts working in the other thread - if (isSynchronous()) { - emit startHttpRequestSynchronously(); // This one is BlockingQueuedConnection, so it will return when all work is done + if (synchronous) { + emit q->startHttpRequestSynchronously(); // This one is BlockingQueuedConnection, so it will return when all work is done if (delegate->incomingErrorCode != QNetworkReply::NoError) { replyDownloadMetaData @@ -668,41 +837,61 @@ void QNetworkAccessHttpBackend::postRequest() finished(); } else { - emit startHttpRequest(); // Signal to the HTTP thread and go back to user. + emit q->startHttpRequest(); // Signal to the HTTP thread and go back to user. } } -void QNetworkAccessHttpBackend::invalidateCache() +void QNetworkReplyHttpImplPrivate::invalidateCache() { - QAbstractNetworkCache *nc = networkCache(); + QAbstractNetworkCache *nc = managerPrivate->networkCache; if (nc) - nc->remove(url()); + nc->remove(request.url()); } -void QNetworkAccessHttpBackend::open() +void QNetworkReplyHttpImplPrivate::initCacheSaveDevice() { - postRequest(); + Q_Q(QNetworkReplyHttpImpl); + + // The disk cache does not support partial content, so don't even try to + // save any such content into the cache. + if (q->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() == 206) { + cacheEnabled = false; + return; + } + + // save the meta data + QNetworkCacheMetaData metaData; + metaData.setUrl(url); + metaData = fetchCacheMetaData(metaData); + + // save the redirect request also in the cache + QVariant redirectionTarget = q->attribute(QNetworkRequest::RedirectionTargetAttribute); + if (redirectionTarget.isValid()) { + QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes(); + attributes.insert(QNetworkRequest::RedirectionTargetAttribute, redirectionTarget); + metaData.setAttributes(attributes); + } + + cacheSaveDevice = managerPrivate->networkCache->prepare(metaData); + + if (!cacheSaveDevice || (cacheSaveDevice && !cacheSaveDevice->isOpen())) { + if (cacheSaveDevice && !cacheSaveDevice->isOpen()) + qCritical("QNetworkReplyImpl: network cache returned a device that is not open -- " + "class %s probably needs to be fixed", + managerPrivate->networkCache->metaObject()->className()); + + managerPrivate->networkCache->remove(url); + cacheSaveDevice = 0; + cacheEnabled = false; + } } -void QNetworkAccessHttpBackend::closeDownstreamChannel() +void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) { - // FIXME Maybe we can get rid of this whole architecture part -} + Q_Q(QNetworkReplyHttpImpl); -void QNetworkAccessHttpBackend::downstreamReadyWrite() -{ - // FIXME Maybe we can get rid of this whole architecture part -} + qDebug() << "QNetworkReplyHttpImplPrivate::replyDownloadData" << d.size(); -void QNetworkAccessHttpBackend::setDownstreamLimited(bool b) -{ - Q_UNUSED(b); - // We know that readBuffer maximum size limiting is broken since quite a while. - // The task to fix this is QTBUG-15065 -} - -void QNetworkAccessHttpBackend::replyDownloadData(QByteArray d) -{ int pendingSignals = (int)pendingDownloadDataEmissions->fetchAndAddAcquire(-1) - 1; if (pendingSignals > 0) { @@ -722,10 +911,54 @@ void QNetworkAccessHttpBackend::replyDownloadData(QByteArray d) // processEvents() or spin an event loop when this occur. QByteDataBuffer pendingDownloadDataCopy = pendingDownloadData; pendingDownloadData.clear(); - writeDownstreamData(pendingDownloadDataCopy); + + // FIXME + //writeDownstreamData(pendingDownloadDataCopy); + // instead we do: + + // We could be closed + if (!q->isOpen()) + return; + + if (cacheEnabled && !cacheSaveDevice) { + initCacheSaveDevice(); + } + + qint64 bytesWritten = 0; + for (int i = 0; i < pendingDownloadDataCopy.bufferCount(); i++) { + QByteArray const &item = pendingDownloadDataCopy[i]; + + if (cacheSaveDevice) + cacheSaveDevice->write(item.constData(), item.size()); + downloadMultiBuffer.append(item); + + bytesWritten += item.size(); + } + pendingDownloadDataCopy.clear(); + + bytesDownloaded += bytesWritten; + lastBytesDownloaded = bytesDownloaded; + + //appendDownstreamDataSignalEmissions(); + // instead: + QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); + if (preMigrationDownloaded != Q_INT64_C(-1)) + totalSize = totalSize.toLongLong() + preMigrationDownloaded; +// pauseNotificationHandling(); + // important: At the point of this readyRead(), the data parameter list must be empty, + // else implicit sharing will trigger memcpy when the user is reading data! + emit q->readyRead(); + // emit readyRead before downloadProgress incase this will cause events to be + // processed and we get into a recursive call (as in QProgressDialog). + emit q->downloadProgress(bytesDownloaded, + totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); + +// resumeNotificationHandling(); + + } -void QNetworkAccessHttpBackend::replyFinished() +void QNetworkReplyHttpImplPrivate::replyFinished() { // We are already loading from cache, we still however // got this signal because it was posted already @@ -735,8 +968,9 @@ void QNetworkAccessHttpBackend::replyFinished() finished(); } -void QNetworkAccessHttpBackend::checkForRedirect(const int statusCode) +void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode) { + Q_Q(QNetworkReplyHttpImpl); switch (statusCode) { case 301: // Moved Permanently case 302: // Found @@ -745,32 +979,38 @@ void QNetworkAccessHttpBackend::checkForRedirect(const int statusCode) // What do we do about the caching of the HTML note? // The response to a 303 MUST NOT be cached, while the response to // all of the others is cacheable if the headers indicate it to be - QByteArray header = rawHeader("location"); + QByteArray header = q->rawHeader("location"); QUrl url = QUrl::fromEncoded(header); if (!url.isValid()) url = QUrl(QLatin1String(header)); - redirectionRequested(url); + //redirectionRequested(url); + q->setAttribute(QNetworkRequest::RedirectionTargetAttribute, url); } } -void QNetworkAccessHttpBackend::replyDownloadMetaData +void QNetworkReplyHttpImplPrivate::replyDownloadMetaData (QList > hm, int sc,QString rp,bool pu, QSharedPointer db, qint64 contentLength) { + qDebug() << "QNetworkReplyHttpImplPrivate::replyDownloadMetaData" << contentLength << sc; + Q_Q(QNetworkReplyHttpImpl); + statusCode = sc; reasonPhrase = rp; // Download buffer if (!db.isNull()) { - reply->setDownloadBuffer(db, contentLength); - usingZerocopyDownloadBuffer = true; - } else { - usingZerocopyDownloadBuffer = false; + //setDownloadBuffer(db, contentLength); + downloadBufferPointer = db; + downloadZerocopyBuffer = downloadBufferPointer.data(); + downloadBufferCurrentSize = 0; + downloadBufferMaximumSize = contentLength; + q->setAttribute(QNetworkRequest::DownloadBufferAttribute, qVariantFromValue > (downloadBufferPointer)); } - setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, pu); + q->setAttribute(QNetworkRequest::HttpPipeliningWasUsedAttribute, pu); // reconstruct the HTTP header QList > headerMap = hm; @@ -779,7 +1019,7 @@ void QNetworkAccessHttpBackend::replyDownloadMetaData QByteArray header; for (; it != end; ++it) { - QByteArray value = rawHeader(it->first); + QByteArray value = q->rawHeader(it->first); if (!value.isEmpty()) { if (qstricmp(it->first.constData(), "set-cookie") == 0) value += '\n'; @@ -787,19 +1027,19 @@ void QNetworkAccessHttpBackend::replyDownloadMetaData value += ", "; } value += it->second; - setRawHeader(it->first, value); + q->setRawHeader(it->first, value); } - setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode); - setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); + q->setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode); + q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); // is it a redirection? checkForRedirect(statusCode); if (statusCode >= 500 && statusCode < 600) { - QAbstractNetworkCache *nc = networkCache(); + QAbstractNetworkCache *nc = managerPrivate->networkCache; if (nc) { - QNetworkCacheMetaData metaData = nc->metaData(url()); + QNetworkCacheMetaData metaData = nc->metaData(request.url()); QNetworkHeadersPrivate cacheHeaders; cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; @@ -819,9 +1059,9 @@ void QNetworkAccessHttpBackend::replyDownloadMetaData #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "Received a 304 from" << url(); #endif - QAbstractNetworkCache *nc = networkCache(); + QAbstractNetworkCache *nc = managerPrivate->networkCache; if (nc) { - QNetworkCacheMetaData oldMetaData = nc->metaData(url()); + QNetworkCacheMetaData oldMetaData = nc->metaData(request.url()); QNetworkCacheMetaData metaData = fetchCacheMetaData(oldMetaData); if (oldMetaData != metaData) nc->updateMetaData(metaData); @@ -839,8 +1079,10 @@ void QNetworkAccessHttpBackend::replyDownloadMetaData metaDataChanged(); } -void QNetworkAccessHttpBackend::replyDownloadProgressSlot(qint64 received, qint64 total) +void QNetworkReplyHttpImplPrivate::replyDownloadProgressSlot(qint64 bytesReceived, qint64 bytesTotal) { + Q_Q(QNetworkReplyHttpImpl); + // we can be sure here that there is a download buffer int pendingSignals = (int)pendingDownloadProgressEmissions->fetchAndAddAcquire(-1) - 1; @@ -850,32 +1092,63 @@ void QNetworkAccessHttpBackend::replyDownloadProgressSlot(qint64 received, qint return; } - // Now do the actual notification of new bytes - writeDownstreamDataDownloadBuffer(received, total); + if (!q->isOpen()) + return; + + if (cacheEnabled && bytesReceived == bytesTotal) { + // Write everything in one go if we use a download buffer. might be more performant. + initCacheSaveDevice(); + // need to check again if cache enabled and device exists + if (cacheSaveDevice && cacheEnabled) + cacheSaveDevice->write(downloadZerocopyBuffer, bytesTotal); + // FIXME where is it closed? + } + + bytesDownloaded = bytesReceived; + lastBytesDownloaded = bytesReceived; + + downloadBufferCurrentSize = bytesReceived; + + // Only emit readyRead when actual data is there + // emit readyRead before downloadProgress incase this will cause events to be + // processed and we get into a recursive call (as in QProgressDialog). + if (bytesDownloaded > 0) + emit q->readyRead(); + emit q->downloadProgress(bytesDownloaded, bytesTotal); } -void QNetworkAccessHttpBackend::httpAuthenticationRequired(const QHttpNetworkRequest &, +void QNetworkReplyHttpImplPrivate::httpAuthenticationRequired(const QHttpNetworkRequest &, QAuthenticator *auth) { - authenticationRequired(auth); + Q_Q(QNetworkReplyHttpImpl); + managerPrivate->authenticationRequired(auth, q_func(), synchronous, url, &urlForLastAuthentication); } -void QNetworkAccessHttpBackend::httpError(QNetworkReply::NetworkError errorCode, +#ifndef QT_NO_NETWORKPROXY +void QNetworkReplyHttpImplPrivate::proxyAuthenticationRequired(const QNetworkProxy &proxy, + QAuthenticator *authenticator) +{ + managerPrivate->proxyAuthenticationRequired(proxy, synchronous, authenticator, &lastProxyAuthentication); +} +#endif + +void QNetworkReplyHttpImplPrivate::httpError(QNetworkReply::NetworkError errorCode, const QString &errorString) { #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "http error!" << errorCode << errorString; #endif + // FIXME? error(errorCode, errorString); } #ifndef QT_NO_OPENSSL -void QNetworkAccessHttpBackend::replySslErrors( +void QNetworkReplyHttpImplPrivate::replySslErrors( const QList &list, bool *ignoreAll, QList *toBeIgnored) { - // Go to generic backend - sslErrors(list); + Q_Q(QNetworkReplyHttpImpl); + emit q->sslErrors(list); // Check if the callback set any ignore and return this here to http thread if (pendingIgnoreAllSslErrors) *ignoreAll = true; @@ -883,31 +1156,30 @@ void QNetworkAccessHttpBackend::replySslErrors( *toBeIgnored = pendingIgnoreSslErrorsList; } -void QNetworkAccessHttpBackend::replySslConfigurationChanged(const QSslConfiguration &c) +void QNetworkReplyHttpImplPrivate::replySslConfigurationChanged(const QSslConfiguration &sslConfiguration) { // Receiving the used SSL configuration from the HTTP thread - if (pendingSslConfiguration) - *pendingSslConfiguration = c; - else if (!c.isNull()) - pendingSslConfiguration = new QSslConfiguration(c); + this->sslConfiguration = sslConfiguration; } #endif // Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread -void QNetworkAccessHttpBackend::resetUploadDataSlot(bool *r) +void QNetworkReplyHttpImplPrivate::resetUploadDataSlot(bool *r) { *r = uploadByteDevice->reset(); } // Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread -void QNetworkAccessHttpBackend::sentUploadDataSlot(qint64 amount) +void QNetworkReplyHttpImplPrivate::sentUploadDataSlot(qint64 amount) { uploadByteDevice->advanceReadPointer(amount); } // Coming from QNonContiguousByteDeviceThreadForwardImpl in HTTP thread -void QNetworkAccessHttpBackend::wantUploadDataSlot(qint64 maxSize) +void QNetworkReplyHttpImplPrivate::wantUploadDataSlot(qint64 maxSize) { + Q_Q(QNetworkReplyHttpImpl); + // call readPointer qint64 currentUploadDataLength = 0; char *data = const_cast(uploadByteDevice->readPointer(maxSize, currentUploadDataLength)); @@ -915,37 +1187,39 @@ void QNetworkAccessHttpBackend::wantUploadDataSlot(qint64 maxSize) QByteArray dataArray(data, currentUploadDataLength); // Communicate back to HTTP thread - emit haveUploadData(dataArray, uploadByteDevice->atEnd(), uploadByteDevice->size()); + emit q->haveUploadData(dataArray, uploadByteDevice->atEnd(), uploadByteDevice->size()); } /* A simple web page that can be used to test us: http://www.procata.com/cachetest/ */ -bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &metaData) +bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData &metaData) { + Q_Q(QNetworkReplyHttpImpl); + setCachingEnabled(false); if (!metaData.isValid()) return false; - QAbstractNetworkCache *nc = networkCache(); + QAbstractNetworkCache *nc = managerPrivate->networkCache; Q_ASSERT(nc); - QIODevice *contents = nc->data(url()); + QIODevice *contents = nc->data(url); if (!contents) { #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) - qDebug() << "Can not send cache, the contents are 0" << url(); + qDebug() << "Can not send cache, the contents are 0" << url; #endif return false; } - contents->setParent(this); + contents->setParent(q); QNetworkCacheMetaData::AttributesMap attributes = metaData.attributes(); int status = attributes.value(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if (status < 100) status = 200; // fake it - setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status); - setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); - setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true); + q->setAttribute(QNetworkRequest::HttpStatusCodeAttribute, status); + q->setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); + q->setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true); QNetworkCacheMetaData::RawHeaderList rawHeaders = metaData.rawHeaders(); QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(), @@ -955,16 +1229,19 @@ bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &m checkForRedirect(status); + cacheLoadDevice = contents; + q->connect(cacheLoadDevice, SIGNAL(readyRead()), SLOT(_q_cacheLoadReadyRead())); + q->connect(cacheLoadDevice, SIGNAL(readChannelFinished()), SLOT(_q_cacheLoadReadyRead())); + // This needs to be emitted in the event loop because it can be reached at // the direct code path of qnam.get(...) before the user has a chance // to connect any signals. - QMetaObject::invokeMethod(this, "metaDataChanged", Qt::QueuedConnection); - qRegisterMetaType("QIODevice*"); - QMetaObject::invokeMethod(this, "writeDownstreamData", Qt::QueuedConnection, Q_ARG(QIODevice*, contents)); + QMetaObject::invokeMethod(q, "metaDataChanged", Qt::QueuedConnection); + QMetaObject::invokeMethod(q, "_q_cacheLoadReadyRead", Qt::QueuedConnection); #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) - qDebug() << "Successfully sent cache:" << url() << contents->size() << "bytes"; + qDebug() << "Successfully sent cache:" << url << contents->size() << "bytes"; #endif // Set the following flag so we can ignore some signals from HTTP thread @@ -973,50 +1250,49 @@ bool QNetworkAccessHttpBackend::sendCacheContents(const QNetworkCacheMetaData &m return true; } -void QNetworkAccessHttpBackend::copyFinished(QIODevice *dev) -{ - delete dev; - finished(); -} - #ifndef QT_NO_OPENSSL -void QNetworkAccessHttpBackend::ignoreSslErrors() +void QNetworkReplyHttpImpl::ignoreSslErrors() { - pendingIgnoreAllSslErrors = true; + Q_D(QNetworkReplyHttpImpl); + + d->pendingIgnoreAllSslErrors = true; } -void QNetworkAccessHttpBackend::ignoreSslErrors(const QList &errors) +void QNetworkReplyHttpImpl::ignoreSslErrorsImplementation(const QList &errors) { + Q_D(QNetworkReplyHttpImpl); + // the pending list is set if QNetworkReply::ignoreSslErrors(const QList &errors) // is called before QNetworkAccessManager::get() (or post(), etc.) - pendingIgnoreSslErrorsList = errors; + d->pendingIgnoreSslErrorsList = errors; } -void QNetworkAccessHttpBackend::fetchSslConfiguration(QSslConfiguration &config) const -{ - if (pendingSslConfiguration) - config = *pendingSslConfiguration; - else - config = request().sslConfiguration(); -} - -void QNetworkAccessHttpBackend::setSslConfiguration(const QSslConfiguration &newconfig) +void QNetworkReplyHttpImpl::setSslConfigurationImplementation(const QSslConfiguration &newconfig) { // Setting a SSL configuration on a reply is not supported. The user needs to set // her/his QSslConfiguration on the QNetworkRequest. Q_UNUSED(newconfig); } + +QSslConfiguration QNetworkReplyHttpImpl::sslConfigurationImplementation() const +{ + Q_D(const QNetworkReplyHttpImpl); + qDebug() << "sslConfigurationImplementation"; + return d->sslConfiguration; +} #endif -QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const +QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const { + Q_Q(const QNetworkReplyHttpImpl); + QNetworkCacheMetaData metaData = oldMetaData; QNetworkHeadersPrivate cacheHeaders; cacheHeaders.setAllRawHeaders(metaData.rawHeaders()); QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; - QList newHeaders = rawHeaderList(); + QList newHeaders = q->rawHeaderList(); foreach (QByteArray header, newHeaders) { QByteArray originalHeader = header; header = header.toLower(); @@ -1043,7 +1319,7 @@ QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetwo // Don't store Warning 1xx headers if (header == "warning") { - QByteArray v = rawHeader(header); + QByteArray v = q->rawHeader(header); if (v.length() == 3 && v[0] == '1' && v[1] >= '0' && v[1] <= '9' @@ -1076,7 +1352,7 @@ QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetwo qDebug() << "old" << o; } #endif - cacheHeaders.setRawHeader(originalHeader, rawHeader(header)); + cacheHeaders.setRawHeader(originalHeader, q->rawHeader(header)); } metaData.setRawHeaders(cacheHeaders.rawHeaders); @@ -1154,37 +1430,532 @@ QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetwo return metaData; } -bool QNetworkAccessHttpBackend::canResume() const +bool QNetworkReplyHttpImplPrivate::canResume() const { + Q_Q(const QNetworkReplyHttpImpl); + // Only GET operation supports resuming. - if (operation() != QNetworkAccessManager::GetOperation) + if (operation != QNetworkAccessManager::GetOperation) return false; // Can only resume if server/resource supports Range header. QByteArray acceptRangesheaderName("Accept-Ranges"); - if (!hasRawHeader(acceptRangesheaderName) || rawHeader(acceptRangesheaderName) == "none") + if (!q->hasRawHeader(acceptRangesheaderName) || q->rawHeader(acceptRangesheaderName) == "none") return false; // We only support resuming for byte ranges. - if (request().hasRawHeader("Range")) { - QByteArray range = request().rawHeader("Range"); + if (request.hasRawHeader("Range")) { + QByteArray range = request.rawHeader("Range"); if (!range.startsWith("bytes=")) return false; } // If we're using a download buffer then we don't support resuming/migration // right now. Too much trouble. - if (usingZerocopyDownloadBuffer) + if (downloadZerocopyBuffer) return false; return true; } -void QNetworkAccessHttpBackend::setResumeOffset(quint64 offset) +void QNetworkReplyHttpImplPrivate::setResumeOffset(quint64 offset) { resumeOffset = offset; } +/*! + Starts the backend. Returns true if the backend is started. Returns false if the backend + could not be started due to an unopened or roaming session. The caller should recall this + function once the session has been opened or the roaming process has finished. +*/ +bool QNetworkReplyHttpImplPrivate::start() +{ + if (!managerPrivate->networkSession) { + postRequest(); + return true; + } + + // This is not ideal. + const QString host = url.host(); + if (host == QLatin1String("localhost") || + QHostAddress(host) == QHostAddress::LocalHost || + QHostAddress(host) == QHostAddress::LocalHostIPv6) { + // Don't need an open session for localhost access. + postRequest(); + return true; + } + + if (managerPrivate->networkSession->isOpen() && + managerPrivate->networkSession->state() == QNetworkSession::Connected) { + postRequest(); + return true; + } + + return false; +} + +void QNetworkReplyHttpImplPrivate::_q_startOperation() +{ + // ensure this function is only being called once + if (state == Working) { + qDebug("QNetworkReplyImpl::_q_startOperation was called more than once"); + return; + } + state = Working; + +#ifndef QT_NO_BEARERMANAGEMENT + if (!start()) { // ### we should call that method even if bearer is not used + // backend failed to start because the session state is not Connected. + // QNetworkAccessManager will call reply->backend->start() again for us when the session + // state changes. + state = WaitingForSession; + + QNetworkSession *session = managerPrivate->networkSession.data(); + + if (session) { + Q_Q(QNetworkReplyHttpImpl); + + QObject::connect(session, SIGNAL(error(QNetworkSession::SessionError)), + q, SLOT(_q_networkSessionFailed())); + + if (!session->isOpen()) + session->open(); + } else { + qWarning("Backend is waiting for QNetworkSession to connect, but there is none!"); + } + + return; + } +#endif + + if (synchronous) { + state = Finished; + q_func()->setFinished(true); + } else { + if (state != Finished) { +// if (operation == QNetworkAccessManager::GetOperation) +// pendingNotifications.append(NotifyDownstreamReadyWrite); + +// handleNotifications(); + + } + } +} + +void QNetworkReplyHttpImplPrivate::_q_cacheLoadReadyRead() +{ + Q_Q(QNetworkReplyHttpImpl); + + if (state != Working) + return; + if (!cacheLoadDevice || !q->isOpen() || !cacheLoadDevice->bytesAvailable()) + return; + + // FIXME Optimize to use zerocopy download buffer if it is a QBuffer. + // Needs to be done where sendCacheContents() (?) of HTTP is emitting + // metaDataChanged ? + + + // FIXME + lastBytesDownloaded = bytesDownloaded; + QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); + + //pauseNotificationHandling(); + // emit readyRead before downloadProgress incase this will cause events to be + // processed and we get into a recursive call (as in QProgressDialog). + + // This readyRead() goes to the user. The user then may or may not read() anything. + emit q->readyRead(); + emit q->downloadProgress(bytesDownloaded, + totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); + + // If there are still bytes available in the cacheLoadDevice then the user did not read + // in response to the readyRead() signal. This means we have to load from the cacheLoadDevice + // and buffer that stuff. This is needed to be able to properly emit finished() later. + while (cacheLoadDevice->bytesAvailable()) { + downloadMultiBuffer.append(cacheLoadDevice->readAll()); + } + + if (cacheLoadDevice->isSequential()) { + // check if end and we can read the EOF -1 + char c; + qint64 actualCount = cacheLoadDevice->read(&c, 1); + if (actualCount < 0) { + cacheLoadDevice->deleteLater(); + cacheLoadDevice = 0; + QMetaObject::invokeMethod(q, "_q_finished", Qt::QueuedConnection); + } else if (actualCount == 1) { + // This is most probably not happening since most QIODevice returned something proper for bytesAvailable() + // and had already been "emptied". + cacheLoadDevice->ungetChar(c); + } + } else if ((!cacheLoadDevice->isSequential() && cacheLoadDevice->atEnd())) { + // This codepath is in case the cache device is a QBuffer, e.g. from QNetworkDiskCache. + cacheLoadDevice->deleteLater(); + cacheLoadDevice = 0; + QMetaObject::invokeMethod(q, "_q_finished", Qt::QueuedConnection); + } + +} + + +void QNetworkReplyHttpImplPrivate::_q_bufferOutgoingDataFinished() +{ + Q_Q(QNetworkReplyHttpImpl); + + // make sure this is only called once, ever. + //_q_bufferOutgoingData may call it or the readChannelFinished emission + if (state != Buffering) + return; + + // disconnect signals + QObject::disconnect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData())); + QObject::disconnect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished())); + + // finally, start the request + QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); +} + +void QNetworkReplyHttpImplPrivate::_q_bufferOutgoingData() +{ + Q_Q(QNetworkReplyHttpImpl); + + if (!outgoingDataBuffer) { + // first call, create our buffer + outgoingDataBuffer = QSharedPointer(new QRingBuffer()); + + QObject::connect(outgoingData, SIGNAL(readyRead()), q, SLOT(_q_bufferOutgoingData())); + QObject::connect(outgoingData, SIGNAL(readChannelFinished()), q, SLOT(_q_bufferOutgoingDataFinished())); + } + + qint64 bytesBuffered = 0; + qint64 bytesToBuffer = 0; + + // read data into our buffer + forever { + bytesToBuffer = outgoingData->bytesAvailable(); + // unknown? just try 2 kB, this also ensures we always try to read the EOF + if (bytesToBuffer <= 0) + bytesToBuffer = 2*1024; + + char *dst = outgoingDataBuffer->reserve(bytesToBuffer); + bytesBuffered = outgoingData->read(dst, bytesToBuffer); + + if (bytesBuffered == -1) { + // EOF has been reached. + outgoingDataBuffer->chop(bytesToBuffer); + + _q_bufferOutgoingDataFinished(); + break; + } else if (bytesBuffered == 0) { + // nothing read right now, just wait until we get called again + outgoingDataBuffer->chop(bytesToBuffer); + + break; + } else { + // don't break, try to read() again + outgoingDataBuffer->chop(bytesToBuffer - bytesBuffered); + } + } +} + +#ifndef QT_NO_BEARERMANAGEMENT +void QNetworkReplyHttpImplPrivate::_q_networkSessionConnected() +{ + Q_Q(QNetworkReplyHttpImpl); + + if (!manager) + return; + + QNetworkSession *session = managerPrivate->networkSession.data(); + if (!session) + return; + + if (session->state() != QNetworkSession::Connected) + return; + + switch (state) { + case QNetworkReplyImplPrivate::Buffering: + case QNetworkReplyImplPrivate::Working: + case QNetworkReplyImplPrivate::Reconnecting: + // Migrate existing downloads to new network connection. + migrateBackend(); + break; + case QNetworkReplyImplPrivate::WaitingForSession: + // Start waiting requests. + QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); + break; + default: + ; + } +} + +void QNetworkReplyHttpImplPrivate::_q_networkSessionFailed() +{ + // Abort waiting and working replies. + if (state == WaitingForSession || state == Working) { + state = Working; + error(QNetworkReplyImpl::UnknownNetworkError, + QCoreApplication::translate("QNetworkReply", "Network session error.")); + finished(); + } +} +#endif + + +// need to have this function since the reply is a private member variable +// and the special backends need to access this. +void QNetworkReplyHttpImplPrivate::emitReplyUploadProgress(qint64 bytesSent, qint64 bytesTotal) +{ + Q_Q(QNetworkReplyHttpImpl); + if (isFinished) + return; + emit q->uploadProgress(bytesSent, bytesTotal); +} + +QNonContiguousByteDevice* QNetworkReplyHttpImplPrivate::createUploadByteDevice() +{ + Q_Q(QNetworkReplyHttpImpl); + + if (outgoingDataBuffer) + uploadByteDevice = QSharedPointer(QNonContiguousByteDeviceFactory::create(outgoingDataBuffer)); + else if (outgoingData) { + uploadByteDevice = QSharedPointer(QNonContiguousByteDeviceFactory::create(outgoingData)); + } else { + return 0; + } + + bool bufferDisallowed = + request.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute, + QVariant(false)) == QVariant(true); + if (bufferDisallowed) + uploadByteDevice->disableReset(); + + // We want signal emissions only for normal asynchronous uploads + if (synchronous) + QObject::connect(uploadByteDevice.data(), SIGNAL(readProgress(qint64,qint64)), + q, SLOT(emitReplyUploadProgress(qint64,qint64))); + + return uploadByteDevice.data(); +} + +void QNetworkReplyHttpImplPrivate::_q_finished() +{ + // This gets called queued, just forward to real call then + finished(); +} + +void QNetworkReplyHttpImplPrivate::finished() +{ + Q_Q(QNetworkReplyHttpImpl); + + if (state == Finished || state == Aborted || state == WaitingForSession) + return; + + //pauseNotificationHandling(); + QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); + if (preMigrationDownloaded != Q_INT64_C(-1)) + totalSize = totalSize.toLongLong() + preMigrationDownloaded; + + // FIXME why should it be 0 + if (manager) { +#ifndef QT_NO_BEARERMANAGEMENT + QNetworkSession *session = managerPrivate->networkSession.data(); + if (session && session->state() == QNetworkSession::Roaming && + state == Working && errorCode != QNetworkReply::OperationCanceledError) { + // only content with a known size will fail with a temporary network failure error + if (!totalSize.isNull()) { + if (bytesDownloaded != totalSize) { + if (migrateBackend()) { + // either we are migrating or the request is finished/aborted + if (state == Reconnecting || state == WaitingForSession) { + //resumeNotificationHandling(); + return; // exit early if we are migrating. + } + } else { + error(QNetworkReply::TemporaryNetworkFailureError, + QNetworkReply::tr("Temporary network failure.")); + } + } + } + } +#endif + } + //resumeNotificationHandling(); + + state = Finished; + q->setFinished(true); + + //pendingNotifications.clear(); + + //pauseNotificationHandling(); + if (totalSize.isNull() || totalSize == -1) { + emit q->downloadProgress(bytesDownloaded, bytesDownloaded); + } + + if (bytesUploaded == -1 && (outgoingData || outgoingDataBuffer)) + emit q->uploadProgress(0, 0); + //resumeNotificationHandling(); + + // if we don't know the total size of or we received everything save the cache + if (totalSize.isNull() || totalSize == -1 || bytesDownloaded == totalSize) + completeCacheSave(); + + // note: might not be a good idea, since users could decide to delete us + // which would delete the backend too... + // maybe we should protect the backend + //pauseNotificationHandling(); + emit q->readChannelFinished(); + emit q->finished(); + //resumeNotificationHandling(); +} + +void QNetworkReplyHttpImplPrivate::error(QNetworkReplyImpl::NetworkError code, const QString &errorMessage) +{ + Q_Q(QNetworkReplyHttpImpl); + // Can't set and emit multiple errors. + if (errorCode != QNetworkReply::NoError) { + qWarning() << "QNetworkReplyImplPrivate::error: Internal problem, this method must only be called once."; + return; + } + + errorCode = code; + q->setErrorString(errorMessage); + + // note: might not be a good idea, since users could decide to delete us + // which would delete the backend too... + // maybe we should protect the backend + emit q->error(code); +} + +void QNetworkReplyHttpImplPrivate::metaDataChanged() +{ + // FIXME merge this with replyDownloadMetaData(); ? + + Q_Q(QNetworkReplyHttpImpl); + // 1. do we have cookies? + // 2. are we allowed to set them? + if (cookedHeaders.contains(QNetworkRequest::SetCookieHeader) && manager + && (static_cast + (request.attribute(QNetworkRequest::CookieSaveControlAttribute, + QNetworkRequest::Automatic).toInt()) == QNetworkRequest::Automatic)) { + QList cookies = + qvariant_cast >(cookedHeaders.value(QNetworkRequest::SetCookieHeader)); + QNetworkCookieJar *jar = manager->cookieJar(); + if (jar) + jar->setCookiesFromUrl(cookies, url); + } + emit q->metaDataChanged(); +} + +/* + Migrates the backend of the QNetworkReply to a new network connection if required. Returns + true if the reply is migrated or it is not required; otherwise returns false. +*/ +bool QNetworkReplyHttpImplPrivate::migrateBackend() +{ + Q_Q(QNetworkReplyHttpImpl); + + // Network reply is already finished or aborted, don't need to migrate. + if (state == Finished || state == Aborted) + return true; + + // Backend does not support resuming download. + if (!canResume()) + return false; + + // Request has outgoing data, not migrating. + if (outgoingData) + return false; + + // Request is serviced from the cache, don't need to migrate. + if (cacheLoadDevice) + return true; + + state = Reconnecting; + +// if (backend) { +// delete backend; +// backend = 0; +// } + + cookedHeaders.clear(); + rawHeaders.clear(); + + preMigrationDownloaded = bytesDownloaded; + +// backend = manager->d_func()->findBackend(operation, request); + +// if (backend) { +// backend->setParent(q); +// backend->reply = this; +// backend->setResumeOffset(bytesDownloaded); +// } + + // FIXME + Q_ASSERT(0); + // What probably needs to be done is an abort and then re-send? + + QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); + + return true; +} + + +void QNetworkReplyHttpImplPrivate::createCache() +{ + // check if we can save and if we're allowed to + if (!managerPrivate->networkCache + || !request.attribute(QNetworkRequest::CacheSaveControlAttribute, true).toBool() + || request.attribute(QNetworkRequest::CacheLoadControlAttribute, + QNetworkRequest::PreferNetwork).toInt() + == QNetworkRequest::AlwaysNetwork) + return; + cacheEnabled = true; +} + +bool QNetworkReplyHttpImplPrivate::isCachingEnabled() const +{ + return (cacheEnabled && managerPrivate->networkCache != 0); +} + +void QNetworkReplyHttpImplPrivate::setCachingEnabled(bool enable) +{ + if (!enable && !cacheEnabled) + return; // nothing to do + if (enable && cacheEnabled) + return; // nothing to do either! + + if (enable) { + if (bytesDownloaded) { + qDebug() << "x" << bytesDownloaded; + // refuse to enable in this case + qCritical("QNetworkReplyImpl: backend error: caching was enabled after some bytes had been written"); + return; + } + + createCache(); + } else { + // someone told us to turn on, then back off? + // ok... but you should make up your mind + qDebug("QNetworkReplyImpl: setCachingEnabled(true) called after setCachingEnabled(false)"); + managerPrivate->networkCache->remove(url); + cacheSaveDevice = 0; + cacheEnabled = false; + } +} + +void QNetworkReplyHttpImplPrivate::completeCacheSave() +{ + if (cacheEnabled && errorCode != QNetworkReplyImpl::NoError) { + managerPrivate->networkCache->remove(url); + } else if (cacheEnabled && cacheSaveDevice) { + managerPrivate->networkCache->insert(cacheSaveDevice); + } + cacheSaveDevice = 0; + cacheEnabled = false; +} + QT_END_NAMESPACE #endif // QT_NO_HTTP diff --git a/src/network/access/qnetworkreplyhttpimpl_p.h b/src/network/access/qnetworkreplyhttpimpl_p.h index 712dd2ff642..476022d7f7e 100644 --- a/src/network/access/qnetworkreplyhttpimpl_p.h +++ b/src/network/access/qnetworkreplyhttpimpl_p.h @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#ifndef QNETWORKACCESSHTTPBACKEND_P_H -#define QNETWORKACCESSHTTPBACKEND_P_H +#ifndef QNETWORKREPLYHTTPIMPL_P_H +#define QNETWORKREPLYHTTPIMPL_P_H // // W A R N I N G @@ -53,53 +53,83 @@ // We mean it. // -#include "qhttpnetworkconnection_p.h" -#include "qnetworkaccessbackend_p.h" #include "qnetworkrequest.h" #include "qnetworkreply.h" -#include "qabstractsocket.h" #include "QtCore/qpointer.h" #include "QtCore/qdatetime.h" #include "QtCore/qsharedpointer.h" #include "qatomic.h" +#include +#include +#include +#include +#include + +#ifndef QT_NO_OPENSSL +#include +#endif + #ifndef QT_NO_HTTP QT_BEGIN_NAMESPACE -class QNetworkAccessCachedHttpConnection; +class QIODevice; -class QNetworkAccessHttpBackendIODevice; - -class QNetworkAccessHttpBackend: public QNetworkAccessBackend +class QNetworkReplyHttpImplPrivate; +class QNetworkReplyHttpImpl: public QNetworkReply { Q_OBJECT public: - QNetworkAccessHttpBackend(); - virtual ~QNetworkAccessHttpBackend(); + QNetworkReplyHttpImpl(QNetworkAccessManager* const, const QNetworkRequest&, QNetworkAccessManager::Operation&, QIODevice* outgoingData); + virtual ~QNetworkReplyHttpImpl(); - virtual void open(); - virtual void closeDownstreamChannel(); + void close(); + void abort(); + qint64 bytesAvailable() const; + bool isSequential () const; + qint64 size() const; + qint64 readData(char*, qint64); + void setReadBufferSize(qint64 size); + bool canReadLine () const; - virtual void downstreamReadyWrite(); - virtual void setDownstreamLimited(bool b); - - virtual void copyFinished(QIODevice *); #ifndef QT_NO_OPENSSL - virtual void ignoreSslErrors(); - virtual void ignoreSslErrors(const QList &errors); - - virtual void fetchSslConfiguration(QSslConfiguration &configuration) const; - virtual void setSslConfiguration(const QSslConfiguration &configuration); + void ignoreSslErrors(); + Q_INVOKABLE void ignoreSslErrorsImplementation(const QList &errors); + Q_INVOKABLE void setSslConfigurationImplementation(const QSslConfiguration &configuration); + Q_INVOKABLE QSslConfiguration sslConfigurationImplementation() const; #endif - QNetworkCacheMetaData fetchCacheMetaData(const QNetworkCacheMetaData &metaData) const; +// Q_INVOKABLE QSslConfiguration sslConfigurationImplementation() const; +// Q_INVOKABLE void setSslConfigurationImplementation(const QSslConfiguration &configuration); - // we return true since HTTP needs to send PUT/POST data again after having authenticated - bool needsResetableUploadData() { return true; } + Q_DECLARE_PRIVATE(QNetworkReplyHttpImpl) + Q_PRIVATE_SLOT(d_func(), void _q_startOperation()) + Q_PRIVATE_SLOT(d_func(), void _q_cacheLoadReadyRead()) + Q_PRIVATE_SLOT(d_func(), void _q_bufferOutgoingData()) + Q_PRIVATE_SLOT(d_func(), void _q_bufferOutgoingDataFinished()) +#ifndef QT_NO_BEARERMANAGEMENT + Q_PRIVATE_SLOT(d_func(), void _q_networkSessionConnected()) + Q_PRIVATE_SLOT(d_func(), void _q_networkSessionFailed()) +#endif + Q_PRIVATE_SLOT(d_func(), void _q_finished()) + Q_PRIVATE_SLOT(d_func(), void replyDownloadData(QByteArray)) + Q_PRIVATE_SLOT(d_func(), void replyFinished()) + Q_PRIVATE_SLOT(d_func(), void replyDownloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)) + Q_PRIVATE_SLOT(d_func(), void replyDownloadProgressSlot(qint64,qint64)) + Q_PRIVATE_SLOT(d_func(), void httpAuthenticationRequired(const QHttpNetworkRequest &, QAuthenticator *)) + Q_PRIVATE_SLOT(d_func(), void httpError(QNetworkReply::NetworkError, const QString &)) +#ifndef QT_NO_OPENSSL + Q_PRIVATE_SLOT(d_func(), void replySslErrors(const QList &, bool *, QList *)) + Q_PRIVATE_SLOT(d_func(), void replySslConfigurationChanged(const QSslConfiguration&)) +#endif + Q_PRIVATE_SLOT(d_func(), void proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *auth)) + + Q_PRIVATE_SLOT(d_func(), void resetUploadDataSlot(bool *r)) + Q_PRIVATE_SLOT(d_func(), void wantUploadDataSlot(qint64)) + Q_PRIVATE_SLOT(d_func(), void sentUploadDataSlot(qint64)) + Q_PRIVATE_SLOT(d_func(), void emitReplyUploadProgress(qint64, qint64)) - bool canResume() const; - void setResumeOffset(quint64 offset); signals: // To HTTP thread: @@ -109,7 +139,140 @@ signals: void startHttpRequestSynchronously(); void haveUploadData(QByteArray dataArray, bool dataAtEnd, qint64 dataSize); -private slots: +}; + +class QNetworkReplyHttpImplPrivate: public QNetworkReplyPrivate +{ +public: + + static QHttpNetworkRequest::Priority convert(const QNetworkRequest::Priority& prio); + + enum State { + Idle, // The reply is idle. + Buffering, // The reply is buffering outgoing data. + Working, // The reply is uploading/downloading data. + Finished, // The reply has finished. + Aborted, // The reply has been aborted. + WaitingForSession, // The reply is waiting for the session to open before connecting. + Reconnecting // The reply will reconnect to once roaming has completed. + }; + + QNetworkReplyHttpImplPrivate(); + ~QNetworkReplyHttpImplPrivate(); + + bool start(); + void _q_startOperation(); + + void _q_cacheLoadReadyRead(); + + void _q_bufferOutgoingData(); + void _q_bufferOutgoingDataFinished(); + +#ifndef QT_NO_BEARERMANAGEMENT + void _q_networkSessionConnected(); + void _q_networkSessionFailed(); +#endif + void _q_finished(); + + + // ? + void consume(qint64 count); + + void setDownloadBuffer(QSharedPointer sp, qint64 size); + char* getDownloadBuffer(qint64 size); + + // FIXME + void finished(); + void error(QNetworkReply::NetworkError code, const QString &errorString); + void metaDataChanged(); + void redirectionRequested(const QUrl &target); + + + QNetworkAccessManager *manager; + QNetworkAccessManagerPrivate *managerPrivate; + QNetworkRequest request; + QNetworkAccessManager::Operation operation; + + QNonContiguousByteDevice* createUploadByteDevice(); + QSharedPointer uploadByteDevice; + QIODevice *outgoingData; + QSharedPointer outgoingDataBuffer; + void emitReplyUploadProgress(qint64 bytesSent, qint64 bytesTotal); // dup? + + + bool migrateBackend(); + quint64 resumeOffset; + bool canResume() const; + void setResumeOffset(quint64 offset); + qint64 bytesUploaded; + qint64 preMigrationDownloaded; + + void createCache(); + void completeCacheSave(); + void setCachingEnabled(bool enable); + bool isCachingEnabled() const; + void initCacheSaveDevice(); + QAbstractNetworkCache *networkCache() const; + QIODevice *cacheLoadDevice; + bool cacheEnabled; // is this for saving? + QIODevice *cacheSaveDevice; + bool loadingFromCache; + + + QUrl urlForLastAuthentication; +#ifndef QT_NO_NETWORKPROXY + QNetworkProxy lastProxyAuthentication; + QList proxyList; +#endif + + int statusCode; + QString reasonPhrase; + + State state; + + + // Used for normal downloading. For "zero copy" the downloadZerocopyBuffer is used + QByteDataBuffer downloadMultiBuffer; + QByteDataBuffer pendingDownloadData; // For signal compression + qint64 bytesDownloaded; + qint64 lastBytesDownloaded; + + // only used when the "zero copy" style is used. Else downloadMultiBuffer is used. + // Please note that the whole "zero copy" download buffer API is private right now. Do not use it. + qint64 downloadBufferReadPosition; + qint64 downloadBufferCurrentSize; + qint64 downloadBufferMaximumSize; + QSharedPointer downloadBufferPointer; + char* downloadZerocopyBuffer; + + + QHttpNetworkRequest httpRequest; // There is also a copy in the HTTP thread + + // Will be increased by HTTP thread: + QSharedPointer pendingDownloadDataEmissions; + QSharedPointer pendingDownloadProgressEmissions; + + bool synchronous; + +#ifndef QT_NO_OPENSSL + QSslConfiguration sslConfiguration; + bool pendingIgnoreAllSslErrors; + QList pendingIgnoreSslErrorsList; +#endif + + + void validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache); + void invalidateCache(); + bool sendCacheContents(const QNetworkCacheMetaData &metaData); + QNetworkCacheMetaData fetchCacheMetaData(const QNetworkCacheMetaData &metaData) const; + + + void postRequest(); + + + void checkForRedirect(const int statusCode); + +public: // From HTTP thread: void replyDownloadData(QByteArray); void replyFinished(); @@ -121,45 +284,19 @@ private slots: void replySslErrors(const QList &, bool *, QList *); void replySslConfigurationChanged(const QSslConfiguration&); #endif +#ifndef QT_NO_NETWORKPROXY + void proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *auth); +#endif // From QNonContiguousByteDeviceThreadForwardImpl in HTTP thread: void resetUploadDataSlot(bool *r); void wantUploadDataSlot(qint64); void sentUploadDataSlot(qint64); - bool sendCacheContents(const QNetworkCacheMetaData &metaData); -private: - QHttpNetworkRequest httpRequest; // There is also a copy in the HTTP thread - int statusCode; - QString reasonPhrase; - // Will be increased by HTTP thread: - QSharedPointer pendingDownloadDataEmissions; - QSharedPointer pendingDownloadProgressEmissions; - bool loadingFromCache; - QByteDataBuffer pendingDownloadData; - bool usingZerocopyDownloadBuffer; -#ifndef QT_NO_OPENSSL - QSslConfiguration *pendingSslConfiguration; - bool pendingIgnoreAllSslErrors; - QList pendingIgnoreSslErrorsList; -#endif - quint64 resumeOffset; - - void validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache); - void invalidateCache(); - void postRequest(); - void readFromHttp(); - void checkForRedirect(const int statusCode); -}; - -class QNetworkAccessHttpBackendFactory : public QNetworkAccessBackendFactory -{ -public: - virtual QNetworkAccessBackend *create(QNetworkAccessManager::Operation op, - const QNetworkRequest &request) const; + Q_DECLARE_PUBLIC(QNetworkReplyHttpImpl) }; QT_END_NAMESPACE diff --git a/src/network/access/qnetworkreplyimpl.cpp b/src/network/access/qnetworkreplyimpl.cpp index 9eb505d1b7b..78e246315c0 100644 --- a/src/network/access/qnetworkreplyimpl.cpp +++ b/src/network/access/qnetworkreplyimpl.cpp @@ -47,7 +47,6 @@ #include "QtCore/qdatetime.h" #include "QtNetwork/qsslconfiguration.h" #include "QtNetwork/qnetworksession.h" -#include "qnetworkaccesshttpbackend_p.h" #include "qnetworkaccessmanager_p.h" #include @@ -356,7 +355,7 @@ void QNetworkReplyImplPrivate::setup(QNetworkAccessManager::Operation op, const // for HTTP, we want to send out the request as fast as possible to the network, without // invoking methods in a QueuedConnection #ifndef QT_NO_HTTP - if (qobject_cast(backend) || (backend && backend->isSynchronous())) { + if (backend && backend->isSynchronous()) { _q_startOperation(); } else { QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); @@ -1043,11 +1042,7 @@ bool QNetworkReplyImplPrivate::migrateBackend() } #ifndef QT_NO_HTTP - if (qobject_cast(backend)) { - _q_startOperation(); - } else { - QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); - } + QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); #else QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection); #endif // QT_NO_HTTP From 0b7df2875409a1a164ea531b39a0a450dc4e0f02 Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Fri, 8 Apr 2011 13:25:13 +0200 Subject: [PATCH 05/26] fix build --- src/network/access/qnetworkreplyhttpimpl.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index d38850b450b..9971dbb9b3c 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -54,6 +54,7 @@ #include "QtNetwork/qsslconfiguration.h" #include "qhttpthreaddelegate_p.h" #include "qthread.h" +#include "QtCore/qcoreapplication.h" #include "qnetworkcookiejar.h" From 43b6320a5a1e740e7b3dbc8fe623bd83903ad172 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Fri, 8 Apr 2011 13:41:57 +0200 Subject: [PATCH 06/26] QNAM: Fix proxy code after refactoring --- src/network/access/qnetworkreplyhttpimpl.cpp | 10 ++++++++-- src/network/access/qnetworkreplyhttpimpl_p.h | 10 +++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 9971dbb9b3c..5e9d53b7f15 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -600,10 +600,10 @@ void QNetworkReplyHttpImplPrivate::postRequest() if (transparentProxy.type() == QNetworkProxy::DefaultProxy && cacheProxy.type() == QNetworkProxy::DefaultProxy) { // unsuitable proxies - QMetaObject::invokeMethod(q, "error", synchronous ? Qt::DirectConnection : Qt::QueuedConnection, + QMetaObject::invokeMethod(q, "_q_error", synchronous ? Qt::DirectConnection : Qt::QueuedConnection, Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProxyNotFoundError), Q_ARG(QString, q->tr("No suitable proxy found"))); - QMetaObject::invokeMethod(q, "finished", synchronous ? Qt::DirectConnection : Qt::QueuedConnection); + QMetaObject::invokeMethod(q, "_q_finished", synchronous ? Qt::DirectConnection : Qt::QueuedConnection); return; } #endif @@ -1811,6 +1811,12 @@ void QNetworkReplyHttpImplPrivate::finished() //resumeNotificationHandling(); } +void QNetworkReplyHttpImplPrivate::_q_error(QNetworkReplyImpl::NetworkError code, const QString &errorMessage) +{ + this->error(code, errorMessage); +} + + void QNetworkReplyHttpImplPrivate::error(QNetworkReplyImpl::NetworkError code, const QString &errorMessage) { Q_Q(QNetworkReplyHttpImpl); diff --git a/src/network/access/qnetworkreplyhttpimpl_p.h b/src/network/access/qnetworkreplyhttpimpl_p.h index 476022d7f7e..8711a59f7ae 100644 --- a/src/network/access/qnetworkreplyhttpimpl_p.h +++ b/src/network/access/qnetworkreplyhttpimpl_p.h @@ -100,8 +100,6 @@ public: Q_INVOKABLE void setSslConfigurationImplementation(const QSslConfiguration &configuration); Q_INVOKABLE QSslConfiguration sslConfigurationImplementation() const; #endif -// Q_INVOKABLE QSslConfiguration sslConfigurationImplementation() const; -// Q_INVOKABLE void setSslConfigurationImplementation(const QSslConfiguration &configuration); Q_DECLARE_PRIVATE(QNetworkReplyHttpImpl) Q_PRIVATE_SLOT(d_func(), void _q_startOperation()) @@ -113,6 +111,9 @@ public: Q_PRIVATE_SLOT(d_func(), void _q_networkSessionFailed()) #endif Q_PRIVATE_SLOT(d_func(), void _q_finished()) + Q_PRIVATE_SLOT(d_func(), void _q_error(QNetworkReply::NetworkError, const QString &)) + + // From reply Q_PRIVATE_SLOT(d_func(), void replyDownloadData(QByteArray)) Q_PRIVATE_SLOT(d_func(), void replyFinished()) Q_PRIVATE_SLOT(d_func(), void replyDownloadMetaData(QList >,int,QString,bool,QSharedPointer,qint64)) @@ -131,6 +132,7 @@ public: Q_PRIVATE_SLOT(d_func(), void emitReplyUploadProgress(qint64, qint64)) + signals: // To HTTP thread: void startHttpRequest(); @@ -184,6 +186,7 @@ public: // FIXME void finished(); void error(QNetworkReply::NetworkError code, const QString &errorString); + void _q_error(QNetworkReply::NetworkError code, const QString &errorString); void metaDataChanged(); void redirectionRequested(const QUrl &target); @@ -293,9 +296,6 @@ public: void wantUploadDataSlot(qint64); void sentUploadDataSlot(qint64); - - - Q_DECLARE_PUBLIC(QNetworkReplyHttpImpl) }; From 76e04fdf3db06f8109be8d7440ca087ec828ec3a Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Fri, 8 Apr 2011 14:47:21 +0200 Subject: [PATCH 07/26] cache: rename validateCache() to loadFromCacheIfAllowed() --- src/network/access/qnetworkreplyhttpimpl.cpp | 6 +++--- src/network/access/qnetworkreplyhttpimpl_p.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 5e9d53b7f15..8ecea5cfdf9 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -398,7 +398,7 @@ QNetworkReplyHttpImplPrivate::~QNetworkReplyHttpImplPrivate() 2) If we have a cache entry for this url populate headers so the server can return 304 3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true */ -void QNetworkReplyHttpImplPrivate::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache) +void QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest, bool &loadedFromCache) { QNetworkRequest::CacheLoadControl CacheLoadControlAttribute = (QNetworkRequest::CacheLoadControl)request.attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt(); @@ -615,12 +615,12 @@ void QNetworkReplyHttpImplPrivate::postRequest() switch (operation) { case QNetworkAccessManager::GetOperation: httpRequest.setOperation(QHttpNetworkRequest::Get); - validateCache(httpRequest, loadedFromCache); + loadFromCacheIfAllowed(httpRequest, loadedFromCache); break; case QNetworkAccessManager::HeadOperation: httpRequest.setOperation(QHttpNetworkRequest::Head); - validateCache(httpRequest, loadedFromCache); + loadFromCacheIfAllowed(httpRequest, loadedFromCache); break; case QNetworkAccessManager::PostOperation: diff --git a/src/network/access/qnetworkreplyhttpimpl_p.h b/src/network/access/qnetworkreplyhttpimpl_p.h index 8711a59f7ae..adeaa6a4b27 100644 --- a/src/network/access/qnetworkreplyhttpimpl_p.h +++ b/src/network/access/qnetworkreplyhttpimpl_p.h @@ -264,7 +264,7 @@ public: #endif - void validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache); + void loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest, bool &loadedFromCache); void invalidateCache(); bool sendCacheContents(const QNetworkCacheMetaData &metaData); QNetworkCacheMetaData fetchCacheMetaData(const QNetworkCacheMetaData &metaData) const; From 7876177c0c2bf8eab4fc487b85db155548fcc777 Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Fri, 8 Apr 2011 14:52:21 +0200 Subject: [PATCH 08/26] HTTP Reply implementation: make cache validation method return bool --- src/network/access/qnetworkreplyhttpimpl.cpp | 24 +++++++++----------- src/network/access/qnetworkreplyhttpimpl_p.h | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 8ecea5cfdf9..351973bf491 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -398,7 +398,7 @@ QNetworkReplyHttpImplPrivate::~QNetworkReplyHttpImplPrivate() 2) If we have a cache entry for this url populate headers so the server can return 304 3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true */ -void QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest, bool &loadedFromCache) +bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest) { QNetworkRequest::CacheLoadControl CacheLoadControlAttribute = (QNetworkRequest::CacheLoadControl)request.attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt(); @@ -409,24 +409,24 @@ void QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h httpRequest.setHeaderField("Cache-Control", "no-cache"); httpRequest.setHeaderField("Pragma", "no-cache"); } - return; + return false; } // The disk cache API does not currently support partial content retrieval. // That is why we don't use the disk cache for any such requests. if (request.hasRawHeader("Range")) - return; + return false; QAbstractNetworkCache *nc = managerPrivate->networkCache; if (!nc) - return; // no local cache + return false; // no local cache QNetworkCacheMetaData metaData = nc->metaData(request.url()); if (!metaData.isValid()) - return; // not in cache + return false; // not in cache if (!metaData.saveToDisk()) - return; + return false; QNetworkHeadersPrivate cacheHeaders; QNetworkHeadersPrivate::RawHeadersList::ConstIterator it; @@ -445,7 +445,7 @@ void QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h if (it != cacheHeaders.rawHeaders.constEnd()) { QHash cacheControl = parseHttpOptionHeader(it->second); if (cacheControl.contains("must-revalidate")) - return; + return false; } } @@ -517,14 +517,12 @@ void QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h #endif if (!response_is_fresh) - return; + return false; - loadedFromCache = true; #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG) qDebug() << "response_is_fresh" << CacheLoadControlAttribute; #endif - if (!sendCacheContents(metaData)) - loadedFromCache = false; + return sendCacheContents(metaData); } QHttpNetworkRequest::Priority QNetworkReplyHttpImplPrivate::convert(const QNetworkRequest::Priority& prio) @@ -615,12 +613,12 @@ void QNetworkReplyHttpImplPrivate::postRequest() switch (operation) { case QNetworkAccessManager::GetOperation: httpRequest.setOperation(QHttpNetworkRequest::Get); - loadFromCacheIfAllowed(httpRequest, loadedFromCache); + loadedFromCache = loadFromCacheIfAllowed(httpRequest); break; case QNetworkAccessManager::HeadOperation: httpRequest.setOperation(QHttpNetworkRequest::Head); - loadFromCacheIfAllowed(httpRequest, loadedFromCache); + loadedFromCache = loadFromCacheIfAllowed(httpRequest); break; case QNetworkAccessManager::PostOperation: diff --git a/src/network/access/qnetworkreplyhttpimpl_p.h b/src/network/access/qnetworkreplyhttpimpl_p.h index adeaa6a4b27..2e8374af786 100644 --- a/src/network/access/qnetworkreplyhttpimpl_p.h +++ b/src/network/access/qnetworkreplyhttpimpl_p.h @@ -264,7 +264,7 @@ public: #endif - void loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest, bool &loadedFromCache); + bool loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest); void invalidateCache(); bool sendCacheContents(const QNetworkCacheMetaData &metaData); QNetworkCacheMetaData fetchCacheMetaData(const QNetworkCacheMetaData &metaData) const; From 2aba57d58eba4ec6033c8896516b4e24929bbe76 Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Fri, 8 Apr 2011 14:54:45 +0200 Subject: [PATCH 09/26] HTTP Network Reply Impl: return earlier when resource loaded from cache --- src/network/access/qnetworkreplyhttpimpl.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 351973bf491..3acbfff70f0 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -650,6 +650,10 @@ void QNetworkReplyHttpImplPrivate::postRequest() break; // can't happen } + if (loadedFromCache) { + return; // no need to send the request! :) + } + QList headers = request.rawHeaderList(); if (resumeOffset != 0) { if (headers.contains("Range")) { @@ -677,10 +681,6 @@ void QNetworkReplyHttpImplPrivate::postRequest() foreach (const QByteArray &header, headers) httpRequest.setHeaderField(header, request.rawHeader(header)); - if (loadedFromCache) { - return; // no need to send the request! :) - } - if (request.attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool() == true) httpRequest.setPipeliningAllowed(true); From a5562d345b212c722b3b0a9296b0ab9187959ddf Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Fri, 8 Apr 2011 15:08:05 +0200 Subject: [PATCH 10/26] HTTP caching internals: fix logic for PreferNetwork and PreferCache --- src/network/access/qnetworkreplyhttpimpl.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 3acbfff70f0..1d8605810c7 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -441,6 +441,11 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified)); if (CacheLoadControlAttribute == QNetworkRequest::PreferNetwork) { + // PreferNetwork == send request with "If-None-Match" and "If-Modified-Since" header, + // which will return a 304 Not Modifed if resource has not been changed. + // We might read from cache later, if receiving a 304. + return false; + } else if (CacheLoadControlAttribute == QNetworkRequest::PreferCache) { it = cacheHeaders.findRawHeader("Cache-Control"); if (it != cacheHeaders.rawHeaders.constEnd()) { QHash cacheControl = parseHttpOptionHeader(it->second); @@ -1422,7 +1427,7 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe attributes.insert(QNetworkRequest::HttpStatusCodeAttribute, statusCode); attributes.insert(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); } else { - // this is a redirection, keep the attributes intact + // this is the server telling us the resource has not changed, keep the attributes intact attributes = oldMetaData.attributes(); } metaData.setAttributes(attributes); From 5b2b0a7b35c9df42a92aa953fe0c58d208130d51 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Tue, 12 Apr 2011 16:58:46 +0200 Subject: [PATCH 11/26] QNAM HTTP: Fix upload progress signal --- src/network/access/qnetworkreplyhttpimpl.cpp | 2 +- .../auto/qnetworkreply/tst_qnetworkreply.cpp | 34 +++++++++++-------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 1d8605810c7..d240fb8ab23 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -1736,7 +1736,7 @@ QNonContiguousByteDevice* QNetworkReplyHttpImplPrivate::createUploadByteDevice() uploadByteDevice->disableReset(); // We want signal emissions only for normal asynchronous uploads - if (synchronous) + if (!synchronous) QObject::connect(uploadByteDevice.data(), SIGNAL(readProgress(qint64,qint64)), q, SLOT(emitReplyUploadProgress(qint64,qint64))); diff --git a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp index f509ceaad6b..36bb2efdc32 100644 --- a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp +++ b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp @@ -4162,6 +4162,7 @@ public: if (serverSocket->setSocketDescriptor(socketDescriptor)) { connect(serverSocket, SIGNAL(encrypted()), this, SLOT(encryptedSlot())); + connect(serverSocket, SIGNAL(readyRead()), this, SLOT(readyReadSlot())); serverSocket->setProtocol(QSsl::AnyProtocol); connect(serverSocket, SIGNAL(sslErrors(const QList&)), serverSocket, SLOT(ignoreSslErrors())); serverSocket->setLocalCertificate(SRCDIR "/certs/server.pem"); @@ -4178,6 +4179,11 @@ public slots: socket = (QSslSocket*) sender(); emit newEncryptedConnection(); } + void readyReadSlot() { + // for the incoming sockets, not the server socket + //qDebug() << static_cast(sender())->bytesAvailable() << static_cast(sender())->encryptedBytesAvailable(); + } + public: QSslSocket *socket; }; @@ -4185,8 +4191,15 @@ public: // very similar to ioPostToHttpUploadProgress but for SSL void tst_QNetworkReply::ioPostToHttpsUploadProgress() { - QFile sourceFile(SRCDIR "/bigfile"); - QVERIFY(sourceFile.open(QIODevice::ReadOnly)); + //QFile sourceFile(SRCDIR "/bigfile"); + //QVERIFY(sourceFile.open(QIODevice::ReadOnly)); + qint64 wantedSize = 2*1024*1024; // 2 MB + QByteArray sourceFile; + // And in the case of SSL, the compression can fool us and let the + // server send the data much faster than expected. + // So better provide random data that cannot be compressed. + for (int i = 0; i < wantedSize; ++i) + sourceFile += (char)qrand(); // emulate a minimal https server SslServer server; @@ -4195,8 +4208,10 @@ void tst_QNetworkReply::ioPostToHttpsUploadProgress() // create the request QUrl url = QUrl(QString("https://127.0.0.1:%1/").arg(server.serverPort())); QNetworkRequest request(url); + request.setRawHeader("Content-Type", "application/octet-stream"); - QNetworkReplyPtr reply = manager.post(request, &sourceFile); + QNetworkReplyPtr reply = manager.post(request, sourceFile); + QSignalSpy spy(reply, SIGNAL(uploadProgress(qint64,qint64))); connect(&server, SIGNAL(newEncryptedConnection()), &QTestEventLoop::instance(), SLOT(exitLoop())); connect(reply, SIGNAL(sslErrors(const QList&)), reply, SLOT(ignoreSslErrors())); @@ -4215,26 +4230,17 @@ void tst_QNetworkReply::ioPostToHttpsUploadProgress() QVERIFY(!spy.isEmpty()); QList args = spy.last(); QVERIFY(args.at(0).toLongLong() > 0); - + // but not everything! QVERIFY(args.at(0).toLongLong() != sourceFile.size()); - incomingSocket->setReadBufferSize(32*1024); - incomingSocket->read(16*1024); - QTestEventLoop::instance().enterLoop(2); - // some more progress than before - QVERIFY(!spy.isEmpty()); - QList args2 = spy.last(); - QVERIFY(args2.at(0).toLongLong() > args.at(0).toLongLong()); - // set the read buffer to unlimited incomingSocket->setReadBufferSize(0); QTestEventLoop::instance().enterLoop(10); // progress should be finished QVERIFY(!spy.isEmpty()); QList args3 = spy.last(); - QVERIFY(args3.at(0).toLongLong() > args2.at(0).toLongLong()); QCOMPARE(args3.at(0).toLongLong(), args3.at(1).toLongLong()); - QCOMPARE(args3.at(0).toLongLong(), sourceFile.size()); + QCOMPARE(args3.at(0).toLongLong(), qint64(sourceFile.size())); // after sending this, the QNAM should emit finished() connect(reply, SIGNAL(finished()), &QTestEventLoop::instance(), SLOT(exitLoop())); From b68f29720ad040ebeb4a7ed05676b6768fa4d49a Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Tue, 12 Apr 2011 17:09:20 +0200 Subject: [PATCH 12/26] QNAM: TODO comments for Qt 5 --- src/network/access/qnetworkreplyhttpimpl_p.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/network/access/qnetworkreplyhttpimpl_p.h b/src/network/access/qnetworkreplyhttpimpl_p.h index 2e8374af786..2040d4209be 100644 --- a/src/network/access/qnetworkreplyhttpimpl_p.h +++ b/src/network/access/qnetworkreplyhttpimpl_p.h @@ -96,8 +96,11 @@ public: #ifndef QT_NO_OPENSSL void ignoreSslErrors(); + // ### Qt5 Add proper virtual Q_INVOKABLE void ignoreSslErrorsImplementation(const QList &errors); + // ### Qt5 Add proper virtual Q_INVOKABLE void setSslConfigurationImplementation(const QSslConfiguration &configuration); + // ### Qt5 Add proper virtual Q_INVOKABLE QSslConfiguration sslConfigurationImplementation() const; #endif From 6d3044db9578d14b11dc7d156e2d81709c3d0ed7 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Tue, 12 Apr 2011 17:32:18 +0200 Subject: [PATCH 13/26] QNAM: Fix initialization order warnings --- src/network/access/qnetworkreplyhttpimpl.cpp | 50 +++++++++++++------- src/network/access/qnetworkreplyhttpimpl_p.h | 39 ++++++++------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index d240fb8ab23..fcfa1bb7eac 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -244,6 +244,7 @@ QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manage QNetworkReplyHttpImpl::~QNetworkReplyHttpImpl() { + // FIXME? } void QNetworkReplyHttpImpl::close() @@ -361,26 +362,41 @@ bool QNetworkReplyHttpImpl::canReadLine () const QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate() // FIXME order etc : QNetworkReplyPrivate() + + , manager(0) + , managerPrivate(0) + , synchronous(false) + + , state(Idle) + , statusCode(0) + + , outgoingData(0) + + , bytesUploaded(-1) + + + , cacheLoadDevice(0) + , loadingFromCache(false) + + , cacheSaveDevice(0) + , cacheEnabled(false) + + + , resumeOffset(0) + , preMigrationDownloaded(-1) + + , bytesDownloaded(0) + , lastBytesDownloaded(-1) + , downloadBufferReadPosition(0) + , downloadBufferCurrentSize(0) + , downloadBufferMaximumSize(0) + , downloadZerocopyBuffer(0) , pendingDownloadDataEmissions(new QAtomicInt()) , pendingDownloadProgressEmissions(new QAtomicInt()) - , loadingFromCache(false) -#ifndef QT_NO_OPENSSL - , pendingIgnoreAllSslErrors(false) -#endif - , resumeOffset(0) - , outgoingData(0), - cacheLoadDevice(0), - cacheEnabled(false), cacheSaveDevice(0), - // notificationHandlingPaused(false), - bytesDownloaded(0), lastBytesDownloaded(-1), bytesUploaded(-1), preMigrationDownloaded(-1), - //httpStatusCode(0), - state(Idle) - , downloadBufferReadPosition(0) - , downloadBufferCurrentSize(0) - , downloadBufferMaximumSize(0) - , downloadZerocopyBuffer(0) - , synchronous(false) + #ifndef QT_NO_OPENSSL + , pendingIgnoreAllSslErrors(false) + #endif { } diff --git a/src/network/access/qnetworkreplyhttpimpl_p.h b/src/network/access/qnetworkreplyhttpimpl_p.h index 2040d4209be..63ad2b97507 100644 --- a/src/network/access/qnetworkreplyhttpimpl_p.h +++ b/src/network/access/qnetworkreplyhttpimpl_p.h @@ -194,25 +194,30 @@ public: void redirectionRequested(const QUrl &target); + // incoming from user QNetworkAccessManager *manager; QNetworkAccessManagerPrivate *managerPrivate; QNetworkRequest request; - QNetworkAccessManager::Operation operation; + QNetworkAccessManager::Operation operation; // FIXME already in replyprivate? + QHttpNetworkRequest httpRequest; // There is also a copy in the HTTP thread + bool synchronous; + State state; + + // from http thread + int statusCode; + QString reasonPhrase; + + // upload QNonContiguousByteDevice* createUploadByteDevice(); QSharedPointer uploadByteDevice; QIODevice *outgoingData; QSharedPointer outgoingDataBuffer; void emitReplyUploadProgress(qint64 bytesSent, qint64 bytesTotal); // dup? - - - bool migrateBackend(); - quint64 resumeOffset; - bool canResume() const; - void setResumeOffset(quint64 offset); qint64 bytesUploaded; - qint64 preMigrationDownloaded; + + // cache void createCache(); void completeCacheSave(); void setCachingEnabled(bool enable); @@ -220,10 +225,11 @@ public: void initCacheSaveDevice(); QAbstractNetworkCache *networkCache() const; QIODevice *cacheLoadDevice; - bool cacheEnabled; // is this for saving? - QIODevice *cacheSaveDevice; bool loadingFromCache; + QIODevice *cacheSaveDevice; + bool cacheEnabled; // is this for saving? + QUrl urlForLastAuthentication; #ifndef QT_NO_NETWORKPROXY @@ -231,11 +237,12 @@ public: QList proxyList; #endif - int statusCode; - QString reasonPhrase; - - State state; + bool migrateBackend(); + bool canResume() const; + void setResumeOffset(quint64 offset); + quint64 resumeOffset; + qint64 preMigrationDownloaded; // Used for normal downloading. For "zero copy" the downloadZerocopyBuffer is used QByteDataBuffer downloadMultiBuffer; @@ -251,14 +258,10 @@ public: QSharedPointer downloadBufferPointer; char* downloadZerocopyBuffer; - - QHttpNetworkRequest httpRequest; // There is also a copy in the HTTP thread - // Will be increased by HTTP thread: QSharedPointer pendingDownloadDataEmissions; QSharedPointer pendingDownloadProgressEmissions; - bool synchronous; #ifndef QT_NO_OPENSSL QSslConfiguration sslConfiguration; From 04f2c4a7a7c65ff36ad60a26b193fa9d681af45f Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Wed, 13 Apr 2011 11:45:00 +0200 Subject: [PATCH 14/26] Revert "HTTP caching internals: fix logic for PreferNetwork and PreferCache" This reverts commit 11838861a23ace66307306cf9c96b3f1910e09a9. --- src/network/access/qnetworkreplyhttpimpl.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index fcfa1bb7eac..eb4adb8cd07 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -457,11 +457,6 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified)); if (CacheLoadControlAttribute == QNetworkRequest::PreferNetwork) { - // PreferNetwork == send request with "If-None-Match" and "If-Modified-Since" header, - // which will return a 304 Not Modifed if resource has not been changed. - // We might read from cache later, if receiving a 304. - return false; - } else if (CacheLoadControlAttribute == QNetworkRequest::PreferCache) { it = cacheHeaders.findRawHeader("Cache-Control"); if (it != cacheHeaders.rawHeaders.constEnd()) { QHash cacheControl = parseHttpOptionHeader(it->second); @@ -1443,7 +1438,7 @@ QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNe attributes.insert(QNetworkRequest::HttpStatusCodeAttribute, statusCode); attributes.insert(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase); } else { - // this is the server telling us the resource has not changed, keep the attributes intact + // this is a redirection, keep the attributes intact attributes = oldMetaData.attributes(); } metaData.setAttributes(attributes); From 5fdd72042587267dc1814f7fbebd3496cfa46b60 Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Wed, 13 Apr 2011 12:15:26 +0200 Subject: [PATCH 15/26] HTTP caching documentation: add some comments --- src/network/access/qnetworkrequest.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/network/access/qnetworkrequest.cpp b/src/network/access/qnetworkrequest.cpp index 338969a9096..a9a1e49572e 100644 --- a/src/network/access/qnetworkrequest.cpp +++ b/src/network/access/qnetworkrequest.cpp @@ -247,10 +247,13 @@ QT_BEGIN_NAMESPACE \value AlwaysNetwork always load from network and do not check if the cache has a valid entry (similar to the - "Reload" feature in browsers) + "Reload" feature in browsers); in addition, force intermediate + caches to re-validate. \value PreferNetwork default value; load from the network - if the cached entry is older than the network entry + if the cached entry is older than the network entry. This will never + return stale data from the cache, but revalidate resources that + have become stale. \value PreferCache load from cache if available, otherwise load from network. Note that this can return possibly From aef6239e7d7cff1ed162408998a897cf112be47e Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Wed, 13 Apr 2011 12:29:01 +0200 Subject: [PATCH 16/26] Revert "HTTP caching internals: fix logic for PreferNetwork and PreferCache" This reverts commit e5d27e7aeac984e46f3aa8de20160cc00fc63155. do not change the cache logic fundamentally. --- tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp b/tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp index 76e671103ec..3e6ad24e0b9 100644 --- a/tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp +++ b/tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp @@ -137,6 +137,7 @@ static bool AlwaysFalse = false; Q_DECLARE_METATYPE(QNetworkRequest::CacheLoadControl) + void tst_QAbstractNetworkCache::initTestCase() { #ifndef QT_NO_BEARERMANAGEMENT @@ -150,6 +151,7 @@ void tst_QAbstractNetworkCache::initTestCase() #endif } + void tst_QAbstractNetworkCache::expires_data() { QTest::addColumn("cacheLoadControl"); From ea4df8586e16efd82eb3ae5b71857c950cbb717f Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Wed, 13 Apr 2011 14:10:41 +0200 Subject: [PATCH 17/26] QNAM: Function re-ordering --- src/network/access/qnetworkreplyhttpimpl.cpp | 64 ++++++++++---------- src/network/access/qnetworkreplyhttpimpl_p.h | 11 ++-- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index eb4adb8cd07..5034176a3f2 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -359,6 +359,38 @@ bool QNetworkReplyHttpImpl::canReadLine () const return d->downloadMultiBuffer.canReadLine(); } +#ifndef QT_NO_OPENSSL +void QNetworkReplyHttpImpl::ignoreSslErrors() +{ + Q_D(QNetworkReplyHttpImpl); + + d->pendingIgnoreAllSslErrors = true; +} + +void QNetworkReplyHttpImpl::ignoreSslErrorsImplementation(const QList &errors) +{ + Q_D(QNetworkReplyHttpImpl); + + // the pending list is set if QNetworkReply::ignoreSslErrors(const QList &errors) + // is called before QNetworkAccessManager::get() (or post(), etc.) + d->pendingIgnoreSslErrorsList = errors; +} + +void QNetworkReplyHttpImpl::setSslConfigurationImplementation(const QSslConfiguration &newconfig) +{ + // Setting a SSL configuration on a reply is not supported. The user needs to set + // her/his QSslConfiguration on the QNetworkRequest. + Q_UNUSED(newconfig); +} + +QSslConfiguration QNetworkReplyHttpImpl::sslConfigurationImplementation() const +{ + Q_D(const QNetworkReplyHttpImpl); + qDebug() << "sslConfigurationImplementation"; + return d->sslConfiguration; +} +#endif + QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate() // FIXME order etc : QNetworkReplyPrivate() @@ -1265,38 +1297,6 @@ bool QNetworkReplyHttpImplPrivate::sendCacheContents(const QNetworkCacheMetaData return true; } -#ifndef QT_NO_OPENSSL -void QNetworkReplyHttpImpl::ignoreSslErrors() -{ - Q_D(QNetworkReplyHttpImpl); - - d->pendingIgnoreAllSslErrors = true; -} - -void QNetworkReplyHttpImpl::ignoreSslErrorsImplementation(const QList &errors) -{ - Q_D(QNetworkReplyHttpImpl); - - // the pending list is set if QNetworkReply::ignoreSslErrors(const QList &errors) - // is called before QNetworkAccessManager::get() (or post(), etc.) - d->pendingIgnoreSslErrorsList = errors; -} - -void QNetworkReplyHttpImpl::setSslConfigurationImplementation(const QSslConfiguration &newconfig) -{ - // Setting a SSL configuration on a reply is not supported. The user needs to set - // her/his QSslConfiguration on the QNetworkRequest. - Q_UNUSED(newconfig); -} - -QSslConfiguration QNetworkReplyHttpImpl::sslConfigurationImplementation() const -{ - Q_D(const QNetworkReplyHttpImpl); - qDebug() << "sslConfigurationImplementation"; - return d->sslConfiguration; -} -#endif - QNetworkCacheMetaData QNetworkReplyHttpImplPrivate::fetchCacheMetaData(const QNetworkCacheMetaData &oldMetaData) const { Q_Q(const QNetworkReplyHttpImpl); diff --git a/src/network/access/qnetworkreplyhttpimpl_p.h b/src/network/access/qnetworkreplyhttpimpl_p.h index 63ad2b97507..0420ffd3134 100644 --- a/src/network/access/qnetworkreplyhttpimpl_p.h +++ b/src/network/access/qnetworkreplyhttpimpl_p.h @@ -180,18 +180,14 @@ public: void _q_finished(); - // ? - void consume(qint64 count); - - void setDownloadBuffer(QSharedPointer sp, qint64 size); - char* getDownloadBuffer(qint64 size); - // FIXME void finished(); void error(QNetworkReply::NetworkError code, const QString &errorString); void _q_error(QNetworkReply::NetworkError code, const QString &errorString); void metaDataChanged(); + void redirectionRequested(const QUrl &target); + void checkForRedirect(const int statusCode); // incoming from user @@ -249,6 +245,8 @@ public: QByteDataBuffer pendingDownloadData; // For signal compression qint64 bytesDownloaded; qint64 lastBytesDownloaded; + void setDownloadBuffer(QSharedPointer sp, qint64 size); + char* getDownloadBuffer(qint64 size); // only used when the "zero copy" style is used. Else downloadMultiBuffer is used. // Please note that the whole "zero copy" download buffer API is private right now. Do not use it. @@ -279,7 +277,6 @@ public: void postRequest(); - void checkForRedirect(const int statusCode); public: // From HTTP thread: From 718657d8b24ebd8f9004f41e79e6bd79c385b316 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Wed, 13 Apr 2011 14:16:15 +0200 Subject: [PATCH 18/26] QNAM HTTP: Comments --- src/network/access/qnetworkreplyhttpimpl.cpp | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 5034176a3f2..7e6f2d4b5e4 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -244,7 +244,7 @@ QNetworkReplyHttpImpl::QNetworkReplyHttpImpl(QNetworkAccessManager* const manage QNetworkReplyHttpImpl::~QNetworkReplyHttpImpl() { - // FIXME? + // Most work is done in private destructor } void QNetworkReplyHttpImpl::close() @@ -264,35 +264,31 @@ void QNetworkReplyHttpImpl::abort() qint64 QNetworkReplyHttpImpl::bytesAvailable() const { Q_D(const QNetworkReplyHttpImpl); - qDebug() << "QNetworkReplyHttpImpl::bytesAvailable()"; - // FIXME cache device + // if we load from cache device if (d->cacheLoadDevice) { return QNetworkReply::bytesAvailable() + d->cacheLoadDevice->bytesAvailable() + d->downloadMultiBuffer.byteAmount(); } - // FIXME 0-copy buffer + // zerocopy buffer if (d->downloadZerocopyBuffer) { return QNetworkReply::bytesAvailable() + d->downloadBufferCurrentSize - d->downloadBufferReadPosition; } - // FIXME normal buffer - qDebug() << "QNetworkReplyHttpImpl::bytesAvailable() ==" << QNetworkReply::bytesAvailable() + d->downloadMultiBuffer.byteAmount(); + // normal buffer return QNetworkReply::bytesAvailable() + d->downloadMultiBuffer.byteAmount(); - - // FIXME } bool QNetworkReplyHttpImpl::isSequential () const { - // FIXME Maybe not for cache or 0-copy buffer + // FIXME In the cache of a cached load or the zero-copy buffer we could actually be non-sequential. + // FIXME however this requires us to implement stuff like seek() too. return true; } qint64 QNetworkReplyHttpImpl::size() const { - Q_D(const QNetworkReplyHttpImpl); - //return -1; + // FIXME At some point, this could return a proper value, e.g. if we're non-sequential. return QNetworkReply::size(); } From 9aa2feec0e579c7e1bfcf990aae07a82daaad647 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Wed, 13 Apr 2011 14:33:11 +0200 Subject: [PATCH 19/26] QNAM HTTP: Implement abort() and close() --- src/network/access/qnetworkreplyhttpimpl.cpp | 41 +++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 7e6f2d4b5e4..359fb563e5c 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -250,15 +250,40 @@ QNetworkReplyHttpImpl::~QNetworkReplyHttpImpl() void QNetworkReplyHttpImpl::close() { Q_D(QNetworkReplyHttpImpl); + + if (d->state == QNetworkReplyHttpImplPrivate::Aborted || + d->state == QNetworkReplyHttpImplPrivate::Finished) + return; + + // According to the documentation close only stops the download + // by closing we can ignore the download part and continue uploading. QNetworkReply::close(); - // FIXME + + // call finished which will emit signals + // FIXME shouldn't this be emitted Queued? + d->error(OperationCanceledError, tr("Operation canceled")); + d->finished(); } void QNetworkReplyHttpImpl::abort() { Q_D(QNetworkReplyHttpImpl); - QNetworkReply::close(); // FIXME + if (d->state == QNetworkReplyHttpImplPrivate::Finished || d->state == QNetworkReplyHttpImplPrivate::Aborted) + return; + + QNetworkReply::close(); + + if (d->state != QNetworkReplyHttpImplPrivate::Finished) { + // call finished which will emit signals + // FIXME shouldn't this be emitted Queued? + d->error(OperationCanceledError, tr("Operation canceled")); + d->finished(); + } + + d->state = QNetworkReplyHttpImplPrivate::Aborted; + + emit abortHttpRequest(); } qint64 QNetworkReplyHttpImpl::bytesAvailable() const @@ -935,6 +960,10 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) qDebug() << "QNetworkReplyHttpImplPrivate::replyDownloadData" << d.size(); + // If we're closed just ignore this data + if (!q->isOpen()) + return; + int pendingSignals = (int)pendingDownloadDataEmissions->fetchAndAddAcquire(-1) - 1; if (pendingSignals > 0) { @@ -959,10 +988,6 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) //writeDownstreamData(pendingDownloadDataCopy); // instead we do: - // We could be closed - if (!q->isOpen()) - return; - if (cacheEnabled && !cacheSaveDevice) { initCacheSaveDevice(); } @@ -1126,6 +1151,10 @@ void QNetworkReplyHttpImplPrivate::replyDownloadProgressSlot(qint64 bytesReceive { Q_Q(QNetworkReplyHttpImpl); + // If we're closed just ignore this data + if (!q->isOpen()) + return; + // we can be sure here that there is a download buffer int pendingSignals = (int)pendingDownloadProgressEmissions->fetchAndAddAcquire(-1) - 1; From 100b7565aafbd6d264e906eba77509891e607831 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Wed, 13 Apr 2011 15:11:09 +0200 Subject: [PATCH 20/26] QNAM HTTP: More comments --- src/network/access/qnetworkreplyhttpimpl.cpp | 37 ++++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 359fb563e5c..122c213f638 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -320,9 +320,8 @@ qint64 QNetworkReplyHttpImpl::size() const qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen) { Q_D(QNetworkReplyHttpImpl); - qDebug() << "QNetworkReplyHttpImpl::readData()" << maxlen; - // FIXME cacheload device + // cacheload device if (d->cacheLoadDevice) { // FIXME bytesdownloaded, position etc? @@ -336,9 +335,10 @@ qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen) return ret; } - // FIXME 0-copy buffer + // zerocopy buffer if (d->downloadZerocopyBuffer) { - // bla + // FIXME bytesdownloaded, position etc? + qint64 howMuch = qMin(maxlen, (d->downloadBufferCurrentSize - d->downloadBufferReadPosition)); memcpy(data, d->downloadZerocopyBuffer + d->downloadBufferReadPosition, howMuch); d->downloadBufferReadPosition += howMuch; @@ -346,12 +346,13 @@ qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen) } - // FIXME normal buffer - if (d->downloadMultiBuffer.isEmpty()) - return d->state == d->Finished ? -1 : 0; - // FIXME what about "Aborted" state? + // normal buffer + if (d->downloadMultiBuffer.isEmpty()) { + if (d->state == d->Finished || d->state == d->Aborted) + return -1; + return 0; + } - //d->backendNotify(QNetworkReplyImplPrivate::NotifyDownstreamReadyWrite); if (maxlen == 1) { // optimization for getChar() *data = d->downloadMultiBuffer.getChar(); @@ -364,7 +365,8 @@ qint64 QNetworkReplyHttpImpl::readData(char* data, qint64 maxlen) void QNetworkReplyHttpImpl::setReadBufferSize(qint64 size) { - return; // FIXME, unsupported right now + // FIXME, unsupported right now + return; } bool QNetworkReplyHttpImpl::canReadLine () const @@ -377,6 +379,8 @@ bool QNetworkReplyHttpImpl::canReadLine () const if (d->cacheLoadDevice) return d->cacheLoadDevice->canReadLine() || d->downloadMultiBuffer.canReadLine(); + // FIXME zerocopy buffer? + return d->downloadMultiBuffer.canReadLine(); } @@ -407,38 +411,25 @@ void QNetworkReplyHttpImpl::setSslConfigurationImplementation(const QSslConfigur QSslConfiguration QNetworkReplyHttpImpl::sslConfigurationImplementation() const { Q_D(const QNetworkReplyHttpImpl); - qDebug() << "sslConfigurationImplementation"; return d->sslConfiguration; } #endif QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate() -// FIXME order etc : QNetworkReplyPrivate() - , manager(0) , managerPrivate(0) , synchronous(false) - , state(Idle) - , statusCode(0) - , outgoingData(0) - , bytesUploaded(-1) - - , cacheLoadDevice(0) , loadingFromCache(false) - , cacheSaveDevice(0) , cacheEnabled(false) - - , resumeOffset(0) , preMigrationDownloaded(-1) - , bytesDownloaded(0) , lastBytesDownloaded(-1) , downloadBufferReadPosition(0) From 8e9aa019bb2183d0a8f56c3fdbaf829f0a4efff6 Mon Sep 17 00:00:00 2001 From: Markus Goetz Date: Thu, 14 Apr 2011 10:27:07 +0200 Subject: [PATCH 21/26] QNAM HTTP: Removes some comments --- src/network/access/qnetworkreplyhttpimpl.cpp | 21 ++++---------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 122c213f638..4b5d310cf0c 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -439,7 +439,7 @@ QNetworkReplyHttpImplPrivate::QNetworkReplyHttpImplPrivate() , pendingDownloadDataEmissions(new QAtomicInt()) , pendingDownloadProgressEmissions(new QAtomicInt()) #ifndef QT_NO_OPENSSL - , pendingIgnoreAllSslErrors(false) + , pendingIgnoreAllSslErrors(false) #endif { @@ -605,7 +605,6 @@ void QNetworkReplyHttpImplPrivate::postRequest() QThread *thread = 0; if (synchronous) { // A synchronous HTTP request uses its own thread - qDebug() << "sync!"; thread = new QThread(); QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); thread->start(); @@ -949,8 +948,6 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) { Q_Q(QNetworkReplyHttpImpl); - qDebug() << "QNetworkReplyHttpImplPrivate::replyDownloadData" << d.size(); - // If we're closed just ignore this data if (!q->isOpen()) return; @@ -975,10 +972,6 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) QByteDataBuffer pendingDownloadDataCopy = pendingDownloadData; pendingDownloadData.clear(); - // FIXME - //writeDownstreamData(pendingDownloadDataCopy); - // instead we do: - if (cacheEnabled && !cacheSaveDevice) { initCacheSaveDevice(); } @@ -998,23 +991,17 @@ void QNetworkReplyHttpImplPrivate::replyDownloadData(QByteArray d) bytesDownloaded += bytesWritten; lastBytesDownloaded = bytesDownloaded; - //appendDownstreamDataSignalEmissions(); - // instead: + QVariant totalSize = cookedHeaders.value(QNetworkRequest::ContentLengthHeader); if (preMigrationDownloaded != Q_INT64_C(-1)) totalSize = totalSize.toLongLong() + preMigrationDownloaded; -// pauseNotificationHandling(); - // important: At the point of this readyRead(), the data parameter list must be empty, - // else implicit sharing will trigger memcpy when the user is reading data! + emit q->readyRead(); // emit readyRead before downloadProgress incase this will cause events to be // processed and we get into a recursive call (as in QProgressDialog). emit q->downloadProgress(bytesDownloaded, totalSize.isNull() ? Q_INT64_C(-1) : totalSize.toLongLong()); -// resumeNotificationHandling(); - - } void QNetworkReplyHttpImplPrivate::replyFinished() @@ -1042,6 +1029,7 @@ void QNetworkReplyHttpImplPrivate::checkForRedirect(const int statusCode) QUrl url = QUrl::fromEncoded(header); if (!url.isValid()) url = QUrl(QLatin1String(header)); + // FIXME? //redirectionRequested(url); q->setAttribute(QNetworkRequest::RedirectionTargetAttribute, url); } @@ -1053,7 +1041,6 @@ void QNetworkReplyHttpImplPrivate::replyDownloadMetaData QSharedPointer db, qint64 contentLength) { - qDebug() << "QNetworkReplyHttpImplPrivate::replyDownloadMetaData" << contentLength << sc; Q_Q(QNetworkReplyHttpImpl); statusCode = sc; From 863de43dcad501a6c9675823217d0abf0269f802 Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Wed, 27 Apr 2011 11:48:53 +0200 Subject: [PATCH 22/26] QNetworkCookie: allow spaces in unquoted values We should follow http://tools.ietf.org/html/draft-ietf-httpstate-cookie-23 , which says parse the value until reaching the next ';' or the end of the line. Other cookie implementations allow spaces in unquoted values as well. Reviewed-by: Martin Petersson Task-number: QTBUG-18876 --- src/network/access/qnetworkcookie.cpp | 9 +++++---- tests/auto/qnetworkcookie/tst_qnetworkcookie.cpp | 8 ++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/network/access/qnetworkcookie.cpp b/src/network/access/qnetworkcookie.cpp index 52eb3453b87..eec850797b2 100644 --- a/src/network/access/qnetworkcookie.cpp +++ b/src/network/access/qnetworkcookie.cpp @@ -395,8 +395,8 @@ static QPair nextField(const QByteArray &text, int &posi // qdtext = > // quoted-pair = "\" CHAR - // If its NAME=VALUE, retain the value as is - // refer to ttp://bugreports.qt.nokia.com/browse/QTBUG-17746 + // If it is NAME=VALUE, retain the value as is + // refer to http://bugreports.qt.nokia.com/browse/QTBUG-17746 if (isNameValue) second += '"'; ++i; @@ -432,7 +432,9 @@ static QPair nextField(const QByteArray &text, int &posi position = i; for ( ; i < length; ++i) { register char c = text.at(i); - if (c == ',' || c == ';' || isLWS(c)) + // for name value pairs, we want to parse until reaching the next ';' + // and not break when reaching a space char + if (c == ',' || c == ';' || ((isNameValue && (c == '\n' || c == '\r')) || (!isNameValue && isLWS(c)))) break; } @@ -487,7 +489,6 @@ QByteArray QNetworkCookie::toRawForm(RawForm form) const result += '='; if ((d->value.contains(';') || d->value.contains(',') || - d->value.contains(' ') || d->value.contains('"')) && (!d->value.startsWith('"') && !d->value.endsWith('"'))) { diff --git a/tests/auto/qnetworkcookie/tst_qnetworkcookie.cpp b/tests/auto/qnetworkcookie/tst_qnetworkcookie.cpp index e0c477b2df3..9a58482de07 100644 --- a/tests/auto/qnetworkcookie/tst_qnetworkcookie.cpp +++ b/tests/auto/qnetworkcookie/tst_qnetworkcookie.cpp @@ -182,6 +182,14 @@ void tst_QNetworkCookie::parseSingleCookie_data() cookie.setValue("\"\\\"a, b; c\\\"\""); QTest::newRow("with-value-with-special5") << "a = \"\\\"a, b; c\\\"\"" << cookie; + cookie.setValue("b c"); + QTest::newRow("with-value-with-whitespace") << "a = b c" << cookie; + + cookie.setValue("\"b\""); + QTest::newRow("quoted-value") << "a = \"b\"" << cookie; + cookie.setValue("\"b c\""); + QTest::newRow("quoted-value-with-whitespace") << "a = \"b c\"" << cookie; + cookie.setValue("b"); cookie.setSecure(true); QTest::newRow("secure") << "a=b;secure" << cookie; From 0c9f327102d30fa3b37559469445da6f7de1f6ff Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Wed, 4 May 2011 14:33:21 +0200 Subject: [PATCH 23/26] HTTP reply: do not load resources from cache that must be revalidated The header field "Cache-Control: must-revalidate" is a strict requirement for loading the resource from the server, and not reading it from the cache without revalidating first. With this patch, PreferCache will load such from the network instead of loading them from the cache, and AlwaysCache will throw a ContentNotFound error. Reviewed-by: Markus Goetz Task-number: QTBUG-18983 --- src/network/access/qnetworkreplyhttpimpl.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/network/access/qnetworkreplyhttpimpl.cpp b/src/network/access/qnetworkreplyhttpimpl.cpp index 4b5d310cf0c..cd6d8fb87e0 100644 --- a/src/network/access/qnetworkreplyhttpimpl.cpp +++ b/src/network/access/qnetworkreplyhttpimpl.cpp @@ -500,13 +500,11 @@ bool QNetworkReplyHttpImplPrivate::loadFromCacheIfAllowed(QHttpNetworkRequest &h if (lastModified.isValid()) httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified)); - if (CacheLoadControlAttribute == QNetworkRequest::PreferNetwork) { - it = cacheHeaders.findRawHeader("Cache-Control"); - if (it != cacheHeaders.rawHeaders.constEnd()) { - QHash cacheControl = parseHttpOptionHeader(it->second); - if (cacheControl.contains("must-revalidate")) - return false; - } + it = cacheHeaders.findRawHeader("Cache-Control"); + if (it != cacheHeaders.rawHeaders.constEnd()) { + QHash cacheControl = parseHttpOptionHeader(it->second); + if (cacheControl.contains("must-revalidate")) + return false; } QDateTime currentDateTime = QDateTime::currentDateTime(); From 33ce5392e8e6ed2aa729e2268c1e58cdb1d32fad Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Wed, 4 May 2011 13:49:51 +0200 Subject: [PATCH 24/26] HTTP cache backend: do not load resources that must be revalidated The header field "Cache-Control: must-revalidate" is a strict requirement for loading the resource from the server, and not reading it from the cache without revalidating first. With this patch, PreferCache will load such from the network instead of loading them from the cache, and AlwaysCache will throw a ContentNotFound error. Reviewed-by: Markus Goetz Task-number: QTBUG-18983 --- src/network/access/qnetworkaccesscachebackend.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/network/access/qnetworkaccesscachebackend.cpp b/src/network/access/qnetworkaccesscachebackend.cpp index 13f4cd9cbb6..c585848c988 100644 --- a/src/network/access/qnetworkaccesscachebackend.cpp +++ b/src/network/access/qnetworkaccesscachebackend.cpp @@ -66,6 +66,7 @@ void QNetworkAccessCacheBackend::open() QString msg = QCoreApplication::translate("QNetworkAccessCacheBackend", "Error opening %1") .arg(this->url().toString()); error(QNetworkReply::ContentNotFoundError, msg); + } else { setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true); } finished(); @@ -85,14 +86,18 @@ bool QNetworkAccessCacheBackend::sendCacheContents() QNetworkCacheMetaData::AttributesMap attributes = item.attributes(); setAttribute(QNetworkRequest::HttpStatusCodeAttribute, attributes.value(QNetworkRequest::HttpStatusCodeAttribute)); setAttribute(QNetworkRequest::HttpReasonPhraseAttribute, attributes.value(QNetworkRequest::HttpReasonPhraseAttribute)); - setAttribute(QNetworkRequest::SourceIsFromCacheAttribute, true); // set the raw headers QNetworkCacheMetaData::RawHeaderList rawHeaders = item.rawHeaders(); QNetworkCacheMetaData::RawHeaderList::ConstIterator it = rawHeaders.constBegin(), end = rawHeaders.constEnd(); - for ( ; it != end; ++it) + for ( ; it != end; ++it) { + if (it->first.toLower() == "cache-control" && + it->second.toLower().contains("must-revalidate")) { + return false; + } setRawHeader(it->first, it->second); + } // handle a possible redirect QVariant redirectionTarget = attributes.value(QNetworkRequest::RedirectionTargetAttribute); From a4e3b2bd5ed8c80b7735c21aa406236340553dbe Mon Sep 17 00:00:00 2001 From: Peter Hartmann Date: Wed, 4 May 2011 13:50:40 +0200 Subject: [PATCH 25/26] HTTP auto tests: do not load resources from cache that must be revalidtd The header field "Cache-Control: must-revalidate" is a strict requirement for loading the resource from the server, and not reading it from the cache without revalidating first. With this patch, PreferCache will load such from the network instead of loading them from the cache, and AlwaysCache will throw a ContentNotFound error. Reviewed-by: Markus Goetz Task-number: QTBUG-18983 --- .../qabstractnetworkcache/tst_qabstractnetworkcache.cpp | 4 ++-- tests/auto/qnetworkreply/tst_qnetworkreply.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp b/tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp index 3e6ad24e0b9..4777f4e4688 100644 --- a/tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp +++ b/tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp @@ -263,14 +263,14 @@ void tst_QAbstractNetworkCache::cacheControl_data() QTest::newRow("200-2") << QNetworkRequest::AlwaysNetwork << "httpcachetest_cachecontrol.cgi?no-cache" << AlwaysFalse; QTest::newRow("200-3") << QNetworkRequest::PreferNetwork << "httpcachetest_cachecontrol.cgi?no-cache" << false; - QTest::newRow("200-4") << QNetworkRequest::AlwaysCache << "httpcachetest_cachecontrol.cgi?no-cache" << false;//AlwaysTrue; + QTest::newRow("200-4") << QNetworkRequest::AlwaysCache << "httpcachetest_cachecontrol.cgi?no-cache" << false; QTest::newRow("200-5") << QNetworkRequest::PreferCache << "httpcachetest_cachecontrol.cgi?no-cache" << false; QTest::newRow("304-0") << QNetworkRequest::PreferNetwork << "httpcachetest_cachecontrol.cgi?max-age=1000" << true; QTest::newRow("304-1") << QNetworkRequest::AlwaysNetwork << "httpcachetest_cachecontrol.cgi?max-age=1000, must-revalidate" << AlwaysFalse; QTest::newRow("304-2") << QNetworkRequest::PreferNetwork << "httpcachetest_cachecontrol.cgi?max-age=1000, must-revalidate" << true; - QTest::newRow("304-3") << QNetworkRequest::AlwaysCache << "httpcachetest_cachecontrol.cgi?max-age=1000, must-revalidate" << AlwaysTrue; + QTest::newRow("304-3") << QNetworkRequest::AlwaysCache << "httpcachetest_cachecontrol.cgi?max-age=1000, must-revalidate" << false; QTest::newRow("304-4") << QNetworkRequest::PreferCache << "httpcachetest_cachecontrol.cgi?max-age=1000, must-revalidate" << true; // see QTBUG-7060 diff --git a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp index 36bb2efdc32..f7365dfbfd5 100644 --- a/tests/auto/qnetworkreply/tst_qnetworkreply.cpp +++ b/tests/auto/qnetworkreply/tst_qnetworkreply.cpp @@ -3237,16 +3237,16 @@ void tst_QNetworkReply::ioGetFromHttpWithCache_data() QTest::newRow("must-revalidate,200,prefer-network") << reply200 << "Reloaded" << content << int(QNetworkRequest::PreferNetwork) << QStringList() << false << true; QTest::newRow("must-revalidate,200,prefer-cache") - << reply200 << "Not-reloaded" << content << int(QNetworkRequest::PreferCache) << QStringList() << true << false; + << reply200 << "Reloaded" << content << int(QNetworkRequest::PreferCache) << QStringList() << false << true; QTest::newRow("must-revalidate,200,always-cache") - << reply200 << "Not-reloaded" << content << int(QNetworkRequest::AlwaysCache) << QStringList() << true << false; + << reply200 << "" << content << int(QNetworkRequest::AlwaysCache) << QStringList() << false << false; QTest::newRow("must-revalidate,304,prefer-network") << reply304 << "Not-reloaded" << content << int(QNetworkRequest::PreferNetwork) << QStringList() << true << true; QTest::newRow("must-revalidate,304,prefer-cache") - << reply304 << "Not-reloaded" << content << int(QNetworkRequest::PreferCache) << QStringList() << true << false; + << reply304 << "Not-reloaded" << content << int(QNetworkRequest::PreferCache) << QStringList() << true << true; QTest::newRow("must-revalidate,304,always-cache") - << reply304 << "Not-reloaded" << content << int(QNetworkRequest::AlwaysCache) << QStringList() << true << false; + << reply304 << "" << content << int(QNetworkRequest::AlwaysCache) << QStringList() << false << false; // // Partial content From c3ad797d8af6a7f1cbd51553430393a4317a23b9 Mon Sep 17 00:00:00 2001 From: Taito Silvola Date: Thu, 5 May 2011 12:09:32 +0300 Subject: [PATCH 26/26] QSslConfiguration documentation fix setPeerVerifyMode() and peerVerifyMode() comments fix Merge-request: 1220 Reviewed-by: Peter Hartmann --- src/network/ssl/qsslconfiguration.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/network/ssl/qsslconfiguration.cpp b/src/network/ssl/qsslconfiguration.cpp index 70d7dd8df16..ab4d7f1d764 100644 --- a/src/network/ssl/qsslconfiguration.cpp +++ b/src/network/ssl/qsslconfiguration.cpp @@ -233,7 +233,7 @@ void QSslConfiguration::setProtocol(QSsl::SslProtocol protocol) client), and whether it should require that this certificate is valid. The default mode is AutoVerifyPeer, which tells QSslSocket to use - VerifyPeer for clients, QueryPeer for clients. + VerifyPeer for clients, QueryPeer for servers. \sa setPeerVerifyMode() */ @@ -249,7 +249,7 @@ QSslSocket::PeerVerifyMode QSslConfiguration::peerVerifyMode() const client), and whether it should require that this certificate is valid. The default mode is AutoVerifyPeer, which tells QSslSocket to use - VerifyPeer for clients, QueryPeer for clients. + VerifyPeer for clients, QueryPeer for servers. \sa peerVerifyMode() */