Add more verification when parsing http headers and add tests

Adding tests from QtWebSockets that will reuse QHttpHeaderParser

Task-number: QTBUG-80700
Change-Id: I76294a9156173314a3cf09160d0ca4e0d7c6ef3a
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
This commit is contained in:
Øystein Heskestad 2021-11-12 17:33:35 +01:00
parent d62e9d3c5b
commit 18aff2b424
2 changed files with 136 additions and 24 deletions

View File

@ -39,8 +39,16 @@
#include "qhttpheaderparser_p.h"
#include <algorithm>
QT_BEGIN_NAMESPACE
// both constants are taken from the default settings of Apache
// see: http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfieldsize and
// http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfields
static const int MAX_HEADER_FIELD_SIZE = 8 * 1024;
static const int MAX_HEADER_FIELDS = 100;
QHttpHeaderParser::QHttpHeaderParser()
: statusCode(100) // Required by tst_QHttpNetworkConnection::ignoresslerror(failure)
, majorVersion(0)
@ -57,36 +65,66 @@ void QHttpHeaderParser::clear()
fields.clear();
}
static bool fieldNameCheck(QByteArrayView name)
{
static constexpr QByteArrayView otherCharacters("!#$%&'*+-.^_`|~");
static const auto fieldNameChar = [](char c) {
return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9')
|| otherCharacters.contains(c);
};
return name.size() > 0 && std::all_of(name.begin(), name.end(), fieldNameChar);
}
bool QHttpHeaderParser::parseHeaders(QByteArrayView header)
{
// see rfc2616, sec 4 for information about HTTP/1.1 headers.
// allows relaxed parsing here, accepts both CRLF & LF line endings
int i = 0;
while (i < header.size()) {
int j = header.indexOf(':', i); // field-name
if (j == -1)
break;
QByteArrayView field = header.sliced(i, j - i).trimmed();
j++;
// any number of LWS is allowed before and after the value
QByteArray value;
do {
i = header.indexOf('\n', j);
if (i == -1)
break;
if (!value.isEmpty())
value += ' ';
// check if we have CRLF or only LF
bool hasCR = i && header[i - 1] == '\r';
int length = i - (hasCR ? 1: 0) - j;
value += header.sliced(j, length).trimmed();
j = ++i;
} while (i < header.size() && (header.at(i) == ' ' || header.at(i) == '\t'));
if (i == -1)
return false; // something is wrong
Q_ASSERT(fields.isEmpty());
const auto hSpaceStart = [](QByteArrayView h) {
return h.startsWith(' ') || h.startsWith('\t');
};
// Headers, if non-empty, start with a non-space and end with a newline:
if (hSpaceStart(header) || (header.size() && !header.endsWith('\n')))
return false;
fields.append(qMakePair(field.toByteArray(), value));
while (int tail = header.endsWith("\n\r\n") ? 2 : header.endsWith("\n\n") ? 1 : 0)
header.chop(tail);
QList<QPair<QByteArray, QByteArray>> result;
while (header.size()) {
const int colon = header.indexOf(':');
if (colon == -1) // if no colon check if empty headers
return result.size() == 0 && (header == "\n" || header == "\r\n");
if (result.size() >= MAX_HEADER_FIELDS)
return false;
QByteArrayView name = header.first(colon);
if (!fieldNameCheck(name))
return false;
header = header.sliced(colon + 1);
QByteArray value;
int valueSpace = MAX_HEADER_FIELD_SIZE - name.size() - 1;
do {
const int endLine = header.indexOf('\n');
Q_ASSERT(endLine != -1);
auto line = header.first(endLine); // includes space
valueSpace -= line.size() - (line.endsWith('\r') ? 1 : 0);
if (valueSpace < 0)
return false;
line = line.trimmed();
if (line.size()) {
if (value.size())
value += ' ' + line.toByteArray();
else
value = line.toByteArray();
}
header = header.sliced(endLine + 1);
} while (hSpaceStart(header));
Q_ASSERT(name.size() + 1 + value.size() <= MAX_HEADER_FIELD_SIZE);
result.append(qMakePair(name.toByteArray(), value));
}
fields = result;
return true;
}

View File

@ -40,6 +40,9 @@ private Q_SLOTS:
void parseHeader_data();
void parseHeader();
void parseHeaderVerification_data();
void parseHeaderVerification();
void parseEndOfHeader_data();
void parseEndOfHeader();
};
@ -50,6 +53,7 @@ void tst_QHttpNetworkReply::parseHeader_data()
QTest::addColumn<QStringList>("fields");
QTest::addColumn<QStringList>("values");
QTest::newRow("no-fields") << QByteArray("\r\n") << QStringList() << QStringList();
QTest::newRow("empty-field") << QByteArray("Set-Cookie: \r\n")
<< (QStringList() << "Set-Cookie")
<< (QStringList() << "");
@ -60,6 +64,9 @@ void tst_QHttpNetworkReply::parseHeader_data()
" charset=utf-8\r\n")
<< (QStringList() << "Content-Type")
<< (QStringList() << "text/html; charset=utf-8");
QTest::newRow("single-field-on-five-lines")
<< QByteArray("Name:\r\n first\r\n \r\n \r\n last\r\n") << (QStringList() << "Name")
<< (QStringList() << "first last");
QTest::newRow("multi-field") << QByteArray("Content-Type: text/html; charset=utf-8\r\n"
"Content-Length: 1024\r\n"
@ -101,6 +108,73 @@ void tst_QHttpNetworkReply::parseHeader()
}
}
// both constants are taken from the default settings of Apache
// see: http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfieldsize and
// http://httpd.apache.org/docs/2.2/mod/core.html#limitrequestfields
const int MAX_HEADER_FIELD_SIZE = 8 * 1024;
const int MAX_HEADER_FIELDS = 100;
void tst_QHttpNetworkReply::parseHeaderVerification_data()
{
QTest::addColumn<QByteArray>("headers");
QTest::addColumn<bool>("success");
QTest::newRow("no-header-fields") << QByteArray("\r\n") << true;
QTest::newRow("starting-with-space") << QByteArray(" Content-Encoding: gzip\r\n") << false;
QTest::newRow("starting-with-tab") << QByteArray("\tContent-Encoding: gzip\r\n") << false;
QTest::newRow("only-colon") << QByteArray(":\r\n") << false;
QTest::newRow("colon-and-value") << QByteArray(": only-value\r\n") << false;
QTest::newRow("name-with-space") << QByteArray("Content Length: 10\r\n") << false;
QTest::newRow("missing-colon-1") << QByteArray("Content-Encoding\r\n") << false;
QTest::newRow("missing-colon-2")
<< QByteArray("Content-Encoding\r\nContent-Length: 10\r\n") << false;
QTest::newRow("missing-colon-3")
<< QByteArray("Content-Encoding: gzip\r\nContent-Length\r\n") << false;
QTest::newRow("header-field-too-long")
<< (QByteArray("Content-Type: ") + QByteArray(MAX_HEADER_FIELD_SIZE, 'a')
+ QByteArray("\r\n"))
<< false;
QByteArray name = "Content-Type: ";
QTest::newRow("max-header-field-size")
<< (name + QByteArray(MAX_HEADER_FIELD_SIZE - name.size(), 'a') + QByteArray("\r\n"))
<< true;
QByteArray tooManyHeaders = QByteArray("Content-Type: text/html; charset=utf-8\r\n")
.repeated(MAX_HEADER_FIELDS + 1);
QTest::newRow("too-many-headers") << tooManyHeaders << false;
QByteArray maxHeaders =
QByteArray("Content-Type: text/html; charset=utf-8\r\n").repeated(MAX_HEADER_FIELDS);
QTest::newRow("max-headers") << maxHeaders << true;
QByteArray firstValue(MAX_HEADER_FIELD_SIZE / 2, 'a');
constexpr int obsFold = 1;
QTest::newRow("max-continuation-size")
<< (name + firstValue + QByteArray("\r\n ")
+ QByteArray(MAX_HEADER_FIELD_SIZE - name.size() - firstValue.size() - obsFold, 'b')
+ QByteArray("\r\n"))
<< true;
QTest::newRow("too-long-continuation-size")
<< (name + firstValue + QByteArray("\r\n ")
+ QByteArray(MAX_HEADER_FIELD_SIZE - name.size() - firstValue.size() - obsFold + 1,
'b')
+ QByteArray("\r\n"))
<< false;
}
void tst_QHttpNetworkReply::parseHeaderVerification()
{
QFETCH(QByteArray, headers);
QFETCH(bool, success);
QHttpNetworkReply reply;
reply.parseHeader(headers);
if (success && QByteArrayView(headers).trimmed().size())
QVERIFY(reply.header().size() > 0);
else
QCOMPARE(reply.header().size(), 0);
}
class TestHeaderSocket : public QAbstractSocket
{
public: