1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

QBenchlib: add support for a measurer reporting multiple results

Browse Source 
Implemented for the Linux Perf measurer, with four measurements by
default.

RESULT : tst_MyClass::QString_toInt():
     149.574444 CPU cycles per iteration (total: 149,574,445, iterations: 1000000)
RESULT : tst_MyClass::QString_toInt():
     620.000181 instructions per iteration (total: 620,000,182, iterations: 1000000)
RESULT : tst_MyClass::QString_toInt():
     131.000046 branch instructions per iteration (total: 131,000,047, iterations: 1000000)
RESULT : tst_MyClass::QString_toInt():
     32.118771 nsecs per iteration (total: 32,118,771, iterations: 1000000)

Change-Id: I3c79b7e08fa346988dfefffd17202cda3df8431b
Reviewed-by: Fabian Kosmale <fabian.kosmale@qt.io>
This commit is contained in:
Thiago Macieira 2022-10-21 12:39:37 -07:00
parent b5b00e7790
commit 4731baf6d3
12 changed files with 131 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/testlib/qbenchmark.cpp
View File

@ -96,10 +96,13 @@ int QBenchmarkTestMethodData::adjustIterationCount(int suggestion)
return iterationCount;
}
void QBenchmarkTestMethodData::setResult(QBenchmarkMeasurerBase::Measurement m,
bool setByMacro)
void QBenchmarkTestMethodData::setResults(const QList<QBenchmarkMeasurerBase::Measurement> &list,
bool setByMacro)
{
bool accepted = false;
QBenchmarkMeasurerBase::Measurement firstMeasurement = {};
if (!list.isEmpty())
firstMeasurement = list.constFirst();
// Always accept the result if the iteration count has been
// specified on the command line with -iterations.
@ -114,9 +117,9 @@ void QBenchmarkTestMethodData::setResult(QBenchmarkMeasurerBase::Measurement m,
// Test the result directly without calling the measurer if the minimum time
// has been specified on the command line with -minimumvalue.
else if (QBenchmarkGlobalData::current->walltimeMinimum != -1)
accepted = (m.value > QBenchmarkGlobalData::current->walltimeMinimum);
accepted = (firstMeasurement.value > QBenchmarkGlobalData::current->walltimeMinimum);
else
accepted = QBenchmarkGlobalData::current->measurer->isMeasurementAccepted(m);
accepted = QBenchmarkGlobalData::current->measurer->isMeasurementAccepted(firstMeasurement);
// Accept the result or double the number of iterations.
if (accepted)
@ -124,8 +127,10 @@ void QBenchmarkTestMethodData::setResult(QBenchmarkMeasurerBase::Measurement m,
else
iterationCount *= 2;
this->result = QBenchmarkResult(QBenchmarkGlobalData::current->context, m,
iterationCount, setByMacro);
valid = true;
results.reserve(list.size());
for (auto m : list)
results.emplaceBack(QBenchmarkGlobalData::current->context, m, iterationCount, setByMacro);
}
/*!
@ -157,8 +162,7 @@ QTest::QBenchmarkIterationController::QBenchmarkIterationController()
*/
QTest::QBenchmarkIterationController::~QBenchmarkIterationController()
{
QBenchmarkMeasurerBase::Measurement measurement = QTest::endBenchmarkMeasurement();
QBenchmarkTestMethodData::current->setResult(measurement);
QBenchmarkTestMethodData::current->setResults(QTest::endBenchmarkMeasurement());
}
/*! \internal
@ -209,7 +213,7 @@ void QTest::beginBenchmarkMeasurement()
/*! \internal
*/
QBenchmarkMeasurerBase::Measurement QTest::endBenchmarkMeasurement()
QList<QBenchmarkMeasurerBase::Measurement> QTest::endBenchmarkMeasurement()
{
// the clock is ticking before the line below, don't add code here.
return QBenchmarkGlobalData::current->measurer->stop();

13
src/testlib/qbenchmark_p.h
View File

@ -64,7 +64,6 @@ public:
QBenchmarkMeasurerBase::Measurement measurement = { -1, QTest::FramesPerSecond };
int iterations = -1;
bool setByMacro = true;
bool valid = false;
QBenchmarkResult() = default;
@ -75,7 +74,6 @@ public:
, measurement(m)
, iterations(iterations)
, setByMacro(setByMacro)
, valid(true)
{ }
bool operator<(const QBenchmarkResult &other) const
@ -134,12 +132,15 @@ public:
void beginDataRun();
void endDataRun();
bool isBenchmark() const { return result.valid; }
bool isBenchmark() const { return valid; }
bool resultsAccepted() const { return resultAccepted; }
int adjustIterationCount(int suggestion);
void setResult(QBenchmarkMeasurerBase::Measurement m, bool setByMacro = true);
void setResults(const QList<QBenchmarkMeasurerBase::Measurement> &m, bool setByMacro = true);
void setResult(QBenchmarkMeasurerBase::Measurement m, bool setByMacro = true)
{ setResults({ m }, setByMacro); }
QBenchmarkResult result;
QList<QBenchmarkResult> results;
bool valid = false;
bool resultAccepted = false;
bool runOnce = false;
int iterationCount = -1;
@ -153,7 +154,7 @@ namespace QTest
void setIterationCount(int count);
void beginBenchmarkMeasurement();
QBenchmarkMeasurerBase::Measurement endBenchmarkMeasurement();
QList<QBenchmarkMeasurerBase::Measurement> endBenchmarkMeasurement();
}
QT_END_NAMESPACE

4
src/testlib/qbenchmarkevent.cpp
View File

@ -18,10 +18,10 @@ void QBenchmarkEvent::start()
QAbstractEventDispatcher::instance()->installNativeEventFilter(this);
}
QBenchmarkMeasurerBase::Measurement QBenchmarkEvent::stop()
QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkEvent::stop()
{
QAbstractEventDispatcher::instance()->removeNativeEventFilter(this);
return { qreal(eventCounter), QTest::Events };
return { { qreal(eventCounter), QTest::Events } };
}
// It's very tempting to simply reject a measurement if 0 events

2
src/testlib/qbenchmarkevent_p.h
View File

@ -28,7 +28,7 @@ public:
QBenchmarkEvent();
~QBenchmarkEvent();
void start() override;
Measurement stop() override;
QList<Measurement> stop() override;
bool isMeasurementAccepted(Measurement measurement) override;
int adjustIterationCount(int suggestion) override;
int adjustMedianCount(int suggestion) override;

8
src/testlib/qbenchmarkmeasurement.cpp
View File

@ -16,9 +16,9 @@ void QBenchmarkTimeMeasurer::start()
time.start();
}
QBenchmarkMeasurerBase::Measurement QBenchmarkTimeMeasurer::stop()
QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkTimeMeasurer::stop()
{
return { qreal(time.elapsed()), QTest::WalltimeMilliseconds };
return { { qreal(time.elapsed()), QTest::WalltimeMilliseconds } };
}
bool QBenchmarkTimeMeasurer::isMeasurementAccepted(Measurement measurement)
@ -48,10 +48,10 @@ void QBenchmarkTickMeasurer::start()
startTicks = getticks();
}
QBenchmarkMeasurerBase::Measurement QBenchmarkTickMeasurer::stop()
QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkTickMeasurer::stop()
{
CycleCounterTicks now = getticks();
return { elapsed(now, startTicks), QTest::CPUTicks };
return { { elapsed(now, startTicks), QTest::CPUTicks } };
}
bool QBenchmarkTickMeasurer::isMeasurementAccepted(QBenchmarkMeasurerBase::Measurement)

3
src/testlib/qbenchmarkmeasurement_p.h
View File

@ -16,6 +16,7 @@
//
#include <QtTest/qbenchmark.h>
#include <QtCore/qlist.h>
#include <QtCore/private/qglobal_p.h>
QT_BEGIN_NAMESPACE
@ -31,7 +32,7 @@ public:
virtual ~QBenchmarkMeasurerBase() = default;
virtual void init() {}
virtual void start() = 0;
virtual Measurement stop() = 0;
virtual QList<Measurement> stop() = 0;
virtual bool isMeasurementAccepted(Measurement m) = 0;
virtual int adjustIterationCount(int suggestion) = 0;
virtual int adjustMedianCount(int suggestion) = 0;

93
src/testlib/qbenchmarkperfevents.cpp
View File

@ -47,7 +47,13 @@
QT_BEGIN_NAMESPACE
struct PerfEvent
{
quint32 type;
quint64 config;
};
static perf_event_attr attr;
Q_GLOBAL_STATIC(QList<PerfEvent>, eventTypes);
static void initPerf()
{
@ -62,14 +68,20 @@ static void initPerf()
attr.inherit_stat = true; // aggregate all the info from child processes
attr.task = true; // trace fork/exits
// set a default performance counter: CPU cycles
attr.type = PERF_TYPE_HARDWARE;
attr.config = PERF_COUNT_HW_CPU_CYCLES; // default
done = true;
}
}
static QList<PerfEvent> defaultCounters()
{
return {
{ .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_TASK_CLOCK },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS },
};
}
// This class does not exist in the API so it's qdoc comment marker was removed.
/*
@ -383,11 +395,11 @@ static const Events eventlist[] = {
};
/* -- END GENERATED CODE -- */
QTest::QBenchmarkMetric QBenchmarkPerfEventsMeasurer::metricForEvent(quint32 type, quint64 event_id)
static QTest::QBenchmarkMetric metricForEvent(PerfEvent counter)
{
const Events *ptr = eventlist;
for ( ; ptr->type != PERF_TYPE_MAX; ++ptr) {
if (ptr->type == type && ptr->event_id == event_id)
if (ptr->type == counter.type && ptr->event_id == counter.config)
return ptr->metric;
}
return QTest::Events;
@ -396,6 +408,7 @@ QTest::QBenchmarkMetric QBenchmarkPerfEventsMeasurer::metricForEvent(quint32 typ
void QBenchmarkPerfEventsMeasurer::setCounter(const char *name)
{
initPerf();
eventTypes->clear();
const char *colon = strchr(name, ':');
int n = colon ? colon - name : strlen(name);
const Events *ptr = eventlist;
@ -409,8 +422,7 @@ void QBenchmarkPerfEventsMeasurer::setCounter(const char *name)
}
}
attr.type = ptr->type;
attr.config = ptr->event_id;
*eventTypes = { { ptr->type, ptr->event_id } };
// We used to support attributes, but our code was the opposite of what
// perf(1) does, plus QBenchlib isn't exactly expected to be used to
@ -441,7 +453,8 @@ QBenchmarkPerfEventsMeasurer::QBenchmarkPerfEventsMeasurer() = default;
QBenchmarkPerfEventsMeasurer::~QBenchmarkPerfEventsMeasurer()
{
qt_safe_close(fd);
for (int fd : std::as_const(fds))
qt_safe_close(fd);
}
void QBenchmarkPerfEventsMeasurer::init()
@ -451,34 +464,54 @@ void QBenchmarkPerfEventsMeasurer::init()
void QBenchmarkPerfEventsMeasurer::start()
{
initPerf();
if (fd == -1) {
QList<PerfEvent> &counters = *eventTypes;
if (counters.isEmpty())
counters = defaultCounters();
if (fds.isEmpty()) {
pid_t pid = 0; // attach to the current process only
int cpu = -1; // on any CPU
int group_fd = -1;
int flags = PERF_FLAG_FD_CLOEXEC;
fd = perf_event_open(&attr, pid, cpu, group_fd, flags);
if (fd == -1) {
// probably a paranoid kernel (/proc/sys/kernel/perf_event_paranoid)
attr.exclude_kernel = true;
attr.exclude_hv = true;
fd = perf_event_open(&attr, pid, cpu, group_fd, flags);
}
if (fd == -1) {
perror("QBenchmarkPerfEventsMeasurer::start: perf_event_open");
exit(1);
fds.reserve(counters.size());
for (PerfEvent counter : std::as_const(counters)) {
attr.type = counter.type;
attr.config = counter.config;
int fd = perf_event_open(&attr, pid, cpu, group_fd, flags);
if (fd == -1) {
// probably a paranoid kernel (/proc/sys/kernel/perf_event_paranoid)
attr.exclude_kernel = true;
attr.exclude_hv = true;
fd = perf_event_open(&attr, pid, cpu, group_fd, flags);
}
if (fd == -1) {
perror("QBenchmarkPerfEventsMeasurer::start: perf_event_open");
exit(1);
}
fds.append(fd);
}
}
// enable the counter
::ioctl(fd, PERF_EVENT_IOC_RESET);
::ioctl(fd, PERF_EVENT_IOC_ENABLE);
// enable the counters
for (int fd : std::as_const(fds))
::ioctl(fd, PERF_EVENT_IOC_RESET);
for (int fd : std::as_const(fds))
::ioctl(fd, PERF_EVENT_IOC_ENABLE);
}
QBenchmarkMeasurerBase::Measurement QBenchmarkPerfEventsMeasurer::stop()
QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkPerfEventsMeasurer::stop()
{
// disable the counter
::ioctl(fd, PERF_EVENT_IOC_DISABLE);
return readValue();
// disable the counters
for (int fd : std::as_const(fds))
::ioctl(fd, PERF_EVENT_IOC_DISABLE);
const QList<PerfEvent> &counters = *eventTypes;
QList<Measurement> result(counters.size(), {});
for (qsizetype i = 0; i < counters.size(); ++i) {
result[i] = readValue(i);
}
return result;
}
bool QBenchmarkPerfEventsMeasurer::isMeasurementAccepted(Measurement)
@ -531,10 +564,10 @@ static quint64 rawReadValue(int fd)
return results.value * (double(results.time_running) / double(results.time_enabled));
}
QBenchmarkMeasurerBase::Measurement QBenchmarkPerfEventsMeasurer::readValue()
QBenchmarkMeasurerBase::Measurement QBenchmarkPerfEventsMeasurer::readValue(qsizetype idx)
{
quint64 raw = rawReadValue(fd);
return { qreal(qint64(raw)), metricForEvent(attr.type, attr.config) };
quint64 raw = rawReadValue(fds.at(idx));
return { qreal(qint64(raw)), metricForEvent(eventTypes->at(idx)) };
}
QT_END_NAMESPACE

7
src/testlib/qbenchmarkperfevents_p.h
View File

@ -26,20 +26,19 @@ public:
~QBenchmarkPerfEventsMeasurer();
void init() override;
void start() override;
Measurement stop() override;
QList<Measurement> stop() override;
bool isMeasurementAccepted(Measurement measurement) override;
int adjustIterationCount(int suggestion) override;
int adjustMedianCount(int suggestion) override;
bool needsWarmupIteration() override { return true; }
static bool isAvailable();
static QTest::QBenchmarkMetric metricForEvent(quint32 type, quint64 event_id);
static void setCounter(const char *name);
static void listCounters();
private:
int fd = -1;
QList<int> fds;
Measurement readValue();
Measurement readValue(qsizetype idx = 0);
};
QT_END_NAMESPACE

4
src/testlib/qbenchmarktimemeasurers_p.h
View File

@ -25,7 +25,7 @@ class QBenchmarkTimeMeasurer : public QBenchmarkMeasurerBase
{
public:
void start() override;
Measurement stop() override;
QList<Measurement> stop() override;
bool isMeasurementAccepted(Measurement measurement) override;
int adjustIterationCount(int sugestion) override;
int adjustMedianCount(int suggestion) override;
@ -40,7 +40,7 @@ class QBenchmarkTickMeasurer : public QBenchmarkMeasurerBase
{
public:
void start() override;
Measurement stop() override;
QList<Measurement> stop() override;
bool isMeasurementAccepted(Measurement measurement) override;
int adjustIterationCount(int) override;
int adjustMedianCount(int suggestion) override;

4
src/testlib/qbenchmarkvalgrind.cpp
View File

@ -170,11 +170,11 @@ void QBenchmarkCallgrindMeasurer::start()
CALLGRIND_ZERO_STATS;
}
QBenchmarkMeasurerBase::Measurement QBenchmarkCallgrindMeasurer::stop()
QList<QBenchmarkMeasurerBase::Measurement> QBenchmarkCallgrindMeasurer::stop()
{
CALLGRIND_DUMP_STATS;
const qint64 result = QBenchmarkValgrindUtils::extractLastResult();
return { qreal(result), QTest::InstructionReads };
return { { qreal(result), QTest::InstructionReads } };
}
bool QBenchmarkCallgrindMeasurer::isMeasurementAccepted(Measurement measurement)

2
src/testlib/qbenchmarkvalgrind_p.h
View File

@ -41,7 +41,7 @@ class QBenchmarkCallgrindMeasurer : public QBenchmarkMeasurerBase
{
public:
void start() override;
Measurement stop() override;
QList<Measurement> stop() override;
bool isMeasurementAccepted(Measurement measurement) override;
int adjustIterationCount(int) override;
int adjustMedianCount(int) override;

54
src/testlib/qtestcase.cpp
View File

@ -1067,17 +1067,20 @@ Q_TESTLIB_EXPORT void qtest_qParseArgs(int argc, char *argv[], bool qml) {
qtest_qParseArgs(argc, const_cast<const char *const *>(argv), qml);
}
QBenchmarkResult qMedian(const QList<QBenchmarkResult> &container)
static QList<QBenchmarkResult> qMedian(const QList<QList<QBenchmarkResult>> &container)
{
const int count = container.size();
if (count == 0)
return QBenchmarkResult();
return {};
if (count == 1)
return container.front();
QList<QBenchmarkResult> containerCopy = container;
std::sort(containerCopy.begin(), containerCopy.end());
QList<QList<QBenchmarkResult>> containerCopy = container;
std::sort(containerCopy.begin(), containerCopy.end(),
[](const QList<QBenchmarkResult> &a, const QList<QBenchmarkResult> &b) {
return a.first() < b.first();
});
const int middle = count / 2;
@ -1104,7 +1107,7 @@ void TestMethods::invokeTestOnData(int index) const
bool isBenchmark = false;
int i = (QBenchmarkGlobalData::current->measurer->needsWarmupIteration()) ? -1 : 0;
QList<QBenchmarkResult> results;
QList<QList<QBenchmarkResult>> resultsList;
bool minimumTotalReached = false;
do {
QBenchmarkTestMethodData::current->beginDataRun();
@ -1121,8 +1124,9 @@ void TestMethods::invokeTestOnData(int index) const
const bool initQuit =
QTestResult::skipCurrentTest() || QTestResult::currentTestFailed();
if (!initQuit) {
QBenchmarkTestMethodData::current->result = QBenchmarkResult();
QBenchmarkTestMethodData::current->results.clear();
QBenchmarkTestMethodData::current->resultAccepted = false;
QBenchmarkTestMethodData::current->valid = false;
QBenchmarkGlobalData::current->context.tag = QLatin1StringView(
QTestResult::currentDataTag() ? QTestResult::currentDataTag() : "");
@ -1164,29 +1168,29 @@ void TestMethods::invokeTestOnData(int index) const
QBenchmarkTestMethodData::current->endDataRun();
if (!QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed()) {
if (i > -1) // iteration -1 is the warmup iteration.
results.append(QBenchmarkTestMethodData::current->result);
resultsList.append(QBenchmarkTestMethodData::current->results);
if (isBenchmark && QBenchmarkGlobalData::current->verboseOutput) {
if (i == -1) {
QTestLog::info(qPrintable(
QString::fromLatin1("warmup stage result : %1")
.arg(QBenchmarkTestMethodData::current->result.measurement.value)), nullptr, 0);
} else {
QTestLog::info(qPrintable(
QString::fromLatin1("accumulation stage result: %1")
.arg(QBenchmarkTestMethodData::current->result.measurement.value)), nullptr, 0);
}
if (isBenchmark && QBenchmarkGlobalData::current->verboseOutput &&
!QBenchmarkTestMethodData::current->results.isEmpty()) {
// we only print the first result
const QBenchmarkResult &first = QBenchmarkTestMethodData::current->results.constFirst();
QString pattern = i < 0 ? "warmup stage result : %1"_L1
: "accumulation stage result: %1"_L1;
QTestLog::info(qPrintable(pattern.arg(first.measurement.value)), nullptr, 0);
}
}
// Verify if the minimum total measurement is reached, if it was specified:
// Verify if the minimum total measurement (for the first measurement)
// was reached, if it was specified:
if (QBenchmarkGlobalData::current->minimumTotal == -1) {
minimumTotalReached = true;
} else {
auto addResult = [](qreal current, const QBenchmarkResult& r) {
return current + r.measurement.value;
auto addResult = [](qreal current, const QList<QBenchmarkResult> &r) {
if (!r.isEmpty())
current += r.first().measurement.value;
return current;
};
const qreal total = std::accumulate(results.begin(), results.end(), 0.0, addResult);
const qreal total = std::accumulate(resultsList.begin(), resultsList.end(), 0.0, addResult);
minimumTotalReached = (total >= QBenchmarkGlobalData::current->minimumTotal);
}
} while (isBenchmark
@ -1198,8 +1202,12 @@ void TestMethods::invokeTestOnData(int index) const
bool testPassed = !QTestResult::skipCurrentTest() && !QTestResult::currentTestFailed();
QTestResult::finishedCurrentTestDataCleanup();
// Only report benchmark figures if the test passed
if (testPassed && QBenchmarkTestMethodData::current->resultsAccepted())
QTestLog::addBenchmarkResult(qMedian(results));
if (testPassed && QBenchmarkTestMethodData::current->resultsAccepted()) {
const QList<QBenchmarkResult> median = qMedian(resultsList);
for (auto m : median) {
QTestLog::addBenchmarkResult(m);
}
}
}
}