1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Modify the baseline testing framework to show new test items

Browse Source 
If a commit adds new baseline test items, they would not be reported
to the server when doing a LanceBot test on that commit, and so not be
visible in the report either. This commit fixes the client side of
that issue; the server is already updated.

Change-Id: I60c2958def5e7b54ddc2789cf5727edcbb1a90e1
Reviewed-by: Eskil Abrahamsen Blomfeldt <eskil.abrahamsen-blomfeldt@qt.io>
Reviewed-by: Jonas Karlsson <jonas.karlsson@qt.io>
This commit is contained in:
Eirik Aavitsland 2024-01-31 13:45:02 +01:00
parent ec38f0002d
commit 930535fe30
1 changed files with 11 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
tests/baseline/shared/qbaselinetest.cpp
View File

@ -249,6 +249,7 @@ void modifyImage(QImage *img)
bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, bool *error) bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, bool *error)
{ {
*error = false;
ImageItem item = baseline; ImageItem item = baseline;
if (simfail) { if (simfail) {
// Simulate test failure by forcing image mismatch; for testing purposes // Simulate test failure by forcing image mismatch; for testing purposes
@ -259,6 +260,7 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
} else { } else {
item.image = img; item.image = img;
} }
bool isNewItem = false;
item.imageChecksums.clear(); item.imageChecksums.clear();
item.imageChecksums.prepend(ImageItem::computeChecksum(item.image)); item.imageChecksums.prepend(ImageItem::computeChecksum(item.image));
QByteArray srvMsg; QByteArray srvMsg;
@ -270,9 +272,11 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
return true; return true;
break; break;
case ImageItem::BaselineNotFound: case ImageItem::BaselineNotFound:
if (!customInfo.overrides().isEmpty() || baselinePolicy == UploadNone) { if (!customInfo.overrides().isEmpty())
qWarning() << "Cannot compare to baseline: No such baseline found on server.";
return true; return true;
if (baselinePolicy == UploadNone) {
isNewItem = true;
break;
} }
if (proto.submitNewBaseline(item, &srvMsg)) if (proto.submitNewBaseline(item, &srvMsg))
qDebug() << msg->constData() << "Baseline not found on server. New baseline uploaded."; qDebug() << msg->constData() << "Baseline not found on server. New baseline uploaded.";
@ -285,7 +289,6 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
return true; return true;
break; break;
} }
*error = false;
// The actual comparison of the given image with the baseline: // The actual comparison of the given image with the baseline:
if (baseline.imageChecksums.contains(item.imageChecksums.at(0))) { if (baseline.imageChecksums.contains(item.imageChecksums.at(0))) {
if (!proto.submitMatch(item, &srvMsg)) if (!proto.submitMatch(item, &srvMsg))
@ -306,7 +309,11 @@ bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg,
qInfo() << "Baseline server reports:" << srvMsg; qInfo() << "Baseline server reports:" << srvMsg;
return true; // The server decides: a fuzzy match means no mismatch return true; // The server decides: a fuzzy match means no mismatch
} }
*msg += "Mismatch. See report:\n " + srvMsg; if (isNewItem)
*msg += "No baseline on server, so cannot compare.";
else
*msg += "Mismatch.";
*msg += " See report:\n " + srvMsg;
if (dryRunMode) { if (dryRunMode) {
qDebug() << "Dryrun, so ignoring" << *msg; qDebug() << "Dryrun, so ignoring" << *msg;
return true; return true;