Fix 64-bit integer support in QtJSON

Fixes parsing writing and pass-through of integers with
higher precision than double can handle.

Note this adds extra precision compared to JavaScript, but the
JSON files read and written this way are still valid, and the extra
precision in reading and writing this way is used by many JSON
libraries.

Fixes: QTBUG-28560
Change-Id: I30b2415c928d1c34c8cb4e4c6218602095e7e8aa
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

src/corelib/global/qnumeric_p.h

@@ -202,7 +202,7 @@ namespace {
     This function works for v containing infinities, but not NaN. It's the
     caller's responsibility to exclude that possibility before calling it.
 */
-template <typename T> static inline bool convertDoubleTo(double v, T *value)
+template <typename T> static inline bool convertDoubleTo(double v, T *value, bool allow_precision_upgrade = true)
 {
     Q_STATIC_ASSERT(std::numeric_limits<T>::is_integer);
 
@@ -227,6 +227,10 @@ template <typename T> static inline bool convertDoubleTo(double v, T *value)
         supremum = -2.0 * std::numeric_limits<ST>::min();   // -2 * (-2^63) = 2^64, exact (for T = quint64)
         v = fabs(v);
     }
+    if (std::is_integral<T>::value && sizeof(T) > 4 && !allow_precision_upgrade) {
+        if (v > double(Q_INT64_C(1)<<53) || v < double(-((Q_INT64_C(1)<<53) + 1)))
+            return false;
+    }
 
     *value = std::numeric_limits<T>::max();
     if (v >= supremum)

src/corelib/serialization/qjsoncbor.cpp

@@ -262,8 +262,7 @@ QJsonValue qt_convertToJson(QCborContainerPrivate *d, qsizetype idx)
     const auto &e = d->elements.at(idx);
     switch (e.type) {
     case QCborValue::Integer:
-        return QJsonPrivate::Value::fromTrustedCbor(e.value);
+        return QJsonValue(e.value);
-
     case QCborValue::ByteArray:
     case QCborValue::String:
     case QCborValue::SimpleType:
@@ -370,7 +369,7 @@ QJsonValue QCborValue::toJsonValue() const
         return false;
 
     case Integer:
-        return QJsonPrivate::Value::fromTrustedCbor(n);
+        return QJsonPrivate::Value::fromTrustedCbor(*this);
 
     case True:
         return true;
@@ -615,11 +614,9 @@ QCborValue QCborValue::fromJsonValue(const QJsonValue &v)
     case QJsonValue::Bool:
         return v.toBool();
     case QJsonValue::Double: {
-        qint64 i;
+        if (v.t == Integer)
-        const double dbl = v.toDouble();
+            return v.toInteger();
-        if (convertDoubleTo(dbl, &i))
+        return v.toDouble();
-            return i;
-        return dbl;
     }
     case QJsonValue::String:
         return v.toString();
@@ -667,9 +664,7 @@ static void appendVariant(QCborContainerPrivate *d, const QVariant &variant)
       \row  \li \c bool                     \li Bool
       \row  \li \c std::nullptr_t           \li Null
       \row  \li \c short, \c ushort, \c int, \c uint, \l qint64  \li Integer
-      \row  \li \l quint64                  \li Integer, but they are cast to \c qint64 first so
+      \row  \li \l quint64                  \li Integer, or Double if outside the range of qint64
-                                                values higher than 2\sup{63}-1 (\c INT64_MAX) will
-                                                be wrapped to negative
       \row  \li \c float, \c double         \li Double
       \row  \li \l QByteArray               \li ByteArray
       \row  \li \l QDateTime                \li DateTime
@@ -713,9 +708,12 @@ QCborValue QCborValue::fromVariant(const QVariant &variant)
     case QMetaType::UShort:
     case QVariant::Int:
     case QVariant::LongLong:
-    case QVariant::ULongLong:
     case QVariant::UInt:
         return variant.toLongLong();
+    case QVariant::ULongLong:
+        if (variant.toULongLong() <= static_cast<uint64_t>(std::numeric_limits<qint64>::max()))
+            return variant.toLongLong();
+        Q_FALLTHROUGH();
     case QMetaType::Float:
     case QVariant::Double:
         return variant.toDouble();

src/corelib/serialization/qjsonparser.cpp

@@ -709,10 +709,11 @@ bool Parser::parseNumber()
 
     // frac = decimal-point 1*DIGIT
     if (json < end && *json == '.') {
-        isInt = false;
         ++json;
-        while (json < end && *json >= '0' && *json <= '9')
+        while (json < end && *json >= '0' && *json <= '9') {
+            isInt = isInt && *json == '0';
             ++json;
+        }
     }
 
     // exp = e [ minus / plus ] 1*DIGIT

src/corelib/serialization/qjsonvalue.cpp

@@ -154,7 +154,9 @@ QJsonValue::QJsonValue(bool b)
 QJsonValue::QJsonValue(double v)
     : d(nullptr)
 {
-    if (convertDoubleTo(v, &n)) {
+    // Convert to integer if the number is an integer and changing wouldn't
+    // introduce additional digit precision not present in the double.
+    if (convertDoubleTo<qint64>(v, &n, false /* allow_precision_upgrade */)) {
         t = QCborValue::Integer;
     } else {
         memcpy(&n, &v, sizeof(n));
@@ -449,12 +451,18 @@ QJsonValue QJsonValue::fromVariant(const QVariant &variant)
         return QJsonValue(Null);
     case QVariant::Bool:
         return QJsonValue(variant.toBool());
+    case QMetaType::Short:
+    case QMetaType::UShort:
     case QVariant::Int:
+    case QVariant::UInt:
+    case QVariant::LongLong:
+        return QJsonValue(variant.toLongLong());
+    case QVariant::ULongLong:
+        if (variant.toULongLong() <= static_cast<uint64_t>(std::numeric_limits<qint64>::max()))
+            return QJsonValue(variant.toLongLong());
+        Q_FALLTHROUGH();
     case QMetaType::Float:
     case QVariant::Double:
-    case QVariant::LongLong:
-    case QVariant::ULongLong:
-    case QVariant::UInt:
         return QJsonValue(variant.toDouble());
     case QVariant::String:
         return QJsonValue(variant.toString());
@@ -504,7 +512,7 @@ QJsonValue QJsonValue::fromVariant(const QVariant &variant)
 
     \value Null     QMetaType::Nullptr
     \value Bool     QMetaType::Bool
-    \value Double   QMetaType::Double
+    \value Double   QMetaType::Double or QMetaType::LongLong
     \value String   QString
     \value Array    QVariantList
     \value Object   QVariantMap
@@ -520,6 +528,7 @@ QVariant QJsonValue::toVariant() const
     case QCborValue::False:
         return false;
     case QCborValue::Integer:
+        return toInteger();
     case QCborValue::Double:
         return toDouble();
     case QCborValue::String:
@@ -548,7 +557,8 @@ QVariant QJsonValue::toVariant() const
 
     \value Null     A Null value
     \value Bool     A boolean value. Use toBool() to convert to a bool.
-    \value Double   A double. Use toDouble() to convert to a double.
+    \value Double   A number value. Use toDouble() to convert to a double,
+                    or toInteger() to convert to a qint64.
     \value String   A string. Use toString() to convert to a QString.
     \value Array    An array. Use toArray() to convert to a QJsonArray.
     \value Object   An object. Use toObject() to convert to a QJsonObject.
@@ -613,18 +623,43 @@ int QJsonValue::toInt(int defaultValue) const
 {
     switch (t) {
     case QCborValue::Double: {
-        const double dbl = toDouble();
         int dblInt;
-        convertDoubleTo<int>(dbl, &dblInt);
+        if (convertDoubleTo<int>(toDouble(), &dblInt))
-        return dbl == dblInt ? dblInt : defaultValue;
+            return dblInt;
+        break;
     }
     case QCborValue::Integer:
-        return (n <= qint64(std::numeric_limits<int>::max())
+        if (qint64(int(n)) == n)
-                && n >= qint64(std::numeric_limits<int>::min()))
+            return int(n);
-                ? n : defaultValue;
+        break;
     default:
-        return defaultValue;
+        break;
     }
+    return defaultValue;
+}
+
+/*!
+    \since 6.0
+    Converts the value to an integer and returns it.
+
+    If type() is not Double or the value is not a whole number
+    representable as qint64, the \a defaultValue will be returned.
+ */
+qint64 QJsonValue::toInteger(qint64 defaultValue) const
+{
+    switch (t) {
+    case QCborValue::Integer:
+        return n;
+    case QCborValue::Double: {
+        qint64 dblInt;
+        if (convertDoubleTo<qint64>(toDouble(), &dblInt))
+            return dblInt;
+        break;
+    }
+    default:
+        break;
+    }
+    return defaultValue;
 }
 
 /*!
@@ -641,7 +676,7 @@ double QJsonValue::toDouble(double defaultValue) const
         return d;
     }
     case QCborValue::Integer:
-        return n;
+        return double(n);
     default:
         return defaultValue;
     }
@@ -787,8 +822,13 @@ const QJsonValue QJsonValue::operator[](int i) const
  */
 bool QJsonValue::operator==(const QJsonValue &other) const
 {
-    if (t != other.t)
+    if (t != other.t) {
+        if (isDouble() && other.isDouble()) {
+            // One value Cbor integer, one Cbor double, should interact as doubles.
+            return toDouble() == other.toDouble();
+        }
         return false;
+    }
 
     switch (t) {
     case QCborValue::Undefined:
@@ -929,32 +969,38 @@ uint qHash(const QJsonValue &value, uint seed)
 QDebug operator<<(QDebug dbg, const QJsonValue &o)
 {
     QDebugStateSaver saver(dbg);
-    switch (o.type()) {
+    switch (o.t) {
-    case QJsonValue::Undefined:
+    case QCborValue::Undefined:
         dbg << "QJsonValue(undefined)";
         break;
-    case QJsonValue::Null:
+    case QCborValue::Null:
         dbg << "QJsonValue(null)";
         break;
-    case QJsonValue::Bool:
+    case QCborValue::True:
+    case QCborValue::False:
         dbg.nospace() << "QJsonValue(bool, " << o.toBool() << ')';
         break;
-    case QJsonValue::Double:
+    case QCborValue::Integer:
+        dbg.nospace() << "QJsonValue(double, " << o.toInteger() << ')';
+        break;
+    case QCborValue::Double:
         dbg.nospace() << "QJsonValue(double, " << o.toDouble() << ')';
         break;
-    case QJsonValue::String:
+    case QCborValue::String:
         dbg.nospace() << "QJsonValue(string, " << o.toString() << ')';
         break;
-    case QJsonValue::Array:
+    case QCborValue::Array:
         dbg.nospace() << "QJsonValue(array, ";
         dbg << o.toArray();
         dbg << ')';
         break;
-    case QJsonValue::Object:
+    case QCborValue::Map:
         dbg.nospace() << "QJsonValue(object, ";
         dbg << o.toObject();
         dbg << ')';
         break;
+    default:
+        Q_UNREACHABLE();
     }
     return dbg;
 }

src/corelib/serialization/qjsonvalue.h

@@ -112,6 +112,7 @@ public:
 
     bool toBool(bool defaultValue = false) const;
     int toInt(int defaultValue = 0) const;
+    qint64 toInteger(qint64 defaultValue = 0) const;
     double toDouble(double defaultValue = 0) const;
     QString toString() const;
     QString toString(const QString &defaultValue) const;

src/corelib/serialization/qjsonwriter.cpp

@@ -138,15 +138,14 @@ static void valueToJson(const QCborValue &v, QByteArray &json, int indent, bool
         json += "false";
         break;
     case QCborValue::Integer:
+        json += QByteArray::number(v.toInteger());
+        break;
     case QCborValue::Double: {
         const double d = v.toDouble();
-        if (qIsFinite(d)) {
+        if (qIsFinite(d))
-            quint64 absInt;
+            json += QByteArray::number(d, 'g', QLocale::FloatingPointShortest);
-            json += QByteArray::number(d, convertDoubleTo(std::abs(d), &absInt) ? 'f' : 'g',
+        else
-                                       QLocale::FloatingPointShortest);
-        } else {
             json += "null"; // +INF || -INF || NaN (see RFC4627#section2.4)
-        }
         break;
     }
     case QCborValue::String:

tests/auto/corelib/serialization/json/tst_qtjson.cpp

@@ -52,6 +52,8 @@ private Q_SLOTS:
     void testNumbers_3();
     void testNumbers_4();
 
+    void testNumberComparisons();
+
     void testObjectSimple();
     void testObjectSmallKeys();
     void testArraySimple();
@@ -218,26 +220,32 @@ void tst_QtJson::testNumbers()
     {
         int numbers[] = {
             0,
-            -1,
             1,
+            2,
+            -1,
+            -2,
+            (1<<25),
             (1<<26),
             (1<<27),
             (1<<28),
+            -(1<<25),
             -(1<<26),
             -(1<<27),
             -(1<<28),
             (1<<26) - 1,
             (1<<27) - 1,
             (1<<28) - 1,
+            (1<<29) - 1,
             -((1<<26) - 1),
             -((1<<27) - 1),
-            -((1<<28) - 1)
+            -((1<<28) - 1),
+            -((1<<29) - 1)
         };
         int n = sizeof(numbers)/sizeof(int);
 
         QJsonArray array;
         for (int i = 0; i < n; ++i)
-            array.append((double)numbers[i]);
+            array.append(numbers[i]);
 
         QByteArray serialized = QJsonDocument(array).toJson();
         QJsonDocument json = QJsonDocument::fromJson(serialized);
@@ -246,8 +254,10 @@ void tst_QtJson::testNumbers()
         QCOMPARE(array.size(), array2.size());
         for (int i = 0; i < array.size(); ++i) {
             QCOMPARE(array.at(i).type(), QJsonValue::Double);
+            QCOMPARE(array.at(i).toInt(), numbers[i]);
             QCOMPARE(array.at(i).toDouble(), (double)numbers[i]);
             QCOMPARE(array2.at(i).type(), QJsonValue::Double);
+            QCOMPARE(array2.at(i).toInt(), numbers[i]);
             QCOMPARE(array2.at(i).toDouble(), (double)numbers[i]);
         }
     }
@@ -255,8 +265,10 @@ void tst_QtJson::testNumbers()
     {
         qint64 numbers[] = {
             0,
-            -1,
             1,
+            2,
+            -1,
+            -2,
             (1ll<<54),
             (1ll<<55),
             (1ll<<56),
@@ -266,15 +278,21 @@ void tst_QtJson::testNumbers()
             (1ll<<54) - 1,
             (1ll<<55) - 1,
             (1ll<<56) - 1,
+            (1ll<<57) - 1,
+            (1ll<<58) - 1,
+            (1ll<<59) + 1001,
             -((1ll<<54) - 1),
             -((1ll<<55) - 1),
-            -((1ll<<56) - 1)
+            -((1ll<<56) - 1),
+            -((1ll<<57) - 1),
+            -((1ll<<58) - 1),
+            -((1ll<<59) + 1001),
         };
         int n = sizeof(numbers)/sizeof(qint64);
 
         QJsonArray array;
         for (int i = 0; i < n; ++i)
-            array.append((double)numbers[i]);
+            array.append(QJsonValue(numbers[i]));
 
         QByteArray serialized = QJsonDocument(array).toJson();
         QJsonDocument json = QJsonDocument::fromJson(serialized);
@@ -283,8 +301,10 @@ void tst_QtJson::testNumbers()
         QCOMPARE(array.size(), array2.size());
         for (int i = 0; i < array.size(); ++i) {
             QCOMPARE(array.at(i).type(), QJsonValue::Double);
+            QCOMPARE(array.at(i).toInteger(), numbers[i]);
             QCOMPARE(array.at(i).toDouble(), (double)numbers[i]);
             QCOMPARE(array2.at(i).type(), QJsonValue::Double);
+            QCOMPARE(array2.at(i).toInteger(), numbers[i]);
             QCOMPARE(array2.at(i).toDouble(), (double)numbers[i]);
         }
     }
@@ -422,6 +442,46 @@ void tst_QtJson::testNumbers_4()
             "    -18446744073709552000\n"
             "]\n";
     QCOMPARE(json, expected);
+
+    QJsonArray array2;
+    array2 << QJsonValue(Q_INT64_C(+1000000000000000));
+    array2 << QJsonValue(Q_INT64_C(-1000000000000000));
+    array2 << QJsonValue(Q_INT64_C(+9007199254740992));
+    array2 << QJsonValue(Q_INT64_C(-9007199254740992));
+    array2 << QJsonValue(Q_INT64_C(+9223372036854775807));
+    array2 << QJsonValue(Q_INT64_C(-9223372036854775807));
+    const QByteArray json2(QJsonDocument(array2).toJson());
+    const QByteArray expected2 =
+            "[\n"
+            "    1000000000000000,\n"
+            "    -1000000000000000,\n"
+            "    9007199254740992,\n"
+            "    -9007199254740992,\n"
+            "    9223372036854775807,\n"
+            "    -9223372036854775807\n"
+            "]\n";
+    QCOMPARE(json2, expected2);
+}
+
+void tst_QtJson::testNumberComparisons()
+{
+    // QJsonValues created using doubles only have double precision
+    QJsonValue llMinDbl(-9223372036854775807.0);
+    QJsonValue llMinPlus1Dbl(-9223372036854775806.0);
+    QCOMPARE(llMinDbl == llMinPlus1Dbl, -9223372036854775807.0 == -9223372036854775806.0); // true
+
+    // QJsonValues created using qint64 have full qint64 precision
+    QJsonValue llMin(Q_INT64_C(-9223372036854775807));
+    QJsonValue llMinPlus1(Q_INT64_C(-9223372036854775806));
+    QCOMPARE(llMin == llMinPlus1, Q_INT64_C(-9223372036854775807) == Q_INT64_C(-9223372036854775806)); // false
+
+    // The different storage formats should be able to compare as their C++ versions (all true)
+    QCOMPARE(llMin == llMinDbl, Q_INT64_C(-9223372036854775807) == -9223372036854775807.0);
+    QCOMPARE(llMinDbl == llMin, -9223372036854775807.0 == Q_INT64_C(-9223372036854775807));
+    QCOMPARE(llMinPlus1 == llMinPlus1Dbl, Q_INT64_C(-9223372036854775806) == -9223372036854775806.0);
+    QCOMPARE(llMinPlus1Dbl == llMinPlus1, -9223372036854775806.0 == Q_INT64_C(-9223372036854775806));
+    QCOMPARE(llMinPlus1 == llMinDbl, Q_INT64_C(-9223372036854775806) == -9223372036854775807.0);
+    QCOMPARE(llMinPlus1Dbl == llMin, -9223372036854775806.0 == Q_INT64_C(-9223372036854775807));
 }
 
 void tst_QtJson::testObjectSimple()
@@ -1160,8 +1220,8 @@ void tst_QtJson::fromVariant_data()
     bool boolValue = true;
     int intValue = -1;
     uint uintValue = 1;
-    long long longlongValue = -2;
+    qlonglong longlongValue = -2;
-    unsigned long long ulonglongValue = 2;
+    qulonglong ulonglongValue = 2;
     float floatValue = 3.3f;
     double doubleValue = 4.4;
     QString stringValue("str");
@@ -1207,7 +1267,7 @@ void tst_QtJson::fromVariant_data()
     QTest::newRow("nullptr") << QVariant::fromValue(nullptr) <<  QJsonValue(QJsonValue::Null);
     QTest::newRow("bool") << QVariant(boolValue) <<  QJsonValue(boolValue);
     QTest::newRow("int") << QVariant(intValue) <<  QJsonValue(intValue);
-    QTest::newRow("uint") << QVariant(uintValue) <<  QJsonValue(static_cast<double>(uintValue));
+    QTest::newRow("uint") << QVariant(uintValue) <<  QJsonValue(static_cast<qint64>(uintValue));
     QTest::newRow("longlong") << QVariant(longlongValue) <<  QJsonValue(longlongValue);
     QTest::newRow("ulonglong") << QVariant(ulonglongValue) <<  QJsonValue(static_cast<double>(ulonglongValue));
     QTest::newRow("float") << QVariant(floatValue) <<  QJsonValue(floatValue);

tests/auto/corelib/serialization/qcborvalue_json/tst_qcborvalue_json.cpp

@@ -97,9 +97,12 @@ void tst_QCborValue_Json::toVariant_data()
     add(1, 1, 1);
     add(-1, -1, -1);
     add(0., 0., 0.);
+    add(2., 2., 2.);
     add(1.25, 1.25, 1.25);
     add(-1.25, -1.25, -1.25);
     add("Hello", "Hello", "Hello");
+    add(std::numeric_limits<qint64>::max(), std::numeric_limits<qint64>::max(), std::numeric_limits<qint64>::max());
+    add(std::numeric_limits<qint64>::min(), std::numeric_limits<qint64>::min(), std::numeric_limits<qint64>::min());
 
     // converts to string in JSON:
     add(QByteArray("Hello"), QByteArray("Hello"), "SGVsbG8");
@@ -123,14 +126,6 @@ void tst_QCborValue_Json::toVariant_data()
                                 << QVariant(qQNaN())
                                 << QJsonValue();
 
-    // large integral values lose precision in JSON
-    QTest::newRow("Integer:max") << QCborValue(std::numeric_limits<qint64>::max())
-                                 << QVariant(std::numeric_limits<qint64>::max())
-                                 << QJsonValue(std::numeric_limits<qint64>::max());
-    QTest::newRow("Integer:min") << QCborValue(std::numeric_limits<qint64>::min())
-                                 << QVariant(std::numeric_limits<qint64>::min())
-                                 << QJsonValue(std::numeric_limits<qint64>::min());
-
     // empty arrays and maps
     add(QCborArray(), QVariantList(), QJsonArray());
     add(QCborMap(), QVariantMap(), QJsonObject());
@@ -257,6 +252,10 @@ void tst_QCborValue_Json::fromJson_data()
     QTest::newRow("0") << QCborValue(0) << QJsonValue(0.);
     QTest::newRow("1") << QCborValue(1) << QJsonValue(1);
     QTest::newRow("1.5") << QCborValue(1.5) << QJsonValue(1.5);
+    QTest::newRow("Integer:max") << QCborValue(std::numeric_limits<qint64>::max())
+                                 << QJsonValue(std::numeric_limits<qint64>::max());
+    QTest::newRow("Integer:min") << QCborValue(std::numeric_limits<qint64>::min())
+                                 << QJsonValue(std::numeric_limits<qint64>::min());
     QTest::newRow("string") << QCborValue("Hello") << QJsonValue("Hello");
     QTest::newRow("array") << QCborValue(QCborValue::Array) << QJsonValue(QJsonValue::Array);
     QTest::newRow("map") << QCborValue(QCborValue::Map) << QJsonValue(QJsonValue::Object);