From e309f0b013987013e910f9b1e260a6b85250b4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20K=C3=B6hne?= Date: Tue, 28 May 2024 18:16:50 +0200 Subject: [PATCH] Core: Move SipHash implementation into separate file Isolate the code inspired by the SipHash reference C implementation in a separate file. This makes it clearer what code is available under the CC0 license, and which not. [ChangeLog][Third-Party Code] Adapted copyright information for the SipHash Algorithm (used in Qt Core). Change-Id: I9b8fc27a4e791c0f1ccbdfa6244d4fa47c7a219b Reviewed-by: Thiago Macieira (cherry picked from commit da2d3e914c1b3f9da17c40502c8e7c1463d35612) Reviewed-by: Qt Cherry-pick Bot --- .../siphash}/qt_attribution.json | 8 +- src/3rdparty/siphash/siphash.cpp | 262 ++++++++++++++++++ src/corelib/tools/LICENSE.siphash | 116 -------- src/corelib/tools/qhash.cpp | 245 +--------------- 4 files changed, 271 insertions(+), 360 deletions(-) rename src/{corelib/tools => 3rdparty/siphash}/qt_attribution.json (66%) create mode 100644 src/3rdparty/siphash/siphash.cpp delete mode 100644 src/corelib/tools/LICENSE.siphash diff --git a/src/corelib/tools/qt_attribution.json b/src/3rdparty/siphash/qt_attribution.json similarity index 66% rename from src/corelib/tools/qt_attribution.json rename to src/3rdparty/siphash/qt_attribution.json index 928ff537ca3..9c74513e499 100644 --- a/src/corelib/tools/qt_attribution.json +++ b/src/3rdparty/siphash/qt_attribution.json @@ -3,6 +3,7 @@ "Name": "SipHash Algorithm", "QDocModule": "qtcore", "QtUsage": "Used in Qt Core (QHash)", + "Files": "siphash.cpp", "Description": "Implements the SipHash algorithm.", "Homepage": "https://131002.net/siphash/", @@ -10,6 +11,9 @@ "License": "Creative Commons Zero v1.0 Universal", "LicenseId": "CC0-1.0", - "LicenseFile": "LICENSE.siphash", - "Copyright": "(C) 2012-2014 Jean-Philippe Aumasson, (C) 2012-2014 Daniel J. Bernstein " + "Copyright": [ + "Copyright (C) 2012-2014 Jean-Philippe Aumasson", + "Copyright (C) 2012-2014 Daniel J. Bernstein ", + "Copyright (C) 2016 Intel Corporation" + ] } diff --git a/src/3rdparty/siphash/siphash.cpp b/src/3rdparty/siphash/siphash.cpp new file mode 100644 index 00000000000..63002c788a1 --- /dev/null +++ b/src/3rdparty/siphash/siphash.cpp @@ -0,0 +1,262 @@ +// Copyright (C) 2012-2014 Jean-Philippe Aumasson +// Copyright (C) 2012-2014 Daniel J. Bernstein +// Copyright (C) 2016 Intel Corporation. +// SPDX-License-Identifier: CC0-1.0 + +#include +#include +#include + +#ifdef Q_CC_GNU +# define DECL_HOT_FUNCTION __attribute__((hot)) +#else +# define DECL_HOT_FUNCTION +#endif + +#include + +QT_USE_NAMESPACE + +namespace { + +// This is an inlined version of the SipHash implementation that is +// trying to avoid some memcpy's from uint64 to uint8[] and back. + +#define ROTL(x, b) (((x) << (b)) | ((x) >> (sizeof(x) * 8 - (b)))) + +#define SIPROUND \ +do { \ + v0 += v1; \ + v1 = ROTL(v1, 13); \ + v1 ^= v0; \ + v0 = ROTL(v0, 32); \ + v2 += v3; \ + v3 = ROTL(v3, 16); \ + v3 ^= v2; \ + v0 += v3; \ + v3 = ROTL(v3, 21); \ + v3 ^= v0; \ + v2 += v1; \ + v1 = ROTL(v1, 17); \ + v1 ^= v2; \ + v2 = ROTL(v2, 32); \ +} while (0) + +template struct SipHash64 +{ + /* "somepseudorandomlygeneratedbytes" */ + uint64_t v0 = 0x736f6d6570736575ULL; + uint64_t v1 = 0x646f72616e646f6dULL; + uint64_t v2 = 0x6c7967656e657261ULL; + uint64_t v3 = 0x7465646279746573ULL; + uint64_t b; + uint64_t k0; + uint64_t k1; + + inline SipHash64(uint64_t fulllen, uint64_t seed, uint64_t seed2); + inline void addBlock(const uint8_t *in, size_t inlen); + inline uint64_t finalize(const uint8_t *in, size_t left); +}; + +template +SipHash64::SipHash64(uint64_t inlen, uint64_t seed, uint64_t seed2) +{ + b = inlen << 56; + k0 = seed; + k1 = seed2; + v3 ^= k1; + v2 ^= k0; + v1 ^= k1; + v0 ^= k0; +} + +template DECL_HOT_FUNCTION void +SipHash64::addBlock(const uint8_t *in, size_t inlen) +{ + Q_ASSERT((inlen & 7ULL) == 0); + int i; + const uint8_t *end = in + inlen; + for (; in != end; in += 8) { + uint64_t m = qFromUnaligned(in); + v3 ^= m; + + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= m; + } +} + +template DECL_HOT_FUNCTION uint64_t +SipHash64::finalize(const uint8_t *in, size_t left) +{ + int i; + switch (left) { + case 7: + b |= ((uint64_t)in[6]) << 48; + Q_FALLTHROUGH(); + case 6: + b |= ((uint64_t)in[5]) << 40; + Q_FALLTHROUGH(); + case 5: + b |= ((uint64_t)in[4]) << 32; + Q_FALLTHROUGH(); + case 4: + b |= ((uint64_t)in[3]) << 24; + Q_FALLTHROUGH(); + case 3: + b |= ((uint64_t)in[2]) << 16; + Q_FALLTHROUGH(); + case 2: + b |= ((uint64_t)in[1]) << 8; + Q_FALLTHROUGH(); + case 1: + b |= ((uint64_t)in[0]); + break; + case 0: + break; + } + + v3 ^= b; + + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= b; + + v2 ^= 0xff; + + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + return b; +} +#undef SIPROUND + +// This is a "SipHash" implementation adopted for 32bit platforms. It performs +// basically the same operations as the 64bit version using 4 byte at a time +// instead of 8. +// +// To make this work, we also need to change the constants for the mixing +// rotations in ROTL. We're simply using half of the 64bit constants, rounded up +// for odd numbers. +// +// For the v0-v4 constants, simply use the first four bytes of the 64 bit versions. +// + +#define SIPROUND \ +do { \ + v0 += v1; \ + v1 = ROTL(v1, 7); \ + v1 ^= v0; \ + v0 = ROTL(v0, 16); \ + v2 += v3; \ + v3 = ROTL(v3, 8); \ + v3 ^= v2; \ + v0 += v3; \ + v3 = ROTL(v3, 11); \ + v3 ^= v0; \ + v2 += v1; \ + v1 = ROTL(v1, 9); \ + v1 ^= v2; \ + v2 = ROTL(v2, 16); \ +} while (0) + +template struct SipHash32 +{ + /* "somepseudorandomlygeneratedbytes" */ + uint v0 = 0x736f6d65U; + uint v1 = 0x646f7261U; + uint v2 = 0x6c796765U; + uint v3 = 0x74656462U; + uint b; + uint k0; + uint k1; + + inline SipHash32(size_t fulllen, uint seed, uint seed2); + inline void addBlock(const uint8_t *in, size_t inlen); + inline uint finalize(const uint8_t *in, size_t left); +}; + +template inline + SipHash32::SipHash32(size_t inlen, uint seed, uint seed2) +{ + uint k0 = seed; + uint k1 = seed2; + b = inlen << 24; + v3 ^= k1; + v2 ^= k0; + v1 ^= k1; + v0 ^= k0; +} + +template inline DECL_HOT_FUNCTION void +SipHash32::addBlock(const uint8_t *in, size_t inlen) +{ + Q_ASSERT((inlen & 3ULL) == 0); + int i; + const uint8_t *end = in + inlen; + for (; in != end; in += 4) { + uint m = qFromUnaligned(in); + v3 ^= m; + + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= m; + } +} + +template inline DECL_HOT_FUNCTION uint +SipHash32::finalize(const uint8_t *in, size_t left) +{ + int i; + switch (left) { + case 3: + b |= ((uint)in[2]) << 16; + Q_FALLTHROUGH(); + case 2: + b |= ((uint)in[1]) << 8; + Q_FALLTHROUGH(); + case 1: + b |= ((uint)in[0]); + break; + case 0: + break; + } + + v3 ^= b; + + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= b; + + v2 ^= 0xff; + + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + return b; +} +#undef SIPROUND +#undef ROTL + +// Use SipHash-1-2, which has similar performance characteristics as +// Qt 4's hash implementation, instead of the SipHash-2-4 default +template +using SipHash = std::conditional_t, SipHash32>; + +} // unnamed namespace + +Q_NEVER_INLINE DECL_HOT_FUNCTION +static size_t siphash(const uint8_t *in, size_t inlen, size_t seed, size_t seed2) +{ + constexpr size_t TailSizeMask = sizeof(void *) - 1; + SipHash<> hasher(inlen, seed, seed2); + hasher.addBlock(in, inlen & ~TailSizeMask); + return hasher.finalize(in + (inlen & ~TailSizeMask), inlen & TailSizeMask); +} diff --git a/src/corelib/tools/LICENSE.siphash b/src/corelib/tools/LICENSE.siphash deleted file mode 100644 index 670154e3538..00000000000 --- a/src/corelib/tools/LICENSE.siphash +++ /dev/null @@ -1,116 +0,0 @@ -CC0 1.0 Universal - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator and -subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for the -purpose of contributing to a commons of creative, cultural and scientific -works ("Commons") that the public can reliably and without fear of later -claims of infringement build upon, modify, incorporate in other works, reuse -and redistribute as freely as possible in any form whatsoever and for any -purposes, including without limitation commercial purposes. These owners may -contribute to the Commons to promote the ideal of a free culture and the -further production of creative, cultural and scientific works, or to gain -reputation or greater distribution for their Work in part through the use and -efforts of others. - -For these and/or other purposes and motivations, and without any expectation -of additional consideration or compensation, the person associating CC0 with a -Work (the "Affirmer"), to the extent that he or she is an owner of Copyright -and Related Rights in the Work, voluntarily elects to apply CC0 to the Work -and publicly distribute the Work under its terms, with knowledge of his or her -Copyright and Related Rights in the Work and the meaning and intended legal -effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not limited -to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, communicate, - and translate a Work; - - ii. moral rights retained by the original author(s) and/or performer(s); - - iii. publicity and privacy rights pertaining to a person's image or likeness - depicted in a Work; - - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - - v. rights protecting the extraction, dissemination, use and reuse of data in - a Work; - - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation thereof, - including any amended or successor version of such directive); and - - vii. other similar, equivalent or corresponding rights throughout the world - based on applicable law or treaty, and any national implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention of, -applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and -unconditionally waives, abandons, and surrenders all of Affirmer's Copyright -and Related Rights and associated claims and causes of action, whether now -known or unknown (including existing as well as future claims and causes of -action), in the Work (i) in all territories worldwide, (ii) for the maximum -duration provided by applicable law or treaty (including future time -extensions), (iii) in any current or future medium and for any number of -copies, and (iv) for any purpose whatsoever, including without limitation -commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes -the Waiver for the benefit of each member of the public at large and to the -detriment of Affirmer's heirs and successors, fully intending that such Waiver -shall not be subject to revocation, rescission, cancellation, termination, or -any other legal or equitable action to disrupt the quiet enjoyment of the Work -by the public as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason be -judged legally invalid or ineffective under applicable law, then the Waiver -shall be preserved to the maximum extent permitted taking into account -Affirmer's express Statement of Purpose. In addition, to the extent the Waiver -is so judged Affirmer hereby grants to each affected person a royalty-free, -non transferable, non sublicensable, non exclusive, irrevocable and -unconditional license to exercise Affirmer's Copyright and Related Rights in -the Work (i) in all territories worldwide, (ii) for the maximum duration -provided by applicable law or treaty (including future time extensions), (iii) -in any current or future medium and for any number of copies, and (iv) for any -purpose whatsoever, including without limitation commercial, advertising or -promotional purposes (the "License"). The License shall be deemed effective as -of the date CC0 was applied by Affirmer to the Work. Should any part of the -License for any reason be judged legally invalid or ineffective under -applicable law, such partial invalidity or ineffectiveness shall not -invalidate the remainder of the License, and in such case Affirmer hereby -affirms that he or she will not (i) exercise any of his or her remaining -Copyright and Related Rights in the Work or (ii) assert any associated claims -and causes of action with respect to the Work, in either case contrary to -Affirmer's express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - - b. Affirmer offers the Work as-is and makes no representations or warranties - of any kind concerning the Work, express, implied, statutory or otherwise, - including without limitation warranties of title, merchantability, fitness - for a particular purpose, non infringement, or the absence of latent or - other defects, accuracy, or the present or absence of errors, whether or not - discoverable, all to the greatest extent permissible under applicable law. - - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without limitation - any person's Copyright and Related Rights in the Work. Further, Affirmer - disclaims responsibility for obtaining any necessary consents, permissions - or other rights required for any use of the Work. - - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to this - CC0 or use of the Work. - -For more information, please see - diff --git a/src/corelib/tools/qhash.cpp b/src/corelib/tools/qhash.cpp index 12e90daecff..40e16482ee8 100644 --- a/src/corelib/tools/qhash.cpp +++ b/src/corelib/tools/qhash.cpp @@ -33,6 +33,9 @@ #include #endif // QT_BOOTSTRAPPED +// Implementation of SipHash algorithm +#include "../../3rdparty/siphash/siphash.cpp" + #include #include @@ -285,248 +288,6 @@ static inline uint64_t murmurhash(const void *key, uint64_t len, uint64_t seed) #endif -namespace { -// This is an inlined version of the SipHash implementation that is -// trying to avoid some memcpy's from uint64 to uint8[] and back. - -#define ROTL(x, b) (((x) << (b)) | ((x) >> (sizeof(x) * 8 - (b)))) - -#define SIPROUND \ - do { \ - v0 += v1; \ - v1 = ROTL(v1, 13); \ - v1 ^= v0; \ - v0 = ROTL(v0, 32); \ - v2 += v3; \ - v3 = ROTL(v3, 16); \ - v3 ^= v2; \ - v0 += v3; \ - v3 = ROTL(v3, 21); \ - v3 ^= v0; \ - v2 += v1; \ - v1 = ROTL(v1, 17); \ - v1 ^= v2; \ - v2 = ROTL(v2, 32); \ - } while (0) - -template struct SipHash64 -{ - /* "somepseudorandomlygeneratedbytes" */ - uint64_t v0 = 0x736f6d6570736575ULL; - uint64_t v1 = 0x646f72616e646f6dULL; - uint64_t v2 = 0x6c7967656e657261ULL; - uint64_t v3 = 0x7465646279746573ULL; - uint64_t b; - uint64_t k0; - uint64_t k1; - - inline SipHash64(uint64_t fulllen, uint64_t seed, uint64_t seed2); - inline void addBlock(const uint8_t *in, size_t inlen); - inline uint64_t finalize(const uint8_t *in, size_t left); -}; - -template -SipHash64::SipHash64(uint64_t inlen, uint64_t seed, uint64_t seed2) -{ - b = inlen << 56; - k0 = seed; - k1 = seed2; - v3 ^= k1; - v2 ^= k0; - v1 ^= k1; - v0 ^= k0; -} - -template Q_DECL_HOT_FUNCTION void -SipHash64::addBlock(const uint8_t *in, size_t inlen) -{ - Q_ASSERT((inlen & 7ULL) == 0); - int i; - const uint8_t *end = in + inlen; - for (; in != end; in += 8) { - uint64_t m = qFromUnaligned(in); - v3 ^= m; - - for (i = 0; i < cROUNDS; ++i) - SIPROUND; - - v0 ^= m; - } -} - -template Q_DECL_HOT_FUNCTION uint64_t -SipHash64::finalize(const uint8_t *in, size_t left) -{ - int i; - switch (left) { - case 7: - b |= ((uint64_t)in[6]) << 48; - Q_FALLTHROUGH(); - case 6: - b |= ((uint64_t)in[5]) << 40; - Q_FALLTHROUGH(); - case 5: - b |= ((uint64_t)in[4]) << 32; - Q_FALLTHROUGH(); - case 4: - b |= ((uint64_t)in[3]) << 24; - Q_FALLTHROUGH(); - case 3: - b |= ((uint64_t)in[2]) << 16; - Q_FALLTHROUGH(); - case 2: - b |= ((uint64_t)in[1]) << 8; - Q_FALLTHROUGH(); - case 1: - b |= ((uint64_t)in[0]); - break; - case 0: - break; - } - - v3 ^= b; - - for (i = 0; i < cROUNDS; ++i) - SIPROUND; - - v0 ^= b; - - v2 ^= 0xff; - - for (i = 0; i < dROUNDS; ++i) - SIPROUND; - - b = v0 ^ v1 ^ v2 ^ v3; - return b; -} -#undef SIPROUND - -// This is a "SipHash" implementation adopted for 32bit platforms. It performs -// basically the same operations as the 64bit version using 4 byte at a time -// instead of 8. -// -// To make this work, we also need to change the constants for the mixing -// rotations in ROTL. We're simply using half of the 64bit constants, rounded up -// for odd numbers. -// -// For the v0-v4 constants, simply use the first four bytes of the 64 bit versions. -// - -#define SIPROUND \ - do { \ - v0 += v1; \ - v1 = ROTL(v1, 7); \ - v1 ^= v0; \ - v0 = ROTL(v0, 16); \ - v2 += v3; \ - v3 = ROTL(v3, 8); \ - v3 ^= v2; \ - v0 += v3; \ - v3 = ROTL(v3, 11); \ - v3 ^= v0; \ - v2 += v1; \ - v1 = ROTL(v1, 9); \ - v1 ^= v2; \ - v2 = ROTL(v2, 16); \ - } while (0) - -template struct SipHash32 -{ - /* "somepseudorandomlygeneratedbytes" */ - uint v0 = 0x736f6d65U; - uint v1 = 0x646f7261U; - uint v2 = 0x6c796765U; - uint v3 = 0x74656462U; - uint b; - uint k0; - uint k1; - - inline SipHash32(size_t fulllen, uint seed, uint seed2); - inline void addBlock(const uint8_t *in, size_t inlen); - inline uint finalize(const uint8_t *in, size_t left); -}; - -template inline -SipHash32::SipHash32(size_t inlen, uint seed, uint seed2) -{ - uint k0 = seed; - uint k1 = seed2; - b = inlen << 24; - v3 ^= k1; - v2 ^= k0; - v1 ^= k1; - v0 ^= k0; -} - -template inline Q_DECL_HOT_FUNCTION void -SipHash32::addBlock(const uint8_t *in, size_t inlen) -{ - Q_ASSERT((inlen & 3ULL) == 0); - int i; - const uint8_t *end = in + inlen; - for (; in != end; in += 4) { - uint m = qFromUnaligned(in); - v3 ^= m; - - for (i = 0; i < cROUNDS; ++i) - SIPROUND; - - v0 ^= m; - } -} - -template inline Q_DECL_HOT_FUNCTION uint -SipHash32::finalize(const uint8_t *in, size_t left) -{ - int i; - switch (left) { - case 3: - b |= ((uint)in[2]) << 16; - Q_FALLTHROUGH(); - case 2: - b |= ((uint)in[1]) << 8; - Q_FALLTHROUGH(); - case 1: - b |= ((uint)in[0]); - break; - case 0: - break; - } - - v3 ^= b; - - for (i = 0; i < cROUNDS; ++i) - SIPROUND; - - v0 ^= b; - - v2 ^= 0xff; - - for (i = 0; i < dROUNDS; ++i) - SIPROUND; - - b = v0 ^ v1 ^ v2 ^ v3; - return b; -} -#undef SIPROUND -#undef ROTL - -// Use SipHash-1-2, which has similar performance characteristics as -// stablehash() above, instead of the SipHash-2-4 default -template -using SipHash = std::conditional_t, SipHash32>; -} // unnamed namespace - -Q_NEVER_INLINE Q_DECL_HOT_FUNCTION -static size_t siphash(const uint8_t *in, size_t inlen, size_t seed, size_t seed2) -{ - constexpr size_t TailSizeMask = sizeof(void *) - 1; - SipHash<> hasher(inlen, seed, seed2); - hasher.addBlock(in, inlen & ~TailSizeMask); - return hasher.finalize(in + (inlen & ~TailSizeMask), inlen & TailSizeMask); -} - enum ZeroExtension { None = 0, ByteToWord = 1,