1berry/MariaDB-server
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

MDEV-17958 Make bug-endian innodb_checksum_algorithm=crc32 optional

Browse Source 
In MySQL 5.7, it was noticed that files are not portable between
big-endian and little-endian processor architectures
(such as SPARC and x86), because the original implementation of
innodb_checksum_algorithm=crc32 was not byte order agnostic.

A byte order agnostic implementation of innodb_checksum_algorithm=crc32
was only added to MySQL 5.7, not backported to 5.6. Consequently,
MariaDB Server versions 10.0 and 10.1 only contain the CRC-32C
implementation that works incorrectly on big-endian architectures,
and MariaDB Server 10.2.2 got the byte-order agnostic CRC-32C
implementation from MySQL 5.7.

MySQL 5.7 introduced a "legacy crc32" variant that is functionally
equivalent to the big-endian version of the original crc32 implementation.
Thanks to this variant, old data files can be transferred from big-endian
systems to newer versions.

Introducing new variants of checksum algorithms (without introducing
new names for them, or something on the pages themselves to identify
the algorithm) generally is a bad idea, because each checksum algorithm
is like a lottery ticket. The more algorithms you try, the more likely
it will be for the checksum to match on a corrupted page.

So, essentially MySQL 5.7 weakened innodb_checksum_algorithm=crc32,
and MariaDB 10.2.2 inherited this weakening.

We introduce a build option that together with MDEV-17957
makes innodb_checksum_algorithm=strict_crc32 strict again
by only allowing one variant of the checksum to match.

WITH_INNODB_BUG_ENDIAN_CRC32: A new cmake option for enabling the
bug-compatible "legacy crc32" checksum. This is only enabled on
big-endian systems by default, to facilitate an upgrade from
MariaDB 10.0 or 10.1. Checked by #ifdef INNODB_BUG_ENDIAN_CRC32.

ut_crc32_byte_by_byte: Remove (unused function).

legacy_big_endian_checksum: Remove. This variable seems to have
unnecessarily complicated the logic. When the weakening is enabled,
we must always fall back to the buggy checksum.

buf_page_check_crc32(): A helper function to compute one or
two CRC-32C variants.
This commit is contained in:
Marko Mäkelä 2018-12-13 17:57:10 +02:00
parent 2e5aea4bab
commit 1a780eefc9
10 changed files with 211 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
storage/innobase/buf/buf0buf.cc
View File

@ -756,17 +756,14 @@ buf_page_is_zeroes(
@param[in] read_buf database page @param[in] read_buf database page
@param[in] checksum_field1 new checksum field @param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field @param[in] checksum_field2 old checksum field
@param[in] use_legacy_big_endian use legacy big endian algorithm
@return true if the page is in crc32 checksum format. */ @return true if the page is in crc32 checksum format. */
bool bool
buf_page_is_checksum_valid_crc32( buf_page_is_checksum_valid_crc32(
const byte* read_buf, const byte* read_buf,
ulint checksum_field1, ulint checksum_field1,
ulint checksum_field2, ulint checksum_field2)
bool use_legacy_big_endian)
{ {
const uint32_t crc32 = buf_calc_page_crc32(read_buf, const uint32_t crc32 = buf_calc_page_crc32(read_buf);
use_legacy_big_endian);
#ifdef UNIV_INNOCHECKSUM #ifdef UNIV_INNOCHECKSUM
if (log_file if (log_file
@ -783,17 +780,11 @@ buf_page_is_checksum_valid_crc32(
return false; return false;
} }
if (checksum_field1 == crc32) { return checksum_field1 == crc32
return(true); #ifdef INNODB_BUG_ENDIAN_CRC32
} else { || checksum_field1 == buf_calc_page_crc32(read_buf, true)
const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf, true); #endif
;
if (checksum_field1 == crc32_legacy) {
return(true);
}
}
return(false);
} }
/** Checks if the page is in innodb checksum format. /** Checks if the page is in innodb checksum format.
@ -922,6 +913,29 @@ buf_page_is_checksum_valid_none(
&& checksum_field1 == BUF_NO_CHECKSUM_MAGIC); && checksum_field1 == BUF_NO_CHECKSUM_MAGIC);
} }
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Validate the CRC-32C checksum of a page.
@param[in] page buffer page (srv_page_size bytes)
@param[in] checksum CRC-32C checksum stored on page
@return computed checksum */
static uint32_t buf_page_check_crc32(const byte* page, uint32_t checksum)
{
uint32_t crc32 = buf_calc_page_crc32(page);
if (checksum != crc32) {
crc32 = buf_calc_page_crc32(page, true);
}
return crc32;
}
#else /* INNODB_BUG_ENDIAN_CRC32 */
/** Validate the CRC-32C checksum of a page.
@param[in] page buffer page (srv_page_size bytes)
@param[in] checksum CRC-32C checksum stored on page
@return computed checksum */
# define buf_page_check_crc32(page, checksum) buf_calc_page_crc32(page)
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Check if a page is corrupt. /** Check if a page is corrupt.
@param[in] check_lsn whether the LSN should be checked @param[in] check_lsn whether the LSN should be checked
@param[in] read_buf database page @param[in] read_buf database page
@ -1037,26 +1051,20 @@ buf_page_is_corrupted(
&& *reinterpret_cast<const ib_uint64_t*>( && *reinterpret_cast<const ib_uint64_t*>(
read_buf + FIL_PAGE_LSN) == 0) { read_buf + FIL_PAGE_LSN) == 0) {
ulint i;
/* make sure that the page is really empty */ /* make sure that the page is really empty */
for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) { for (ulint i = 0; i < page_size.logical(); i++) {
if (read_buf[i] != 0) { if (read_buf[i] != 0) {
return(true); return(true);
} }
} }
#ifdef UNIV_INNOCHECKSUM #ifdef UNIV_INNOCHECKSUM
if (i >= page_size.logical()) { if (log_file) {
if (log_file) { fprintf(log_file, "Page::%llu"
fprintf(log_file, "Page::%llu" " is empty and uncorrupted\n",
" is empty and uncorrupted\n", cur_page_num);
cur_page_num);
}
return(false);
} }
#else
return(i < page_size.logical());
#endif /* UNIV_INNOCHECKSUM */ #endif /* UNIV_INNOCHECKSUM */
return(false);
} }
const srv_checksum_algorithm_t curr_algo = const srv_checksum_algorithm_t curr_algo =
@ -1065,10 +1073,7 @@ buf_page_is_corrupted(
switch (curr_algo) { switch (curr_algo) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
return !buf_page_is_checksum_valid_crc32( return !buf_page_is_checksum_valid_crc32(
read_buf, checksum_field1, checksum_field2, false) read_buf, checksum_field1, checksum_field2);
&& !buf_page_is_checksum_valid_crc32(
read_buf, checksum_field1, checksum_field2,
true);
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
return !buf_page_is_checksum_valid_innodb( return !buf_page_is_checksum_valid_innodb(
read_buf, checksum_field1, checksum_field2); read_buf, checksum_field1, checksum_field2);
@ -1111,19 +1116,10 @@ buf_page_is_corrupted(
if (srv_checksum_algorithm if (srv_checksum_algorithm
== SRV_CHECKSUM_ALGORITHM_CRC32) { == SRV_CHECKSUM_ALGORITHM_CRC32) {
crc32 = buf_page_check_crc32(read_buf,
crc32 = buf_calc_page_crc32( checksum_field2);
read_buf, legacy_big_endian_checksum);
crc32_inited = true; crc32_inited = true;
if (!legacy_big_endian_checksum
&& checksum_field2 != crc32) {
crc32 = buf_calc_page_crc32(read_buf,
true);
legacy_big_endian_checksum =
checksum_field2 == crc32;
}
if (checksum_field2 != crc32 if (checksum_field2 != crc32
&& checksum_field2 && checksum_field2
!= buf_calc_page_old_checksum(read_buf)) { != buf_calc_page_old_checksum(read_buf)) {
@ -1135,19 +1131,10 @@ buf_page_is_corrupted(
if (checksum_field2 if (checksum_field2
!= buf_calc_page_old_checksum(read_buf)) { != buf_calc_page_old_checksum(read_buf)) {
crc32 = buf_calc_page_crc32( crc32 = buf_page_check_crc32(
read_buf, read_buf, checksum_field2);
legacy_big_endian_checksum);
crc32_inited = true; crc32_inited = true;
if (!legacy_big_endian_checksum
&& checksum_field2 != crc32) {
crc32 = buf_calc_page_crc32(
read_buf, true);
legacy_big_endian_checksum =
checksum_field2 == crc32;
}
if (checksum_field2 != crc32) { if (checksum_field2 != crc32) {
return true; return true;
} }
@ -1161,18 +1148,9 @@ buf_page_is_corrupted(
== SRV_CHECKSUM_ALGORITHM_CRC32) { == SRV_CHECKSUM_ALGORITHM_CRC32) {
if (!crc32_inited) { if (!crc32_inited) {
crc32 = buf_calc_page_crc32( crc32 = buf_page_check_crc32(
read_buf, read_buf, checksum_field2);
legacy_big_endian_checksum);
crc32_inited = true; crc32_inited = true;
if (!legacy_big_endian_checksum
&& checksum_field2 != crc32) {
crc32 = buf_calc_page_crc32(
read_buf, true);
legacy_big_endian_checksum =
checksum_field2 == crc32;
}
} }
if (checksum_field1 != crc32 if (checksum_field1 != crc32
@ -1188,18 +1166,9 @@ buf_page_is_corrupted(
!= buf_calc_page_new_checksum(read_buf)) { != buf_calc_page_new_checksum(read_buf)) {
if (!crc32_inited) { if (!crc32_inited) {
crc32 = buf_calc_page_crc32( crc32 = buf_page_check_crc32(
read_buf, read_buf, checksum_field2);
legacy_big_endian_checksum);
crc32_inited = true; crc32_inited = true;
if (!legacy_big_endian_checksum
&& checksum_field2 != crc32) {
crc32 = buf_calc_page_crc32(
read_buf, true);
legacy_big_endian_checksum =
checksum_field2 == crc32;
}
} }
if (checksum_field1 != crc32) { if (checksum_field1 != crc32) {
@ -1255,10 +1224,12 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
<< page_zip_calc_checksum( << page_zip_calc_checksum(
read_buf, page_size.physical(), read_buf, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_CRC32) SRV_CHECKSUM_ALGORITHM_CRC32)
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/" << "/"
<< page_zip_calc_checksum( << page_zip_calc_checksum(
read_buf, page_size.physical(), read_buf, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_CRC32, true) SRV_CHECKSUM_ALGORITHM_CRC32, true)
#endif
<< ", " << ", "
<< buf_checksum_algorithm_name( << buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_INNODB) SRV_CHECKSUM_ALGORITHM_INNODB)
@ -1284,9 +1255,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
} else { } else {
const uint32_t crc32 = buf_calc_page_crc32(read_buf); const uint32_t crc32 = buf_calc_page_crc32(read_buf);
#ifdef INNODB_BUG_ENDIAN_CRC32
const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf, const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf,
true); true);
#endif /* INNODB_BUG_ENDIAN_CRC32 */
ulint page_type = fil_page_get_type(read_buf); ulint page_type = fil_page_get_type(read_buf);
ib::info() << "Uncompressed page, stored checksum in field1 " ib::info() << "Uncompressed page, stored checksum in field1 "
@ -1295,7 +1267,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
<< ", calculated checksums for field1: " << ", calculated checksums for field1: "
<< buf_checksum_algorithm_name( << buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_CRC32) << " " SRV_CHECKSUM_ALGORITHM_CRC32) << " "
<< crc32 << "/" << crc32_legacy << crc32
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/" << crc32_legacy
#endif
<< ", " << ", "
<< buf_checksum_algorithm_name( << buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_INNODB) << " " SRV_CHECKSUM_ALGORITHM_INNODB) << " "
@ -1312,7 +1287,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
<< ", calculated checksums for field2: " << ", calculated checksums for field2: "
<< buf_checksum_algorithm_name( << buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_CRC32) << " " SRV_CHECKSUM_ALGORITHM_CRC32) << " "
<< crc32 << "/" << crc32_legacy << crc32
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/" << crc32_legacy
#endif
<< ", " << ", "
<< buf_checksum_algorithm_name( << buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_INNODB) << " " SRV_CHECKSUM_ALGORITHM_INNODB) << " "
@ -3950,10 +3928,12 @@ buf_zip_decompress(
<< ", crc32: " << ", crc32: "
<< page_zip_calc_checksum( << page_zip_calc_checksum(
frame, size, SRV_CHECKSUM_ALGORITHM_CRC32) frame, size, SRV_CHECKSUM_ALGORITHM_CRC32)
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/" << "/"
<< page_zip_calc_checksum( << page_zip_calc_checksum(
frame, size, SRV_CHECKSUM_ALGORITHM_CRC32, frame, size, SRV_CHECKSUM_ALGORITHM_CRC32,
true) true)
#endif
<< " innodb: " << " innodb: "
<< page_zip_calc_checksum( << page_zip_calc_checksum(
frame, size, SRV_CHECKSUM_ALGORITHM_INNODB) frame, size, SRV_CHECKSUM_ALGORITHM_INNODB)

82
storage/innobase/buf/buf0checksum.cc
View File

@ -39,44 +39,56 @@ ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to
'long unsigned int*' in initialization */ 'long unsigned int*' in initialization */
ulong srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; ulong srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
/** set if we have found pages matching legacy big endian checksum */ #ifdef INNODB_BUG_ENDIAN_CRC32
bool legacy_big_endian_checksum = false; /** Calculate the CRC32 checksum of a page. The value is stored to the page
/** Calculates the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from when it is written to a file and also checked for a match when reading from
the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian the file. Note that we must be careful to calculate the same value on all
variants. Note that we must be careful to calculate the same value on 32-bit architectures.
and 64-bit architectures. @param[in] page buffer page (srv_page_size bytes)
@param[in] page buffer page (UNIV_PAGE_SIZE bytes) @param[in] bug_endian whether to use big endian byteorder
@param[in] use_legacy_big_endian if true then use big endian when converting byte strings to integers, for bug-compatibility with
byteorder when converting byte strings to integers big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1
@return checksum */ @return CRC-32C */
uint32_t uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian)
buf_calc_page_crc32(
const byte* page,
bool use_legacy_big_endian /* = false */)
{ {
/* Since the field FIL_PAGE_FILE_FLUSH_LSN, and in versions <= 4.1.x return bug_endian
FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, are written outside the buffer pool ? ut_crc32_legacy_big_endian(
to the first pages of data files, we have to skip them in the page page + FIL_PAGE_OFFSET,
checksum calculation. FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
We must also skip the field FIL_PAGE_SPACE_OR_CHKSUM where the - FIL_PAGE_OFFSET)
checksum is stored, and also the last 8 bytes of page because ^ ut_crc32_legacy_big_endian(page + FIL_PAGE_DATA,
there we store the old formula checksum. */ srv_page_size
- (FIL_PAGE_DATA
ut_crc32_func_t crc32_func = use_legacy_big_endian + FIL_PAGE_END_LSN_OLD_CHKSUM))
? ut_crc32_legacy_big_endian : ut_crc32(page + FIL_PAGE_OFFSET,
: ut_crc32; FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
- FIL_PAGE_OFFSET)
const uint32_t c1 = crc32_func( ^ ut_crc32(page + FIL_PAGE_DATA,
page + FIL_PAGE_OFFSET, srv_page_size
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - FIL_PAGE_OFFSET); - (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
const uint32_t c2 = crc32_func(
page + FIL_PAGE_DATA,
UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_END_LSN_OLD_CHKSUM);
return(c1 ^ c2);
} }
#else
/** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from
the file. Note that we must be careful to calculate the same value on all
architectures.
@param[in] page buffer page (srv_page_size bytes)
@return CRC-32C */
uint32_t buf_calc_page_crc32(const byte* page)
{
/* Note: innodb_checksum_algorithm=crc32 could and should have
included the entire page in the checksum, and CRC-32 values
should be combined with the CRC-32 function, not with
exclusive OR. We stick to the current algorithm in order to
remain compatible with old data files. */
return ut_crc32(page + FIL_PAGE_OFFSET,
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
- FIL_PAGE_OFFSET)
^ ut_crc32(page + FIL_PAGE_DATA,
srv_page_size
- (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
}
#endif
/** Calculate a checksum which is stored to the page when it is written /** Calculate a checksum which is stored to the page when it is written
to a file. Note that we must be careful to calculate the same value on to a file. Note that we must be careful to calculate the same value on

17
storage/innobase/fil/fil0crypt.cc
View File

@ -2636,10 +2636,8 @@ fil_space_verify_crypt_checksum(
srv_checksum_algorithm); srv_checksum_algorithm);
switch (algorithm) { switch (algorithm) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
/* We never supported upgrade from the "legacy crc32"
on big endian systems from MariaDB 10.1 to later. */
valid = buf_page_is_checksum_valid_crc32( valid = buf_page_is_checksum_valid_crc32(
page, checksum1, checksum2, false); page, checksum1, checksum2);
break; break;
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
valid = buf_page_is_checksum_valid_innodb( valid = buf_page_is_checksum_valid_innodb(
@ -2649,13 +2647,11 @@ fil_space_verify_crypt_checksum(
case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_NONE: case SRV_CHECKSUM_ALGORITHM_NONE:
/* We never supported upgrade from the "legacy crc32" /* never supported
on big endian systems from MariaDB 10.1 to later.
We also never supported
innodb_checksum_algorithm=none or strict_none innodb_checksum_algorithm=none or strict_none
for encrypted pages. */ for encrypted pages. */
valid = buf_page_is_checksum_valid_crc32( valid = buf_page_is_checksum_valid_crc32(
page, checksum1, checksum2, false) page, checksum1, checksum2)
|| buf_page_is_checksum_valid_innodb( || buf_page_is_checksum_valid_innodb(
page, checksum1, checksum2); page, checksum1, checksum2);
break; break;
@ -2684,8 +2680,11 @@ fil_space_verify_crypt_checksum(
ib::info() ib::info()
<< "If unencrypted: stored checksum [" << checksum1 << "If unencrypted: stored checksum [" << checksum1
<< ":" << checksum2 << "] calculated crc32 [" << ":" << checksum2 << "] calculated crc32 ["
<< buf_calc_page_crc32(page, false) << ":" << buf_calc_page_crc32(page)
<< buf_calc_page_crc32(page, true) << "] innodb [" # ifdef INNODB_BUG_ENDIAN_CRC32
<< ":" << buf_calc_page_crc32(page, true)
# endif /* INNODB_BUG_ENDIAN_CRC32 */
<< "] innodb ["
<< buf_calc_page_old_checksum(page) << ":" << buf_calc_page_old_checksum(page) << ":"
<< buf_calc_page_new_checksum(page) << "] LSN " << buf_calc_page_new_checksum(page) << "] LSN "
<< mach_read_from_4(page + FIL_PAGE_LSN); << mach_read_from_4(page + FIL_PAGE_LSN);

4
storage/innobase/include/buf0buf.h
View File

@ -716,14 +716,12 @@ buf_block_unfix(
@param[in] read_buf database page @param[in] read_buf database page
@param[in] checksum_field1 new checksum field @param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field @param[in] checksum_field2 old checksum field
@param[in] use_legacy_big_endian use legacy big endian algorithm
@return true if the page is in crc32 checksum format. */ @return true if the page is in crc32 checksum format. */
bool bool
buf_page_is_checksum_valid_crc32( buf_page_is_checksum_valid_crc32(
const byte* read_buf, const byte* read_buf,
ulint checksum_field1, ulint checksum_field1,
ulint checksum_field2, ulint checksum_field2)
bool use_legacy_big_endian)
MY_ATTRIBUTE((nonnull(1), warn_unused_result)); MY_ATTRIBUTE((nonnull(1), warn_unused_result));
/** Checks if the page is in innodb checksum format. /** Checks if the page is in innodb checksum format.

32
storage/innobase/include/buf0checksum.h
View File

@ -1,7 +1,7 @@
/***************************************************************************** /*****************************************************************************
Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2017, MariaDB Corporation. Copyright (c) 2017, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
@ -29,19 +29,26 @@ Created Aug 11, 2011 Vasil Dimov
#include "buf0types.h" #include "buf0types.h"
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate the CRC32 checksum of a page. The value is stored to the page /** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from when it is written to a file and also checked for a match when reading from
the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian the file. Note that we must be careful to calculate the same value on all
variants. Note that we must be careful to calculate the same value on 32-bit architectures.
and 64-bit architectures. @param[in] page buffer page (srv_page_size bytes)
@param[in] page buffer page (UNIV_PAGE_SIZE bytes) @param[in] bug_endian whether to use big endian byteorder
@param[in] use_legacy_big_endian if true then use big endian when converting byte strings to integers, for bug-compatibility with
byteorder when converting byte strings to integers big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1
@return checksum */ @return CRC-32C */
uint32_t uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian = false);
buf_calc_page_crc32( #else
const byte* page, /** Calculate the CRC32 checksum of a page. The value is stored to the page
bool use_legacy_big_endian = false); when it is written to a file and also checked for a match when reading from
the file. Note that we must be careful to calculate the same value on all
architectures.
@param[in] page buffer page (srv_page_size bytes)
@return CRC-32C */
uint32_t buf_calc_page_crc32(const byte* page);
#endif
/** Calculate a checksum which is stored to the page when it is written /** Calculate a checksum which is stored to the page when it is written
to a file. Note that we must be careful to calculate the same value on to a file. Note that we must be careful to calculate the same value on
@ -69,6 +76,5 @@ const char*
buf_checksum_algorithm_name(srv_checksum_algorithm_t algo); buf_checksum_algorithm_name(srv_checksum_algorithm_t algo);
extern ulong srv_checksum_algorithm; extern ulong srv_checksum_algorithm;
extern bool legacy_big_endian_checksum;
#endif /* buf0checksum_h */ #endif /* buf0checksum_h */

11
storage/innobase/include/page0zip.h
View File

@ -492,16 +492,17 @@ page_zip_parse_compress(
@param[in] data compressed page @param[in] data compressed page
@param[in] size size of compressed page @param[in] size size of compressed page
@param[in] algo algorithm to use @param[in] algo algorithm to use
@param[in] use_legacy_big_endian only used if algo is
SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true
then use big endian byteorder when converting byte strings to integers.
@return page checksum */ @return page checksum */
uint32_t uint32_t
page_zip_calc_checksum( page_zip_calc_checksum(
const void* data, const void* data,
ulint size, ulint size,
srv_checksum_algorithm_t algo, srv_checksum_algorithm_t algo
bool use_legacy_big_endian = false); #ifdef INNODB_BUG_ENDIAN_CRC32
/** for crc32, use the big-endian bug-compatible crc32 variant */
, bool use_legacy_big_endian = false
#endif
);
/**********************************************************************//** /**********************************************************************//**
Verify a compressed page's checksum. Verify a compressed page's checksum.

8
storage/innobase/include/ut0crc32.h
View File

@ -1,7 +1,7 @@
/***************************************************************************** /*****************************************************************************
Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2016, MariaDB Corporation. Copyright (c) 2016, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
@ -47,13 +47,11 @@ typedef uint32_t (*ut_crc32_func_t)(const byte* ptr, ulint len);
/** Pointer to CRC32 calculation function. */ /** Pointer to CRC32 calculation function. */
extern ut_crc32_func_t ut_crc32; extern ut_crc32_func_t ut_crc32;
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Pointer to CRC32 calculation function, which uses big-endian byte order /** Pointer to CRC32 calculation function, which uses big-endian byte order
when converting byte strings to integers internally. */ when converting byte strings to integers internally. */
extern ut_crc32_func_t ut_crc32_legacy_big_endian; extern ut_crc32_func_t ut_crc32_legacy_big_endian;
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Pointer to CRC32-byte-by-byte calculation function (byte order agnostic,
but very slow). */
extern ut_crc32_func_t ut_crc32_byte_by_byte;
extern const char* ut_crc32_implementation; extern const char* ut_crc32_implementation;

7
storage/innobase/innodb.cmake
View File

@ -25,6 +25,7 @@ INCLUDE(lzma.cmake)
INCLUDE(bzip2.cmake) INCLUDE(bzip2.cmake)
INCLUDE(snappy.cmake) INCLUDE(snappy.cmake)
INCLUDE(numa) INCLUDE(numa)
INCLUDE(TestBigEndian)
MYSQL_CHECK_LZ4() MYSQL_CHECK_LZ4()
MYSQL_CHECK_LZO() MYSQL_CHECK_LZO()
@ -32,6 +33,7 @@ MYSQL_CHECK_LZMA()
MYSQL_CHECK_BZIP2() MYSQL_CHECK_BZIP2()
MYSQL_CHECK_SNAPPY() MYSQL_CHECK_SNAPPY()
MYSQL_CHECK_NUMA() MYSQL_CHECK_NUMA()
TEST_BIG_ENDIAN(IS_BIG_ENDIAN)
IF(CMAKE_CROSSCOMPILING) IF(CMAKE_CROSSCOMPILING)
# Use CHECK_C_SOURCE_COMPILES instead of CHECK_C_SOURCE_RUNS when # Use CHECK_C_SOURCE_COMPILES instead of CHECK_C_SOURCE_RUNS when
@ -123,6 +125,11 @@ ELSEIF(WITH_INNODB_ROOT_GUESS)
ADD_DEFINITIONS(-DBTR_CUR_ADAPT) ADD_DEFINITIONS(-DBTR_CUR_ADAPT)
ENDIF() ENDIF()
OPTION(WITH_INNODB_BUG_ENDIAN_CRC32 "Weaken innodb_checksum_algorithm=crc32 by supporting upgrade from big-endian systems running 5.6/10.0/10.1" ${IS_BIG_ENDIAN})
IF(WITH_INNODB_BUG_ENDIAN_CRC32)
ADD_DEFINITIONS(-DINNODB_BUG_ENDIAN_CRC32)
ENDIF()
OPTION(WITH_INNODB_EXTRA_DEBUG "Enable extra InnoDB debug checks" OFF) OPTION(WITH_INNODB_EXTRA_DEBUG "Enable extra InnoDB debug checks" OFF)
IF(WITH_INNODB_EXTRA_DEBUG) IF(WITH_INNODB_EXTRA_DEBUG)
IF(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") IF(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")

101
storage/innobase/page/page0zip.cc
View File

@ -4909,18 +4909,17 @@ corrupt:
@param[in] data compressed page @param[in] data compressed page
@param[in] size size of compressed page @param[in] size size of compressed page
@param[in] algo algorithm to use @param[in] algo algorithm to use
@param[in] use_legacy_big_endian only used if algo is
SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true
then use big endian byteorder when converting byte strings to integers.
SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true
then use big endian byteorder when converting byte strings to integers.
@return page checksum */ @return page checksum */
uint32_t uint32_t
page_zip_calc_checksum( page_zip_calc_checksum(
const void* data, const void* data,
ulint size, ulint size,
srv_checksum_algorithm_t algo, srv_checksum_algorithm_t algo
bool use_legacy_big_endian /* = false */) #ifdef INNODB_BUG_ENDIAN_CRC32
/** for crc32, use the big-endian bug-compatible crc32 variant */
, bool use_legacy_big_endian
#endif
)
{ {
uLong adler; uLong adler;
const Bytef* s = static_cast<const byte*>(data); const Bytef* s = static_cast<const byte*>(data);
@ -4931,25 +4930,25 @@ page_zip_calc_checksum(
switch (algo) { switch (algo) {
case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
{ ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); #ifdef INNODB_BUG_ENDIAN_CRC32
if (use_legacy_big_endian) {
ut_crc32_func_t crc32_func = use_legacy_big_endian return ut_crc32_legacy_big_endian(s + FIL_PAGE_OFFSET,
? ut_crc32_legacy_big_endian FIL_PAGE_LSN
: ut_crc32; - FIL_PAGE_OFFSET)
^ ut_crc32_legacy_big_endian(
const uint32_t crc32
= crc32_func(
s + FIL_PAGE_OFFSET,
FIL_PAGE_LSN - FIL_PAGE_OFFSET)
^ crc32_func(
s + FIL_PAGE_TYPE, 2) s + FIL_PAGE_TYPE, 2)
^ crc32_func( ^ ut_crc32_legacy_big_endian(
s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID,
size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); size
- FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
return(crc32);
} }
#endif
return ut_crc32(s + FIL_PAGE_OFFSET,
FIL_PAGE_LSN - FIL_PAGE_OFFSET)
^ ut_crc32(s + FIL_PAGE_TYPE, 2)
^ ut_crc32(s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID,
size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
@ -4984,13 +4983,8 @@ page_zip_verify_checksum(
const void* data, /*!< in: compressed page */ const void* data, /*!< in: compressed page */
ulint size) /*!< in: size of compressed page */ ulint size) /*!< in: size of compressed page */
{ {
ib_uint32_t stored; const uint32_t stored = mach_read_from_4(
ib_uint32_t calc; static_cast<const byte*>(data) + FIL_PAGE_SPACE_OR_CHKSUM);
ib_uint32_t crc32 = 0;
ib_uint32_t innodb = 0;
stored = static_cast<ib_uint32_t>(mach_read_from_4(
static_cast<const unsigned char*>(data) + FIL_PAGE_SPACE_OR_CHKSUM));
#if FIL_PAGE_LSN % 8 #if FIL_PAGE_LSN % 8
#error "FIL_PAGE_LSN must be 64 bit aligned" #error "FIL_PAGE_LSN must be 64 bit aligned"
@ -5034,8 +5028,7 @@ page_zip_verify_checksum(
return(TRUE); return(TRUE);
} }
calc = static_cast<ib_uint32_t>(page_zip_calc_checksum( uint32_t calc = page_zip_calc_checksum(data, size, curr_algo);
data, size, curr_algo));
#ifdef UNIV_INNOCHECKSUM #ifdef UNIV_INNOCHECKSUM
if (log_file) { if (log_file) {
@ -5070,13 +5063,11 @@ page_zip_verify_checksum(
switch (curr_algo) { switch (curr_algo) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
calc = page_zip_calc_checksum(data, size, curr_algo, true); #ifdef INNODB_BUG_ENDIAN_CRC32
if (calc == stored) { return stored == page_zip_calc_checksum(data, size, curr_algo,
legacy_big_endian_checksum = true; true);
return TRUE; #endif
} /* fall through */
return FALSE;
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
return FALSE; return FALSE;
@ -5085,29 +5076,29 @@ page_zip_verify_checksum(
return(TRUE); return(TRUE);
} }
calc = page_zip_calc_checksum(data, size, curr_algo, true); return
crc32 = calc; #ifdef INNODB_BUG_ENDIAN_CRC32
stored == page_zip_calc_checksum(data, size, curr_algo,
if (crc32 == stored) { true) ||
legacy_big_endian_checksum = true; #endif
return TRUE; stored == page_zip_calc_checksum(
} data, size, SRV_CHECKSUM_ALGORITHM_INNODB);
innodb = static_cast<ib_uint32_t>(page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_INNODB));
break;
case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB:
if (stored == BUF_NO_CHECKSUM_MAGIC) { if (stored == BUF_NO_CHECKSUM_MAGIC) {
return TRUE; return TRUE;
} }
crc32 = static_cast<ib_uint32_t>(page_zip_calc_checksum( return stored == page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_CRC32)); data, size, SRV_CHECKSUM_ALGORITHM_CRC32)
innodb = calc; #ifdef INNODB_BUG_ENDIAN_CRC32
break; || stored == page_zip_calc_checksum(
data, size,
SRV_CHECKSUM_ALGORITHM_CRC32, true)
#endif
;
case SRV_CHECKSUM_ALGORITHM_NONE: case SRV_CHECKSUM_ALGORITHM_NONE:
return TRUE; return TRUE;
} }
return (stored == crc32 || stored == innodb); return FALSE;
} }

64
storage/innobase/ut/ut0crc32.cc
View File

@ -2,7 +2,7 @@
Copyright (c) 2009, 2010 Facebook, Inc. All Rights Reserved. Copyright (c) 2009, 2010 Facebook, Inc. All Rights Reserved.
Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2016, MariaDB Corporation. Copyright (c) 2016, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
@ -92,13 +92,11 @@ mysys/my_perf.c, contributed by Facebook under the following license.
/** Pointer to CRC32 calculation function. */ /** Pointer to CRC32 calculation function. */
ut_crc32_func_t ut_crc32; ut_crc32_func_t ut_crc32;
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Pointer to CRC32 calculation function, which uses big-endian byte order /** Pointer to CRC32 calculation function, which uses big-endian byte order
when converting byte strings to integers internally. */ when converting byte strings to integers internally. */
ut_crc32_func_t ut_crc32_legacy_big_endian; ut_crc32_func_t ut_crc32_legacy_big_endian;
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Pointer to CRC32-byte-by-byte calculation function (byte order agnostic,
but very slow). */
ut_crc32_func_t ut_crc32_byte_by_byte;
/** Text description of CRC32 implementation */ /** Text description of CRC32 implementation */
const char* ut_crc32_implementation; const char* ut_crc32_implementation;
@ -278,6 +276,7 @@ ut_crc32_64_hw(
*len -= 8; *len -= 8;
} }
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate CRC32 over 64-bit byte string using a hardware/CPU instruction. /** Calculate CRC32 over 64-bit byte string using a hardware/CPU instruction.
The byte string is converted to a 64-bit integer using big endian byte order. The byte string is converted to a 64-bit integer using big endian byte order.
@param[in,out] crc crc32 checksum so far when this function is called, @param[in,out] crc crc32 checksum so far when this function is called,
@ -308,6 +307,7 @@ ut_crc32_64_legacy_big_endian_hw(
*data += 8; *data += 8;
*len -= 8; *len -= 8;
} }
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Calculates CRC32 using hardware/CPU instructions. /** Calculates CRC32 using hardware/CPU instructions.
@param[in] buf data over which to calculate CRC32 @param[in] buf data over which to calculate CRC32
@ -396,6 +396,7 @@ ut_crc32_hw(
return(~crc); return(~crc);
} }
# ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculates CRC32 using hardware/CPU instructions. /** Calculates CRC32 using hardware/CPU instructions.
This function uses big endian byte ordering when converting byte sequence to This function uses big endian byte ordering when converting byte sequence to
integers. integers.
@ -445,26 +446,7 @@ ut_crc32_legacy_big_endian_hw(
return(~crc); return(~crc);
} }
# endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Calculates CRC32 using hardware/CPU instructions.
This function processes one byte at a time (very slow) and thus it does
not depend on the byte order of the machine.
@param[in] buf data over which to calculate CRC32
@param[in] len data length
@return CRC-32C (polynomial 0x11EDC6F41) */
uint32_t
ut_crc32_byte_by_byte_hw(
const byte* buf,
ulint len)
{
uint32_t crc = 0xFFFFFFFFU;
while (len > 0) {
ut_crc32_8_hw(&crc, &buf, &len);
}
return(~crc);
}
#endif /* defined(__GNUC__) && defined(__x86_64__) || (_WIN64) */ #endif /* defined(__GNUC__) && defined(__x86_64__) || (_WIN64) */
/* CRC32 software implementation. */ /* CRC32 software implementation. */
@ -577,6 +559,7 @@ ut_crc32_64_sw(
*len -= 8; *len -= 8;
} }
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate CRC32 over 64-bit byte string using a software implementation. /** Calculate CRC32 over 64-bit byte string using a software implementation.
The byte string is converted to a 64-bit integer using big endian byte order. The byte string is converted to a 64-bit integer using big endian byte order.
@param[in,out] crc crc32 checksum so far when this function is called, @param[in,out] crc crc32 checksum so far when this function is called,
@ -602,6 +585,7 @@ ut_crc32_64_legacy_big_endian_sw(
*data += 8; *data += 8;
*len -= 8; *len -= 8;
} }
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Calculates CRC32 in software, without using CPU instructions. /** Calculates CRC32 in software, without using CPU instructions.
@param[in] buf data over which to calculate CRC32 @param[in] buf data over which to calculate CRC32
@ -653,6 +637,7 @@ ut_crc32_sw(
return(~crc); return(~crc);
} }
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculates CRC32 in software, without using CPU instructions. /** Calculates CRC32 in software, without using CPU instructions.
This function uses big endian byte ordering when converting byte sequence to This function uses big endian byte ordering when converting byte sequence to
integers. integers.
@ -704,28 +689,7 @@ ut_crc32_legacy_big_endian_sw(
return(~crc); return(~crc);
} }
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Calculates CRC32 in software, without using CPU instructions.
This function processes one byte at a time (very slow) and thus it does
not depend on the byte order of the machine.
@param[in] buf data over which to calculate CRC32
@param[in] len data length
@return CRC-32C (polynomial 0x11EDC6F41) */
uint32_t
ut_crc32_byte_by_byte_sw(
const byte* buf,
ulint len)
{
uint32_t crc = 0xFFFFFFFFU;
ut_a(ut_crc32_slice8_table_initialized);
while (len > 0) {
ut_crc32_8_sw(&crc, &buf, &len);
}
return(~crc);
}
/********************************************************************//** /********************************************************************//**
Initializes the data structures used by ut_crc32*(). Does not do any Initializes the data structures used by ut_crc32*(). Does not do any
@ -736,8 +700,9 @@ ut_crc32_init()
{ {
ut_crc32_slice8_table_init(); ut_crc32_slice8_table_init();
ut_crc32 = ut_crc32_sw; ut_crc32 = ut_crc32_sw;
#ifdef INNODB_BUG_ENDIAN_CRC32
ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_sw; ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_sw;
ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_sw; #endif /* INNODB_BUG_ENDIAN_CRC32 */
ut_crc32_implementation = "Using generic crc32 instructions"; ut_crc32_implementation = "Using generic crc32 instructions";
#if (defined(__GNUC__) && defined(__x86_64__)) || defined(_MSC_VER) #if (defined(__GNUC__) && defined(__x86_64__)) || defined(_MSC_VER)
@ -770,8 +735,9 @@ ut_crc32_init()
if (features_ecx & 1 << 20) { if (features_ecx & 1 << 20) {
ut_crc32 = ut_crc32_hw; ut_crc32 = ut_crc32_hw;
#ifdef INNODB_BUG_ENDIAN_CRC32
ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_hw; ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_hw;
ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_hw; #endif /* INNODB_BUG_ENDIAN_CRC32 */
ut_crc32_implementation = "Using SSE2 crc32 instructions"; ut_crc32_implementation = "Using SSE2 crc32 instructions";
} }