From d7be886cb8e43b2df8ad5d01f7ba91f0296057cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 14 Aug 2019 10:11:30 +0300 Subject: [PATCH 1/3] Fix main.opt_trace This was broken in 2dbe472ed011a951b28434ae8e67945e964d2030 --- mysql-test/main/opt_trace.result | 1 + mysql-test/main/opt_trace.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result index 87e4d41a395..e315b0212f7 100644 --- a/mysql-test/main/opt_trace.result +++ b/mysql-test/main/opt_trace.result @@ -6983,4 +6983,5 @@ JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans')) } ] ] +drop table t0, one_k; set optimizer_trace='enabled=off'; diff --git a/mysql-test/main/opt_trace.test b/mysql-test/main/opt_trace.test index 503e92057af..e5d635db042 100644 --- a/mysql-test/main/opt_trace.test +++ b/mysql-test/main/opt_trace.test @@ -534,5 +534,6 @@ set join_cache_level=@tmp_jcl; --echo # This shows post-join selectivity explain select * from t0 A, one_k B where A.a=B.b and B.a<800; select JSON_DETAILED(JSON_EXTRACT(trace, '$**.considered_execution_plans')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; +drop table t0, one_k; set optimizer_trace='enabled=off'; From dc8a20f3d00ec4ce4c65287b31696796d43bdfa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 14 Aug 2019 09:29:25 +0300 Subject: [PATCH 2/3] MDEV-19781: Adapt the test for full_crc32 --- .../suite/innodb/t/page_id_innochecksum.test | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/mysql-test/suite/innodb/t/page_id_innochecksum.test b/mysql-test/suite/innodb/t/page_id_innochecksum.test index 807185437d5..106da09df6d 100644 --- a/mysql-test/suite/innodb/t/page_id_innochecksum.test +++ b/mysql-test/suite/innodb/t/page_id_innochecksum.test @@ -1,5 +1,6 @@ --source include/have_innodb.inc --source include/not_embedded.inc +--source include/innodb_checksum_algorithm.inc --echo # Set the environmental variables let MYSQLD_BASEDIR= `SELECT @@basedir`; let MYSQLD_DATADIR= `SELECT @@datadir`; @@ -22,22 +23,32 @@ my $page_size = $ENV{INNODB_PAGE_SIZE}; sysopen IBD_FILE, "$ENV{MYSQLD_DATADIR}/test/t1.ibd", O_RDWR || die "Cannot open t1.ibd\n"; -sysread(IBD_FILE, $_, 38) || die "Cannot read t1.ibd\n"; -my $space = unpack("x[34]N", $_); +sysread(IBD_FILE, $_, 58) || die "Cannot read t1.ibd\n"; +my ($space,$flags) = unpack("x[34]Nx[16]N", $_); +my $full_crc32 = $flags & 0x10; sysseek(IBD_FILE, $page_size * 3, SEEK_SET) || die "Cannot seek t1.ibd\n"; -my $head = pack("Nx[18]", 4); # better to have a valid page number -my $body = chr(0) x ($page_size - 38 - 8); +my $page= pack("NNx[18]NNN", 0, 4, 1, 0, $space).(chr(0) x ($page_size - 38)); -# Calculate innodb_checksum_algorithm=crc32 for the unencrypted page. -# The following bytes are excluded: -# bytes 0..3 (the checksum is stored there) -# bytes 26..37 (encryption key version, post-encryption checksum, tablespace id) -# bytes $page_size-8..$page_size-1 (checksum, LSB of FIL_PAGE_LSN) +# Calculate innodb_checksum_algorithm=crc32 or full_crc32. my $polynomial = 0x82f63b78; # CRC-32C -my $ck = mycrc32($head, 0, $polynomial) ^ mycrc32($body, 0, $polynomial); +if ($full_crc32) +{ + my $ck = mycrc32(substr($page, 0, $page_size-4), 0, $polynomial); + substr($page, $page_size-4, 4) = pack("N", $ck); +} +else +{ + # The following bytes are excluded: + # 0..3 (the checksum is stored there) + # 26..37 (encryption key version, post-encryption checksum, tablespace id) + # $page_size-8..$page_size-1 (checksum, LSB of FIL_PAGE_LSN) + my $ck= pack("N",mycrc32(substr($page, 4, 22), 0, $polynomial) ^ + mycrc32(substr($page, 38, $page_size-38-8), 0, $polynomial)); + substr($page,0,4)=$ck; + substr($page,$page_size-8,4)=$ck; +} -my $page= pack("N",$ck).$head.pack("NNN",1,$ck,$space).$body.pack("Nx[4]",$ck); die unless syswrite(IBD_FILE, $page, $page_size) == $page_size; close IBD_FILE; EOF From 7772c7cd945cf674b212b82f4d156099c67344a2 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Wed, 14 Aug 2019 12:01:40 +0530 Subject: [PATCH 3/3] MDEV-20340 Encrypted temporary tables cannot be read with full_crc32 Problem: ======== Checksum for the encrypted temporary tablespace is not stored in the page for full crc32 format. Solution: ======== Made temporary tablespace in full crc32 format irrespective of encryption parameter. buf_tmp_page_encrypt(), buf_tmp_page_decrypt() - Both follows full_crc32 format. --- ...nodb_encrypt_temporary_tables.combinations | 5 +++ storage/innobase/buf/buf0buf.cc | 37 ++++++++----------- storage/innobase/handler/ha_innodb.cc | 12 ++---- 3 files changed, 24 insertions(+), 30 deletions(-) create mode 100644 mysql-test/suite/encryption/t/innodb_encrypt_temporary_tables.combinations diff --git a/mysql-test/suite/encryption/t/innodb_encrypt_temporary_tables.combinations b/mysql-test/suite/encryption/t/innodb_encrypt_temporary_tables.combinations new file mode 100644 index 00000000000..729380593f3 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_encrypt_temporary_tables.combinations @@ -0,0 +1,5 @@ +[strict_crc32] +--innodb-checksum-algorithm=strict_crc32 + +[strict_full_crc32] +--innodb-checksum-algorithm=strict_full_crc32 diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 0701ec31f5f..86b669209cb 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -484,7 +484,7 @@ static bool buf_tmp_page_decrypt(byte* tmp_frame, byte* src_frame) } /* read space & lsn */ - uint header_len = FIL_PAGE_DATA; + uint header_len = FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; /* Copy FIL page header, it is not encrypted */ memcpy(tmp_frame, src_frame, header_len); @@ -493,7 +493,7 @@ static bool buf_tmp_page_decrypt(byte* tmp_frame, byte* src_frame) const byte* src = src_frame + header_len; byte* dst = tmp_frame + header_len; uint srclen = uint(srv_page_size) - - header_len - FIL_PAGE_DATA_END; + - (header_len + FIL_PAGE_FCRC32_CHECKSUM); ulint offset = mach_read_from_4(src_frame + FIL_PAGE_OFFSET); if (!log_tmp_block_decrypt(src, srclen, dst, @@ -501,9 +501,9 @@ static bool buf_tmp_page_decrypt(byte* tmp_frame, byte* src_frame) return false; } - memcpy(tmp_frame + srv_page_size - FIL_PAGE_DATA_END, - src_frame + srv_page_size - FIL_PAGE_DATA_END, - FIL_PAGE_DATA_END); + memcpy(tmp_frame + srv_page_size - FIL_PAGE_FCRC32_CHECKSUM, + src_frame + srv_page_size - FIL_PAGE_FCRC32_CHECKSUM, + FIL_PAGE_FCRC32_CHECKSUM); memcpy(src_frame, tmp_frame, srv_page_size); srv_stats.pages_decrypted.inc(); @@ -5976,13 +5976,15 @@ static dberr_t buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) && !bpage->encrypted && fil_space_verify_crypt_checksum(dst_frame, bpage->zip_size()); + ut_ad(space->purpose != FIL_TYPE_TEMPORARY || space->full_crc32()); if (!still_encrypted) { /* If traditional checksums match, we assume that page is not anymore encrypted. */ if (space->full_crc32() && !buf_page_is_zeroes(dst_frame, space->physical_size()) - && (key_version || space->is_compressed())) { + && (key_version || space->is_compressed() + || space->purpose == FIL_TYPE_TEMPORARY)) { corrupted = buf_page_full_crc32_is_corrupted( space->id, dst_frame, space->is_compressed()); @@ -7427,28 +7429,21 @@ static byte* buf_tmp_page_encrypt( byte* src_frame, byte* dst_frame) { - uint header_len = FIL_PAGE_DATA; - /* FIL page header is not encrypted */ - memcpy(dst_frame, src_frame, header_len); - /* Calculate the start offset in a page */ - uint unencrypted_bytes = header_len + FIL_PAGE_DATA_END; - uint srclen = srv_page_size - unencrypted_bytes; - const byte* src = src_frame + header_len; - byte* dst = dst_frame + header_len; + uint srclen = srv_page_size - (FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + FIL_PAGE_FCRC32_CHECKSUM); + const byte* src = src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; + byte* dst = dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; + + memcpy(dst_frame, src_frame, FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); if (!log_tmp_block_encrypt(src, srclen, dst, (offset * srv_page_size), true)) { return NULL; } - memcpy(dst_frame + srv_page_size - FIL_PAGE_DATA_END, - src_frame + srv_page_size - FIL_PAGE_DATA_END, - FIL_PAGE_DATA_END); - - /* Handle post encryption checksum */ - mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, - buf_calc_page_crc32(dst_frame)); + const ulint payload = srv_page_size - FIL_PAGE_FCRC32_CHECKSUM; + mach_write_to_4(dst_frame + payload, ut_crc32(dst_frame, payload)); srv_stats.pages_encrypted.inc(); srv_stats.n_temp_blocks_encrypted.inc(); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 7b0ddceca0d..aa2f3805bd6 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3876,15 +3876,9 @@ static int innodb_init_params() srv_tmp_space.set_name("innodb_temporary"); srv_tmp_space.set_path(srv_data_home); - switch (srv_checksum_algorithm) { - case SRV_CHECKSUM_ALGORITHM_FULL_CRC32: - case SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32: - srv_tmp_space.set_flags(FSP_FLAGS_FCRC32_MASK_MARKER - | FSP_FLAGS_FCRC32_PAGE_SSIZE()); - break; - default: - srv_tmp_space.set_flags(FSP_FLAGS_PAGE_SSIZE()); - } + /* Temporary tablespace is in full crc32 format. */ + srv_tmp_space.set_flags(FSP_FLAGS_FCRC32_MASK_MARKER + | FSP_FLAGS_FCRC32_PAGE_SSIZE()); if (!srv_tmp_space.parse_params(innobase_temp_data_file_path, false)) { ib::error() << "Unable to parse innodb_temp_data_file_path="