From fa2701c6f7b028782cf231565f578b2fc0f10d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 14 Sep 2017 09:23:20 +0300 Subject: [PATCH 01/30] =?UTF-8?q?MDEV-12634:=20Uninitialised=20ROW=5FMERGE?= =?UTF-8?q?=5FRESERVE=5FSIZE=20bytes=20written=20to=20tem=E2=80=A6=20?= =?UTF-8?q?=E2=80=A6porary=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed by removing writing key version to start of every block that was encrypted. Instead we will use single key version from log_sys crypt info. After this MDEV also blocks writen to row log are encrypted and blocks read from row log aren decrypted if encryption is configured for the table. innodb_status_variables[], struct srv_stats_t Added status variables for merge block and row log block encryption and decryption amounts. Removed ROW_MERGE_RESERVE_SIZE define. row_merge_fts_doc_tokenize Remove ROW_MERGE_RESERVE_SIZE row_log_t Add index, crypt_tail, crypt_head to be used in case of encryption. row_log_online_op, row_log_table_close_func Before writing a block encrypt it if encryption is enabled row_log_table_apply_ops, row_log_apply_ops After reading a block decrypt it if encryption is enabled row_log_allocate Allocate temporary buffers crypt_head and crypt_tail if needed. row_log_free Free temporary buffers crypt_head and crypt_tail if they exist. row_merge_encrypt_buf, row_merge_decrypt_buf Removed. row_merge_buf_create, row_merge_buf_write Remove ROW_MERGE_RESERVE_SIZE row_merge_build_indexes Allocate temporary buffer used in decryption and encryption if needed. log_tmp_blocks_crypt, log_tmp_block_encrypt, log_temp_block_decrypt New functions used in block encryption and decryption log_tmp_is_encrypted New function to check is encryption enabled. Added test case innodb-rowlog to force creating a row log and verify that operations are done using introduced status variables. --- .../suite/encryption/r/innodb-rowlog.result | 30 ++ .../suite/encryption/t/innodb-rowlog.opt | 7 + .../suite/encryption/t/innodb-rowlog.test | 90 ++++++ storage/innobase/handler/ha_innodb.cc | 12 + storage/innobase/include/log0crypt.h | 44 ++- storage/innobase/include/row0ftsort.h | 3 +- storage/innobase/include/row0merge.h | 26 +- storage/innobase/include/srv0srv.h | 17 ++ storage/innobase/log/log0crypt.cc | 125 +++++++- storage/innobase/row/row0ftsort.cc | 33 +-- storage/innobase/row/row0log.cc | 114 ++++++- storage/innobase/row/row0merge.cc | 271 ++++++----------- storage/innobase/srv/srv0srv.cc | 4 + storage/xtradb/handler/ha_innodb.cc | 12 + storage/xtradb/include/log0crypt.h | 44 ++- storage/xtradb/include/row0ftsort.h | 3 +- storage/xtradb/include/row0merge.h | 30 +- storage/xtradb/include/srv0srv.h | 17 ++ storage/xtradb/log/log0crypt.cc | 125 +++++++- storage/xtradb/row/row0ftsort.cc | 33 +-- storage/xtradb/row/row0log.cc | 114 ++++++- storage/xtradb/row/row0merge.cc | 280 ++++++------------ storage/xtradb/srv/srv0srv.cc | 4 + 23 files changed, 971 insertions(+), 467 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innodb-rowlog.result create mode 100644 mysql-test/suite/encryption/t/innodb-rowlog.opt create mode 100644 mysql-test/suite/encryption/t/innodb-rowlog.test diff --git a/mysql-test/suite/encryption/r/innodb-rowlog.result b/mysql-test/suite/encryption/r/innodb-rowlog.result new file mode 100644 index 00000000000..873041ab421 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-rowlog.result @@ -0,0 +1,30 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; +SET GLOBAL innodb_encryption_rotate_key_age = 1; +create table t2(id int) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +drop table t2; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; +variable_value > 0 +1 +drop table t1; +SET GLOBAL innodb_file_format=Antelope; +SET GLOBAL innodb_encryption_rotate_key_age=15; diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.opt b/mysql-test/suite/encryption/t/innodb-rowlog.opt new file mode 100644 index 00000000000..6338ddbde35 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-rowlog.opt @@ -0,0 +1,7 @@ +--aria-encrypt-tables +--encrypt-tmp-disk-tables +--innodb-encrypt-tables +--innodb-encrypt-log +--innodb-encryption-rotate-key-age=15 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.test b/mysql-test/suite/encryption/t/innodb-rowlog.test new file mode 100644 index 00000000000..135293ef4b9 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-rowlog.test @@ -0,0 +1,90 @@ +-- source include/have_innodb.inc +-- source include/have_example_key_management_plugin.inc +# needs dbug_dbug +-- source include/have_debug.inc + +--disable_warnings +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $key_age = `SELECT @@innodb_encryption_rotate_key_age`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +--enable_warnings + +let $MYSQLD_DATADIR = `SELECT @@datadir`; +let $MYSQLD_TMPDIR = `SELECT @@tmpdir`; + +# +# Create a table that will be encrypted and put some sensitive data to it (credit card numbers) +# + +create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; + +let $rows = 15000; +--disable_query_log +begin; +while ($rows) +{ + eval insert into t1 values(NULL, '0000-0000-0000-0000','private_data'); + dec $rows; +} +commit; +--enable_query_log + +--let $wait_timeout= 600 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--source include/wait_condition.inc + +# +# Now we create secondary index for credit_card column in parallel we create new rows +# forcing alter table to wait so that row log is used. +# +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +send alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; +# +# Force key rotation and create second index for same table +# +connect (con2,localhost,root,,); +SET GLOBAL innodb_encryption_rotate_key_age = 1; +create table t2(id int) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +send alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; +# +# Create new rows to row log +# +connect (con1,localhost,root,,); +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; + +connection default; +reap; +disconnect con1; + +connection con2; +reap; +drop table t2; + +connection default; +disconnect con2; + +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; +--source include/wait_condition.inc + +# +# Verify that both merge blocks and row log blocks are encryted and decrypted +# + +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; + +drop table t1; + +--disable_warnings +eval SET GLOBAL innodb_file_format=$innodb_file_format_orig; +eval SET GLOBAL innodb_encryption_rotate_key_age=$key_age; +--enable_warnings diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4733c33a83f..afc320de517 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1087,6 +1087,18 @@ static SHOW_VAR innodb_status_variables[]= { {"encryption_key_rotation_list_length", (char*)&export_vars.innodb_key_rotation_list_length, SHOW_LONGLONG}, + {"encryption_n_merge_blocks_encrypted", + (char*)&export_vars.innodb_n_merge_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_merge_blocks_decrypted", + (char*)&export_vars.innodb_n_merge_blocks_decrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_encrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_decrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_decrypted, + SHOW_LONGLONG}, /* scrubing */ {"scrub_background_page_reorganizations", diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index 6b164e90d6e..0ad7e7d7037 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +29,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "univ.i" #include "ut0byte.h" #include "my_crypt.h" +#include "os0file.h" typedef int Crypt_result; @@ -125,4 +126,45 @@ log_crypt_print_checkpoint_keys( /*============================*/ const byte* log_block); +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() MY_ATTRIBUTE((warn_unused_result)); #endif // log0crypt.h diff --git a/storage/innobase/include/row0ftsort.h b/storage/innobase/include/row0ftsort.h index 00bd3317de3..e784fae78b9 100644 --- a/storage/innobase/include/row0ftsort.h +++ b/storage/innobase/include/row0ftsort.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -72,7 +72,6 @@ struct fts_psort_common_t { store Doc ID during sort, if Doc ID will not be big enough to use 8 bytes value */ - fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */ }; struct fts_psort_t { diff --git a/storage/innobase/include/row0merge.h b/storage/innobase/include/row0merge.h index 04d4010ad48..fea6bd99799 100644 --- a/storage/innobase/include/row0merge.h +++ b/storage/innobase/include/row0merge.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +41,6 @@ Created 13/06/2005 Jan Lindstrom #include "lock0types.h" #include "srv0srv.h" -/* Reserve free space from every block for key_version */ -#define ROW_MERGE_RESERVE_SIZE 4 - /* Cluster index read task is mandatory */ #define COST_READ_CLUSTERED_INDEX 1.0 @@ -352,17 +349,16 @@ row_merge_buf_sort( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ - + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Empty a sort buffer. @return sort buffer */ @@ -400,10 +396,10 @@ row_merge_sort( const bool update_progress, /*!< in: update progress status variable or not */ const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,5))); + MY_ATTRIBUTE((warn_unused_result)); + /*********************************************************************//** Allocate a sort buffer. @return own: sort buffer */ @@ -433,7 +429,7 @@ row_merge_file_destroy( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -441,9 +437,9 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Read a merge record. @@ -462,8 +458,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,6,7,8), warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); + #endif /* row0merge.h */ diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 79cf9cdd2ab..09af0b2cdd2 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -137,6 +137,14 @@ struct srv_stats_t { ulint_ctr_64_t pages_encrypted; /* Number of pages decrypted */ ulint_ctr_64_t pages_decrypted; + /* Number of merge blocks encrypted */ + ulint_ctr_64_t n_merge_blocks_encrypted; + /* Number of merge blocks decrypted */ + ulint_ctr_64_t n_merge_blocks_decrypted; + /* Number of row log blocks encrypted */ + ulint_ctr_64_t n_rowlog_blocks_encrypted; + /* Number of row log blocks decrypted */ + ulint_ctr_64_t n_rowlog_blocks_decrypted; /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1033,6 +1041,15 @@ struct export_var_t{ ib_int64_t innodb_pages_decrypted; /*!< Number of pages decrypted */ + /*!< Number of merge blocks encrypted */ + ib_int64_t innodb_n_merge_blocks_encrypted; + /*!< Number of merge blocks decrypted */ + ib_int64_t innodb_n_merge_blocks_decrypted; + /*!< Number of row log blocks encrypted */ + ib_int64_t innodb_n_rowlog_blocks_encrypted; + /*!< Number of row log blocks decrypted */ + ib_int64_t innodb_n_rowlog_blocks_decrypted; + ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ ulint innodb_sec_rec_cluster_reads_avoided;/*!< srv_sec_rec_cluster_reads_avoided */ diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index af9b2349187..fe1b41202ea 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -237,6 +237,129 @@ next: return rc; } +/** Encrypt/decrypt temporary log blocks. + +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] what ENCRYPTION_FLAG_ENCRYPT or + ENCRYPTION_FLAG_DECRYPT +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +static +bool +log_tmp_blocks_crypt( + const byte* src_block, + ulint size, + byte* dst_block, + int what, + os_offset_t offs, + ulint space_id) +{ + Crypt_result rc = MY_AES_OK; + uint dst_len; + byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; + byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; + const crypt_info_t* info = static_cast(&crypt_info[0]); + + // AES_CTR_COUNTER = space_id + offs + + bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); + mach_write_to_8(aes_ctr_counter, space_id); + mach_write_to_8(aes_ctr_counter + 8, offs); + + rc = encryption_crypt(src_block, size, + dst_block, &dst_len, + (unsigned char*)(info->crypt_key), 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + what | ENCRYPTION_FLAG_NOPAD, + LOG_DEFAULT_ENCRYPTION_KEY, + info->key_version); + + if (rc != MY_AES_OK) { + ib_logf(IB_LOG_LEVEL_ERROR, + "%s failed for temporary log file with rc = %d", + is_encrypt ? "Encryption" : "Decryption", + rc); + return false; + } + + return true; +} + +/** Get crypt info +@return pointer to log crypt info or NULL +*/ +inline +const crypt_info_t* +get_crypt_info() +{ + mutex_enter(&log_sys->mutex); + const crypt_info_t* info = get_crypt_info(log_sys->next_checkpoint_no); + mutex_exit(&log_sys->mutex); + + return info; +} + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() +{ + const crypt_info_t* info = get_crypt_info(); + + if (info == NULL || info->key_version == UNENCRYPTED_KEY_VER) { + return false; + } + + return true; +} + +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_ENCRYPT, offs, space_id)); +} + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_DECRYPT, offs, space_id)); +} + /*********************************************************************//** Generate crypt key from crypt msg. @return true if successfull, false if not. */ diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 62271e766cf..40ddb3a6a87 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -39,7 +39,7 @@ Created 10/13/2010 Jimmy Yang b[N] = row_merge_read_rec( \ block[N], buf[N], b[N], index, \ fd[N], &foffs[N], &mrec[N], offsets[N], \ - crypt_data, crypt_block[N], space); \ + crypt_block[N], space); \ if (UNIV_UNLIKELY(!b[N])) { \ if (mrec[N]) { \ goto exit; \ @@ -191,7 +191,6 @@ row_fts_psort_info_init( fts_psort_t* merge_info = NULL; ulint block_size; ibool ret = TRUE; - fil_space_crypt_t* crypt_data = NULL; bool encrypted = false; block_size = 3 * srv_sort_buf_size; @@ -222,21 +221,8 @@ row_fts_psort_info_init( common_info->merge_event = os_event_create(); common_info->opt_doc_id_size = opt_doc_id_size; - /* Theoretically the tablespace can be dropped straight away. - In practice, the DDL completion will wait for this thread to - finish. */ - if (fil_space_t* space = fil_space_acquire(new_table->space)) { - crypt_data = space->crypt_data; - fil_space_release(space); - } - - if (crypt_data && crypt_data->should_encrypt()) { - common_info->crypt_data = crypt_data; + if (log_tmp_is_encrypted()) { encrypted = true; - } else { - /* Not needed */ - common_info->crypt_data = NULL; - crypt_data = NULL; } ut_ad(trx->mysql_thd != NULL); @@ -574,11 +560,9 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* Reserve one byte for the end marker of row_merge_block_t - and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for - encryption key_version in the beginning of the buffer. */ + /* Reserve one byte for the end marker of row_merge_block_t */ if (buf->total_size + data_size[idx] + cur_len - >= (srv_sort_buf_size - 1 - ROW_MERGE_RESERVE_SIZE)) { + >= (srv_sort_buf_size - 1)) { buf_full = TRUE; break; @@ -672,7 +656,6 @@ fts_parallel_tokenization( fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; - fil_space_crypt_t* crypt_data = NULL; ut_ad(psort_info->psort_common->trx->mysql_thd != NULL); @@ -693,7 +676,6 @@ fts_parallel_tokenization( block = psort_info->merge_block; crypt_block = psort_info->crypt_block; - crypt_data = psort_info->psort_common->crypt_data; zip_size = dict_table_zip_size(table); row_merge_fts_get_next_doc_item(psort_info, &doc_item); @@ -788,7 +770,6 @@ loop: if (!row_merge_write(merge_file[t_ctx.buf_used]->fd, merge_file[t_ctx.buf_used]->offset++, block[t_ctx.buf_used], - crypt_data, crypt_block[t_ctx.buf_used], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -884,7 +865,6 @@ exit: if (!row_merge_write(merge_file[i]->fd, merge_file[i]->offset++, block[i], - crypt_data, crypt_block[i], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -924,7 +904,7 @@ exit: psort_info->psort_common->dup, merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */, - crypt_data, crypt_block[i], table->space); + crypt_block[i], table->space); if (error != DB_SUCCESS) { close(tmpfd[i]); @@ -1436,7 +1416,6 @@ row_fts_merge_insert( ulint start; fts_psort_insert_t ins_ctx; ulint count_diag = 0; - fil_space_crypt_t* crypt_data = NULL; ulint space; ut_ad(index); @@ -1450,7 +1429,6 @@ row_fts_merge_insert( ins_ctx.trx->op_info = "inserting index entries"; ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size; - crypt_data = psort_info[0].psort_common->crypt_data; heap = mem_heap_create(500 + sizeof(mrec_buf_t)); @@ -1544,7 +1522,6 @@ row_fts_merge_insert( && (!row_merge_read( fd[i], foffs[i], (row_merge_block_t*) block[i], - crypt_data, (row_merge_block_t*) crypt_block[i], space))) { error = DB_CORRUPTION; diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 88398c20342..ec6b4a60680 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -182,6 +182,7 @@ struct row_log_t { dict_table_t* table; /*!< table that is being rebuilt, or NULL when this is a secondary index that is being created online */ + dict_index_t* index; /*!< index to be build */ bool same_pk;/*!< whether the definition of the PRIMARY KEY has remained the same */ const dtuple_t* add_cols; @@ -197,8 +198,14 @@ struct row_log_t { row_log_buf_t tail; /*!< writer context; protected by mutex and index->lock S-latch, or by index->lock X-latch only */ + byte* crypt_tail; /*!< writer context; + temporary buffer used in encryption, + decryption or NULL*/ row_log_buf_t head; /*!< reader context; protected by MDL only; modifiable by row_log_apply_ops() */ + byte* crypt_head; /*!< reader context; + temporary buffer used in encryption, + decryption or NULL */ const char* path; /*!< where to create temporary file during log operation */ }; @@ -349,6 +356,7 @@ row_log_online_op( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -368,11 +376,29 @@ row_log_online_op( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: /* We set the flag directly instead of invoking @@ -380,7 +406,9 @@ write_failed: because the index is not "public" yet. */ index->type |= DICT_CORRUPT; } + UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); + memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); log->tail.bytes = mrec_size - avail_size; @@ -465,6 +493,7 @@ row_log_table_close_func( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -484,11 +513,29 @@ row_log_table_close_func( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + log->index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: log->error = DB_ONLINE_LOG_TOO_BIG; @@ -2622,10 +2669,29 @@ all_done: goto func_exit; } + byte * buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for table %s\n", index->table_name); @@ -2932,9 +2998,22 @@ row_log_allocate( log->head.blocks = log->head.bytes = 0; log->head.total = 0; log->path = path; + log->crypt_tail = log->crypt_head = NULL; + log->index = index; dict_index_set_online_status(index, ONLINE_INDEX_CREATION); index->online_log = log; + if (log_tmp_is_encrypted()) { + ulint size = srv_sort_buf_size; + log->crypt_head = static_cast(os_mem_alloc_large(&size)); + log->crypt_tail = static_cast(os_mem_alloc_large(&size)); + + if (!log->crypt_head || !log->crypt_tail) { + row_log_free(log); + DBUG_RETURN(false); + } + } + /* While we might be holding an exclusive data dictionary lock here, in row_log_abort_sec() we will not always be holding it. Use atomic operations in both cases. */ @@ -2957,6 +3036,15 @@ row_log_free( row_log_block_free(log->tail); row_log_block_free(log->head); row_merge_file_destroy_low(log->fd); + + if (log->crypt_head) { + os_mem_free_large(log->crypt_head, srv_sort_buf_size); + } + + if (log->crypt_tail) { + os_mem_free_large(log->crypt_tail, srv_sort_buf_size); + } + mutex_free(&log->mutex); ut_free(log); log = 0; @@ -3451,11 +3539,29 @@ all_done: goto func_exit; } + byte* buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for index %s\n", index->name + 1); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 65569eabf57..a91c0f67218 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -79,88 +79,6 @@ UNIV_INTERN char srv_disable_sort_file_cache; /* Maximum pending doc memory limit in bytes for a fts tokenization thread */ #define FTS_PENDING_DOC_MEMORY_LIMIT 1000000 - -/******************************************************//** -Encrypt a merge block. */ -static -void -row_merge_encrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - key_version = encryption_key_get_latest_version(crypt_data->key_id); - - /* Store key_version at the begining of the input buffer */ - mach_write_to_4((byte *)crypted_buf, key_version); - - int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } -} - -/******************************************************//** -Decrypt a merge block. */ -static -bool -row_merge_decrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - /* Read key_version from begining of the buffer */ - key_version = mach_read_from_4((byte *)input_buf); - - if (key_version == 0) { - /* block not encrypted */ - return false; - } - - int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } - - return true; -} - #ifdef UNIV_DEBUG /******************************************************//** Display a merge tuple. */ @@ -279,7 +197,7 @@ row_merge_buf_create( ulint buf_size; mem_heap_t* heap; - max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE) + max_tuples = (srv_sort_buf_size) / ut_max(1, dict_index_get_min_size(index)); buf_size = (sizeof *buf); @@ -695,7 +613,7 @@ row_merge_buf_add( ut_ad(data_size < srv_sort_buf_size); /* Reserve bytes for the end marker of row_merge_block_t. */ - if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) { + if (buf->total_size + data_size >= srv_sort_buf_size) { DBUG_RETURN(0); } @@ -816,7 +734,7 @@ UT_SORT_FUNCTION_BODY(). /**********************************************************************//** Merge sort the tuple buffer in main memory. */ -static MY_ATTRIBUTE((nonnull(4,5))) +static void row_merge_tuple_sort( /*=================*/ @@ -867,7 +785,7 @@ row_merge_buf_write( { const dict_index_t* index = buf->index; ulint n_fields= dict_index_get_n_fields(index); - byte* b = &block[ROW_MERGE_RESERVE_SIZE]; + byte* b = &block[0]; for (ulint i = 0; i < buf->n_tuples; i++) { const mtuple_t* entry = &buf->tuples[i]; @@ -886,7 +804,7 @@ row_merge_buf_write( /* Write an "end-of-chunk" marker. */ ut_a(b < &block[srv_sort_buf_size]); - ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE); + ut_a(b == &block[0] + buf->total_size); *b++ = 0; #ifdef UNIV_DEBUG_VALGRIND /* The rest of the block is uninitialized. Initialize it @@ -936,7 +854,7 @@ row_merge_heap_create( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -944,12 +862,11 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size; - ibool success; + bool success; DBUG_EXECUTE_IF("row_merge_read_failure", return(FALSE);); @@ -964,10 +881,14 @@ row_merge_read( ofs, srv_sort_buf_size); /* For encrypted tables, decrypt data after reading and copy data */ - if (crypt_data && crypt_buf) { - if (row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) { - memcpy(buf, crypt_buf, srv_sort_buf_size); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, srv_sort_buf_size, + crypt_buf, ofs, space)) { + return (FALSE); } + + srv_stats.n_merge_blocks_decrypted.inc(); + memcpy(buf, crypt_buf, srv_sort_buf_size); } #ifdef POSIX_FADV_DONTNEED @@ -989,31 +910,32 @@ row_merge_read( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { size_t buf_len = srv_sort_buf_size; os_offset_t ofs = buf_len * (os_offset_t) offset; - ibool ret; + bool ret; void* out_buf = (void *)buf; DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE);); /* For encrypted tables, encrypt data before writing */ - if (crypt_data && crypt_buf) { - row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt((const byte *)buf, buf_len, + (byte *)crypt_buf, ofs, space)) { + return (FALSE); + } + + srv_stats.n_merge_blocks_encrypted.inc(); out_buf = crypt_buf; - } else { - /* Mark block unencrypted */ - mach_write_to_4((byte *)out_buf, 0); } ret = os_file_write_int_fd("(merge)", fd, out_buf, ofs, buf_len); @@ -1051,9 +973,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint data_size; @@ -1065,10 +986,6 @@ row_merge_read_rec( ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - extra_size = *b++; if (UNIV_UNLIKELY(!extra_size)) { @@ -1090,7 +1007,8 @@ row_merge_read_rec( if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) { if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { err_exit: /* Signal I/O error. */ *mrec = b; @@ -1098,7 +1016,7 @@ err_exit: } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; } extra_size = (extra_size & 0x7f) << 8; @@ -1120,13 +1038,14 @@ err_exit: memcpy(*buf, b, avail_size); if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the record. */ memcpy(*buf + avail_size, b, extra_size - avail_size); @@ -1182,13 +1101,14 @@ err_exit: #endif /* UNIV_DEBUG */ if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the rest of the record. */ memcpy(*buf + avail_size, b, extra_size + data_size - avail_size); @@ -1265,7 +1185,6 @@ row_merge_write_rec( ulint* foffs, /*!< in/out: file offset */ const mrec_t* mrec, /*!< in: record to write */ const ulint* offsets,/*!< in: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1288,10 +1207,6 @@ row_merge_write_rec( size = extra_size + (extra_size >= 0x80) + rec_offs_data_size(offsets); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) { /* The record spans two blocks. Copy it to the temporary buffer first. */ @@ -1307,14 +1222,15 @@ row_merge_write_rec( memcpy(b, buf[0], avail_size); if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); /* Copy the rest. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; memcpy(b, buf[0] + avail_size, size - avail_size); b += size - avail_size; } else { @@ -1337,7 +1253,6 @@ row_merge_write_eof( byte* b, /*!< in: pointer to end of block */ int fd, /*!< in: file descriptor */ ulint* foffs, /*!< in/out: file offset */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1352,10 +1267,6 @@ row_merge_write_eof( } #endif /* UNIV_DEBUG */ - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - *b++ = 0; UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]); UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size); @@ -1367,7 +1278,8 @@ row_merge_write_eof( #endif /* UNIV_DEBUG_VALGRIND */ if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } @@ -1445,10 +1357,9 @@ containing the index entries for the indexes to be built. @param[in,out] block file buffer @param[in,out] tmpfd temporary file handle @param[in] pct_cost percent of task weight out of total alter job -@param[in] crypt_data crypt data or NULL @param[in,out] crypt_block crypted file buffer -return DB_SUCCESS or error */ -static MY_ATTRIBUTE((nonnull(1,2,3,4,6,9,10,16), warn_unused_result)) +@return DB_SUCCESS or error */ +static MY_ATTRIBUTE((warn_unused_result)) dberr_t row_merge_read_clustered_index( trx_t* trx, @@ -1469,7 +1380,6 @@ row_merge_read_clustered_index( row_merge_block_t* block, int* tmpfd, float pct_cost, - fil_space_crypt_t* crypt_data, row_merge_block_t* crypt_block) { dict_index_t* clust_index; /* Clustered index */ @@ -2004,8 +1914,8 @@ write_buffers: row_merge_buf_write(buf, file, block); if (!row_merge_write(file->fd, file->offset++, - block, crypt_data, crypt_block, - new_table->space)) { + block, crypt_block, + new_table->space)) { err = DB_TEMP_FILE_WRITE_FAILURE; trx->error_key_num = i; break; @@ -2054,7 +1964,7 @@ write_buffers: if(read_rows % 1000 == 0) { /* Update progress for each 1000 rows */ curr_progress = (read_rows >= table_total_rows) ? - pct_cost : + pct_cost : ((pct_cost * read_rows) / table_total_rows); /* presenting 10.12% as 1012 integer */ onlineddl_pct_progress = curr_progress * 100; @@ -2183,9 +2093,8 @@ wait_again: &buf[2], b2, \ of->fd, &of->offset, \ mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL , \ - space); \ + space); \ if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \ goto corrupt; \ } \ @@ -2193,7 +2102,6 @@ wait_again: &buf[N], b##N, INDEX, \ file->fd, foffs##N, \ &mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[N * srv_sort_buf_size] : NULL, \ space); \ \ @@ -2208,7 +2116,7 @@ wait_again: /*************************************************************//** Merge two blocks of records on disk and write a bigger block. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_blocks( /*=============*/ @@ -2222,7 +2130,6 @@ row_merge_blocks( ulint* foffs1, /*!< in/out: offset of second source list in the file */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ row_merge_block_t* crypt_block,/*!< in: in/out: crypted file buffer */ ulint space) /*!< in: space id */ @@ -2258,9 +2165,11 @@ row_merge_blocks( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space) - || !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space) || + !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space)) { corrupt: mem_heap_free(heap); return(DB_CORRUPTION); @@ -2273,13 +2182,15 @@ corrupt: b0 = row_merge_read_rec( &block[0], &buf[0], b0, dup->index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); b1 = row_merge_read_rec( &block[srv_sort_buf_size], &buf[srv_sort_buf_size], b1, dup->index, file->fd, foffs1, &mrec1, offsets1, - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0) || UNIV_UNLIKELY(!b1 && mrec1)) { @@ -2325,7 +2236,8 @@ done1: b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space); return(b2 ? DB_SUCCESS : DB_CORRUPTION); } @@ -2333,8 +2245,8 @@ done1: /*************************************************************//** Copy a block of index entries. @return TRUE on success, FALSE on failure */ -static __attribute__((nonnull(1,2,3,4,5), warn_unused_result)) -ibool +static __attribute__((warn_unused_result)) +bool row_merge_blocks_copy( /*==================*/ const dict_index_t* index, /*!< in: index being created */ @@ -2342,9 +2254,8 @@ row_merge_blocks_copy( row_merge_block_t* block, /*!< in/out: 3 buffers */ ulint* foffs0, /*!< in/out: input file offset */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */ @@ -2372,7 +2283,8 @@ row_merge_blocks_copy( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space)) { corrupt: mem_heap_free(heap); return(FALSE); @@ -2384,7 +2296,8 @@ corrupt: b0 = row_merge_read_rec(&block[0], &buf[0], b0, index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0)) { @@ -2407,15 +2320,15 @@ done0: return(row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, - crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space) + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space) != NULL); } /*************************************************************//** Merge disk files. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6,7))) +static dberr_t row_merge( /*======*/ @@ -2431,9 +2344,8 @@ row_merge( ulint* run_offset, /*!< in/out: Array contains the first offset number for each merge run */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint foffs0; /*!< first input offset */ ulint foffs1; /*!< second input offset */ @@ -2481,7 +2393,8 @@ row_merge( error = row_merge_blocks(dup, file, block, &foffs0, &foffs1, &of, - crypt_data, crypt_block, space); + crypt_block, + space); if (error != DB_SUCCESS) { return(error); @@ -2502,7 +2415,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs0, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2520,7 +2434,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs1, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2576,9 +2491,8 @@ row_merge_sort( /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const ulint half = file->offset / 2; ulint num_runs; @@ -2646,7 +2560,8 @@ row_merge_sort( error = row_merge(trx, dup, file, block, tmpfd, &num_runs, run_offset, - crypt_data, crypt_block, space); + crypt_block, + space); if(update_progress) { merge_count++; @@ -2723,7 +2638,7 @@ row_merge_copy_blobs( Read sorted file containing index data tuples and insert these data tuples to the index @return DB_SUCCESS or error number */ -static __attribute__((nonnull(2,3,5), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_insert_index_tuples( /*==========================*/ @@ -2736,9 +2651,8 @@ row_merge_insert_index_tuples( const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const byte* b; mem_heap_t* heap; @@ -2772,7 +2686,8 @@ row_merge_insert_index_tuples( b = &block[0]; if (!row_merge_read(fd, foffs, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { error = DB_CORRUPTION; } else { buf = static_cast( @@ -2789,7 +2704,9 @@ row_merge_insert_index_tuples( b = row_merge_read_rec(block, buf, b, index, fd, &foffs, &mrec, offsets, - crypt_data, crypt_block, space); + crypt_block, + space); + if (UNIV_UNLIKELY(!b)) { /* End of list, or I/O error */ if (mrec) { @@ -3974,7 +3891,6 @@ row_merge_build_indexes( fts_psort_t* merge_info = NULL; ib_int64_t sig_count = 0; bool fts_psort_initiated = false; - fil_space_crypt_t * crypt_data = NULL; float total_static_cost = 0; float total_dynamic_cost = 0; @@ -4003,25 +3919,20 @@ row_merge_build_indexes( from concurrent DDL (e.g. drop table) by MDL-locks. */ fil_space_t* space = fil_space_acquire(new_table->space); - if (space) { - crypt_data = space->crypt_data; - } else { + if (!space) { DBUG_RETURN(DB_TABLESPACE_NOT_FOUND); } /* If tablespace is encrypted, allocate additional buffer for encryption/decryption. */ - if (crypt_data && crypt_data->should_encrypt()) { + if (log_tmp_is_encrypted()) { crypt_block = static_cast( - os_mem_alloc_large(&block_size)); + os_mem_alloc_large(&block_size)); if (crypt_block == NULL) { fil_space_release(space); DBUG_RETURN(DB_OUT_OF_MEMORY); } - } else { - /* Not needed */ - crypt_data = NULL; } trx_start_if_not_started_xa(trx); @@ -4103,7 +4014,7 @@ row_merge_build_indexes( fts_sort_idx, psort_info, merge_files, key_numbers, n_indexes, add_cols, col_map, add_autoinc, sequence, block, &tmpfd, - pct_cost, crypt_data, crypt_block); + pct_cost, crypt_block); pct_progress += pct_cost; @@ -4240,7 +4151,8 @@ wait_again: trx, &dup, &merge_files[i], block, &tmpfd, true, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, + new_table->space); pct_progress += pct_cost; @@ -4278,7 +4190,8 @@ wait_again: trx->id, sort_idx, old_table, merge_files[i].fd, block, merge_files[i].n_rec, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, new_table->space); + pct_progress += pct_cost; if (global_system_variables.log_warnings > 2) { diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 7ddd934a9a1..1431211279b 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1619,6 +1619,10 @@ srv_export_innodb_status(void) export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; export_vars.innodb_pages_encrypted = srv_stats.pages_encrypted; + export_vars.innodb_n_merge_blocks_encrypted = srv_stats.n_merge_blocks_encrypted; + export_vars.innodb_n_merge_blocks_decrypted = srv_stats.n_merge_blocks_decrypted; + export_vars.innodb_n_rowlog_blocks_encrypted = srv_stats.n_rowlog_blocks_encrypted; + export_vars.innodb_n_rowlog_blocks_decrypted = srv_stats.n_rowlog_blocks_decrypted; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index caf7a3ef412..4af523b1018 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1309,6 +1309,18 @@ static SHOW_VAR innodb_status_variables[]= { {"encryption_key_rotation_list_length", (char*)&export_vars.innodb_key_rotation_list_length, SHOW_LONGLONG}, + {"encryption_n_merge_blocks_encrypted", + (char*)&export_vars.innodb_n_merge_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_merge_blocks_decrypted", + (char*)&export_vars.innodb_n_merge_blocks_decrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_encrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_decrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_decrypted, + SHOW_LONGLONG}, /* Scrubing feature */ {"scrub_background_page_reorganizations", diff --git a/storage/xtradb/include/log0crypt.h b/storage/xtradb/include/log0crypt.h index 6b164e90d6e..0ad7e7d7037 100644 --- a/storage/xtradb/include/log0crypt.h +++ b/storage/xtradb/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +29,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "univ.i" #include "ut0byte.h" #include "my_crypt.h" +#include "os0file.h" typedef int Crypt_result; @@ -125,4 +126,45 @@ log_crypt_print_checkpoint_keys( /*============================*/ const byte* log_block); +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() MY_ATTRIBUTE((warn_unused_result)); #endif // log0crypt.h diff --git a/storage/xtradb/include/row0ftsort.h b/storage/xtradb/include/row0ftsort.h index 7c9ed23645c..b2dd90e7e3b 100644 --- a/storage/xtradb/include/row0ftsort.h +++ b/storage/xtradb/include/row0ftsort.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -72,7 +72,6 @@ struct fts_psort_common_t { store Doc ID during sort, if Doc ID will not be big enough to use 8 bytes value */ - fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */ }; struct fts_psort_t { diff --git a/storage/xtradb/include/row0merge.h b/storage/xtradb/include/row0merge.h index 04d4010ad48..152a51dafc6 100644 --- a/storage/xtradb/include/row0merge.h +++ b/storage/xtradb/include/row0merge.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +41,6 @@ Created 13/06/2005 Jan Lindstrom #include "lock0types.h" #include "srv0srv.h" -/* Reserve free space from every block for key_version */ -#define ROW_MERGE_RESERVE_SIZE 4 - /* Cluster index read task is mandatory */ #define COST_READ_CLUSTERED_INDEX 1.0 @@ -352,17 +349,16 @@ row_merge_buf_sort( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ - + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Empty a sort buffer. @return sort buffer */ @@ -400,10 +396,9 @@ row_merge_sort( const bool update_progress, /*!< in: update progress status variable or not */ const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,5))); + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /*********************************************************************//** Allocate a sort buffer. @return own: sort buffer */ @@ -433,7 +428,7 @@ row_merge_file_destroy( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -441,10 +436,9 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ - + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Read a merge record. @return pointer to next record, or NULL on I/O error or end of list */ @@ -462,8 +456,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,6,7,8), warn_unused_result)); + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); + #endif /* row0merge.h */ diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 30c125ea269..e7f6350987e 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -137,6 +137,14 @@ struct srv_stats_t { ulint_ctr_64_t pages_encrypted; /* Number of pages decrypted */ ulint_ctr_64_t pages_decrypted; + /* Number of merge blocks encrypted */ + ulint_ctr_64_t n_merge_blocks_encrypted; + /* Number of merge blocks decrypted */ + ulint_ctr_64_t n_merge_blocks_decrypted; + /* Number of row log blocks encrypted */ + ulint_ctr_64_t n_rowlog_blocks_encrypted; + /* Number of row log blocks decrypted */ + ulint_ctr_64_t n_rowlog_blocks_decrypted; /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1276,6 +1284,15 @@ struct export_var_t{ ib_int64_t innodb_pages_decrypted; /*!< Number of pages decrypted */ + /*!< Number of merge blocks encrypted */ + ib_int64_t innodb_n_merge_blocks_encrypted; + /*!< Number of merge blocks decrypted */ + ib_int64_t innodb_n_merge_blocks_decrypted; + /*!< Number of row log blocks encrypted */ + ib_int64_t innodb_n_rowlog_blocks_encrypted; + /*!< Number of row log blocks decrypted */ + ib_int64_t innodb_n_rowlog_blocks_decrypted; + ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ ulint innodb_sec_rec_cluster_reads_avoided;/*!< srv_sec_rec_cluster_reads_avoided */ diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index f6c1416d81a..9509797dde3 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -236,6 +236,129 @@ next: return rc; } +/** Encrypt/decrypt temporary log blocks. + +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] what ENCRYPTION_FLAG_ENCRYPT or + ENCRYPTION_FLAG_DECRYPT +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +static +bool +log_tmp_blocks_crypt( + const byte* src_block, + ulint size, + byte* dst_block, + int what, + os_offset_t offs, + ulint space_id) +{ + Crypt_result rc = MY_AES_OK; + uint dst_len; + byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; + byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; + const crypt_info_t* info = static_cast(&crypt_info[0]); + + // AES_CTR_COUNTER = space_id + offs + + bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); + mach_write_to_8(aes_ctr_counter, space_id); + mach_write_to_8(aes_ctr_counter + 8, offs); + + rc = encryption_crypt(src_block, size, + dst_block, &dst_len, + (unsigned char*)(info->crypt_key), 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + what | ENCRYPTION_FLAG_NOPAD, + LOG_DEFAULT_ENCRYPTION_KEY, + info->key_version); + + if (rc != MY_AES_OK) { + ib_logf(IB_LOG_LEVEL_ERROR, + "%s failed for temporary log file with rc = %d", + is_encrypt ? "Encryption" : "Decryption", + rc); + return false; + } + + return true; +} + +/** Get crypt info +@return pointer to log crypt info or NULL +*/ +inline +const crypt_info_t* +get_crypt_info() +{ + mutex_enter(&log_sys->mutex); + const crypt_info_t* info = get_crypt_info(log_sys->next_checkpoint_no); + mutex_exit(&log_sys->mutex); + + return info; +} + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() +{ + const crypt_info_t* info = get_crypt_info(); + + if (info == NULL || info->key_version == UNENCRYPTED_KEY_VER) { + return false; + } + + return true; +} + +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_ENCRYPT, offs, space_id)); +} + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_DECRYPT, offs, space_id)); +} + /*********************************************************************//** Generate crypt key from crypt msg. @return true if successfull, false if not. */ diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc index 7ffcc59dc5f..0f9c15f6a69 100644 --- a/storage/xtradb/row/row0ftsort.cc +++ b/storage/xtradb/row/row0ftsort.cc @@ -40,7 +40,7 @@ Created 10/13/2010 Jimmy Yang b[N] = row_merge_read_rec( \ block[N], buf[N], b[N], index, \ fd[N], &foffs[N], &mrec[N], offsets[N], \ - crypt_data, crypt_block[N], space); \ + crypt_block[N], space); \ if (UNIV_UNLIKELY(!b[N])) { \ if (mrec[N]) { \ goto exit; \ @@ -194,7 +194,6 @@ row_fts_psort_info_init( fts_psort_t* merge_info = NULL; ulint block_size; ibool ret = TRUE; - fil_space_crypt_t* crypt_data = NULL; bool encrypted = false; block_size = 3 * srv_sort_buf_size; @@ -225,21 +224,8 @@ row_fts_psort_info_init( common_info->merge_event = os_event_create(); common_info->opt_doc_id_size = opt_doc_id_size; - /* Theoretically the tablespace can be dropped straight away. - In practice, the DDL completion will wait for this thread to - finish. */ - if (fil_space_t* space = fil_space_acquire(new_table->space)) { - crypt_data = space->crypt_data; - fil_space_release(space); - } - - if (crypt_data && crypt_data->should_encrypt()) { - common_info->crypt_data = crypt_data; + if (log_tmp_is_encrypted()) { encrypted = true; - } else { - /* Not needed */ - common_info->crypt_data = NULL; - crypt_data = NULL; } ut_ad(trx->mysql_thd != NULL); @@ -577,11 +563,9 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* Reserve one byte for the end marker of row_merge_block_t - and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for - encryption key_version in the beginning of the buffer. */ + /* Reserve one byte for the end marker of row_merge_block_t */ if (buf->total_size + data_size[idx] + cur_len - >= (srv_sort_buf_size - 1 - ROW_MERGE_RESERVE_SIZE)) { + >= (srv_sort_buf_size - 1)) { buf_full = TRUE; break; @@ -675,7 +659,6 @@ fts_parallel_tokenization( fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; - fil_space_crypt_t* crypt_data = NULL; ut_ad(psort_info->psort_common->trx->mysql_thd != NULL); @@ -696,7 +679,6 @@ fts_parallel_tokenization( block = psort_info->merge_block; crypt_block = psort_info->crypt_block; - crypt_data = psort_info->psort_common->crypt_data; zip_size = dict_table_zip_size(table); row_merge_fts_get_next_doc_item(psort_info, &doc_item); @@ -791,7 +773,6 @@ loop: if (!row_merge_write(merge_file[t_ctx.buf_used]->fd, merge_file[t_ctx.buf_used]->offset++, block[t_ctx.buf_used], - crypt_data, crypt_block[t_ctx.buf_used], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -887,7 +868,6 @@ exit: if (!row_merge_write(merge_file[i]->fd, merge_file[i]->offset++, block[i], - crypt_data, crypt_block[i], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -927,7 +907,7 @@ exit: psort_info->psort_common->dup, merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */, - crypt_data, crypt_block[i], table->space); + crypt_block[i], table->space); if (error != DB_SUCCESS) { close(tmpfd[i]); @@ -1439,7 +1419,6 @@ row_fts_merge_insert( ulint start; fts_psort_insert_t ins_ctx; ulint count_diag = 0; - fil_space_crypt_t* crypt_data = NULL; ulint space; ut_ad(index); @@ -1453,7 +1432,6 @@ row_fts_merge_insert( ins_ctx.trx->op_info = "inserting index entries"; ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size; - crypt_data = psort_info[0].psort_common->crypt_data; heap = mem_heap_create(500 + sizeof(mrec_buf_t)); @@ -1547,7 +1525,6 @@ row_fts_merge_insert( && (!row_merge_read( fd[i], foffs[i], (row_merge_block_t*) block[i], - crypt_data, (row_merge_block_t*) crypt_block[i], space))) { error = DB_CORRUPTION; diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc index 2cd663fd600..f49ff2f1530 100644 --- a/storage/xtradb/row/row0log.cc +++ b/storage/xtradb/row/row0log.cc @@ -182,6 +182,7 @@ struct row_log_t { dict_table_t* table; /*!< table that is being rebuilt, or NULL when this is a secondary index that is being created online */ + dict_index_t* index; /*!< index to be build */ bool same_pk;/*!< whether the definition of the PRIMARY KEY has remained the same */ const dtuple_t* add_cols; @@ -197,8 +198,14 @@ struct row_log_t { row_log_buf_t tail; /*!< writer context; protected by mutex and index->lock S-latch, or by index->lock X-latch only */ + byte* crypt_tail; /*!< writer context; + temporary buffer used in encryption, + decryption or NULL*/ row_log_buf_t head; /*!< reader context; protected by MDL only; modifiable by row_log_apply_ops() */ + byte* crypt_head; /*!< reader context; + temporary buffer used in encryption, + decryption or NULL */ const char* path; /*!< where to create temporary file during log operation */ }; @@ -349,6 +356,7 @@ row_log_online_op( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -368,11 +376,29 @@ row_log_online_op( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: /* We set the flag directly instead of invoking @@ -380,7 +406,9 @@ write_failed: because the index is not "public" yet. */ index->type |= DICT_CORRUPT; } + UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); + memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); log->tail.bytes = mrec_size - avail_size; @@ -465,6 +493,7 @@ row_log_table_close_func( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -484,11 +513,29 @@ row_log_table_close_func( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + log->index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: log->error = DB_ONLINE_LOG_TOO_BIG; @@ -2619,10 +2666,29 @@ all_done: goto func_exit; } + byte * buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for table %s\n", index->table_name); @@ -2929,9 +2995,22 @@ row_log_allocate( log->head.blocks = log->head.bytes = 0; log->head.total = 0; log->path = path; + log->crypt_tail = log->crypt_head = NULL; + log->index = index; dict_index_set_online_status(index, ONLINE_INDEX_CREATION); index->online_log = log; + if (log_tmp_is_encrypted()) { + ulint size = srv_sort_buf_size; + log->crypt_head = static_cast(os_mem_alloc_large(&size)); + log->crypt_tail = static_cast(os_mem_alloc_large(&size)); + + if (!log->crypt_head || !log->crypt_tail) { + row_log_free(log); + DBUG_RETURN(false); + } + } + /* While we might be holding an exclusive data dictionary lock here, in row_log_abort_sec() we will not always be holding it. Use atomic operations in both cases. */ @@ -2954,6 +3033,15 @@ row_log_free( row_log_block_free(log->tail); row_log_block_free(log->head); row_merge_file_destroy_low(log->fd); + + if (log->crypt_head) { + os_mem_free_large(log->crypt_head, srv_sort_buf_size); + } + + if (log->crypt_tail) { + os_mem_free_large(log->crypt_tail, srv_sort_buf_size); + } + mutex_free(&log->mutex); ut_free(log); log = 0; @@ -3445,11 +3533,29 @@ all_done: goto func_exit; } + byte* buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for index %s\n", index->name + 1); diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 408e94b3997..28b43d6abef 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -79,88 +79,6 @@ UNIV_INTERN char srv_disable_sort_file_cache; /* Maximum pending doc memory limit in bytes for a fts tokenization thread */ #define FTS_PENDING_DOC_MEMORY_LIMIT 1000000 - -/******************************************************//** -Encrypt a merge block. */ -static -void -row_merge_encrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - key_version = encryption_key_get_latest_version(crypt_data->key_id); - - /* Store key_version at the begining of the input buffer */ - mach_write_to_4((byte *)crypted_buf, key_version); - - int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } -} - -/******************************************************//** -Decrypt a merge block. */ -static -bool -row_merge_decrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - /* Read key_version from begining of the buffer */ - key_version = mach_read_from_4((byte *)input_buf); - - if (key_version == 0) { - /* block not encrypted */ - return false; - } - - int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } - - return (true); -} - #ifdef UNIV_DEBUG /******************************************************//** Display a merge tuple. */ @@ -279,7 +197,7 @@ row_merge_buf_create( ulint buf_size; mem_heap_t* heap; - max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE) + max_tuples = (srv_sort_buf_size) / ut_max(1, dict_index_get_min_size(index)); buf_size = (sizeof *buf); @@ -702,7 +620,7 @@ row_merge_buf_add( ut_ad(data_size < srv_sort_buf_size); /* Reserve bytes for the end marker of row_merge_block_t. */ - if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) { + if (buf->total_size + data_size >= srv_sort_buf_size) { DBUG_RETURN(0); } @@ -823,7 +741,7 @@ UT_SORT_FUNCTION_BODY(). /**********************************************************************//** Merge sort the tuple buffer in main memory. */ -static MY_ATTRIBUTE((nonnull(4,5))) +static void row_merge_tuple_sort( /*=================*/ @@ -874,7 +792,7 @@ row_merge_buf_write( { const dict_index_t* index = buf->index; ulint n_fields= dict_index_get_n_fields(index); - byte* b = &block[ROW_MERGE_RESERVE_SIZE]; + byte* b = &block[0]; for (ulint i = 0; i < buf->n_tuples; i++) { const mtuple_t* entry = &buf->tuples[i]; @@ -893,7 +811,7 @@ row_merge_buf_write( /* Write an "end-of-chunk" marker. */ ut_a(b < &block[srv_sort_buf_size]); - ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE); + ut_a(b == &block[0] + buf->total_size); *b++ = 0; #ifdef UNIV_DEBUG_VALGRIND /* The rest of the block is uninitialized. Initialize it @@ -943,7 +861,7 @@ row_merge_heap_create( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -951,12 +869,11 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size; - ibool success; + bool success; DBUG_EXECUTE_IF("row_merge_read_failure", return(FALSE);); @@ -971,10 +888,14 @@ row_merge_read( ofs, srv_sort_buf_size); /* For encrypted tables, decrypt data after reading and copy data */ - if (crypt_data && crypt_buf) { - if( row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) { - memcpy(buf, crypt_buf, srv_sort_buf_size); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, srv_sort_buf_size, + crypt_buf, ofs, space)) { + return (FALSE); } + + srv_stats.n_merge_blocks_decrypted.inc(); + memcpy(buf, crypt_buf, srv_sort_buf_size); } #ifdef POSIX_FADV_DONTNEED @@ -996,31 +917,32 @@ row_merge_read( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { size_t buf_len = srv_sort_buf_size; os_offset_t ofs = buf_len * (os_offset_t) offset; - ibool ret; + bool ret; void* out_buf = (void *)buf; DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE);); /* For encrypted tables, encrypt data before writing */ - if (crypt_data && crypt_buf) { - row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt((const byte *)buf, buf_len, + (byte *)crypt_buf, ofs, space)) { + return (FALSE); + } + + srv_stats.n_merge_blocks_encrypted.inc(); out_buf = crypt_buf; - } else { - /* Mark block unencrypted */ - mach_write_to_4((byte *)out_buf, 0); } ret = os_file_write_int_fd("(merge)", fd, out_buf, ofs, buf_len); @@ -1058,9 +980,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint data_size; @@ -1072,10 +993,6 @@ row_merge_read_rec( ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - extra_size = *b++; if (UNIV_UNLIKELY(!extra_size)) { @@ -1097,7 +1014,8 @@ row_merge_read_rec( if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) { if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { err_exit: /* Signal I/O error. */ *mrec = b; @@ -1105,7 +1023,7 @@ err_exit: } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; } extra_size = (extra_size & 0x7f) << 8; @@ -1127,13 +1045,14 @@ err_exit: memcpy(*buf, b, avail_size); if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the record. */ memcpy(*buf + avail_size, b, extra_size - avail_size); @@ -1189,13 +1108,14 @@ err_exit: #endif /* UNIV_DEBUG */ if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the rest of the record. */ memcpy(*buf + avail_size, b, extra_size + data_size - avail_size); @@ -1272,9 +1192,8 @@ row_merge_write_rec( ulint* foffs, /*!< in/out: file offset */ const mrec_t* mrec, /*!< in: record to write */ const ulint* offsets,/*!< in: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint size; @@ -1295,10 +1214,6 @@ row_merge_write_rec( size = extra_size + (extra_size >= 0x80) + rec_offs_data_size(offsets); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) { /* The record spans two blocks. Copy it to the temporary buffer first. */ @@ -1314,14 +1229,15 @@ row_merge_write_rec( memcpy(b, buf[0], avail_size); if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); /* Copy the rest. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; memcpy(b, buf[0] + avail_size, size - avail_size); b += size - avail_size; } else { @@ -1344,7 +1260,6 @@ row_merge_write_eof( byte* b, /*!< in: pointer to end of block */ int fd, /*!< in: file descriptor */ ulint* foffs, /*!< in/out: file offset */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1359,10 +1274,6 @@ row_merge_write_eof( } #endif /* UNIV_DEBUG */ - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - *b++ = 0; UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]); UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size); @@ -1374,7 +1285,8 @@ row_merge_write_eof( #endif /* UNIV_DEBUG_VALGRIND */ if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } @@ -1451,8 +1363,10 @@ containing the index entries for the indexes to be built. @param[in,out] sequence autoinc sequence @param[in,out] block file buffer @param[in,out] tmpfd temporary file handle -return DB_SUCCESS or error */ -static MY_ATTRIBUTE((nonnull(1,2,3,4,6,9,10,16), warn_unused_result)) +@param[in] pct_cost percent of task weight out of total alter job +@param[in,out] crypt_block crypted file buffer +@return DB_SUCCESS or error */ +static MY_ATTRIBUTE((warn_unused_result)) dberr_t row_merge_read_clustered_index( trx_t* trx, @@ -1472,11 +1386,8 @@ row_merge_read_clustered_index( ib_sequence_t& sequence, row_merge_block_t* block, int* tmpfd, - float pct_cost, /*!< in: percent of task weight - out of total alter job */ - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ - row_merge_block_t* crypt_block)/*!< in: in/out: crypted file - buffer */ + float pct_cost, + row_merge_block_t* crypt_block) { dict_index_t* clust_index; /* Clustered index */ mem_heap_t* row_heap; /* Heap memory to create @@ -2014,7 +1925,7 @@ write_buffers: row_merge_buf_write(buf, file, block); if (!row_merge_write(file->fd, file->offset++, - block, crypt_data, crypt_block, + block, crypt_block, new_table->space)) { err = DB_TEMP_FILE_WRITE_FAILURE; trx->error_key_num = i; @@ -2066,7 +1977,7 @@ write_buffers: if(read_rows % 1000 == 0) { /* Update progress for each 1000 rows */ curr_progress = (read_rows >= table_total_rows) ? - pct_cost : + pct_cost : ((pct_cost * read_rows) / table_total_rows); /* presenting 10.12% as 1012 integer */ onlineddl_pct_progress = curr_progress * 100; @@ -2195,9 +2106,8 @@ wait_again: &buf[2], b2, \ of->fd, &of->offset, \ mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL , \ - space); \ + space); \ if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \ goto corrupt; \ } \ @@ -2205,7 +2115,6 @@ wait_again: &buf[N], b##N, INDEX, \ file->fd, foffs##N, \ &mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[N * srv_sort_buf_size] : NULL, \ space); \ \ @@ -2220,7 +2129,7 @@ wait_again: /*************************************************************//** Merge two blocks of records on disk and write a bigger block. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_blocks( /*=============*/ @@ -2234,7 +2143,6 @@ row_merge_blocks( ulint* foffs1, /*!< in/out: offset of second source list in the file */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ row_merge_block_t* crypt_block,/*!< in: in/out: crypted file buffer */ ulint space) /*!< in: space id */ @@ -2270,9 +2178,11 @@ row_merge_blocks( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space) - || !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space) || + !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space)) { corrupt: mem_heap_free(heap); return(DB_CORRUPTION); @@ -2285,13 +2195,15 @@ corrupt: b0 = row_merge_read_rec( &block[0], &buf[0], b0, dup->index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); b1 = row_merge_read_rec( &block[srv_sort_buf_size], &buf[srv_sort_buf_size], b1, dup->index, file->fd, foffs1, &mrec1, offsets1, - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0) || UNIV_UNLIKELY(!b1 && mrec1)) { @@ -2337,7 +2249,8 @@ done1: b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space); return(b2 ? DB_SUCCESS : DB_CORRUPTION); } @@ -2345,8 +2258,8 @@ done1: /*************************************************************//** Copy a block of index entries. @return TRUE on success, FALSE on failure */ -static __attribute__((nonnull(1,2,3,4,5), warn_unused_result)) -ibool +static __attribute__((warn_unused_result)) +bool row_merge_blocks_copy( /*==================*/ const dict_index_t* index, /*!< in: index being created */ @@ -2354,9 +2267,8 @@ row_merge_blocks_copy( row_merge_block_t* block, /*!< in/out: 3 buffers */ ulint* foffs0, /*!< in/out: input file offset */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */ @@ -2384,7 +2296,8 @@ row_merge_blocks_copy( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space)) { corrupt: mem_heap_free(heap); return(FALSE); @@ -2396,7 +2309,8 @@ corrupt: b0 = row_merge_read_rec(&block[0], &buf[0], b0, index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0)) { @@ -2419,15 +2333,15 @@ done0: return(row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, - crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space) + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space) != NULL); } /*************************************************************//** Merge disk files. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6,7))) +static dberr_t row_merge( /*======*/ @@ -2443,9 +2357,8 @@ row_merge( ulint* run_offset, /*!< in/out: Array contains the first offset number for each merge run */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint foffs0; /*!< first input offset */ ulint foffs1; /*!< second input offset */ @@ -2493,7 +2406,8 @@ row_merge( error = row_merge_blocks(dup, file, block, &foffs0, &foffs1, &of, - crypt_data, crypt_block, space); + crypt_block, + space); if (error != DB_SUCCESS) { return(error); @@ -2514,7 +2428,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs0, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2532,7 +2447,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs1, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2588,9 +2504,8 @@ row_merge_sort( /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const ulint half = file->offset / 2; ulint num_runs; @@ -2650,7 +2565,8 @@ row_merge_sort( error = row_merge(trx, dup, file, block, tmpfd, &num_runs, run_offset, - crypt_data, crypt_block, space); + crypt_block, + space); if(update_progress) { merge_count++; @@ -2725,7 +2641,7 @@ row_merge_copy_blobs( Read sorted file containing index data tuples and insert these data tuples to the index @return DB_SUCCESS or error number */ -static __attribute__((nonnull(2,3,5), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_insert_index_tuples( /*==========================*/ @@ -2738,9 +2654,8 @@ row_merge_insert_index_tuples( const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const byte* b; mem_heap_t* heap; @@ -2774,7 +2689,8 @@ row_merge_insert_index_tuples( b = &block[0]; if (!row_merge_read(fd, foffs, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { error = DB_CORRUPTION; } else { buf = static_cast( @@ -2791,7 +2707,9 @@ row_merge_insert_index_tuples( b = row_merge_read_rec(block, buf, b, index, fd, &foffs, &mrec, offsets, - crypt_data, crypt_block, space); + crypt_block, + space); + if (UNIV_UNLIKELY(!b)) { /* End of list, or I/O error */ if (mrec) { @@ -3976,7 +3894,6 @@ row_merge_build_indexes( fts_psort_t* merge_info = NULL; ib_int64_t sig_count = 0; bool fts_psort_initiated = false; - fil_space_crypt_t * crypt_data = NULL; float total_static_cost = 0; float total_dynamic_cost = 0; @@ -4005,25 +3922,20 @@ row_merge_build_indexes( from concurrent DDL (e.g. drop table) by MDL-locks. */ fil_space_t* space = fil_space_acquire(new_table->space); - if (space) { - crypt_data = space->crypt_data; - } else { + if (!space) { DBUG_RETURN(DB_TABLESPACE_NOT_FOUND); } - /* If tablespace is encrypted, allocate additional buffer for + /* If temporal log file is encrypted allocate memory for encryption/decryption. */ - if (crypt_data && crypt_data->should_encrypt()) { + if (log_tmp_is_encrypted()) { crypt_block = static_cast( - os_mem_alloc_large(&block_size)); + os_mem_alloc_large(&block_size)); if (crypt_block == NULL) { fil_space_release(space); DBUG_RETURN(DB_OUT_OF_MEMORY); } - } else { - /* Not needed */ - crypt_data = NULL; } trx_start_if_not_started_xa(trx); @@ -4105,7 +4017,7 @@ row_merge_build_indexes( fts_sort_idx, psort_info, merge_files, key_numbers, n_indexes, add_cols, col_map, add_autoinc, sequence, block, &tmpfd, pct_cost, - crypt_data, crypt_block); + crypt_block); pct_progress += pct_cost; @@ -4242,7 +4154,8 @@ wait_again: trx, &dup, &merge_files[i], block, &tmpfd, true, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, + new_table->space); pct_progress += pct_cost; @@ -4280,7 +4193,8 @@ wait_again: trx->id, sort_idx, old_table, merge_files[i].fd, block, merge_files[i].n_rec, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, new_table->space); + pct_progress += pct_cost; if (global_system_variables.log_warnings > 2) { diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 1f1fef6bea6..d6070107432 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -2060,6 +2060,10 @@ srv_export_innodb_status(void) export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; export_vars.innodb_pages_encrypted = srv_stats.pages_encrypted; + export_vars.innodb_n_merge_blocks_encrypted = srv_stats.n_merge_blocks_encrypted; + export_vars.innodb_n_merge_blocks_decrypted = srv_stats.n_merge_blocks_decrypted; + export_vars.innodb_n_rowlog_blocks_encrypted = srv_stats.n_rowlog_blocks_encrypted; + export_vars.innodb_n_rowlog_blocks_decrypted = srv_stats.n_rowlog_blocks_decrypted; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; From c94fb7b7c18ce058060d377980aa269c24537207 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Thu, 14 Sep 2017 11:16:40 +0400 Subject: [PATCH 02/30] Compiler warning avoided. --- strings/json_lib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/strings/json_lib.c b/strings/json_lib.c index 25e5e81e6f9..1743fce83d0 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -1447,7 +1447,7 @@ int json_find_paths_next(json_engine_t *je, json_find_paths_t *state) /* Path already failed. */ continue; if (state->paths[p_c].steps[state->cur_depth].type & - (je->state == JST_OBJ_START) ? JSON_PATH_KEY : JSON_PATH_ARRAY) + ((je->state == JST_OBJ_START) ? JSON_PATH_KEY : JSON_PATH_ARRAY)) state->path_depths[p_c]++; } state->cur_depth++; From e2b9f6762cc55f8723efaf50098e09edb3ee9570 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Thu, 14 Sep 2017 14:30:24 +0400 Subject: [PATCH 03/30] Make compiler happy with the 'fall through' comments. --- strings/json_lib.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/strings/json_lib.c b/strings/json_lib.c index 1743fce83d0..f666d95985b 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -1139,21 +1139,21 @@ int json_path_setup(json_path_t *p, case PS_EKY: p->last_step->key_end= p->s.c_str - c_len; state= PS_KEY; - /* Note no 'continue' here. */ + /* fall through */ case PS_KEY: p->last_step++; if (p->last_step - p->steps >= JSON_DEPTH_LIMIT) return p->s.error= JE_DEPTH; p->types_used|= p->last_step->type= JSON_PATH_KEY | double_wildcard; double_wildcard= JSON_PATH_KEY_NULL; - /* Note no 'continue' here. */ + /* fall through */ case PS_KEYX: p->last_step->key= p->s.c_str; continue; case PS_EAR: p->last_step->key_end= p->s.c_str - c_len; state= PS_AR; - /* Note no 'continue' here. */ + /* fall through */ case PS_AR: p->last_step++; if (p->last_step - p->steps >= JSON_DEPTH_LIMIT) @@ -1705,6 +1705,7 @@ int json_get_path_next(json_engine_t *je, json_path_t *p) return 1; /* Now we have je.state == JST_VALUE, so let's handle it. */ + /* fall through */ case JST_VALUE: if (json_read_value(je)) return 1; From e9d2f37abd766e5b02d223215bc069530ebb10b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 14 Sep 2017 16:07:10 +0300 Subject: [PATCH 04/30] MDEV-13626 Merge InnoDB test cases from MySQL 5.7 (part 2) Import the changes to innodb.innodb-index innodb.innodb-index-debug Note: As noted in MDEV-13613, due to the behaviour change in MDEV-11114, DROP COLUMN will not imply DROP/ADD PRIMARY/UNIQUE KEY, like it does in MySQL. The tests have been adjusted accordingly. --- .../innodb/include/alter_table_pk_no_sort.inc | 265 +++++++ .../suite/innodb/r/innodb-index-debug.result | 265 ++++++- mysql-test/suite/innodb/r/innodb-index.result | 664 ++++++++++++++++++ .../suite/innodb/t/innodb-index-debug.test | 89 ++- mysql-test/suite/innodb/t/innodb-index.test | 530 ++++++++++++++ 5 files changed, 1784 insertions(+), 29 deletions(-) create mode 100644 mysql-test/suite/innodb/include/alter_table_pk_no_sort.inc diff --git a/mysql-test/suite/innodb/include/alter_table_pk_no_sort.inc b/mysql-test/suite/innodb/include/alter_table_pk_no_sort.inc new file mode 100644 index 00000000000..6a2fcd15be0 --- /dev/null +++ b/mysql-test/suite/innodb/include/alter_table_pk_no_sort.inc @@ -0,0 +1,265 @@ +# skip sort for prefix change +# pk(o1(2)) to pk(o1(3)) +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1(3)), lock=none; +drop table t1; + +# pk(o1(2)) to pk(o1) +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1), lock=none; +drop table t1; + +# pk(o1(2)) to pk(o1(3),n1) +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(3), n1), lock=none; +drop table t1; + +# pk(o1(2)) to pk(o1,n1) +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 add n1 int not null, drop primary key, add primary key(o1, n1), lock=none; +drop table t1; + +# pk(o1(2)) to pk(o1(3), o2) +create table t1(o1 varchar(10), o2 int not null, primary key(o1(2))) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(3), o2), lock=none; +drop table t1; + +# pk(o1(2)) to pk(o1, o2) +create table t1(o1 varchar(10), o2 int not null, primary key(o1(2))) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1, o2), lock=none; +drop table t1; + +# pk(o1(3)) to pk(o1(2)) +create table t1(o1 varchar(10), primary key(o1(3))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; + +# pk(o1) to pk(o1(2)) +create table t1(o1 varchar(10), primary key(o1)) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; + +# pk(o1(3),o2) to pk(o1(2)) +create table t1(o1 varchar(10), o2 int, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1(2)) +create table t1(o1 varchar(10), o2 int, primary key(o1, o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; + +# pk(o1(3),o2) to pk(o1(2),n1) +create table t1(o1 varchar(10), o2 int, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(2),n1), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1(2),n1) +create table t1(o1 varchar(10), o2 int, primary key(o1, o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(2),n1), lock=none; +drop table t1; + +# pk(o1(3),o2) to pk(o1(3),n1) +create table t1(o1 varchar(10), o2 int, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(3),n1), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1,n1) +create table t1(o1 varchar(10), o2 int, primary key(o1, o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1), lock=none; +drop table t1; + +# pk(o1,o2(3)) to pk(o1,o2(2)) +create table t1(o1 int, o2 varchar(10), primary key(o1,o2(3))) engine = innodb; +insert into t1 values(1,'abd'), (2,'acd'); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1,o2(2)) +create table t1(o1 int, o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values(1,'abd'), (2,'acd'); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; + +# pk(o1,o2(2)) to pk(o1,o2(3)) +create table t1(o1 int, o2 varchar(10), primary key(o1,o2(2))) engine = innodb; +insert into t1 values(1, 'abd'), (2, 'acd'); +alter table t1 drop primary key, add primary key(o1,o2(3)), lock=none; +drop table t1; + +# pk(o1,o2(2)) to pk(o1,o2) +create table t1(o1 int, o2 varchar(10), primary key(o1,o2(2))) engine = innodb; +insert into t1 values(1, 'abd'), (2, 'acd'); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; + +# pk(o1,o2(3),o3) to pk(o1,o2(2)) +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2(3),o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (2, 'acd', 2); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o1,o2(2)) +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (2, 'acd', 2); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; + +# pk(o1(3),o2(3)) to pk(o1(3),o2(2)) +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(3))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2(2)), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1,o2(2)) +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; + +# pk(o1(3),o2(2)) to pk(o1(3),o2(3)) +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(2))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2(3)), lock=none; +drop table t1; + +# pk(o1,o2(2)) to pk(o1,o2) +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2(2))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; + +# pk(o1(3),o2,o3(2)) to pk(o1(3),o2,o3(3)) +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1(3),o2,o3(2))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2,o3(3)), lock=none; +drop table t1; + +# pk(o1,o2,o3(2)) to pk(o1,o2,o3) +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1,o2,o3(2))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1,o2,o3), lock=none; +drop table t1; + +# pk(o1(3),o2,o3(3)) to pk(o1(3),o2,o3(2)) +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1(3),o2,o3(3))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2,o3(2)), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o1,o2,o3(2)) +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1,o2,o3(3))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1,o2,o3(2)), lock=none; +drop table t1; + +# skip sort for adding existing columns/newly added columns, dropping pk columns at the end. +# pk(o1) to pk(o1,o2) +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; + +# pk(o1) to pk(o1,n1) +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1), lock=none; +drop table t1; + +# pk(o1) to pk(n1,o1) +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(n1,o1), lock=none; +drop table t1; + +# pk(o1) to pk(n1,o1,n2) +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, add n2 int not null, drop primary key, add primary key(n1,o1,n2), lock=none; +drop table t1; + +# pk(o1) to pk(n1,n2,o1) +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, add n2 int not null, drop primary key, add primary key(n1,n2,o1), lock=none; +drop table t1; + +# pk(o1) to pk(o1,n1,n2) +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, add n2 int not null, drop primary key, add primary key(o1,n1,n2), lock=none; +drop table t1; + +# pk(o1) to pk(o1,o2,n1) +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,o2,n1), lock=none; +drop table t1; + +# pk(o1) to pk(o1,n1,o2) +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1,o2), lock=none; +drop table t1; + +# pk(o1) to pk(n1,o1,o2) +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(n1,o1,o2), lock=none; +drop table t1; + +# pk(o1) to pk(o1,o2,o3) +create table t1(o1 int, o2 int not null, o3 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1,o2,o3), lock=none; +drop table t1; + +# pk(o1) to pk(o1,o3,o2) +create table t1(o1 int, o2 int not null, o3 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1,o3,o2), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o1,o2) +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o1,o2,o3,o4) +create table t1(o1 int, o2 int, o3 int, o4 int not null, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2,2),(2,2,1,1); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,o2,o3,o4), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o1,o2,n1) +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,o2,n1), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o1,n1,o2) +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1,o2), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o1) +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1), lock=none; +drop table t1; diff --git a/mysql-test/suite/innodb/r/innodb-index-debug.result b/mysql-test/suite/innodb/r/innodb-index-debug.result index 09c13e37cf2..172e4ebf454 100644 --- a/mysql-test/suite/innodb/r/innodb-index-debug.result +++ b/mysql-test/suite/innodb/r/innodb-index-debug.result @@ -34,9 +34,8 @@ CREATE UNIQUE INDEX ui ON bug13861218(c1); SET DEBUG_DBUG = @saved_debug_dbug; DROP TABLE bug13861218; # -# Bug #21762319 ADDING INDEXES ON EMPTY TABLE IS SLOW -# WITH LARGE INNODB_SORT_BUFFER_SIZE. -call mtr.add_suppression("InnoDB: Cannot create temporary merge file"); +# Bug #17657223 EXCESSIVE TEMPORARY FILE USAGE IN ALTER TABLE +# create table t480(a serial)engine=innodb; insert into t480 values(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(), @@ -47,14 +46,252 @@ insert into t480 select 0 from t480; insert into t480 select 0 from t480; create table t1(f1 int auto_increment not null, f2 char(200) not null, f3 char(200) not null, -f4 char(200) not null,primary key(f1))engine=innodb; -insert into t1 select NULL,'aaa','bbb','ccc' from t480; -insert into t1 select NULL,'aaaa','bbbb','cccc' from t480; -insert into t1 select NULL,'aaaaa','bbbbb','ccccc' from t480; -insert into t1 select NULL,'aaaaaa','bbbbbb','cccccc' from t480; -insert into t1 select NULL,'aaaaaaa','bbbbbbb','ccccccc' from t480; -insert into t1 select NULL,'aaaaaaaa','bbbbbbbb','cccccccc' from t480; -select count(*) from t1; -count(*) -2880 -drop table t1, t480; +primary key(f1,f2,f3), key(f1))engine=innodb; +insert into t1 select NULL,'aaa','bbb' from t480; +insert into t1 select NULL,'aaaa','bbbb' from t480; +insert into t1 select NULL,'aaaaa','bbbbb' from t480; +insert into t1 select NULL,'aaaaaa','bbbbbb' from t480; +SET DEBUG_DBUG = '+d,row_merge_write_failure'; +alter table t1 drop primary key,add primary key(f2,f1); +ERROR HY000: Temporary file write failure +SET DEBUG_DBUG = @saved_debug_dbug; +drop table t1; +connect con1,localhost,root; +create table t1(k1 int auto_increment primary key, +k2 char(200),k3 char(200))engine=innodb; +insert into t1 values(NULL,'a','b'), (NULL,'aa','bb'); +SET DEBUG_SYNC= 'row_merge_after_scan +SIGNAL opened WAIT_FOR flushed'; +optimize table t1; +connection default; +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +INSERT INTO t1 select NULL,'aaa','bbb' from t480; +SET DEBUG_SYNC= 'now SIGNAL flushed'; +connection con1; +/*con1 reap*/ Optimize table t1; +Table Op Msg_type Msg_text +test.t1 optimize note Table does not support optimize, doing recreate + analyze instead +test.t1 optimize status OK +affected rows: 2 +SELECT COUNT(k1),k2,k3 FROM t1 GROUP BY k2,k3; +COUNT(k1) k2 k3 +1 a b +1 aa bb +480 aaa bbb +drop table t1; +create table t1(k1 int auto_increment primary key, +k2 char(200),k3 char(200))engine=innodb; +SET DEBUG_SYNC= 'row_merge_after_scan +SIGNAL opened WAIT_FOR flushed'; +ALTER TABLE t1 ADD COLUMN k4 int; +connection default; +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +SET debug = '+d,row_log_tmpfile_fail'; +Warnings: +Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +INSERT INTO t1 select NULL,'aaa','bbb' from t480; +INSERT INTO t1 select NULL,'aaaa','bbbb' from t480; +SET DEBUG_SYNC= 'now SIGNAL flushed'; +SET DEBUG_DBUG = @saved_debug_dbug; +connection con1; +/*con1 reap*/ ALTER TABLE t1 ADD COLUMN k4 int; +ERROR HY000: Out of memory; check if mysqld or some other process uses all available memory; if not, you may have to use 'ulimit' to allow mysqld to use more memory or you can add more swap space +SELECT COUNT(k1),k2,k3 FROM t1 GROUP BY k2,k3; +COUNT(k1) k2 k3 +480 aaa bbb +480 aaaa bbbb +disconnect con1; +connection default; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `k1` int(11) NOT NULL AUTO_INCREMENT, + `k2` char(200) DEFAULT NULL, + `k3` char(200) DEFAULT NULL, + PRIMARY KEY (`k1`) +) ENGINE=InnoDB AUTO_INCREMENT=1023 DEFAULT CHARSET=latin1 +drop table t1; +drop table t480; +SET DEBUG_SYNC='RESET'; +# +# BUG#21612714 ALTER TABLE SORTING SKIPPED WHEN CHANGE PK AND DROP +# LAST COLUMN OF OLD PK +# +SET DEBUG_DBUG = '+d,innodb_alter_table_pk_assert_no_sort'; +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1(3)), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(3), n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 add n1 int not null, drop primary key, add primary key(o1, n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int not null, primary key(o1(2))) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(3), o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int not null, primary key(o1(2))) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1, o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1(3))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1)) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1, o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(2),n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1, o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(2),n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(3),n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1, o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), primary key(o1,o2(3))) engine = innodb; +insert into t1 values(1,'abd'), (2,'acd'); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values(1,'abd'), (2,'acd'); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), primary key(o1,o2(2))) engine = innodb; +insert into t1 values(1, 'abd'), (2, 'acd'); +alter table t1 drop primary key, add primary key(o1,o2(3)), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), primary key(o1,o2(2))) engine = innodb; +insert into t1 values(1, 'abd'), (2, 'acd'); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2(3),o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (2, 'acd', 2); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (2, 'acd', 2); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(3))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(2))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2(3)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2(2))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1(3),o2,o3(2))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2,o3(3)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1,o2,o3(2))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1,o2,o3), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1(3),o2,o3(3))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2,o3(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1,o2,o3(3))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1,o2,o3(2)), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(n1,o1), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, add n2 int not null, drop primary key, add primary key(n1,o1,n2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, add n2 int not null, drop primary key, add primary key(n1,n2,o1), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, add n2 int not null, drop primary key, add primary key(o1,n1,n2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,o2,n1), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(n1,o1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, o3 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1,o2,o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, o3 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1,o3,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, o4 int not null, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2,2),(2,2,1,1); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,o2,o3,o4), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,o2,n1), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1), lock=none; +drop table t1; +SET DEBUG_DBUG = @saved_debug_dbug; diff --git a/mysql-test/suite/innodb/r/innodb-index.result b/mysql-test/suite/innodb/r/innodb-index.result index 1b7f44fa8c4..54ad4e8a927 100644 --- a/mysql-test/suite/innodb/r/innodb-index.result +++ b/mysql-test/suite/innodb/r/innodb-index.result @@ -1184,3 +1184,667 @@ t2c CREATE TABLE `t2c` ( KEY `t2a` (`a`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 DROP TABLE t1,t2,t2c,t2i; +# +# Bug #17657223 EXCESSIVE TEMPORARY FILE USAGE IN ALTER TABLE +# +SET GLOBAL innodb_monitor_enable = module_ddl; +create table t1(f1 int not null, f2 blob)engine=innodb; +insert into t1 values(1, repeat('a',20000)); +# Skip sort +# Reusing the same pk +alter table t1 force; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +drop table t1; +create table t1(f1 int not null, f2 int not null, +primary key(f1))engine=innodb; +insert into t1 values(1,2), (3,4); +# Add Secondary index. +# Skip temp file usage due to small table size +alter table t1 add key(f2); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +drop table t1; +create table t480(a serial)engine=innodb; +insert into t480 +values(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(), +(),(),(),(),(),(),(),(); +insert into t480 select 0 from t480; +insert into t480 select 0 from t480; +insert into t480 select 0 from t480; +insert into t480 select 0 from t480; +create table t1(f1 int auto_increment not null, +f2 char(200) not null, f3 char(200) not null, +f4 char(200) not null,primary key(f1))engine=innodb; +insert into t1 select NULL,'aaa','bbb','ccc' from t480; +insert into t1 select NULL,'aaaa','bbbb','cccc' from t480; +insert into t1 select NULL,'aaaaa','bbbbb','ccccc' from t480; +insert into t1 select NULL,'aaaaaa','bbbbbb','cccccc' from t480; +insert into t1 select NULL,'aaaaaaa','bbbbbbb','ccccccc' from t480; +insert into t1 select NULL,'aaaaaaaa','bbbbbbbb','cccccccc' from t480; +select count(*) from t1; +count(*) +2880 +# Skip sort +# Change PK from (f1) to (f1,f2,f3,f4) +alter table t1 drop primary key, add primary key(f1,f2,f3,f4); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Change PK from (f1,f2,f3,f4) to (f1,f2,added_columns) +alter table t1 drop primary key,add column f5 int not null, +add column f6 int not null,add primary key(f1,f2,f5,f6); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Change PK from (f1,f2,f5,f6) to (f1,f2,f5) +alter table t1 drop column f6; +ERROR 42000: Key column 'f6' doesn't exist in table +alter table t1 drop column f6, drop primary key, add primary key(f1,f2,f5); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Reusing the same PK +alter table t1 add column f6 int; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Reusing the same pk +alter table t1 drop column f6; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Must sort +# Change PK from (f1,f2,f5) to (f1,f5) +alter table t1 drop column f2; +ERROR 42000: Key column 'f2' doesn't exist in table +alter table t1 drop column f2, drop primary key, add primary key(f1,f5); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +ddl_sort_file_alter_table 2 +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Reusing the same pk +alter table t1 add column f2n int after f1, drop primary key, add +primary key (f1,f5,f2n); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Reusing the same pk +alter table t1 change f5 f2n int not null,change f2n f5 int not null, +add column f8 int not null; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Change PK from (f1,f4,f2n) to (f1,f4,added_column,f2n) +alter table t1 add column f7 int, drop primary key, +add primary key (f1,f5,f7,f2n); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +ddl_sort_file_alter_table 2 +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Reusing the same pk +alter table t1 force; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Reusing the same pk +alter table t1 row_format=compact; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Reusing the same pk +alter table t1 engine=innodb; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Optimize table +optimize table t1; +Table Op Msg_type Msg_text +test.t1 optimize note Table does not support optimize, doing recreate + analyze instead +test.t1 optimize status OK +affected rows: 2 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Sort files used for adding secondary index +alter table t1 drop primary key, add primary key(f1,f5,f7), add index +i(f3); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# No sort files used for dropping secondary index +alter table t1 drop primary key, add primary key(f1,f5),drop index i; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Change PK(f1,f5) to (f1,added_columns) and drop f5 +alter table t1 drop primary key, add primary key(f1,f12), +drop column f5, add column f12 int not null; +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Must sort +# Change PK(f1,f12) to (f1,existing_columns) +alter table t1 drop primary key, add primary key(f1,f3); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +ddl_sort_file_alter_table 2 +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort +# Change PK(f1,f3) to (f1,added_column,f3,added_column) +alter table t1 drop primary key, add column f3n int, +add column f4n int, add primary key(f1,f3n,f3,f4n); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Adding Secondary index alone. +alter table t1 add key(f1); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Must sort +# Change PK(f1,f3) to (existing_column,f1) +alter table t1 drop primary key, add primary key(f4,f1); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +ddl_sort_file_alter_table 3 +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort for PK. +# Change PK(f4,f1) to (added_columns,f4,f1) +# Secondary index rebuild happens +alter table t1 drop primary key, add column f5n int, +add column f6n int, add primary key(f5n,f6n,f4,f1); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +# Skip sort for PK. +# Change PK(f5n,f6n,f4,f1) to +# (added_columns,f5n,added_column,f6n,f4,f1) +# Secondary index rebuild happens +alter table t1 drop primary key, add column f7n int, +add column f8n int, add primary key(f7n,f5n,f8n,f6n,f4,f1); +affected rows: 0 +info: Records: 0 Duplicates: 0 Warnings: 0 +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; +name count_reset +SET GLOBAL innodb_monitor_reset = module_ddl; +SET GLOBAL innodb_monitor_disable = module_ddl; +select count(*) from t1; +count(*) +2880 +drop table t1; +SET GLOBAL innodb_monitor_reset = default; +SET GLOBAL innodb_monitor_enable = default; +SET GLOBAL innodb_monitor_disable = default; +# Bug#19163915 INNODB: DUPLICATE RECORDS COULD EXIST +# WHEN SKIPPING SORT FOR CLUSTER INDEX +SELECT @@innodb_sort_buffer_size; +@@innodb_sort_buffer_size +1048576 +create table t1(f1 int auto_increment not null, +f2 char(200) not null, f3 char(200) not null, +f4 char(200) not null,primary key(f1,f2,f3,f4)); +insert into t1 select NULL,'aaa','bbb','ccc' from t480; +insert into t1 values(106, 'aaa','bbb','cccc'); +select count(*) from t1; +count(*) +481 +# Skip sort +# Change PK from (f1,f2,f3,f4) to (f1,f2,f3) +alter table t1 drop primary key, add primary key(f1,f2,f3); +ERROR 23000: ALTER TABLE causes auto_increment resequencing, resulting in duplicate entry '106-aaa-bbb' for key 'PRIMARY' +select count(*) from t1; +count(*) +481 +drop table t1; +create table t1(f1 int auto_increment not null, +f2 char(200) not null, f3 char(200) not null, +f4 char(200) not null,primary key(f1,f2,f3,f4)); +insert into t1 select NULL,'aaa','bbb','ccc' from t480; +insert into t1 values(108,'aaa','bbb','cccc'); +select count(*) from t1; +count(*) +481 +alter table t1 drop primary key, add primary key(f1,f2,f3); +ERROR 23000: ALTER TABLE causes auto_increment resequencing, resulting in duplicate entry '108-aaa-bbb' for key 'PRIMARY' +select count(*) from t1; +count(*) +481 +drop table t1, t480; +# +# Bug #19896922 SORTING SKIPPED WHEN PREFIX LENGTH OF THE PK +# FIELD IS CHANGED +# +create table t1(a int not null, b varchar(30) not null, +primary key (b(10), a)) engine = innodb; +insert into t1 values(0,'khdHps6UxW8Lwaoxa604oK6zkb'),(1,'khdHps6UxW8L'); +select * from t1; +a b +0 khdHps6UxW8Lwaoxa604oK6zkb +1 khdHps6UxW8L +alter table t1 drop primary key, add primary key (b(18),a); +select * from t1; +a b +1 khdHps6UxW8L +0 khdHps6UxW8Lwaoxa604oK6zkb +drop table t1; +create table t1(a int not null, b varchar(30) not null, +primary key (b(10), a)) engine = innodb; +insert into t1 values(0,'khdHps6UxW8Lwaoxa604oK6zkb'),(1,'khdHps6UtW8L'); +select * from t1; +a b +1 khdHps6UtW8L +0 khdHps6UxW8Lwaoxa604oK6zkb +alter table t1 drop primary key, add primary key (b(8),a); +select * from t1; +a b +0 khdHps6UxW8Lwaoxa604oK6zkb +1 khdHps6UtW8L +drop table t1; +# +# Bug #21103101 SORTING SKIPPED WHEN DROPPING THE SINGLE +# COLUMN PRIMARY KEY +# +create table t1(f1 int not null, f2 int not null, +primary key (f1), unique key(f1, f2))engine=innodb; +insert into t1 values(1,3), (2,2); +alter table t1 drop column f1; +ERROR 42000: Key column 'f1' doesn't exist in table +alter table t1 drop column f1, drop primary key; +ERROR 42000: Key column 'f1' doesn't exist in table +alter table t1 drop column f1, drop key f1; +drop table t1; +create table t1(f1 int not null, f2 int not null, +primary key (f1), unique key(f1, f2))engine=innodb; +insert into t1 values(1,3), (2,2); +alter table t1 drop primary key, lock=none; +ERROR 0A000: LOCK=NONE is not supported. Reason: Dropping a primary key is not allowed without also adding a new primary key. Try LOCK=SHARED +drop table t1; +# +# BUG#21612714 ALTER TABLE SORTING SKIPPED WHEN CHANGE PK AND DROP +# LAST COLUMN OF OLD PK +# +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o1,o3), drop o2, lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop o1, drop o2, add primary key(o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o1,o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 add column n1 int not null, drop primary key, add primary key(n1,o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 add column n1 int not null, drop primary key, add primary key(o3,n1), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop primary key, add primary key(o2, o1), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop primary key, add primary key(o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop primary key, add primary key(o2,o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o2,o1)) engine = innodb; +insert into t1 values(1,1,2),(2,1,1); +alter table t1 drop primary key, add primary key(o2,o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o1,o3,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop primary key, add primary key(o3,o1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o1,o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop o1, lock=none; +ERROR 42000: Key column 'o1' doesn't exist in table +alter table t1 drop o1, drop primary key, add primary key(o2,o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop o2, lock=none; +ERROR 42000: Key column 'o2' doesn't exist in table +alter table t1 drop o2, drop primary key, add primary key(o1,o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop o1, drop o2, lock=none; +ERROR 42000: Key column 'o2' doesn't exist in table +alter table t1 drop o1, drop o2,drop primary key,add primary key(o3),lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(2), o2)) engine = innodb; +insert into t1 values('abd', 1, 1), ('abc', 2, 2); +alter table t1 drop primary key, add primary key(o1(3), o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(2), o2)) engine = innodb; +insert into t1 values('abd', 1, 1), ('abc', 2, 2); +alter table t1 drop primary key, add primary key(o1, o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(2), o2)) engine = innodb; +insert into t1 values('abd', 1, 1), ('abc', 2, 2); +alter table t1 drop primary key, add primary key(o1(3), o3), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(2), o2)) engine = innodb; +insert into t1 values('abd', 1, 1), ('abc', 2, 2); +alter table t1 drop primary key, add primary key(o1, o3), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abc', 2, 1), ('abd', 1, 2); +alter table t1 drop primary key, add primary key(o1(2), o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1, o2)) engine = innodb; +insert into t1 values('abc', 2, 1), ('abd', 1, 2); +alter table t1 drop primary key, add primary key(o1(2), o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abc', 2, 2), ('abd', 1, 1); +alter table t1 drop primary key, add primary key(o1(2), o3), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1, o2)) engine = innodb; +insert into t1 values('abc', 2, 2), ('abd', 1, 1); +alter table t1 drop primary key, add primary key(o1(2), o3), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2(2),o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (1, 'abc', 2); +alter table t1 drop primary key, add primary key(o1,o2(3)), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2(2),o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (1, 'abc', 2); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(3))) engine = innodb; +insert into t1 values('abc', 'acd'), ('abd', 'abd'); +alter table t1 drop primary key, add primary key(o1(2),o2(3)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values('abc', 'acd'), ('abd', 'abd'); +alter table t1 drop primary key, add primary key(o1(2),o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(3))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o2(3),o1(3)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o2,o1), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1(3)), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(3), n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1(2))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 add n1 int not null, drop primary key, add primary key(o1, n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int not null, primary key(o1(2))) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(3), o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int not null, primary key(o1(2))) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1, o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1(3))) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), primary key(o1)) engine = innodb; +insert into t1 values('abd'), ('acd'); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1, o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 drop primary key, add primary key(o1(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(2),n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1, o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(2),n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1(3),n1), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, primary key(o1, o2)) engine = innodb; +insert into t1 values('abd', 1), ('acd', 2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), primary key(o1,o2(3))) engine = innodb; +insert into t1 values(1,'abd'), (2,'acd'); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values(1,'abd'), (2,'acd'); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), primary key(o1,o2(2))) engine = innodb; +insert into t1 values(1, 'abd'), (2, 'acd'); +alter table t1 drop primary key, add primary key(o1,o2(3)), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), primary key(o1,o2(2))) engine = innodb; +insert into t1 values(1, 'abd'), (2, 'acd'); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2(3),o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (2, 'acd', 2); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (2, 'acd', 2); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(3))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1,o2(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(2))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2(3)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2(2))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1(3),o2,o3(2))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2,o3(3)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1,o2,o3(2))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1,o2,o3), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1(3),o2,o3(3))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1(3),o2,o3(2)), lock=none; +drop table t1; +create table t1(o1 varchar(10), o2 int, o3 varchar(10), primary key(o1,o2,o3(3))) engine = innodb; +insert into t1 values('abd', 1, 'acd'), ('acd', 2, 'abd'); +alter table t1 drop primary key, add primary key(o1,o2,o3(2)), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(n1,o1), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, add n2 int not null, drop primary key, add primary key(n1,o1,n2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, add n2 int not null, drop primary key, add primary key(n1,n2,o1), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, add n2 int not null, drop primary key, add primary key(o1,n1,n2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,o2,n1), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1),(2,2); +alter table t1 add n1 int not null, drop primary key, add primary key(n1,o1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, o3 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1,o2,o3), lock=none; +drop table t1; +create table t1(o1 int, o2 int not null, o3 int not null, primary key(o1)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1,o3,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, o4 int not null, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2,2),(2,2,1,1); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,o2,o3,o4), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,o2,n1), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 add n1 int not null, drop primary key, add primary key(o1,n1,o2), lock=none; +drop table t1; +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 drop primary key, add primary key(o1), lock=none; +drop table t1; diff --git a/mysql-test/suite/innodb/t/innodb-index-debug.test b/mysql-test/suite/innodb/t/innodb-index-debug.test index d4da0c14246..6927120fd5b 100644 --- a/mysql-test/suite/innodb/t/innodb-index-debug.test +++ b/mysql-test/suite/innodb/t/innodb-index-debug.test @@ -1,5 +1,7 @@ -- source include/have_debug.inc -- source include/have_innodb.inc +-- source include/count_sessions.inc +-- source include/have_debug_sync.inc let $MYSQLD_DATADIR= `select @@datadir`; @@ -42,13 +44,10 @@ SET DEBUG_DBUG = @saved_debug_dbug; DROP TABLE bug13861218; --echo # ---echo # Bug #21762319 ADDING INDEXES ON EMPTY TABLE IS SLOW ---echo # WITH LARGE INNODB_SORT_BUFFER_SIZE. - -call mtr.add_suppression("InnoDB: Cannot create temporary merge file"); - -# Table with large data which is greater than sort buffer +--echo # Bug #17657223 EXCESSIVE TEMPORARY FILE USAGE IN ALTER TABLE +--echo # +# Error during file creation in alter operation create table t480(a serial)engine=innodb; insert into t480 values(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(), @@ -57,14 +56,74 @@ insert into t480 select 0 from t480; insert into t480 select 0 from t480; insert into t480 select 0 from t480; insert into t480 select 0 from t480; + +# Error during file write in alter operation. create table t1(f1 int auto_increment not null, f2 char(200) not null, f3 char(200) not null, - f4 char(200) not null,primary key(f1))engine=innodb; -insert into t1 select NULL,'aaa','bbb','ccc' from t480; -insert into t1 select NULL,'aaaa','bbbb','cccc' from t480; -insert into t1 select NULL,'aaaaa','bbbbb','ccccc' from t480; -insert into t1 select NULL,'aaaaaa','bbbbbb','cccccc' from t480; -insert into t1 select NULL,'aaaaaaa','bbbbbbb','ccccccc' from t480; -insert into t1 select NULL,'aaaaaaaa','bbbbbbbb','cccccccc' from t480; -select count(*) from t1; -drop table t1, t480; + primary key(f1,f2,f3), key(f1))engine=innodb; +insert into t1 select NULL,'aaa','bbb' from t480; +insert into t1 select NULL,'aaaa','bbbb' from t480; +insert into t1 select NULL,'aaaaa','bbbbb' from t480; +insert into t1 select NULL,'aaaaaa','bbbbbb' from t480; +SET DEBUG_DBUG = '+d,row_merge_write_failure'; +--error ER_TEMP_FILE_WRITE_FAILURE +alter table t1 drop primary key,add primary key(f2,f1); +SET DEBUG_DBUG = @saved_debug_dbug; +drop table t1; + +# Optimize table via inplace algorithm +connect (con1,localhost,root); +create table t1(k1 int auto_increment primary key, +k2 char(200),k3 char(200))engine=innodb; +insert into t1 values(NULL,'a','b'), (NULL,'aa','bb'); +SET DEBUG_SYNC= 'row_merge_after_scan +SIGNAL opened WAIT_FOR flushed'; +send optimize table t1; +connection default; +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +INSERT INTO t1 select NULL,'aaa','bbb' from t480; +SET DEBUG_SYNC= 'now SIGNAL flushed'; +connection con1; +--enable_info +--echo /*con1 reap*/ Optimize table t1; +reap; +--disable_info +SELECT COUNT(k1),k2,k3 FROM t1 GROUP BY k2,k3; +drop table t1; + +# Log file creation failure. +create table t1(k1 int auto_increment primary key, +k2 char(200),k3 char(200))engine=innodb; +SET DEBUG_SYNC= 'row_merge_after_scan +SIGNAL opened WAIT_FOR flushed'; +send ALTER TABLE t1 ADD COLUMN k4 int; +connection default; +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +SET debug = '+d,row_log_tmpfile_fail'; +INSERT INTO t1 select NULL,'aaa','bbb' from t480; +INSERT INTO t1 select NULL,'aaaa','bbbb' from t480; +SET DEBUG_SYNC= 'now SIGNAL flushed'; +SET DEBUG_DBUG = @saved_debug_dbug; +connection con1; +--echo /*con1 reap*/ ALTER TABLE t1 ADD COLUMN k4 int; +--error ER_OUT_OF_RESOURCES +reap; +SELECT COUNT(k1),k2,k3 FROM t1 GROUP BY k2,k3; +disconnect con1; +connection default; +show create table t1; +drop table t1; +drop table t480; +SET DEBUG_SYNC='RESET'; +--source include/wait_until_count_sessions.inc + +--echo # +--echo # BUG#21612714 ALTER TABLE SORTING SKIPPED WHEN CHANGE PK AND DROP +--echo # LAST COLUMN OF OLD PK +--echo # + +SET DEBUG_DBUG = '+d,innodb_alter_table_pk_assert_no_sort'; + +--source suite/innodb/include/alter_table_pk_no_sort.inc + +SET DEBUG_DBUG = @saved_debug_dbug; diff --git a/mysql-test/suite/innodb/t/innodb-index.test b/mysql-test/suite/innodb/t/innodb-index.test index 4549b3b6b47..721808c038c 100644 --- a/mysql-test/suite/innodb/t/innodb-index.test +++ b/mysql-test/suite/innodb/t/innodb-index.test @@ -546,3 +546,533 @@ show create table t2c; --disable_info DROP TABLE t1,t2,t2c,t2i; +--echo # +--echo # Bug #17657223 EXCESSIVE TEMPORARY FILE USAGE IN ALTER TABLE +--echo # + +SET GLOBAL innodb_monitor_enable = module_ddl; +let $innodb_metrics_select= +SELECT name, count_reset FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE +subsystem = 'ddl' and count_reset > 0; + +# Table with Blob data. +create table t1(f1 int not null, f2 blob)engine=innodb; +insert into t1 values(1, repeat('a',20000)); +--echo # Skip sort +--echo # Reusing the same pk +--enable_info +alter table t1 force; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; +drop table t1; + +# Table with small data. +create table t1(f1 int not null, f2 int not null, +primary key(f1))engine=innodb; +insert into t1 values(1,2), (3,4); +--echo # Add Secondary index. +--echo # Skip temp file usage due to small table size +--enable_info +alter table t1 add key(f2); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; +drop table t1; + +# Table with large data which is greater than sort buffer +create table t480(a serial)engine=innodb; +insert into t480 +values(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(), +(),(),(),(),(),(),(),(); +insert into t480 select 0 from t480; +insert into t480 select 0 from t480; +insert into t480 select 0 from t480; +insert into t480 select 0 from t480; +create table t1(f1 int auto_increment not null, + f2 char(200) not null, f3 char(200) not null, + f4 char(200) not null,primary key(f1))engine=innodb; +insert into t1 select NULL,'aaa','bbb','ccc' from t480; +insert into t1 select NULL,'aaaa','bbbb','cccc' from t480; +insert into t1 select NULL,'aaaaa','bbbbb','ccccc' from t480; +insert into t1 select NULL,'aaaaaa','bbbbbb','cccccc' from t480; +insert into t1 select NULL,'aaaaaaa','bbbbbbb','ccccccc' from t480; +insert into t1 select NULL,'aaaaaaaa','bbbbbbbb','cccccccc' from t480; +select count(*) from t1; + +--echo # Skip sort +--echo # Change PK from (f1) to (f1,f2,f3,f4) +--enable_info +alter table t1 drop primary key, add primary key(f1,f2,f3,f4); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + + +# Temp file not used during clustered index rebuild +# for the following alter table commands. + +--echo # Skip sort +--echo # Change PK from (f1,f2,f3,f4) to (f1,f2,added_columns) +--enable_info +alter table t1 drop primary key,add column f5 int not null, +add column f6 int not null,add primary key(f1,f2,f5,f6); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Change PK from (f1,f2,f5,f6) to (f1,f2,f5) +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 drop column f6; +--enable_info +alter table t1 drop column f6, drop primary key, add primary key(f1,f2,f5); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Reusing the same PK +--enable_info +alter table t1 add column f6 int; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Reusing the same pk +--enable_info +alter table t1 drop column f6; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Must sort +--echo # Change PK from (f1,f2,f5) to (f1,f5) +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 drop column f2; +--enable_info +alter table t1 drop column f2, drop primary key, add primary key(f1,f5); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Reusing the same pk +--enable_info +alter table t1 add column f2n int after f1, drop primary key, add +primary key (f1,f5,f2n); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Reusing the same pk +--enable_info +alter table t1 change f5 f2n int not null,change f2n f5 int not null, +add column f8 int not null; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Change PK from (f1,f4,f2n) to (f1,f4,added_column,f2n) +--enable_info +alter table t1 add column f7 int, drop primary key, +add primary key (f1,f5,f7,f2n); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Reusing the same pk +--enable_info +alter table t1 force; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Reusing the same pk +--enable_info +alter table t1 row_format=compact; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Reusing the same pk +--enable_info +alter table t1 engine=innodb; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Optimize table +--enable_info +optimize table t1; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Sort files used for adding secondary index +--enable_info +alter table t1 drop primary key, add primary key(f1,f5,f7), add index +i(f3); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # No sort files used for dropping secondary index +--enable_info +alter table t1 drop primary key, add primary key(f1,f5),drop index i; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Change PK(f1,f5) to (f1,added_columns) and drop f5 +--enable_info +alter table t1 drop primary key, add primary key(f1,f12), +drop column f5, add column f12 int not null; +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Must sort +--echo # Change PK(f1,f12) to (f1,existing_columns) +--enable_info +alter table t1 drop primary key, add primary key(f1,f3); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort +--echo # Change PK(f1,f3) to (f1,added_column,f3,added_column) +--enable_info +alter table t1 drop primary key, add column f3n int, +add column f4n int, add primary key(f1,f3n,f3,f4n); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Adding Secondary index alone. +--enable_info +alter table t1 add key(f1); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Must sort +--echo # Change PK(f1,f3) to (existing_column,f1) +--enable_info +alter table t1 drop primary key, add primary key(f4,f1); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort for PK. +--echo # Change PK(f4,f1) to (added_columns,f4,f1) +--echo # Secondary index rebuild happens +--enable_info +alter table t1 drop primary key, add column f5n int, +add column f6n int, add primary key(f5n,f6n,f4,f1); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +--echo # Skip sort for PK. +--echo # Change PK(f5n,f6n,f4,f1) to +--echo # (added_columns,f5n,added_column,f6n,f4,f1) +--echo # Secondary index rebuild happens +--enable_info +alter table t1 drop primary key, add column f7n int, +add column f8n int, add primary key(f7n,f5n,f8n,f6n,f4,f1); +--disable_info +eval $innodb_metrics_select; +SET GLOBAL innodb_monitor_reset = module_ddl; + +SET GLOBAL innodb_monitor_disable = module_ddl; +select count(*) from t1; +drop table t1; +--disable_warnings +SET GLOBAL innodb_monitor_reset = default; +SET GLOBAL innodb_monitor_enable = default; +SET GLOBAL innodb_monitor_disable = default; +--enable_warnings + +--echo # Bug#19163915 INNODB: DUPLICATE RECORDS COULD EXIST +--echo # WHEN SKIPPING SORT FOR CLUSTER INDEX + +# last mtuple in previous buffer and first mtuple in next buffer +# are equal. +SELECT @@innodb_sort_buffer_size; +create table t1(f1 int auto_increment not null, + f2 char(200) not null, f3 char(200) not null, + f4 char(200) not null,primary key(f1,f2,f3,f4)); +insert into t1 select NULL,'aaa','bbb','ccc' from t480; +insert into t1 values(106, 'aaa','bbb','cccc'); +select count(*) from t1; +--echo # Skip sort +--echo # Change PK from (f1,f2,f3,f4) to (f1,f2,f3) +--error ER_DUP_ENTRY +alter table t1 drop primary key, add primary key(f1,f2,f3); +select count(*) from t1; +drop table t1; + +# Duplicates exist with in the buffer +create table t1(f1 int auto_increment not null, + f2 char(200) not null, f3 char(200) not null, + f4 char(200) not null,primary key(f1,f2,f3,f4)); +insert into t1 select NULL,'aaa','bbb','ccc' from t480; +insert into t1 values(108,'aaa','bbb','cccc'); +select count(*) from t1; +--error ER_DUP_ENTRY +alter table t1 drop primary key, add primary key(f1,f2,f3); +select count(*) from t1; +drop table t1, t480; + +--echo # +--echo # Bug #19896922 SORTING SKIPPED WHEN PREFIX LENGTH OF THE PK +--echo # FIELD IS CHANGED +--echo # + +# Prefix length changes for the varchar column. +create table t1(a int not null, b varchar(30) not null, + primary key (b(10), a)) engine = innodb; +insert into t1 values(0,'khdHps6UxW8Lwaoxa604oK6zkb'),(1,'khdHps6UxW8L'); +select * from t1; +alter table t1 drop primary key, add primary key (b(18),a); +select * from t1; +drop table t1; + +create table t1(a int not null, b varchar(30) not null, + primary key (b(10), a)) engine = innodb; +insert into t1 values(0,'khdHps6UxW8Lwaoxa604oK6zkb'),(1,'khdHps6UtW8L'); +select * from t1; +alter table t1 drop primary key, add primary key (b(8),a); +select * from t1; +drop table t1; + + +--echo # +--echo # Bug #21103101 SORTING SKIPPED WHEN DROPPING THE SINGLE +--echo # COLUMN PRIMARY KEY +--echo # + +# Drop primary key column. +create table t1(f1 int not null, f2 int not null, + primary key (f1), unique key(f1, f2))engine=innodb; +insert into t1 values(1,3), (2,2); +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 drop column f1; +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 drop column f1, drop primary key; +# DROP PRIMARY KEY is implied for a single-column PRIMARY KEY +alter table t1 drop column f1, drop key f1; +drop table t1; + +# Drop Primary key when lock is none. +create table t1(f1 int not null, f2 int not null, + primary key (f1), unique key(f1, f2))engine=innodb; +insert into t1 values(1,3), (2,2); +--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON +alter table t1 drop primary key, lock=none; +drop table t1; + +--echo # +--echo # BUG#21612714 ALTER TABLE SORTING SKIPPED WHEN CHANGE PK AND DROP +--echo # LAST COLUMN OF OLD PK +--echo # + +# no skip sort cases +# pk(o1,o2) to pk(o1,o3), drop o2 +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o1,o3), drop o2, lock=none; +drop table t1; + +# pk(o1,o2) to pk(o3), drop o1, o2 +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop o1, drop o2, add primary key(o3), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1,o3) +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o1,o3), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o3) +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o3), lock=none; +drop table t1; + +# pk(o1,o2) to pk(n1,o3) +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 add column n1 int not null, drop primary key, add primary key(n1,o3), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o3,n1) +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(2,2,1); +alter table t1 add column n1 int not null, drop primary key, add primary key(o3,n1), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o2,o1) +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop primary key, add primary key(o2, o1), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o2) +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop primary key, add primary key(o2), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o2,o3) +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop primary key, add primary key(o2,o3), lock=none; +drop table t1; + +# pk(o2,o1) to pk(o2,o3) +create table t1(o1 int, o2 int, o3 int not null, primary key(o2,o1)) engine = innodb; +insert into t1 values(1,1,2),(2,1,1); +alter table t1 drop primary key, add primary key(o2,o3), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1,o3,o2) +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o1,o3,o2), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o3,o1,o2) +create table t1(o1 int, o2 int, o3 int not null, primary key(o1,o2)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +alter table t1 drop primary key, add primary key(o3,o1,o2), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o1,o3) +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +alter table t1 drop primary key, add primary key(o1,o3), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o2,o3) by drop o1 +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 drop o1, lock=none; +alter table t1 drop o1, drop primary key, add primary key(o2,o3), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o1,o3) by drop o2 +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,1,2),(1,2,1); +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 drop o2, lock=none; +alter table t1 drop o2, drop primary key, add primary key(o1,o3), lock=none; +drop table t1; + +# pk(o1,o2,o3) to pk(o3) by drop o1,o2 +create table t1(o1 int, o2 int, o3 int, primary key(o1,o2,o3)) engine = innodb; +insert into t1 values(1,2,2),(2,1,1); +--error ER_KEY_COLUMN_DOES_NOT_EXITS +alter table t1 drop o1, drop o2, lock=none; +alter table t1 drop o1, drop o2,drop primary key,add primary key(o3),lock=none; +drop table t1; + +# no skip sort for prefix change +# pk(o1(2),o2) to pk(o1(3),o2) +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(2), o2)) engine = innodb; +insert into t1 values('abd', 1, 1), ('abc', 2, 2); +alter table t1 drop primary key, add primary key(o1(3), o2), lock=none; +drop table t1; + +# pk(o1(2),o2) to pk(o1,o2) +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(2), o2)) engine = innodb; +insert into t1 values('abd', 1, 1), ('abc', 2, 2); +alter table t1 drop primary key, add primary key(o1, o2), lock=none; +drop table t1; + +# pk(o1(2),o2) to pk(o1(3),o3) +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(2), o2)) engine = innodb; +insert into t1 values('abd', 1, 1), ('abc', 2, 2); +alter table t1 drop primary key, add primary key(o1(3), o3), lock=none; +drop table t1; + +# pk(o1(2),o2) to pk(o1,o3) +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(2), o2)) engine = innodb; +insert into t1 values('abd', 1, 1), ('abc', 2, 2); +alter table t1 drop primary key, add primary key(o1, o3), lock=none; +drop table t1; + +# pk(o1(3),o2) to pk(o1(2),o2) +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abc', 2, 1), ('abd', 1, 2); +alter table t1 drop primary key, add primary key(o1(2), o2), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1(2),o2) +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1, o2)) engine = innodb; +insert into t1 values('abc', 2, 1), ('abd', 1, 2); +alter table t1 drop primary key, add primary key(o1(2), o2), lock=none; +drop table t1; + +# pk(o1(3),o2) to pk(o1(2),o3) +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1(3), o2)) engine = innodb; +insert into t1 values('abc', 2, 2), ('abd', 1, 1); +alter table t1 drop primary key, add primary key(o1(2), o3), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1(2),o3) +create table t1(o1 varchar(10), o2 int, o3 int not null, primary key(o1, o2)) engine = innodb; +insert into t1 values('abc', 2, 2), ('abd', 1, 1); +alter table t1 drop primary key, add primary key(o1(2), o3), lock=none; +drop table t1; + +# pk(o1,o2(2),o3) to pk(o1,o2(3)) +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2(2),o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (1, 'abc', 2); +alter table t1 drop primary key, add primary key(o1,o2(3)), lock=none; +drop table t1; + +# pk(o1,o2(2),o3) to pk(o1,o2) +create table t1(o1 int, o2 varchar(10), o3 int, primary key(o1,o2(2),o3)) engine = innodb; +insert into t1 values(1, 'abd', 1), (1, 'abc', 2); +alter table t1 drop primary key, add primary key(o1,o2), lock=none; +drop table t1; + +# pk(o1(3),o2(3)) to pk(o1(2),o2(3)) +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(3))) engine = innodb; +insert into t1 values('abc', 'acd'), ('abd', 'abd'); +alter table t1 drop primary key, add primary key(o1(2),o2(3)), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o1(2),o2) +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values('abc', 'acd'), ('abd', 'abd'); +alter table t1 drop primary key, add primary key(o1(2),o2), lock=none; +drop table t1; + +# pk(o1(3),o2(3)) to pk(o2(3),o1(3)) +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1(3),o2(3))) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o2(3),o1(3)), lock=none; +drop table t1; + +# pk(o1,o2) to pk(o2,o1) +create table t1(o1 varchar(10), o2 varchar(10), primary key(o1,o2)) engine = innodb; +insert into t1 values('abd', 'acd'), ('acd', 'abd'); +alter table t1 drop primary key, add primary key(o2,o1), lock=none; +drop table t1; + +# no skip sort cases +--source suite/innodb/include/alter_table_pk_no_sort.inc From f3435fc37e7e0cfd5384c609d16b0d6cc3048454 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Tue, 25 Jul 2017 10:56:44 +0300 Subject: [PATCH 05/30] MW-394 * ported the partition.test from MariaDB 10.3 code tree * fix to TOI replicate ALTER TABLE EXCHANGE PARTITION statement --- sql/sql_partition_admin.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql/sql_partition_admin.cc b/sql/sql_partition_admin.cc index 519b7cc8195..48db032fe58 100644 --- a/sql/sql_partition_admin.cc +++ b/sql/sql_partition_admin.cc @@ -89,9 +89,15 @@ bool Sql_cmd_alter_table_exchange_partition::execute(THD *thd) /* Not allowed with EXCHANGE PARTITION */ DBUG_ASSERT(!create_info.data_file_name && !create_info.index_file_name); + WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table); thd->enable_slow_log= opt_log_slow_admin_statements; DBUG_RETURN(exchange_partition(thd, first_table, &alter_info)); +#ifdef WITH_WSREP + error: + /* handle errors in TO_ISOLATION here */ + DBUG_RETURN(true); +#endif /* WITH_WSREP */ } From ed9f68f6ad3417f4e12acd8c108a3fd9cc727111 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Thu, 27 Jul 2017 11:39:31 +0300 Subject: [PATCH 06/30] MW-394 * changed thd_binlog_format to return configured binlog format in wsrep execution, regardless of binlogging setting (i.e. with or without binlogging) * thd_binlog_format is used in innobase::write_row(), and would return confusing result there when log_bin==OFF --- sql/sql_class.cc | 12 +++++++++--- sql/wsrep_mysqld.h | 5 +++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 9aedba3c426..82b2faf6a02 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -4852,9 +4852,15 @@ extern "C" int thd_non_transactional_update(const MYSQL_THD thd) extern "C" int thd_binlog_format(const MYSQL_THD thd) { - if (((WSREP(thd) && wsrep_emulate_bin_log) || mysql_bin_log.is_open()) && - thd->variables.option_bits & OPTION_BIN_LOG) - return (int) thd->wsrep_binlog_format(); +#ifdef WITH_WSREP + if (WSREP(thd)) + { + /* for wsrep binlog format is meaningful also when binlogging is off */ + return (int) WSREP_BINLOG_FORMAT(thd->variables.binlog_format); + } +#endif /* WITH_WSREP */ + if (mysql_bin_log.is_open() && (thd->variables.option_bits & OPTION_BIN_LOG)) + return (int) thd->variables.binlog_format; else return BINLOG_FORMAT_UNSPEC; } diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 3d46e17af99..f154b51516b 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -320,6 +320,10 @@ bool wsrep_create_like_table(THD* thd, TABLE_LIST* table, bool wsrep_node_is_donor(); bool wsrep_node_is_synced(); +#define WSREP_BINLOG_FORMAT(my_format) \ + ((wsrep_forced_binlog_format != BINLOG_FORMAT_UNSPEC) ? \ + wsrep_forced_binlog_format : my_format) + #else /* WITH_WSREP */ #define WSREP(T) (0) @@ -349,6 +353,7 @@ bool wsrep_node_is_synced(); #define wsrep_thr_init() do {} while(0) #define wsrep_thr_deinit() do {} while(0) #define wsrep_running_threads (0) +#define WSREP_BINLOG_FORMAT(my_format) my_format #endif /* WITH_WSREP */ #endif /* WSREP_MYSQLD_H */ From a8b3c603c6fd2977dbdaae5d146b5a9bb3fc7851 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Thu, 27 Jul 2017 11:43:33 +0300 Subject: [PATCH 07/30] MW-394 * recorded new result file, original was copied from MariaDB which produces syntactically a bit different result * semantics are same in MariaDB and ours result file --- sql/sql_partition_admin.cc | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/sql/sql_partition_admin.cc b/sql/sql_partition_admin.cc index 48db032fe58..ff6c05c4215 100644 --- a/sql/sql_partition_admin.cc +++ b/sql/sql_partition_admin.cc @@ -538,21 +538,6 @@ bool Sql_cmd_alter_table_exchange_partition:: &alter_prelocking_strategy)) DBUG_RETURN(true); -#ifdef WITH_WSREP - if (WSREP_ON) - { - if ((!thd->is_current_stmt_binlog_format_row() || - /* TODO: Do we really need to check for temp tables in this case? */ - !thd->find_temporary_table(table_list)) && - wsrep_to_isolation_begin(thd, table_list->db, table_list->table_name, - NULL)) - { - WSREP_WARN("ALTER TABLE EXCHANGE PARTITION isolation failure"); - DBUG_RETURN(TRUE); - } - } -#endif /* WITH_WSREP */ - part_table= table_list->table; swap_table= swap_table_list->table; From bb7ab40521fe1473c6300f1aa5b8bab19631fbc4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 13 Sep 2017 12:10:48 +0000 Subject: [PATCH 08/30] mariabackup : Fix enumerate_ibd_files() to include .isl --- extra/mariabackup/backup_copy.cc | 1 - extra/mariabackup/xtrabackup.cc | 20 +++++++++----------- extra/mariabackup/xtrabackup.h | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index fb84153bdee..19cb768cd01 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -594,7 +594,6 @@ trim_dotslash(const char *path) /************************************************************************ Check if string ends with given suffix. @return true if string ends with given suffix. */ -static bool ends_with(const char *str, const char *suffix) { diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 0027de960c1..448703477c3 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2771,14 +2771,12 @@ static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback) ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir, &dbinfo); while (ret == 0) { - size_t len = strlen(dbinfo.name); /* General tablespaces are always at the first level of the data home dir */ - if (dbinfo.type == OS_FILE_TYPE_FILE && len > 4) { - bool is_isl = !strcmp(dbinfo.name + len - 4, ".isl"); - bool is_ibd = !is_isl - && !strcmp(dbinfo.name + len - 4, ".ibd"); + if (dbinfo.type == OS_FILE_TYPE_FILE) { + bool is_isl = ends_with(dbinfo.name, ".isl"); + bool is_ibd = !is_isl && ends_with(dbinfo.name,".ibd"); if (is_isl || is_ibd) { (*callback)(NULL, dbinfo.name, is_isl); @@ -2794,7 +2792,7 @@ static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback) /* We found a symlink or a directory; try opening it to see if a symlink is a directory */ - len = strlen(fil_path_to_mysql_datadir) + size_t len = strlen(fil_path_to_mysql_datadir) + strlen (dbinfo.name) + 2; if (len > dbpath_len) { dbpath_len = len; @@ -2837,16 +2835,16 @@ static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback) size_t len = strlen(fileinfo.name); /* We found a symlink or a file */ - if (len > 4 - && !strcmp(fileinfo.name + len - 4, - ".ibd")) { - (*callback)(dbinfo.name, fileinfo.name, false); + if (len > 4) { + bool is_isl= false; + if (ends_with(fileinfo.name, ".ibd") || ((is_isl = ends_with(fileinfo.name, ".ibd")))) + (*callback)(dbinfo.name, fileinfo.name, is_isl); } } if (0 != os_file_closedir(dbdir)) { fprintf(stderr, "InnoDB: Warning: could not" - " close database directory %s\n", + " close database directory %s\n", dbpath); err = DB_ERROR; diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index a24626c48bc..c902e7f54ba 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -196,5 +196,5 @@ xb_get_copy_action(const char *dflt = "Copying"); void mdl_lock_init(); void mdl_lock_table(ulint space_id); void mdl_unlock_all(); - +bool ends_with(const char *str, const char *suffix); #endif /* XB_XTRABACKUP_H */ From 74f677fcc222f67a32cafeb5022f340f98d9236e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 14 Sep 2017 16:42:09 +0000 Subject: [PATCH 09/30] MDEV-13802 mariabackup --lock-ddl-per-table fails when table names contain backticks use ut_get_name() for formatting database/table names. --- extra/mariabackup/backup_mysql.cc | 33 +++++++------------ extra/mariabackup/xtrabackup.cc | 14 +++----- .../suite/mariabackup/lock_ddl_per_table.test | 8 +++++ mysql-test/suite/mariabackup/suite.opt | 2 +- 4 files changed, 25 insertions(+), 32 deletions(-) diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 8849e9bd56c..cebbbf3a28c 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -55,6 +55,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "encryption_plugin.h" #include #include +#include char *tool_name; @@ -1665,38 +1666,28 @@ static void check_mdl_lock_works(const char *table_name) free(query); } #endif - -extern void -dict_fs2utf8(const char*, char*, size_t, char*, size_t); - void mdl_lock_table(ulint space_id) { - static const char q[] = "SELECT NAME " + std::ostringstream oss; + oss << "SELECT NAME " "FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES " - "WHERE SPACE = " ULINTPF " AND NAME LIKE '%%/%%'"; - char query[22 + sizeof q]; - snprintf(query, sizeof query, q, space_id); + "WHERE SPACE = " << space_id << " AND NAME LIKE '%%/%%'"; pthread_mutex_lock(&mdl_lock_con_mutex); - MYSQL_RES *mysql_result = xb_mysql_query(mdl_con, query, true, true); + MYSQL_RES *mysql_result = xb_mysql_query(mdl_con, oss.str().c_str(), true, true); + while (MYSQL_ROW row = mysql_fetch_row(mysql_result)) { - char full_table_name[2*FN_REFLEN +2]; - char db_utf8[FN_REFLEN]; - char table_utf8[FN_REFLEN]; - static const char lq[] = "SELECT * FROM %s LIMIT 0"; - char lock_query[sizeof full_table_name + sizeof lq]; + std::string full_table_name = ut_get_name(0,row[0]); + std::ostringstream lock_query; + lock_query << "SELECT * FROM " << full_table_name << " LIMIT 0"; - dict_fs2utf8(row[0], db_utf8, sizeof db_utf8,table_utf8,sizeof table_utf8); - snprintf(full_table_name,sizeof(full_table_name),"`%s`.`%s`",db_utf8,table_utf8); - msg_ts("Locking MDL for %s\n", full_table_name); - snprintf(lock_query, sizeof lock_query, lq, full_table_name); - - xb_mysql_query(mdl_con, lock_query, false, false); + msg_ts("Locking MDL for %s\n", full_table_name.c_str()); + xb_mysql_query(mdl_con, lock_query.str().c_str(), false, false); DBUG_EXECUTE_IF("check_mdl_lock_works", - check_mdl_lock_works(full_table_name);); + check_mdl_lock_works(full_table_name.c_str());); } pthread_mutex_unlock(&mdl_lock_con_mutex); diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 448703477c3..6fc37ead4f6 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1148,17 +1148,12 @@ static void append_export_table(const char *dbname, const char *tablename, bool if(dbname && tablename && !is_remote) { char buf[3*FN_REFLEN]; - char db_utf8[FN_REFLEN]; - char table_utf8[FN_REFLEN]; - snprintf(buf,sizeof(buf),"%s/%s",dbname, tablename); // trim .ibd char *p=strrchr(buf, '.'); if (p) *p=0; - dict_fs2utf8(buf, db_utf8, sizeof(db_utf8),table_utf8,sizeof(table_utf8)); - snprintf(buf,sizeof(buf),"`%s`.`%s`",db_utf8,table_utf8); - tables_for_export.push_back(buf); + tables_for_export.push_back(ut_get_name(0,buf)); } } @@ -2753,6 +2748,7 @@ static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback) os_file_stat_t dbinfo; os_file_stat_t fileinfo; dberr_t err = DB_SUCCESS; + size_t len; /* The datadir of MySQL is always the default directory of mysqld */ @@ -2792,7 +2788,7 @@ static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback) /* We found a symlink or a directory; try opening it to see if a symlink is a directory */ - size_t len = strlen(fil_path_to_mysql_datadir) + len = strlen(fil_path_to_mysql_datadir) + strlen (dbinfo.name) + 2; if (len > dbpath_len) { dbpath_len = len; @@ -2832,10 +2828,8 @@ static dberr_t enumerate_ibd_files(process_single_tablespace_func_t callback) continue; } - size_t len = strlen(fileinfo.name); - /* We found a symlink or a file */ - if (len > 4) { + if (strlen(fileinfo.name) > 4) { bool is_isl= false; if (ends_with(fileinfo.name, ".ibd") || ((is_isl = ends_with(fileinfo.name, ".ibd")))) (*callback)(dbinfo.name, fileinfo.name, is_isl); diff --git a/mysql-test/suite/mariabackup/lock_ddl_per_table.test b/mysql-test/suite/mariabackup/lock_ddl_per_table.test index 5b45aa7c61b..04edb9e05e9 100644 --- a/mysql-test/suite/mariabackup/lock_ddl_per_table.test +++ b/mysql-test/suite/mariabackup/lock_ddl_per_table.test @@ -5,8 +5,16 @@ INSERT INTO t VALUES(1); echo # xtrabackup backup; let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +CREATE TABLE `bobby``tables` (id INT, name VARCHAR(50), purchased DATE) ENGINE INNODB PARTITION BY RANGE( YEAR(purchased) ) ( + PARTITION p0 VALUES LESS THAN (1990), + PARTITION p1 VALUES LESS THAN (1995), + PARTITION p2 VALUES LESS THAN (2000), + PARTITION p3 VALUES LESS THAN (2005) +) ; + --disable_result_log exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --lock-ddl-per-table=1 --dbug=+d,check_mdl_lock_works; --enable_result_log DROP TABLE t; +DROP TABLE `bobby``tables`; rmdir $targetdir; \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/suite.opt b/mysql-test/suite/mariabackup/suite.opt index defba074293..8b1e9c7bcb2 100644 --- a/mysql-test/suite/mariabackup/suite.opt +++ b/mysql-test/suite/mariabackup/suite.opt @@ -1 +1 @@ ---innodb --loose-changed_page_bitmaps --innodb-file-format=Barracuda --innodb-sys-tables \ No newline at end of file +--innodb --loose-changed_page_bitmaps --innodb-file-format=Barracuda --innodb-sys-tables --partition \ No newline at end of file From 65d26d1f31070b5b9c05eba4eb8f4432e56055e0 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 15 Sep 2017 20:59:04 +0400 Subject: [PATCH 10/30] MDEV-10191 non convertible chars convert() resulted in Null instead "?" on Windows The problem was introduced by the patch for MDEV-7661, which (in addition to the fix itself) included an attempt to make CONVERT/CAST work in the same way with fields (i.e. return NULL in strict mode if a non-convertable character found). It appeared to be a bad idea and some users were affected by this behavior change. Changing CONVERT/CAST not depend on sql_mode (restoring pre-10.1.4 behavior). --- mysql-test/r/ctype_utf8.result | 29 ++++++++++++++++++++++-- mysql-test/r/ctype_utf8mb4.result | 4 ++-- mysql-test/r/ctype_utf8mb4_heap.result | 4 ++-- mysql-test/r/ctype_utf8mb4_innodb.result | 4 ++-- mysql-test/r/ctype_utf8mb4_myisam.result | 4 ++-- mysql-test/t/ctype_utf8.test | 12 ++++++++++ sql/item.cc | 4 ++-- sql/item.h | 3 +-- sql/item_timefunc.cc | 2 +- 9 files changed, 51 insertions(+), 15 deletions(-) diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result index 2960b648b84..2b53f43cfc9 100644 --- a/mysql-test/r/ctype_utf8.result +++ b/mysql-test/r/ctype_utf8.result @@ -1945,7 +1945,7 @@ Warnings: Warning 1300 Invalid utf8 character string: 'FD' select convert(char(0xff,0x8f) using utf8); convert(char(0xff,0x8f) using utf8) -NULL +?? Warnings: Warning 1300 Invalid utf8 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8)); @@ -2114,7 +2114,7 @@ Warnings: Warning 1300 Invalid utf8 character string: 'FF' select hex(convert(0xFF using utf8)); hex(convert(0xFF using utf8)) -NULL +3F Warnings: Warning 1300 Invalid utf8 character string: '\xFF' select hex(_utf8 0x616263FF); @@ -10473,5 +10473,30 @@ END DROP PROCEDURE p1; SET @@SQL_MODE=default; # +# MDEV-10191 non convertible chars convert() resulted in Null instead "?" on Windows +# +SET sql_mode='STRICT_TRANS_TABLES'; +SELECT CONVERT(_utf8 0xC499 USING latin1); +CONVERT(_utf8 0xC499 USING latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); +CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +SET sql_mode=default; +SELECT CONVERT(_utf8 0xC499 USING latin1); +CONVERT(_utf8 0xC499 USING latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); +CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +# # End of 10.1 tests # diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result index dd1c5249bc7..bac0fc34f80 100644 --- a/mysql-test/r/ctype_utf8mb4.result +++ b/mysql-test/r/ctype_utf8mb4.result @@ -1970,7 +1970,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -2139,7 +2139,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/r/ctype_utf8mb4_heap.result b/mysql-test/r/ctype_utf8mb4_heap.result index d70e009228e..46b0cc3789f 100644 --- a/mysql-test/r/ctype_utf8mb4_heap.result +++ b/mysql-test/r/ctype_utf8mb4_heap.result @@ -1802,7 +1802,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -1971,7 +1971,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/r/ctype_utf8mb4_innodb.result b/mysql-test/r/ctype_utf8mb4_innodb.result index 7d193f397ac..cfd9bf969ad 100644 --- a/mysql-test/r/ctype_utf8mb4_innodb.result +++ b/mysql-test/r/ctype_utf8mb4_innodb.result @@ -1930,7 +1930,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -2099,7 +2099,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/r/ctype_utf8mb4_myisam.result b/mysql-test/r/ctype_utf8mb4_myisam.result index 28cf36c7492..53ae410046f 100644 --- a/mysql-test/r/ctype_utf8mb4_myisam.result +++ b/mysql-test/r/ctype_utf8mb4_myisam.result @@ -1930,7 +1930,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -2099,7 +2099,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test index 5f1de609df1..e6792922b95 100644 --- a/mysql-test/t/ctype_utf8.test +++ b/mysql-test/t/ctype_utf8.test @@ -1983,6 +1983,18 @@ SET @@SQL_MODE=default; #DROP FUNCTION f1; #SET NAMES utf8; +--echo # +--echo # MDEV-10191 non convertible chars convert() resulted in Null instead "?" on Windows +--echo # + +SET sql_mode='STRICT_TRANS_TABLES'; +SELECT CONVERT(_utf8 0xC499 USING latin1); +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); + +SET sql_mode=default; +SELECT CONVERT(_utf8 0xC499 USING latin1); +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); + --echo # --echo # End of 10.1 tests diff --git a/sql/item.cc b/sql/item.cc index ae77e90fa85..d4f3525c27c 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -5628,7 +5628,7 @@ String_copier_for_item::copy_with_warn(CHARSET_INFO *dstcs, String *dst, srccs == &my_charset_bin ? dstcs->csname : srccs->csname, err.ptr()); - return m_thd->is_strict_mode(); + return false; } if (const char *pos= cannot_convert_error_pos()) { @@ -5641,7 +5641,7 @@ String_copier_for_item::copy_with_warn(CHARSET_INFO *dstcs, String *dst, ER_CANNOT_CONVERT_CHARACTER, ER_THD(m_thd, ER_CANNOT_CONVERT_CHARACTER), srccs->csname, buf, dstcs->csname); - return m_thd->is_strict_mode(); + return false; } return false; } diff --git a/sql/item.h b/sql/item.h index 8ee76a626d4..c338b14e340 100644 --- a/sql/item.h +++ b/sql/item.h @@ -548,8 +548,7 @@ class Item_func_not; class Item_splocal; /** - String_copier that honors the current sql_mode (strict vs non strict) - and can send warnings. + String_copier that sends Item specific warnings. */ class String_copier_for_item: public String_copier { diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index fa74a1903da..d8a37b37315 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -2396,7 +2396,7 @@ String *Item_char_typecast::copy(String *str, CHARSET_INFO *strcs) if (copier.copy_with_warn(cast_cs, &tmp_value, strcs, str->ptr(), str->length(), cast_length)) { - null_value= 1; // In strict mode: malformed data or could not convert + null_value= 1; // EOM return 0; } check_truncation_with_warn(str, copier.source_end_pos() - str->ptr()); From a449f72a4cd60fc74cb7941cfaadfe8aef217520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 15 Sep 2017 17:25:38 +0300 Subject: [PATCH 11/30] Remove the redundant function row_table_got_default_clust_index() Use dict_index_is_auto_gen_clust() instead. --- storage/innobase/handler/ha_innodb.cc | 12 ++++-------- storage/innobase/handler/handler0alter.cc | 6 ++++-- storage/innobase/include/row0mysql.h | 9 --------- storage/innobase/row/row0mysql.cc | 16 ---------------- 4 files changed, 8 insertions(+), 35 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index fe931ce2616..5e7904b2b84 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6689,11 +6689,8 @@ ha_innobase::open(const char* name, int, uint) a row in our table. Note that MySQL may also compare two row references for equality by doing a simple memcmp on the strings of length ref_length! */ - - if (!row_table_got_default_clust_index(ib_table)) { - - m_prebuilt->clust_index_was_generated = FALSE; - + if (!(m_prebuilt->clust_index_was_generated + = dict_index_is_auto_gen_clust(ib_table->indexes.start))) { if (m_primary_key >= MAX_KEY) { ib_table->dict_frm_mismatch = DICT_FRM_NO_PK; @@ -6759,8 +6756,6 @@ ha_innobase::open(const char* name, int, uint) ib_push_frm_error(thd, ib_table, table, 0, true); } - m_prebuilt->clust_index_was_generated = TRUE; - ref_length = DATA_ROW_ID_LEN; /* If we automatically created the clustered index, then @@ -14462,7 +14457,8 @@ innobase_get_mysql_key_number_for_index( i++; } - if (row_table_got_default_clust_index(index->table)) { + if (dict_index_is_clust(index) + && dict_index_is_auto_gen_clust(index)) { ut_a(i > 0); i--; } diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 1140cef8452..352b223ab69 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -689,7 +689,8 @@ ha_innobase::check_if_supported_inplace_alter( /* See if MYSQL table has no pk but we do. */ if (UNIV_UNLIKELY(my_primary_key >= MAX_KEY) - && !row_table_got_default_clust_index(m_prebuilt->table)) { + && !dict_index_is_auto_gen_clust( + dict_table_get_first_index(m_prebuilt->table))) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_PRIMARY_CANT_HAVE_NULL); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); @@ -4448,7 +4449,8 @@ prepare_inplace_alter_table_dict( index_defs = innobase_create_key_defs( ctx->heap, ha_alter_info, altered_table, ctx->num_to_add_index, num_fts_index, - row_table_got_default_clust_index(ctx->new_table), + dict_index_is_auto_gen_clust(dict_table_get_first_index( + ctx->new_table)), fts_doc_id_col, add_fts_doc_id, add_fts_doc_id_idx, old_table); diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 8e93faf7286..2b000603929 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -262,15 +262,6 @@ row_get_prebuilt_update_vector( /*===========================*/ row_prebuilt_t* prebuilt); /*!< in: prebuilt struct in MySQL handle */ -/*********************************************************************//** -Checks if a table is such that we automatically created a clustered -index on it (on row id). -@return TRUE if the clustered index was generated automatically */ -ibool -row_table_got_default_clust_index( -/*==============================*/ - const dict_table_t* table); /*!< in: table */ - /** Does an update or delete of a row for MySQL. @param[in,out] prebuilt prebuilt struct in MySQL handle @return error code or DB_SUCCESS */ diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index db7f906b0cb..4ceef6c918f 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2283,22 +2283,6 @@ no_unlock: trx->op_info = ""; } -/*********************************************************************//** -Checks if a table is such that we automatically created a clustered -index on it (on row id). -@return TRUE if the clustered index was generated automatically */ -ibool -row_table_got_default_clust_index( -/*==============================*/ - const dict_table_t* table) /*!< in: table */ -{ - const dict_index_t* clust_index; - - clust_index = dict_table_get_first_index(table); - - return(dict_index_get_nth_col(clust_index, 0)->mtype == DATA_SYS); -} - /*********************************************************************//** Locks the data dictionary in shared mode from modifications, for performing foreign key check, rollback, or other operation invisible to MySQL. */ From a07b537b396b587460e721689286e38bf793cf5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 15 Sep 2017 20:09:41 +0300 Subject: [PATCH 12/30] Field_varstring: Declare get_data() and get_length() as public The underlying data members are declared public, so there is no point in hiding these const accessors. --- sql/field.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/field.h b/sql/field.h index 18d66ed8891..91c24327b28 100644 --- a/sql/field.h +++ b/sql/field.h @@ -3092,6 +3092,7 @@ private: class Field_varstring :public Field_longstr { +public: uchar *get_data() const { return ptr + length_bytes; @@ -3100,7 +3101,6 @@ class Field_varstring :public Field_longstr { { return length_bytes == 1 ? (uint) *ptr : uint2korr(ptr); } -public: /* The maximum space available in a Field_varstring, in bytes. See length_bytes. From c2815c7c2c7f9dcf4e9f241e10d154faf561103f Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 15 Sep 2017 20:44:03 +0000 Subject: [PATCH 13/30] add updated result file for MDEV-13802 test. --- mysql-test/suite/mariabackup/lock_ddl_per_table.result | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mysql-test/suite/mariabackup/lock_ddl_per_table.result b/mysql-test/suite/mariabackup/lock_ddl_per_table.result index d0137a1e072..434b6852530 100644 --- a/mysql-test/suite/mariabackup/lock_ddl_per_table.result +++ b/mysql-test/suite/mariabackup/lock_ddl_per_table.result @@ -1,4 +1,11 @@ CREATE TABLE t(i INT) ENGINE INNODB; INSERT INTO t VALUES(1); # xtrabackup backup +CREATE TABLE `bobby``tables` (id INT, name VARCHAR(50), purchased DATE) ENGINE INNODB PARTITION BY RANGE( YEAR(purchased) ) ( +PARTITION p0 VALUES LESS THAN (1990), +PARTITION p1 VALUES LESS THAN (1995), +PARTITION p2 VALUES LESS THAN (2000), +PARTITION p3 VALUES LESS THAN (2005) +) ; DROP TABLE t; +DROP TABLE `bobby``tables`; From 8c4df595b8bdd186ee68aee8a7953958fa503d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 16 Sep 2017 09:53:29 +0300 Subject: [PATCH 14/30] MDEV-13807 mariabackup --apply-log-only does generate redo log by performing rollback and possibly other tasks Skip rollback and other redo-log-generating tasks if srv_apply_log_only is set. Instead of assigning the debug variable recv_no_log_write = FALSE, assign it to srv_apply_log_only, so that any unwanted writes are caught. --- .../mariabackup/apply-log-only-incr.result | 29 +++++++++ .../mariabackup/apply-log-only-incr.test | 62 +++++++++++++++++++ .../suite/mariabackup/apply-log-only.result | 10 +++ .../suite/mariabackup/apply-log-only.test | 25 ++++++++ storage/xtradb/log/log0recv.cc | 5 +- storage/xtradb/srv/srv0srv.cc | 2 +- storage/xtradb/srv/srv0start.cc | 10 ++- storage/xtradb/trx/trx0rseg.cc | 2 + storage/xtradb/trx/trx0sys.cc | 7 +-- storage/xtradb/trx/trx0trx.cc | 1 + storage/xtradb/trx/trx0undo.cc | 1 + 11 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 mysql-test/suite/mariabackup/apply-log-only-incr.result create mode 100644 mysql-test/suite/mariabackup/apply-log-only-incr.test create mode 100644 mysql-test/suite/mariabackup/apply-log-only.result create mode 100644 mysql-test/suite/mariabackup/apply-log-only.test diff --git a/mysql-test/suite/mariabackup/apply-log-only-incr.result b/mysql-test/suite/mariabackup/apply-log-only-incr.result new file mode 100644 index 00000000000..2baed8c1db9 --- /dev/null +++ b/mysql-test/suite/mariabackup/apply-log-only-incr.result @@ -0,0 +1,29 @@ +call mtr.add_suppression("InnoDB: New log files created"); +CREATE TABLE t(a INT UNSIGNED PRIMARY KEY) ENGINE INNODB; +INSERT INTO t VALUES(0); +COMMIT; +start transaction; +NOT FOUND /Rollback of trx with id/ in current_test +# expect NOT FOUND +NOT FOUND /Rollback of trx with id/ in current_test +# expect NOT FOUND +commit; +SELECT count(*) FROM t; +count(*) +201 +# Restore and check results +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT COUNT(*) FROM t; +COUNT(*) +1 +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SELECT COUNT(*) FROM t; +COUNT(*) +201 +SELECT * FROM t; +a +0 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/apply-log-only-incr.test b/mysql-test/suite/mariabackup/apply-log-only-incr.test new file mode 100644 index 00000000000..a580d0ffe04 --- /dev/null +++ b/mysql-test/suite/mariabackup/apply-log-only-incr.test @@ -0,0 +1,62 @@ +--source include/have_innodb.inc + +call mtr.add_suppression("InnoDB: New log files created"); + +let $basedir=$MYSQLTEST_VARDIR/tmp/backup; +let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; + +CREATE TABLE t(a INT UNSIGNED PRIMARY KEY) ENGINE INNODB; +INSERT INTO t VALUES(0); +COMMIT; + +start transaction; +--disable_query_log +let $n=100; +while ($n) { +eval INSERT t VALUES(101-$n); +dec $n; +} + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log +let $n=100; +while ($n) { +eval INSERT t VALUES(201-$n); +dec $n; +} +--enable_query_log + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --ftwrl-wait-timeout=5 --ftwrl-wait-threshold=300 --ftwrl-wait-query-type=all --target-dir=$incremental_dir --incremental-basedir=$basedir ; +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir ; +--enable_result_log + +let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test; +--let SEARCH_PATTERN= Rollback of trx with id +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; + +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +commit; +SELECT count(*) FROM t; +echo # Restore and check results; +--let $targetdir=$basedir +--let $restart_parameters= --innodb-force-recovery=3 +--source include/restart_and_restore.inc + +rmdir $basedir; + +SELECT COUNT(*) FROM t; +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SELECT COUNT(*) FROM t; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +SELECT * FROM t; +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/apply-log-only.result b/mysql-test/suite/mariabackup/apply-log-only.result new file mode 100644 index 00000000000..04b9c0d8ee2 --- /dev/null +++ b/mysql-test/suite/mariabackup/apply-log-only.result @@ -0,0 +1,10 @@ +call mtr.add_suppression("InnoDB: New log files created"); +CREATE TABLE t(a varchar(60)) ENGINE INNODB; +start transaction; +INSERT INTO t VALUES(1); +NOT FOUND /Rollback of trx with id/ in current_test +# expect NOT FOUND +SELECT count(*) FROM t; +count(*) +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/apply-log-only.test b/mysql-test/suite/mariabackup/apply-log-only.test new file mode 100644 index 00000000000..96a251fc03b --- /dev/null +++ b/mysql-test/suite/mariabackup/apply-log-only.test @@ -0,0 +1,25 @@ +--source include/have_innodb.inc +call mtr.add_suppression("InnoDB: New log files created"); + +let $basedir=$MYSQLTEST_VARDIR/tmp/backup; + +CREATE TABLE t(a varchar(60)) ENGINE INNODB; +start transaction; +INSERT INTO t VALUES(1); + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log + +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir ; + +let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test; +--let SEARCH_PATTERN= Rollback of trx with id +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +SELECT count(*) FROM t; +DROP TABLE t; + +# Cleanup +rmdir $basedir; diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 75fc7713c5b..1943fb51f37 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -1980,7 +1980,7 @@ recv_apply_hashed_log_recs(bool last_batch) mutex_enter(&(log_sys->mutex)); mutex_enter(&(recv_sys->mutex)); - ut_d(recv_no_log_write = FALSE); + ut_d(recv_no_log_write = srv_apply_log_only); recv_no_ibuf_operations = FALSE; } @@ -3461,7 +3461,8 @@ recv_recovery_from_checkpoint_finish(void) that the data dictionary tables will be free of any locks. The data dictionary latch should guarantee that there is at most one data dictionary transaction active at a time. */ - if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO) { + if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO + && !srv_apply_log_only) { trx_rollback_or_clean_recovered(FALSE); } } diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index d6070107432..cbfe54b1105 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -2470,7 +2470,7 @@ srv_get_active_thread_type(void) { srv_thread_type ret = SRV_NONE; - if (srv_read_only_mode) { + if (srv_read_only_mode || srv_apply_log_only) { return(SRV_NONE); } diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 94e680d0e80..2aa2426ca9c 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -767,7 +767,7 @@ create_log_files( /* Create a log checkpoint. */ mutex_enter(&log_sys->mutex); - ut_d(recv_no_log_write = FALSE); + ut_d(recv_no_log_write = srv_apply_log_only); recv_reset_logs( #ifdef UNIV_LOG_ARCHIVE UT_LIST_GET_FIRST(log_sys->log_groups)->archived_file_no, @@ -2748,6 +2748,11 @@ files_checked: recv_recovery_from_checkpoint_finish(); + if (srv_apply_log_only) { + ut_ad(IS_XTRABACKUP()); + goto skip_processes; + } + if (srv_force_recovery < SRV_FORCE_NO_IBUF_MERGE) { /* The following call is necessary for the insert buffer to work with multiple tablespaces. We must @@ -3156,6 +3161,7 @@ files_checked: os_thread_create(btr_defragment_thread, NULL, NULL); } +skip_processes: srv_was_started = TRUE; return(DB_SUCCESS); @@ -3318,7 +3324,7 @@ innodb_shutdown() srv_misc_tmpfile = 0; } - if (!srv_read_only_mode) { + if (!srv_read_only_mode && !srv_apply_log_only) { dict_stats_thread_deinit(); fil_crypt_threads_cleanup(); btr_scrub_cleanup(); diff --git a/storage/xtradb/trx/trx0rseg.cc b/storage/xtradb/trx/trx0rseg.cc index 16fa334872b..38719a1c771 100644 --- a/storage/xtradb/trx/trx0rseg.cc +++ b/storage/xtradb/trx/trx0rseg.cc @@ -122,9 +122,11 @@ trx_rseg_mem_free( mutex_free(&rseg->mutex); + if (!srv_apply_log_only) { /* There can't be any active transactions. */ ut_a(UT_LIST_GET_LEN(rseg->update_undo_list) == 0); ut_a(UT_LIST_GET_LEN(rseg->insert_undo_list) == 0); + } for (undo = UT_LIST_GET_FIRST(rseg->update_undo_cached); undo != NULL; diff --git a/storage/xtradb/trx/trx0sys.cc b/storage/xtradb/trx/trx0sys.cc index d2b1a8f9f0d..83bb28b3c07 100644 --- a/storage/xtradb/trx/trx0sys.cc +++ b/storage/xtradb/trx/trx0sys.cc @@ -1346,8 +1346,7 @@ trx_sys_close(void) ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx || srv_read_only_mode || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO - || (IS_XTRABACKUP() && srv_apply_log_only)); - + || srv_apply_log_only); while ((trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list)) != NULL) { trx_free_prepared(trx); @@ -1378,7 +1377,7 @@ trx_sys_close(void) UT_LIST_REMOVE(view_list, trx_sys->view_list, prev_view); } - if (!IS_XTRABACKUP() || !srv_apply_log_only) { + if (!srv_apply_log_only) { ut_a(UT_LIST_GET_LEN(trx_sys->view_list) == 0); ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == 0); @@ -1430,7 +1429,7 @@ ulint trx_sys_any_active_transactions(void) /*=================================*/ { - if (IS_XTRABACKUP() && srv_apply_log_only) { + if (srv_apply_log_only) { return(0); } mutex_enter(&trx_sys->mutex); diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index cfc1f83053a..ef6e7d01cc4 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -483,6 +483,7 @@ trx_free_prepared( || (trx_state_eq(trx, TRX_STATE_ACTIVE) && trx->is_recovered && (srv_read_only_mode + || srv_apply_log_only || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO))); ut_a(trx->magic_n == TRX_MAGIC_N); diff --git a/storage/xtradb/trx/trx0undo.cc b/storage/xtradb/trx/trx0undo.cc index 220589dd9ff..cb064e3d485 100644 --- a/storage/xtradb/trx/trx0undo.cc +++ b/storage/xtradb/trx/trx0undo.cc @@ -2037,6 +2037,7 @@ trx_undo_free_prepared( /* lock_trx_release_locks() assigns trx->is_recovered=false */ ut_a(srv_read_only_mode + || srv_apply_log_only || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); break; default: From 93087d5fe74b0078e0b6db4233710648c79043cc Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 15 Sep 2017 15:10:36 +0000 Subject: [PATCH 15/30] Fix some warnings --- storage/innobase/include/ut0timer.ic | 2 +- storage/xtradb/btr/btr0cur.cc | 2 +- storage/xtradb/include/page0zip.ic | 2 +- storage/xtradb/include/ut0timer.ic | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/innobase/include/ut0timer.ic b/storage/innobase/include/ut0timer.ic index 027e89c6279..46dcd0cb718 100644 --- a/storage/innobase/include/ut0timer.ic +++ b/storage/innobase/include/ut0timer.ic @@ -106,7 +106,7 @@ ut_microseconds_to_timer( /*=====================*/ ulonglong when) /*!< in: time where to calculate */ { - double ret = when; + double ret = (double)when; ret *= (double)(ut_timer.frequency); ret /= 1000000.0; return (ulonglong)ret; diff --git a/storage/xtradb/btr/btr0cur.cc b/storage/xtradb/btr/btr0cur.cc index 1a27fd0d6ea..59470b95c35 100644 --- a/storage/xtradb/btr/btr0cur.cc +++ b/storage/xtradb/btr/btr0cur.cc @@ -4379,7 +4379,7 @@ btr_estimate_number_of_different_key_vals( if (index->stat_index_size > 1) { n_sample_pages = (srv_stats_transient_sample_pages < index->stat_index_size) ? ut_min(index->stat_index_size, - log2(index->stat_index_size)*srv_stats_transient_sample_pages) + ulint(log2(index->stat_index_size)*srv_stats_transient_sample_pages)) : index->stat_index_size; } diff --git a/storage/xtradb/include/page0zip.ic b/storage/xtradb/include/page0zip.ic index bee35a48cc2..c6b155b73a0 100644 --- a/storage/xtradb/include/page0zip.ic +++ b/storage/xtradb/include/page0zip.ic @@ -174,7 +174,7 @@ page_zip_rec_needs_ext( ulint zip_size) /*!< in: compressed page size in bytes, or 0 */ { ut_ad(rec_size - > (comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES)); + > ulint((comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES))); ut_ad(ut_is_2pow(zip_size)); ut_ad(comp || !zip_size); diff --git a/storage/xtradb/include/ut0timer.ic b/storage/xtradb/include/ut0timer.ic index 62e17a10fb1..815726e9d0a 100644 --- a/storage/xtradb/include/ut0timer.ic +++ b/storage/xtradb/include/ut0timer.ic @@ -106,7 +106,7 @@ ut_microseconds_to_timer( /*=====================*/ ulonglong when) /*!< in: time where to calculate */ { - double ret = when; + double ret = (double)when; ret *= (double)(ut_timer.frequency); ret /= 1000000.0; return (ulonglong)ret; From ad17e8e518c85a156d05f091874dd5c2ddfafa32 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 16 Sep 2017 09:36:21 +0000 Subject: [PATCH 16/30] MDEV-13821 : mariabackup sometimes could lose ib_logf(FATAL) messages, The messages are getting lost because they are written with fprintf() but without fflush(), so abort() would lose buffered text. Applied fix from 10.2, which makes in_logf() use sql_print_information(), which in turn does proper flush after each message --- extra/mariabackup/xtrabackup.cc | 6 +++++ storage/xtradb/handler/ha_innodb.cc | 36 ++++++++++++----------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index c746b469329..c42a282e203 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -95,6 +95,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include /* TODO: replace with appropriate macros used in InnoDB 5.6 */ #define PAGE_ZIP_MIN_SIZE_SHIFT 10 @@ -6483,6 +6484,11 @@ int main(int argc, char **argv) system_charset_info = &my_charset_utf8_general_ci; key_map_full.set_all(); + logger.init_base(); + logger.set_handlers(LOG_FILE, LOG_NONE, LOG_NONE); + mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log, + MY_MUTEX_INIT_FAST); + handle_options(argc, argv, &client_defaults, &server_defaults); int argc_server; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 4af523b1018..9fb73f98275 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -21949,27 +21949,21 @@ ib_logf( str = static_cast(malloc(BUFSIZ)); my_vsnprintf(str, BUFSIZ, format, args); #endif /* __WIN__ */ - if (!IS_XTRABACKUP()) { - switch (level) { - case IB_LOG_LEVEL_INFO: - sql_print_information("InnoDB: %s", str); - break; - case IB_LOG_LEVEL_WARN: - sql_print_warning("InnoDB: %s", str); - break; - case IB_LOG_LEVEL_ERROR: - sql_print_error("InnoDB: %s", str); - sd_notifyf(0, "STATUS=InnoDB: Error: %s", str); - break; - case IB_LOG_LEVEL_FATAL: - sql_print_error("InnoDB: %s", str); - sd_notifyf(0, "STATUS=InnoDB: Fatal: %s", str); - break; - } - } - else { - /* Don't use server logger for XtraBackup, just print to stderr. */ - fprintf(stderr, "InnoDB: %s\n", str); + switch (level) { + case IB_LOG_LEVEL_INFO: + sql_print_information("InnoDB: %s", str); + break; + case IB_LOG_LEVEL_WARN: + sql_print_warning("InnoDB: %s", str); + break; + case IB_LOG_LEVEL_ERROR: + sql_print_error("InnoDB: %s", str); + sd_notifyf(0, "STATUS=InnoDB: Error: %s", str); + break; + case IB_LOG_LEVEL_FATAL: + sql_print_error("InnoDB: %s", str); + sd_notifyf(0, "STATUS=InnoDB: Fatal: %s", str); + break; } va_end(args); From 1d7bc3b58263153d18198a6bde03b2b2b6519a77 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 16 Sep 2017 09:44:52 +0000 Subject: [PATCH 17/30] Innodb : do not call fflush() in os_get_last_error_low(), if no error message was written. --- storage/innobase/os/os0file.cc | 3 +-- storage/xtradb/os/os0file.cc | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 5a945aae7bc..44093935bc7 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -569,10 +569,9 @@ os_file_get_last_error_low( REFMAN "operating-system-error-codes.html\n"); } + fflush(stderr); } - fflush(stderr); - if (err == ERROR_FILE_NOT_FOUND) { return(OS_FILE_NOT_FOUND); } else if (err == ERROR_DISK_FULL) { diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 32536306320..f3c72576501 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -676,10 +676,9 @@ os_file_get_last_error_low( REFMAN "operating-system-error-codes.html\n"); } + fflush(stderr); } - fflush(stderr); - if (err == ERROR_FILE_NOT_FOUND) { return(OS_FILE_NOT_FOUND); } else if (err == ERROR_DISK_FULL) { From 75dd3bcb4c815202ddb147a1ccffb2b691f2c0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 16 Sep 2017 15:05:22 +0300 Subject: [PATCH 18/30] Clean up after commit 93087d5fe74b0078e0b6db4233710648c79043cc Apply the same changes to both InnoDB and XtraDB. --- storage/innobase/btr/btr0cur.cc | 2 +- storage/innobase/include/page0zip.ic | 2 +- storage/xtradb/include/page0zip.ic | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index d848ee538c6..5597a085525 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -4106,7 +4106,7 @@ btr_estimate_number_of_different_key_vals( if (index->stat_index_size > 1) { n_sample_pages = (srv_stats_transient_sample_pages < index->stat_index_size) ? ut_min(index->stat_index_size, - log2(index->stat_index_size)*srv_stats_transient_sample_pages) + ulint(log2(index->stat_index_size)*srv_stats_transient_sample_pages)) : index->stat_index_size; } diff --git a/storage/innobase/include/page0zip.ic b/storage/innobase/include/page0zip.ic index bee35a48cc2..3fa8a08c4f9 100644 --- a/storage/innobase/include/page0zip.ic +++ b/storage/innobase/include/page0zip.ic @@ -174,7 +174,7 @@ page_zip_rec_needs_ext( ulint zip_size) /*!< in: compressed page size in bytes, or 0 */ { ut_ad(rec_size - > (comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES)); + > ulint(comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES)); ut_ad(ut_is_2pow(zip_size)); ut_ad(comp || !zip_size); diff --git a/storage/xtradb/include/page0zip.ic b/storage/xtradb/include/page0zip.ic index c6b155b73a0..3fa8a08c4f9 100644 --- a/storage/xtradb/include/page0zip.ic +++ b/storage/xtradb/include/page0zip.ic @@ -174,7 +174,7 @@ page_zip_rec_needs_ext( ulint zip_size) /*!< in: compressed page size in bytes, or 0 */ { ut_ad(rec_size - > ulint((comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES))); + > ulint(comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES)); ut_ad(ut_is_2pow(zip_size)); ut_ad(comp || !zip_size); From f24d36ae1e91cb98462139dfae41211b9e9a3a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 16 Sep 2017 21:13:47 +0300 Subject: [PATCH 19/30] Clean up a directory to avoid a failure of another test --- mysql-test/suite/mariabackup/apply-log-only-incr.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/mariabackup/apply-log-only-incr.test b/mysql-test/suite/mariabackup/apply-log-only-incr.test index a580d0ffe04..aa5110cc51e 100644 --- a/mysql-test/suite/mariabackup/apply-log-only-incr.test +++ b/mysql-test/suite/mariabackup/apply-log-only-incr.test @@ -50,6 +50,7 @@ echo # Restore and check results; --source include/restart_and_restore.inc rmdir $basedir; +rmdir $incremental_dir; SELECT COUNT(*) FROM t; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; From 836d4e74d9416a1af839f8068c5a07bee6f4b71e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 16 Sep 2017 16:55:16 +0300 Subject: [PATCH 20/30] Write proper tests for MDEV-12634: Uninitialised ROW_MERGE_RESERVE_SIZE bytes Introduce innodb_encrypt_log.combinations and prove that the encryption and decryption take place during both online ADD INDEX (WL#5266) and online table-rebuilding ALTER (WL#6625). --- .../include/innodb_encrypt_log.combinations | 7 ++ mysql-test/include/innodb_encrypt_log.inc | 4 + .../suite/encryption/r/innodb-rowlog.result | 30 ------ .../suite/encryption/t/innodb-rowlog.opt | 7 -- .../suite/encryption/t/innodb-rowlog.test | 90 ------------------ .../innodb/r/innodb-index-online,crypt.rdiff | 20 ++++ .../suite/innodb/r/innodb-index-online.result | 87 ++++++++++++++++++ .../innodb/r/innodb-table-online,crypt.rdiff | 20 ++++ .../suite/innodb/r/innodb-table-online.result | 91 ++++++++++++++++--- .../suite/innodb/t/innodb-index-online.opt | 2 +- .../suite/innodb/t/innodb-index-online.test | 77 +++++++++++++++- .../innodb/t/innodb-table-online-master.opt | 2 +- .../suite/innodb/t/innodb-table-online.test | 80 ++++++++++++++-- storage/innobase/row/row0merge.cc | 4 - storage/xtradb/row/row0merge.cc | 4 - 15 files changed, 365 insertions(+), 160 deletions(-) create mode 100644 mysql-test/include/innodb_encrypt_log.combinations create mode 100644 mysql-test/include/innodb_encrypt_log.inc delete mode 100644 mysql-test/suite/encryption/r/innodb-rowlog.result delete mode 100644 mysql-test/suite/encryption/t/innodb-rowlog.opt delete mode 100644 mysql-test/suite/encryption/t/innodb-rowlog.test create mode 100644 mysql-test/suite/innodb/r/innodb-index-online,crypt.rdiff create mode 100644 mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff diff --git a/mysql-test/include/innodb_encrypt_log.combinations b/mysql-test/include/innodb_encrypt_log.combinations new file mode 100644 index 00000000000..fd21a57c3c2 --- /dev/null +++ b/mysql-test/include/innodb_encrypt_log.combinations @@ -0,0 +1,7 @@ +[crypt] +innodb_encrypt_log=ON +innodb_encryption_rotate_key_age=1 +plugin_load_add=$DEBUG_KEY_MANAGEMENT_SO + +[clear] +skip_innodb_encrypt_log diff --git a/mysql-test/include/innodb_encrypt_log.inc b/mysql-test/include/innodb_encrypt_log.inc new file mode 100644 index 00000000000..5beebeae81f --- /dev/null +++ b/mysql-test/include/innodb_encrypt_log.inc @@ -0,0 +1,4 @@ +# The goal of including this file is to enable innodb_encrypt_log combinations +# (see include/innodb_encrypt_log.combinations) + +--source include/have_innodb.inc diff --git a/mysql-test/suite/encryption/r/innodb-rowlog.result b/mysql-test/suite/encryption/r/innodb-rowlog.result deleted file mode 100644 index 873041ab421..00000000000 --- a/mysql-test/suite/encryption/r/innodb-rowlog.result +++ /dev/null @@ -1,30 +0,0 @@ -SET GLOBAL innodb_file_format = `Barracuda`; -create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; -SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; -alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; -SET GLOBAL innodb_encryption_rotate_key_age = 1; -create table t2(id int) engine=innodb; -SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; -alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; -insert into t1(credit_card) select credit_card from t1; -insert into t1(credit_card) select credit_card from t1; -insert into t1(credit_card) select credit_card from t1; -drop table t2; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value > 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; -variable_value > 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; -variable_value > 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; -variable_value > 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; -variable_value > 0 -1 -drop table t1; -SET GLOBAL innodb_file_format=Antelope; -SET GLOBAL innodb_encryption_rotate_key_age=15; diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.opt b/mysql-test/suite/encryption/t/innodb-rowlog.opt deleted file mode 100644 index 6338ddbde35..00000000000 --- a/mysql-test/suite/encryption/t/innodb-rowlog.opt +++ /dev/null @@ -1,7 +0,0 @@ ---aria-encrypt-tables ---encrypt-tmp-disk-tables ---innodb-encrypt-tables ---innodb-encrypt-log ---innodb-encryption-rotate-key-age=15 ---innodb-encryption-threads=4 ---innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.test b/mysql-test/suite/encryption/t/innodb-rowlog.test deleted file mode 100644 index 135293ef4b9..00000000000 --- a/mysql-test/suite/encryption/t/innodb-rowlog.test +++ /dev/null @@ -1,90 +0,0 @@ --- source include/have_innodb.inc --- source include/have_example_key_management_plugin.inc -# needs dbug_dbug --- source include/have_debug.inc - ---disable_warnings ---disable_query_log -let $innodb_file_format_orig = `SELECT @@innodb_file_format`; -let $key_age = `SELECT @@innodb_encryption_rotate_key_age`; ---enable_query_log - -SET GLOBAL innodb_file_format = `Barracuda`; ---enable_warnings - -let $MYSQLD_DATADIR = `SELECT @@datadir`; -let $MYSQLD_TMPDIR = `SELECT @@tmpdir`; - -# -# Create a table that will be encrypted and put some sensitive data to it (credit card numbers) -# - -create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; - -let $rows = 15000; ---disable_query_log -begin; -while ($rows) -{ - eval insert into t1 values(NULL, '0000-0000-0000-0000','private_data'); - dec $rows; -} -commit; ---enable_query_log - ---let $wait_timeout= 600 ---let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 ---source include/wait_condition.inc - -# -# Now we create secondary index for credit_card column in parallel we create new rows -# forcing alter table to wait so that row log is used. -# -SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; -send alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; -# -# Force key rotation and create second index for same table -# -connect (con2,localhost,root,,); -SET GLOBAL innodb_encryption_rotate_key_age = 1; -create table t2(id int) engine=innodb; -SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; -send alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; -# -# Create new rows to row log -# -connect (con1,localhost,root,,); -insert into t1(credit_card) select credit_card from t1; -insert into t1(credit_card) select credit_card from t1; -insert into t1(credit_card) select credit_card from t1; - -connection default; -reap; -disconnect con1; - -connection con2; -reap; -drop table t2; - -connection default; -disconnect con2; - -let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; ---source include/wait_condition.inc - -# -# Verify that both merge blocks and row log blocks are encryted and decrypted -# - -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; - -drop table t1; - ---disable_warnings -eval SET GLOBAL innodb_file_format=$innodb_file_format_orig; -eval SET GLOBAL innodb_encryption_rotate_key_age=$key_age; ---enable_warnings diff --git a/mysql-test/suite/innodb/r/innodb-index-online,crypt.rdiff b/mysql-test/suite/innodb/r/innodb-index-online,crypt.rdiff new file mode 100644 index 00000000000..767e36f3bfb --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-index-online,crypt.rdiff @@ -0,0 +1,20 @@ +--- innodb-index-online.result ++++ innodb-index-online,crypt.reject +@@ -260,7 +260,7 @@ + @merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, + @rowlog_encrypt_1>@rowlog_encrypt_0; + sort_balance @merge_encrypt_1>@merge_encrypt_0 @merge_decrypt_1>@merge_decrypt_0 @rowlog_encrypt_1>@rowlog_encrypt_0 +-0 0 0 0 ++0 1 1 1 + SET DEBUG_SYNC = 'now SIGNAL dml2_done'; + ERROR HY000: Creating index 'c2e' required more than 'innodb_online_alter_log_max_size' bytes of modification log. Please try again. + SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; +@@ -364,7 +364,7 @@ + @rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, + @rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; + sort_encrypted sort_decrypted log_encrypted log_decrypted +-0 0 0 0 ++1 1 1 1 + SELECT COUNT(c22f) FROM t1; + COUNT(c22f) + 320 diff --git a/mysql-test/suite/innodb/r/innodb-index-online.result b/mysql-test/suite/innodb/r/innodb-index-online.result index 77b7bc365aa..d6941770cd3 100644 --- a/mysql-test/suite/innodb/r/innodb-index-online.result +++ b/mysql-test/suite/innodb/r/innodb-index-online.result @@ -180,6 +180,15 @@ t1 CREATE TABLE `t1` ( PRIMARY KEY (`c1`), KEY `c2d` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=1 +SET @merge_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); SET DEBUG_SYNC = 'row_log_apply_before SIGNAL c2e_created WAIT_FOR dml2_done'; SET lock_wait_timeout = 10; ALTER TABLE t1 DROP INDEX c2d, ADD INDEX c2e(c2), @@ -209,6 +218,22 @@ BEGIN; UPDATE t1 SET c2 = c2 + 1; DELETE FROM t1; ROLLBACK; +BEGIN; +DELETE FROM t1; +ROLLBACK; +UPDATE t1 SET c2 = c2 + 1; +BEGIN; +UPDATE t1 SET c2 = c2 + 1; +DELETE FROM t1; +ROLLBACK; +BEGIN; +DELETE FROM t1; +ROLLBACK; +UPDATE t1 SET c2 = c2 + 1; +BEGIN; +UPDATE t1 SET c2 = c2 + 1; +DELETE FROM t1; +ROLLBACK; SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; name count ddl_background_drop_indexes 0 @@ -220,6 +245,22 @@ INNER JOIN INFORMATION_SCHEMA.INNODB_SYS_FIELDS sf ON si.index_id = sf.index_id WHERE si.name = '?c2e'; name pos c2 0 +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SELECT +(@merge_encrypt_1-@merge_encrypt_0)- +(@merge_decrypt_1-@merge_decrypt_0) as sort_balance, +@merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, +@rowlog_encrypt_1>@rowlog_encrypt_0; +sort_balance @merge_encrypt_1>@merge_encrypt_0 @merge_decrypt_1>@merge_decrypt_0 @rowlog_encrypt_1>@rowlog_encrypt_0 +0 0 0 0 SET DEBUG_SYNC = 'now SIGNAL dml2_done'; ERROR HY000: Creating index 'c2e' required more than 'innodb_online_alter_log_max_size' bytes of modification log. Please try again. SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; @@ -245,6 +286,18 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 0 ddl_pending_alter_table 0 +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); SET DEBUG_SYNC = 'row_log_apply_before SIGNAL c2f_created WAIT_FOR dml3_done'; ALTER TABLE t1 ADD INDEX c2f(c2); SET DEBUG_SYNC = 'now WAIT_FOR c2f_created'; @@ -262,6 +315,14 @@ BEGIN; UPDATE t1 SET c2 = c2 + 1; DELETE FROM t1; ROLLBACK; +BEGIN; +INSERT INTO t1 SELECT 320 + c1, c2, c3 FROM t1 WHERE c1 > 160; +DELETE FROM t1 WHERE c1 > 320; +ROLLBACK; +BEGIN; +UPDATE t1 SET c2 = c2 + 1; +DELETE FROM t1; +ROLLBACK; SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; name count ddl_background_drop_indexes 0 @@ -278,6 +339,32 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 0 ddl_pending_alter_table 0 +SET @merge_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); +SELECT +(@merge_encrypt_2-@merge_encrypt_1)- +(@merge_decrypt_2-@merge_decrypt_1) as sort_balance, +(@rowlog_encrypt_2-@rowlog_encrypt_1)- +(@rowlog_decrypt_2-@rowlog_decrypt_1) as log_balance; +sort_balance log_balance +0 0 +SELECT +@merge_encrypt_2-@merge_encrypt_1>0 as sort_encrypted, +@merge_decrypt_2-@merge_decrypt_1>0 as sort_decrypted, +@rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, +@rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; +sort_encrypted sort_decrypted log_encrypted log_decrypted +0 0 0 0 SELECT COUNT(c22f) FROM t1; COUNT(c22f) 320 diff --git a/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff b/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff new file mode 100644 index 00000000000..fa36666ba48 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff @@ -0,0 +1,20 @@ +--- innodb-table-online.result ++++ innodb-table-online,crypt.reject +@@ -254,7 +254,7 @@ + @merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, + @rowlog_encrypt_1>@rowlog_encrypt_0; + sort_balance @merge_encrypt_1>@merge_encrypt_0 @merge_decrypt_1>@merge_decrypt_0 @rowlog_encrypt_1>@rowlog_encrypt_0 +-0 0 0 0 ++0 1 1 1 + SET DEBUG_SYNC = 'now SIGNAL dml2_done'; + # session con1 + ERROR HY000: Creating index 'PRIMARY' required more than 'innodb_online_alter_log_max_size' bytes of modification log. Please try again. +@@ -345,7 +345,7 @@ + @rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, + @rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; + sort_encrypted sort_decrypted log_encrypted log_decrypted +-0 0 0 0 ++1 1 1 1 + ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)); + ERROR 23000: Duplicate entry '' for key 'PRIMARY' + UPDATE t1 SET c3 = NULL WHERE c3 = ''; diff --git a/mysql-test/suite/innodb/r/innodb-table-online.result b/mysql-test/suite/innodb/r/innodb-table-online.result index a352f0c2a90..b8cd99659fa 100644 --- a/mysql-test/suite/innodb/r/innodb-table-online.result +++ b/mysql-test/suite/innodb/r/innodb-table-online.result @@ -3,7 +3,7 @@ call mtr.add_suppression("InnoDB: Error: table 'test/t1'"); call mtr.add_suppression("MySQL is trying to open a table handle but the .ibd file for"); SET @global_innodb_file_per_table_orig = @@global.innodb_file_per_table; SET GLOBAL innodb_file_per_table = on; -CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 TEXT NOT NULL) +CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 CHAR(255) NOT NULL) ENGINE = InnoDB; INSERT INTO t1 VALUES (1,1,''), (2,2,''), (3,3,''), (4,4,''), (5,5,''); SET GLOBAL innodb_monitor_enable = module_ddl; @@ -53,7 +53,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, PRIMARY KEY (`c1`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT BEGIN; @@ -81,7 +81,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, UNIQUE KEY `c2` (`c2`), UNIQUE KEY `c2_2` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT @@ -92,7 +92,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, UNIQUE KEY `c2` (`c2`), UNIQUE KEY `c2_2` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT @@ -118,7 +118,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, UNIQUE KEY `c2` (`c2`), UNIQUE KEY `c2_2` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT @@ -173,17 +173,26 @@ INSERT INTO t1 SELECT 20 + c1, c2, c3 FROM t1; INSERT INTO t1 SELECT 40 + c1, c2, c3 FROM t1; EXPLAIN SELECT COUNT(*) FROM t1 WHERE c2 > 3; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ALL NULL NULL NULL NULL 80 Using where +1 SIMPLE t1 ALL NULL NULL NULL NULL ROWS Using where ANALYZE TABLE t1; Table Op Msg_type Msg_text test.t1 analyze status OK +SET @merge_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); # session con1 SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, PRIMARY KEY (`c1`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL rebuilt2 WAIT_FOR dml2_done'; @@ -230,6 +239,22 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 1 ddl_pending_alter_table 1 +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SELECT +(@merge_encrypt_1-@merge_encrypt_0)- +(@merge_decrypt_1-@merge_decrypt_0) as sort_balance, +@merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, +@rowlog_encrypt_1>@rowlog_encrypt_0; +sort_balance @merge_encrypt_1>@merge_encrypt_0 @merge_decrypt_1>@merge_decrypt_0 @rowlog_encrypt_1>@rowlog_encrypt_0 +0 0 0 0 SET DEBUG_SYNC = 'now SIGNAL dml2_done'; # session con1 ERROR HY000: Creating index 'PRIMARY' required more than 'innodb_online_alter_log_max_size' bytes of modification log. Please try again. @@ -239,14 +264,26 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 0 ddl_pending_alter_table 0 +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL rebuilt3 WAIT_FOR dml3_done'; ALTER TABLE t1 ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; ERROR 42000: Multiple primary key defined ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; ERROR 23000: Duplicate entry '5' for key 'PRIMARY' ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c1,c22f,c4(5)), -CHANGE c2 c22f INT, CHANGE c3 c3 TEXT NULL, CHANGE c1 c1 INT AFTER c22f, -ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online'; +CHANGE c2 c22f INT, CHANGE c3 c3 CHAR(255) NULL, CHANGE c1 c1 INT AFTER c22f, +ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online', LOCK=NONE; # session default SET DEBUG_SYNC = 'now WAIT_FOR rebuilt3'; SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; @@ -283,6 +320,32 @@ COUNT(c22f) CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK +SET @merge_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); +SELECT +(@merge_encrypt_2-@merge_encrypt_1)- +(@merge_decrypt_2-@merge_decrypt_1) as sort_balance, +(@rowlog_encrypt_2-@rowlog_encrypt_1)- +(@rowlog_decrypt_2-@rowlog_decrypt_1) as log_balance; +sort_balance log_balance +0 0 +SELECT +@merge_encrypt_2-@merge_encrypt_1>0 as sort_encrypted, +@merge_decrypt_2-@merge_decrypt_1>0 as sort_decrypted, +@rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, +@rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; +sort_encrypted sort_decrypted log_encrypted log_decrypted +0 0 0 0 ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)); ERROR 23000: Duplicate entry '' for key 'PRIMARY' UPDATE t1 SET c3 = NULL WHERE c3 = ''; @@ -294,13 +357,13 @@ SET @@sql_mode = 'STRICT_TRANS_TABLES'; ALTER TABLE t1 DROP COLUMN c22f, DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)), ALGORITHM = INPLACE; ERROR 22004: Invalid use of NULL value -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL; +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL; ERROR 22004: Invalid use of NULL value SET @@sql_mode = @old_sql_mode; -UPDATE t1 SET c3=CONCAT(c1,REPEAT('foo',c1)) WHERE c3 IS NULL; +UPDATE t1 SET c3=LEFT(CONCAT(c1,REPEAT('foo',c1)),255) WHERE c3 IS NULL; SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL c3p5_created0 WAIT_FOR ins_done0'; SET @@sql_mode = 'STRICT_TRANS_TABLES'; -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL, DROP COLUMN c22f, +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL, DROP COLUMN c22f, ADD COLUMN c5 CHAR(5) DEFAULT 'tired' FIRST; # session default SET DEBUG_SYNC = 'now WAIT_FOR c3p5_created0'; @@ -314,7 +377,7 @@ SET @@sql_mode = @old_sql_mode; # session default ROLLBACK; # session con1 -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL; +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL; SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL c3p5_created WAIT_FOR ins_done'; ALTER TABLE t1 DROP PRIMARY KEY, DROP COLUMN c22f, ADD COLUMN c6 VARCHAR(1000) DEFAULT @@ -361,7 +424,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c22f` int(11) NOT NULL, `c1` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, `c4` varchar(6) NOT NULL DEFAULT 'Online', PRIMARY KEY (`c1`,`c22f`,`c4`(5)) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT diff --git a/mysql-test/suite/innodb/t/innodb-index-online.opt b/mysql-test/suite/innodb/t/innodb-index-online.opt index 3f3c063b38f..820778b0004 100644 --- a/mysql-test/suite/innodb/t/innodb-index-online.opt +++ b/mysql-test/suite/innodb/t/innodb-index-online.opt @@ -1,5 +1,5 @@ --innodb-sort-buffer-size=64k ---innodb-online-alter-log-max-size=64k +--innodb-online-alter-log-max-size=128k --innodb-buffer-pool-size=5M --innodb-log-buffer-size=256k --innodb-sys-indexes diff --git a/mysql-test/suite/innodb/t/innodb-index-online.test b/mysql-test/suite/innodb/t/innodb-index-online.test index 24e3cd1ba22..efd700498c6 100644 --- a/mysql-test/suite/innodb/t/innodb-index-online.test +++ b/mysql-test/suite/innodb/t/innodb-index-online.test @@ -1,4 +1,5 @@ --source include/innodb_page_size_small.inc +--source include/innodb_encrypt_log.inc --source include/have_debug_sync.inc let $innodb_metrics_select= @@ -200,6 +201,18 @@ EXPLAIN SELECT COUNT(*) FROM t1 WHERE c2 > 3; SHOW CREATE TABLE t1; +connection default; +SET @merge_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +connection con1; + # Exceed the configured innodb_online_alter_log_max_size. # The actual limit is a multiple of innodb_sort_buf_size, # because that is the size of the in-memory log buffers. @@ -223,7 +236,7 @@ SET DEBUG_SYNC = 'now WAIT_FOR c2e_created'; # At this point, the clustered index scan must have completed, # but the modification log keeps accumulating due to the DEBUG_SYNC. eval $innodb_metrics_select; -let $c= 2; +let $c= 4; while ($c) { BEGIN; @@ -243,6 +256,22 @@ SELECT sf.name, sf.pos FROM INFORMATION_SCHEMA.INNODB_SYS_INDEXES si INNER JOIN INFORMATION_SCHEMA.INNODB_SYS_FIELDS sf ON si.index_id = sf.index_id WHERE si.name = '?c2e'; +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); + +SELECT +(@merge_encrypt_1-@merge_encrypt_0)- +(@merge_decrypt_1-@merge_decrypt_0) as sort_balance, +@merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, +@rowlog_encrypt_1>@rowlog_encrypt_0; + # Release con1. SET DEBUG_SYNC = 'now SIGNAL dml2_done'; @@ -270,6 +299,19 @@ ALTER TABLE t1 COMMENT 'testing if c2e will be dropped'; # Check that the 'zombie' index c2e was dropped. eval $innodb_metrics_select; +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); + connection con1; # Accumulate and apply some modification log. SET DEBUG_SYNC = 'row_log_apply_before SIGNAL c2f_created WAIT_FOR dml3_done'; @@ -282,6 +324,9 @@ connection default; SET DEBUG_SYNC = 'now WAIT_FOR c2f_created'; # Generate some log (delete-mark, delete-unmark, insert etc.) eval $innodb_metrics_select; +let $c= 2; +while ($c) +{ BEGIN; INSERT INTO t1 SELECT 320 + c1, c2, c3 FROM t1 WHERE c1 > 160; DELETE FROM t1 WHERE c1 > 320; @@ -290,6 +335,8 @@ BEGIN; UPDATE t1 SET c2 = c2 + 1; DELETE FROM t1; ROLLBACK; +dec $c; +} eval $innodb_metrics_select; # Release con1. SET DEBUG_SYNC = 'now SIGNAL dml3_done'; @@ -300,6 +347,34 @@ reap; ALTER TABLE t1 CHANGE c2 c22f INT; eval $innodb_metrics_select; + +connection default; + +SET @merge_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); + +SELECT +(@merge_encrypt_2-@merge_encrypt_1)- +(@merge_decrypt_2-@merge_decrypt_1) as sort_balance, +(@rowlog_encrypt_2-@rowlog_encrypt_1)- +(@rowlog_decrypt_2-@rowlog_decrypt_1) as log_balance; +SELECT +@merge_encrypt_2-@merge_encrypt_1>0 as sort_encrypted, +@merge_decrypt_2-@merge_decrypt_1>0 as sort_decrypted, +@rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, +@rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; + +connection con1; SELECT COUNT(c22f) FROM t1; CHECK TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-table-online-master.opt b/mysql-test/suite/innodb/t/innodb-table-online-master.opt index 9ac01ffb5ad..92eea2b0d2e 100644 --- a/mysql-test/suite/innodb/t/innodb-table-online-master.opt +++ b/mysql-test/suite/innodb/t/innodb-table-online-master.opt @@ -1 +1 @@ ---innodb-sort-buffer-size=64k --innodb-online-alter-log-max-size=64k --innodb-buffer-pool-size=5M --innodb-log-buffer-size=256k +--innodb-sort-buffer-size=64k --innodb-online-alter-log-max-size=512k --innodb-buffer-pool-size=5M --innodb-log-buffer-size=256k diff --git a/mysql-test/suite/innodb/t/innodb-table-online.test b/mysql-test/suite/innodb/t/innodb-table-online.test index ff5c55d9cd9..938fd176b1e 100644 --- a/mysql-test/suite/innodb/t/innodb-table-online.test +++ b/mysql-test/suite/innodb/t/innodb-table-online.test @@ -1,4 +1,5 @@ --source include/innodb_page_size_small.inc +--source include/innodb_encrypt_log.inc --source include/have_debug.inc --source include/have_debug_sync.inc @@ -17,7 +18,7 @@ SET GLOBAL innodb_file_per_table = on; # Save the initial number of concurrent sessions. --source include/count_sessions.inc -CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 TEXT NOT NULL) +CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 CHAR(255) NOT NULL) ENGINE = InnoDB; INSERT INTO t1 VALUES (1,1,''), (2,2,''), (3,3,''), (4,4,''), (5,5,''); @@ -183,10 +184,20 @@ INSERT INTO t1 SELECT 10 + c1, c2, c3 FROM t1; INSERT INTO t1 SELECT 20 + c1, c2, c3 FROM t1; INSERT INTO t1 SELECT 40 + c1, c2, c3 FROM t1; # Purge may or may not have cleaned up the DELETE FROM t1 WHERE c1 = 7; ---replace_result 81 80 +--replace_column 9 ROWS EXPLAIN SELECT COUNT(*) FROM t1 WHERE c2 > 3; ANALYZE TABLE t1; +SET @merge_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); + --echo # session con1 connection con1; SHOW CREATE TABLE t1; @@ -228,6 +239,22 @@ while ($c) # Temporary table should exist until the DDL thread notices the overflow. eval $innodb_metrics_select; +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); + +SELECT +(@merge_encrypt_1-@merge_encrypt_0)- +(@merge_decrypt_1-@merge_decrypt_0) as sort_balance, +@merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, +@rowlog_encrypt_1>@rowlog_encrypt_0; + # Release con1. SET DEBUG_SYNC = 'now SIGNAL dml2_done'; @@ -241,6 +268,19 @@ reap; # when the above error was noticed. eval $innodb_metrics_select; +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); + # Accumulate and apply some modification log. SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL rebuilt3 WAIT_FOR dml3_done'; --error ER_MULTIPLE_PRI_KEY @@ -249,8 +289,8 @@ ALTER TABLE t1 ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; --send ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c1,c22f,c4(5)), -CHANGE c2 c22f INT, CHANGE c3 c3 TEXT NULL, CHANGE c1 c1 INT AFTER c22f, -ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online'; +CHANGE c2 c22f INT, CHANGE c3 c3 CHAR(255) NULL, CHANGE c1 c1 INT AFTER c22f, +ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online', LOCK=NONE; --echo # session default connection default; @@ -276,6 +316,30 @@ eval $innodb_metrics_select; SELECT COUNT(c22f) FROM t1; CHECK TABLE t1; +SET @merge_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); + +SELECT +(@merge_encrypt_2-@merge_encrypt_1)- +(@merge_decrypt_2-@merge_decrypt_1) as sort_balance, +(@rowlog_encrypt_2-@rowlog_encrypt_1)- +(@rowlog_decrypt_2-@rowlog_decrypt_1) as log_balance; +SELECT +@merge_encrypt_2-@merge_encrypt_1>0 as sort_encrypted, +@merge_decrypt_2-@merge_decrypt_1>0 as sort_decrypted, +@rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, +@rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; + # Create a column prefix index. --error ER_DUP_ENTRY ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)); @@ -292,15 +356,15 @@ ALTER TABLE t1 DROP COLUMN c22f, DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)), ALGORITHM = INPLACE; --error ER_INVALID_USE_OF_NULL -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL; +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL; SET @@sql_mode = @old_sql_mode; -UPDATE t1 SET c3=CONCAT(c1,REPEAT('foo',c1)) WHERE c3 IS NULL; +UPDATE t1 SET c3=LEFT(CONCAT(c1,REPEAT('foo',c1)),255) WHERE c3 IS NULL; SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL c3p5_created0 WAIT_FOR ins_done0'; # NULL -> NOT NULL only allowed INPLACE if strict sql_mode is on. SET @@sql_mode = 'STRICT_TRANS_TABLES'; --send -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL, DROP COLUMN c22f, +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL, DROP COLUMN c22f, ADD COLUMN c5 CHAR(5) DEFAULT 'tired' FIRST; --echo # session default @@ -324,7 +388,7 @@ ROLLBACK; --echo # session con1 connection con1; -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL; +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL; SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL c3p5_created WAIT_FOR ins_done'; --send ALTER TABLE t1 DROP PRIMARY KEY, DROP COLUMN c22f, diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index a91c0f67218..62dc98a3e30 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -4037,10 +4037,6 @@ row_merge_build_indexes( /* Now we have files containing index entries ready for sorting and inserting. */ - DBUG_EXECUTE_IF( - "ib_merge_wait_after_read", - os_thread_sleep(20000000);); /* 20 sec */ - for (i = 0; i < n_indexes; i++) { dict_index_t* sort_idx = indexes[i]; diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 28b43d6abef..1b0566dd2d3 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -4040,10 +4040,6 @@ row_merge_build_indexes( /* Now we have files containing index entries ready for sorting and inserting. */ - DBUG_EXECUTE_IF( - "ib_merge_wait_after_read", - os_thread_sleep(20000000);); /* 20 sec */ - for (i = 0; i < n_indexes; i++) { dict_index_t* sort_idx = indexes[i]; From d1253e19a1db9aa468035ab126efd50415fc0782 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 16 Sep 2017 22:19:16 +0200 Subject: [PATCH 21/30] Fix compilation in mariabackup Compilation got confused about 2 wsrep.h headers in include path Rename backup's wsrep.h to backup_wsrep.h to fixO --- extra/mariabackup/{wsrep.h => backup_wsrep.h} | 4 ++-- extra/mariabackup/wsrep.cc | 1 + extra/mariabackup/xtrabackup.cc | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) rename extra/mariabackup/{wsrep.h => backup_wsrep.h} (95%) diff --git a/extra/mariabackup/wsrep.h b/extra/mariabackup/backup_wsrep.h similarity index 95% rename from extra/mariabackup/wsrep.h rename to extra/mariabackup/backup_wsrep.h index 7638d1f2b54..5fa261f8db5 100644 --- a/extra/mariabackup/wsrep.h +++ b/extra/mariabackup/backup_wsrep.h @@ -19,8 +19,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA *******************************************************/ -#ifndef WSREP_H -#define WSREP_H +#ifndef MARIABACKUP_WSREP_H +#define MARIABACKUP_WSREP_H /*********************************************************************** Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc index be11e058255..7b196725c07 100644 --- a/extra/mariabackup/wsrep.cc +++ b/extra/mariabackup/wsrep.cc @@ -46,6 +46,7 @@ permission notice: #include #include "common.h" +#include "backup_wsrep.h" #ifdef WITH_WSREP #define WSREP_XID_PREFIX "WSREPXid" #define WSREP_XID_PREFIX_LEN MYSQL_XID_PREFIX_LEN diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index c42a282e203..f07b1c6d3e3 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -85,7 +85,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include "xbstream.h" #include "changed_page_bitmap.h" #include "read_filt.h" -#include "wsrep.h" +#include "backup_wsrep.h" #include "innobackupex.h" #include "backup_mysql.h" #include "backup_copy.h" From d6baf3d364ede93d589015245727100430f1e888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sun, 17 Sep 2017 13:46:51 +0300 Subject: [PATCH 22/30] MDEV-12634 after-merge test fix: Exercise row_merge_write(), row_merge_read() MySQL 5.7 introduced some optimizations to avoid file I/O during ALGORITHM=INPLACE operations. While both innodb-index-online and innodb-table-online will exercise both the merge sort files and the online log files in 10.1, in 10.2 they would only exercise the online log files. Modify one test case in innodb.innodb-table-online so that skip_pk_sort will not hold. In this way, this test case will write and read the merge sort files. The other instrumented tests in innodb-index-online and innodb-table-online will only write and read online_log files. --- .../innodb/r/innodb-table-online,crypt.rdiff | 2 +- .../suite/innodb/r/innodb-table-online.result | 28 +++++++++---------- .../suite/innodb/t/innodb-table-online.test | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff b/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff index cd0904367f9..624af214722 100644 --- a/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff +++ b/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff @@ -14,7 +14,7 @@ @rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; sort_encrypted sort_decrypted log_encrypted log_decrypted -0 0 0 0 -+0 0 1 1 ++1 1 1 1 ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)); ERROR 23000: Duplicate entry '' for key 'PRIMARY' UPDATE t1 SET c3 = NULL WHERE c3 = ''; diff --git a/mysql-test/suite/innodb/r/innodb-table-online.result b/mysql-test/suite/innodb/r/innodb-table-online.result index 49f44c0f655..ec5199e4f52 100644 --- a/mysql-test/suite/innodb/r/innodb-table-online.result +++ b/mysql-test/suite/innodb/r/innodb-table-online.result @@ -321,7 +321,7 @@ ALTER TABLE t1 ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; ERROR 42000: Multiple primary key defined ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; ERROR 23000: Duplicate entry '5' for key 'PRIMARY' -ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c1,c22f,c4(5)), +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c22f,c1,c4(5)), CHANGE c2 c22f INT, CHANGE c3 c3 CHAR(255) NULL, CHANGE c1 c1 INT AFTER c22f, ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online', LOCK=NONE; # session default @@ -333,7 +333,7 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 1 ddl_pending_alter_table 1 -ddl_sort_file_alter_table 0 +ddl_sort_file_alter_table 2 ddl_log_file_alter_table 1 BEGIN; INSERT INTO t1 SELECT 320 + c1, c2, c3 FROM t1 WHERE c1 > 240; @@ -349,7 +349,7 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 1 ddl_pending_alter_table 1 -ddl_sort_file_alter_table 0 +ddl_sort_file_alter_table 2 ddl_log_file_alter_table 2 SET DEBUG_SYNC = 'now SIGNAL dml3_done'; # session con1 @@ -360,7 +360,7 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 0 ddl_pending_alter_table 0 -ddl_sort_file_alter_table 0 +ddl_sort_file_alter_table 2 ddl_log_file_alter_table 2 SELECT COUNT(c22f) FROM t1; COUNT(c22f) @@ -453,7 +453,7 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 0 ddl_pending_alter_table 0 -ddl_sort_file_alter_table 4 +ddl_sort_file_alter_table 6 ddl_log_file_alter_table 2 # session default connection default; @@ -464,15 +464,15 @@ ALTER TABLE t1 ROW_FORMAT=REDUNDANT; SELECT * FROM t1 LIMIT 10; c22f c1 c3 c4 5 1 1foo Online -6 2 2foofoo Online -7 3 3foofoofoo Online -8 4 4foofoofoofoo Online -9 5 5foofoofoofoofoo Online 5 6 6foofoofoofoofoofoo Online -6 7 7foofoofoofoofoofoofoo Online -7 8 8foofoofoofoofoofoofoofoo Online -8 9 9foofoofoofoofoofoofoofoofoo Online -9 10 10foofoofoofoofoofoofoofoofoofoo Online +5 11 11foofoofoofoofoofoofoofoofoofoofoo Online +5 16 16foofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoo Online +5 21 21foofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoo Online +5 26 26foofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoo Online +5 31 31foofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoo Online +5 36 36foofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoo Online +5 41 41foofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoo Online +5 46 46foofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoofoo Online # session con1 connection con1; ALTER TABLE t1 DISCARD TABLESPACE; @@ -487,7 +487,7 @@ t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c3` char(255) NOT NULL, `c4` varchar(6) NOT NULL DEFAULT 'Online', - PRIMARY KEY (`c1`,`c22f`,`c4`(5)) + PRIMARY KEY (`c22f`,`c1`,`c4`(5)) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT SET DEBUG_SYNC = 'RESET'; SET GLOBAL innodb_monitor_disable = module_ddl; diff --git a/mysql-test/suite/innodb/t/innodb-table-online.test b/mysql-test/suite/innodb/t/innodb-table-online.test index dddbbb34117..51f5445693e 100644 --- a/mysql-test/suite/innodb/t/innodb-table-online.test +++ b/mysql-test/suite/innodb/t/innodb-table-online.test @@ -288,7 +288,7 @@ ALTER TABLE t1 ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; --error ER_DUP_ENTRY ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; --send -ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c1,c22f,c4(5)), +ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c22f,c1,c4(5)), CHANGE c2 c22f INT, CHANGE c3 c3 CHAR(255) NULL, CHANGE c1 c1 INT AFTER c22f, ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online', LOCK=NONE; From 372dba097d90e67d19da357d1bf29890489840b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sun, 17 Sep 2017 14:13:32 +0300 Subject: [PATCH 23/30] Silence a -Wimplicit-fallthrough warning --- sql/wsrep_hton.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/wsrep_hton.cc b/sql/wsrep_hton.cc index a8a0d574362..2733a91a3c9 100644 --- a/sql/wsrep_hton.cc +++ b/sql/wsrep_hton.cc @@ -536,6 +536,7 @@ wsrep_run_wsrep_commit(THD *thd, bool all) break; case WSREP_BF_ABORT: DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED); + /* fall through */ case WSREP_TRX_FAIL: WSREP_DEBUG("commit failed for reason: %d", rcode); DBUG_PRINT("wsrep", ("replicating commit fail")); From df24f8469d31cc7eb33b4446e1450b95c12e0122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 18 Sep 2017 09:05:16 +0300 Subject: [PATCH 24/30] MDEV-12893 innodb.log_data_file_size failed in buildbot with InnoDB: Database page corruption The purpose of the test is to ensure that redo log apply will extend data files before applying page-level redo log records. The test intermittently failed, because the doublewrite buffer would sometimes contain data for the pages that the test truncated. When the test truncates data files, it must also remove the doublewrite buffer entries, because under normal operation, the doublewrite buffer would only be written to if the data file already has been extended. --- .../suite/innodb/t/log_data_file_size.test | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/t/log_data_file_size.test b/mysql-test/suite/innodb/t/log_data_file_size.test index 2f1c497595b..877723a3734 100644 --- a/mysql-test/suite/innodb/t/log_data_file_size.test +++ b/mysql-test/suite/innodb/t/log_data_file_size.test @@ -24,13 +24,13 @@ use Fcntl 'SEEK_CUR', 'SEEK_END'; my $page_size = $ENV{'INNODB_PAGE_SIZE'}; my $restart; +open(FILE, "+<", "$ENV{'MYSQLD_DATADIR'}ibdata1") or die; if ($ENV{'MYSQLD_IS_DEBUG'}) { # It is impractical to ensure that CREATE TABLE t will extend ibdata1. # We rely on innodb_system_tablespace_extend_debug=1 # to recover from this fault injection if no size change was redo-logged. my $root = $ENV{'INNODB_ROOT_PAGE'}; - open(FILE, "+<", "$ENV{'MYSQLD_DATADIR'}ibdata1") or die; my $size = sysseek(FILE, 0, SEEK_END) / $page_size; seek(FILE, $page_size * ($root + 1), SEEK_SET) or die; my $empty_tail= 1; @@ -40,8 +40,22 @@ if ($ENV{'MYSQLD_IS_DEBUG'}) $restart = "--innodb-data-file-size-debug=$size"; truncate(FILE, $page_size * $root); } - close FILE; } +# Clear the doublewrite buffer entries for our tables. +sysseek(FILE, 6 * $page_size - 190, 0)||die "Unable to seek ibdata1\n"; +sysread(FILE, $_, 12) == 12||die "Unable to read TRX_SYS\n"; +my($magic,$d1,$d2)=unpack "NNN", $_; +die "magic=$magic, $d1, $d2\n" unless $magic == 536853855 && $d2 >= $d1 + 64; +sysseek(FILE, $d1 * $page_size, 0)||die "Unable to seek ibdata1\n"; +# Find the pages in the doublewrite buffer +for (my $d = $d1; $d < $d2 + 64; $d++) { + sysread(FILE, $_, $page_size)==$page_size||die "Cannot read doublewrite\n"; + my($space_id,$offset)=unpack "x[4]Nx[26]N",$_; + next unless $space_id && $offset > 3; + sysseek(FILE, $d * $page_size, 0)||die "Unable to seek ibdata1\n"; + syswrite(FILE, chr(0) x $page_size)==$page_size||die; +} +close FILE; open(FILE, ">$ENV{MYSQLTEST_VARDIR}/log/start_mysqld.txt") || die; print FILE "--let \$restart_parameters=$restart\n" if $restart; print FILE "--source include/start_mysqld.inc\n"; From efb673fe5f2b59bc383806e2efb24aa7ab6b81d1 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Thu, 24 Aug 2017 10:34:21 +0300 Subject: [PATCH 25/30] MW-402 cascading FK issues * created tests focusing in multi-master conflicts during cascading foreign key processing * in row0upd.cc, calling wsrep_row_ups_check_foreign_constraints only when running in cluster * in row0ins.cc fixed regression from MW-369, which caused crash with MW-402.test --- mysql-test/suite/galera/r/MW-402.result | 152 ++++++++++++++++++++ mysql-test/suite/galera/t/MW-402.test | 184 ++++++++++++++++++++++++ storage/innobase/row/row0ins.cc | 3 +- 3 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/galera/r/MW-402.result create mode 100644 mysql-test/suite/galera/t/MW-402.test diff --git a/mysql-test/suite/galera/r/MW-402.result b/mysql-test/suite/galera/r/MW-402.result new file mode 100644 index 00000000000..4261bc6bac3 --- /dev/null +++ b/mysql-test/suite/galera/r/MW-402.result @@ -0,0 +1,152 @@ +CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, +CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON DELETE CASCADE); +INSERT INTO p VALUES (1, 0); +INSERT INTO p VALUES (2, 0); +INSERT INTO c VALUES (1, 1, 0); +SET AUTOCOMMIT=ON; +START TRANSACTION; +UPDATE c SET f2=1 where f1=1; +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +DELETE FROM p WHERE f1 = 1; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +COMMIT; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; +SET GLOBAL wsrep_provider_options = 'dbug='; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +SELECT * FROM p; +f1 f2 +2 0 +SELECT * FROM c; +f1 p_id f2 +DROP TABLE c; +DROP TABLE p; +CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, +CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON UPDATE CASCADE); +INSERT INTO p VALUES (1, 0); +INSERT INTO p VALUES (2, 0); +INSERT INTO c VALUES (1, 1, 0); +SET AUTOCOMMIT=ON; +START TRANSACTION; +UPDATE c SET f2=2 where f1=1; +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +UPDATE p set f1=11 WHERE f1 = 1; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +COMMIT; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; +SET GLOBAL wsrep_provider_options = 'dbug='; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +SELECT * FROM p; +f1 f2 +2 0 +11 0 +SELECT * FROM c; +f1 p_id f2 +1 11 0 +DROP TABLE c; +DROP TABLE p; +CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, +CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON UPDATE CASCADE); +INSERT INTO p VALUES (1, 0); +INSERT INTO p VALUES (2, 0); +INSERT INTO c VALUES (1, 1, 0); +SET AUTOCOMMIT=ON; +START TRANSACTION; +UPDATE c SET p_id=2 where f1=1; +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +UPDATE p set f1=11 WHERE f1 = 1; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +COMMIT; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; +SET GLOBAL wsrep_provider_options = 'dbug='; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +SELECT * FROM p; +f1 f2 +2 0 +11 0 +SELECT * FROM c; +f1 p_id f2 +1 11 0 +SET AUTOCOMMIT=ON; +START TRANSACTION; +UPDATE p set f1=21 WHERE f1 = 11; +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +UPDATE c SET p_id=2 where f1=1; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +COMMIT; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; +SET GLOBAL wsrep_provider_options = 'dbug='; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +SELECT * FROM p; +f1 f2 +2 0 +11 0 +SELECT * FROM c; +f1 p_id f2 +1 2 0 +DROP TABLE c; +DROP TABLE p; +CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE p2 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER, f2 INTEGER, +CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1) ON DELETE CASCADE, +CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)); +INSERT INTO p1 VALUES (1, 0); +INSERT INTO p2 VALUES (1, 0); +INSERT INTO c VALUES (1, 1, 1, 0); +SET AUTOCOMMIT=ON; +START TRANSACTION; +UPDATE p2 SET f2=2 where f1=1; +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +DELETE FROM p1 WHERE f1 = 1; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +COMMIT; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; +SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; +SET GLOBAL wsrep_provider_options = 'dbug='; +SELECT * FROM p1; +f1 f2 +SELECT * FROM p2; +f1 f2 +1 2 +SELECT * FROM c; +f1 p1_id p2_id f2 +DROP TABLE c; +DROP TABLE p1; +DROP TABLE p2; diff --git a/mysql-test/suite/galera/t/MW-402.test b/mysql-test/suite/galera/t/MW-402.test new file mode 100644 index 00000000000..5713cda5282 --- /dev/null +++ b/mysql-test/suite/galera/t/MW-402.test @@ -0,0 +1,184 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_debug_sync.inc +--source suite/galera/include/galera_have_debug_sync.inc + +# +# we must open connection node_1a here, MW-369.inc will use it later +# +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 + +# +# cascading delete operation is replicated from node2 +# and this conflicts with an update for child table in node1 +# +# As a result, the update should fail for certification error +# +--connection node_1 + +CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, + CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON DELETE CASCADE); + + +INSERT INTO p VALUES (1, 0); +INSERT INTO p VALUES (2, 0); + +INSERT INTO c VALUES (1, 1, 0); + +--let $mw_369_parent_query = UPDATE c SET f2=1 where f1=1 +--let $mw_369_child_query = DELETE FROM p WHERE f1 = 1 + +--connection node_1a +--source MW-369.inc + +# Commit fails +--connection node_1 +--error ER_LOCK_DEADLOCK +--reap + +--connection node_2 +SELECT * FROM p; +SELECT * FROM c; + +DROP TABLE c; +DROP TABLE p; + +# +# cascading update operation is replicated from node2 +# and this conflicts with an update for child table in node1 +# +# As a result, the update should fail for certification error +# +--connection node_1 + +CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, + CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON UPDATE CASCADE); + + +INSERT INTO p VALUES (1, 0); +INSERT INTO p VALUES (2, 0); + +INSERT INTO c VALUES (1, 1, 0); + +--let $mw_369_parent_query = UPDATE c SET f2=2 where f1=1 +--let $mw_369_child_query = UPDATE p set f1=11 WHERE f1 = 1 + +--connection node_1a +--source MW-369.inc + +# Commit fails +--connection node_1 +--error ER_LOCK_DEADLOCK +--reap + +--connection node_2 +SELECT * FROM p; +SELECT * FROM c; + +DROP TABLE c; +DROP TABLE p; + +# +# ON UPDATE CASCADE tests +# Here we update primary key of parent table to cause cascaded update +# on child table +# +# cascading update operation is replicated from node2 +# and this conflicts with an update for child table in node1 +# +# As a result, the update should fail for certification error +# +--connection node_1 + +CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, + CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON UPDATE CASCADE); + + +INSERT INTO p VALUES (1, 0); +INSERT INTO p VALUES (2, 0); + +INSERT INTO c VALUES (1, 1, 0); + +--let $mw_369_parent_query = UPDATE c SET p_id=2 where f1=1 +--let $mw_369_child_query = UPDATE p set f1=11 WHERE f1 = 1 + +--connection node_1a +--source MW-369.inc + +# Commit fails +--connection node_1 +--error ER_LOCK_DEADLOCK +--reap + +# same as previous, but statements in different order +--connection node_2 +SELECT * FROM p; +SELECT * FROM c; + +--let $mw_369_parent_query = UPDATE p set f1=21 WHERE f1 = 11 +--let $mw_369_child_query = UPDATE c SET p_id=2 where f1=1 + +--connection node_1a +--source MW-369.inc + +# Commit fails +--connection node_1 +--error ER_LOCK_DEADLOCK +--reap + + +--connection node_2 +SELECT * FROM p; +SELECT * FROM c; + +DROP TABLE c; +DROP TABLE p; + + +# +# CASCADE DELETE tests with two parent tables +# Here we cause cascaded operation on child table through +# one parent table and have other operation on the other +# parent table +# +# cascading update operation is replicated from node2 +# but this does not conflict with an update for the other parent table in node1 +# +# As a result, the update on p2 should succeed +# +--connection node_1 + +CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE p2 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; +CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER, f2 INTEGER, + CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1) ON DELETE CASCADE, + CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)); + + +INSERT INTO p1 VALUES (1, 0); +INSERT INTO p2 VALUES (1, 0); + +INSERT INTO c VALUES (1, 1, 1, 0); + +--let $mw_369_parent_query = UPDATE p2 SET f2=2 where f1=1 +--let $mw_369_child_query = DELETE FROM p1 WHERE f1 = 1 + +--connection node_1a +--source MW-369.inc + +# Commit succeeds +--connection node_1 +--reap + +# same as previous, but statements in different order +--connection node_2 +SELECT * FROM p1; +SELECT * FROM p2; +SELECT * FROM c; + +DROP TABLE c; +DROP TABLE p1; +DROP TABLE p2; \ No newline at end of file diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 11785dbeb8e..584fd156016 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1442,8 +1442,7 @@ row_ins_foreign_check_on_constraint( foreign, clust_rec, clust_index, - FALSE, - (node) ? TRUE : FALSE); + FALSE, FALSE); if (err != DB_SUCCESS) { fprintf(stderr, "WSREP: foreign key append failed: %d\n", err); From 6a524ca56b2932f38cc4b328abd38aed023b1e8a Mon Sep 17 00:00:00 2001 From: sjaakola Date: Fri, 25 Aug 2017 11:33:13 +0300 Subject: [PATCH 26/30] MW-402 cascading FK issue (5.7 version) Optimizing cascade node list traversal, which could turn out as performance bottleneck Even this current cascade node check could be skipped, but a dedicated mtr test is needed to confirm that --- storage/innobase/row/row0upd.cc | 37 ++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 4468b0660f0..df70efae6a5 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -2434,15 +2434,24 @@ row_upd_sec_index_entry( row_ins_sec_index_entry() below */ if (!rec_get_deleted_flag( rec, dict_table_is_comp(index->table))) { + +#ifdef WITH_WSREP + que_node_t *parent = que_node_get_parent(node); +#endif /* WITH_WSREP */ + err = btr_cur_del_mark_set_sec_rec( flags, btr_cur, TRUE, thr, &mtr); + if (err != DB_SUCCESS) { break; } #ifdef WITH_WSREP - if (!referenced && foreign - && wsrep_must_process_fk(node, trx) - && !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + if (!referenced && foreign && + wsrep_on(trx->mysql_thd) && + !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && + (!parent || (que_node_get_type(parent) != + QUE_NODE_UPDATE) || + ((upd_node_t*)parent)->cascade_upd_nodes->empty())) { ulint* offsets = rec_get_offsets( rec, index, NULL, ULINT_UNDEFINED, &heap); @@ -2661,6 +2670,9 @@ row_upd_clust_rec_by_insert( rec_t* rec; ulint* offsets = NULL; +#ifdef WITH_WSREP + que_node_t *parent = que_node_get_parent(node); +#endif /* WITH_WSREP */ ut_ad(node); ut_ad(dict_index_is_clust(index)); @@ -2748,7 +2760,13 @@ check_fk: goto err_exit; } #ifdef WITH_WSREP - } else if (foreign && wsrep_must_process_fk(node, trx)) { + } else if (foreign && wsrep_on(trx->mysql_thd) && + (!parent || (que_node_get_type(parent) != QUE_NODE_UPDATE) || + ((upd_node_t*)parent)->cascade_upd_nodes->empty())) { + + err = wsrep_row_upd_check_foreign_constraints( + node, pcur, table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: @@ -2956,10 +2974,15 @@ row_upd_del_mark_clust_rec( dberr_t err; rec_t* rec; trx_t* trx = thr_get_trx(thr); + ut_ad(node); ut_ad(dict_index_is_clust(index)); ut_ad(node->is_delete); +#ifdef WITH_WSREP + que_node_t *parent = que_node_get_parent(node); +#endif /* WITH_WSREP */ + pcur = node->pcur; btr_cur = btr_pcur_get_btr_cur(pcur); @@ -2985,9 +3008,13 @@ row_upd_del_mark_clust_rec( err = row_upd_check_references_constraints( node, pcur, index->table, index, offsets, thr, mtr); #ifdef WITH_WSREP - } else if (foreign && wsrep_must_process_fk(node, trx)) { + } else if (trx && wsrep_on(trx->mysql_thd) && + (!parent || (que_node_get_type(parent) != QUE_NODE_UPDATE) || + ((upd_node_t*)parent)->cascade_upd_nodes->empty())) { + err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: From 16b374b978afaff79aba041ea3d3c2ffe53a739a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 13 Sep 2017 19:51:30 +0300 Subject: [PATCH 27/30] MDEV-13678: DELETE with CASCADE takes a long time when Galera is enabled Use wsrep_must_process_fk function to check if foreign key constraint needs to be processed in case of Galera is enabled. --- storage/innobase/row/row0upd.cc | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index df70efae6a5..59626326687 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -464,15 +464,10 @@ static bool wsrep_must_process_fk(const upd_node_t* node, const trx_t* trx) { - if (que_node_get_type(node->common.parent) != QUE_NODE_UPDATE - || !wsrep_on(trx->mysql_thd)) { - return false; - } + const upd_node_t* parent = static_cast(node->common.parent); - const upd_cascade_t& nodes = *static_cast( - node->common.parent)->cascade_upd_nodes; - const upd_cascade_t::const_iterator end = nodes.end(); - return std::find(nodes.begin(), end, node) == end; + return (!parent || (que_node_get_type(parent) != QUE_NODE_UPDATE) || + parent->cascade_upd_nodes->empty()); } #endif /* WITH_WSREP */ @@ -2435,10 +2430,6 @@ row_upd_sec_index_entry( if (!rec_get_deleted_flag( rec, dict_table_is_comp(index->table))) { -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ - err = btr_cur_del_mark_set_sec_rec( flags, btr_cur, TRUE, thr, &mtr); @@ -2449,9 +2440,7 @@ row_upd_sec_index_entry( if (!referenced && foreign && wsrep_on(trx->mysql_thd) && !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && - (!parent || (que_node_get_type(parent) != - QUE_NODE_UPDATE) || - ((upd_node_t*)parent)->cascade_upd_nodes->empty())) { + wsrep_must_process_fk(node, trx)) { ulint* offsets = rec_get_offsets( rec, index, NULL, ULINT_UNDEFINED, &heap); @@ -2670,9 +2659,6 @@ row_upd_clust_rec_by_insert( rec_t* rec; ulint* offsets = NULL; -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ ut_ad(node); ut_ad(dict_index_is_clust(index)); @@ -2761,8 +2747,7 @@ check_fk: } #ifdef WITH_WSREP } else if (foreign && wsrep_on(trx->mysql_thd) && - (!parent || (que_node_get_type(parent) != QUE_NODE_UPDATE) || - ((upd_node_t*)parent)->cascade_upd_nodes->empty())) { + wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, table, index, offsets, thr, mtr); @@ -2979,10 +2964,6 @@ row_upd_del_mark_clust_rec( ut_ad(dict_index_is_clust(index)); ut_ad(node->is_delete); -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ - pcur = node->pcur; btr_cur = btr_pcur_get_btr_cur(pcur); @@ -3009,8 +2990,7 @@ row_upd_del_mark_clust_rec( node, pcur, index->table, index, offsets, thr, mtr); #ifdef WITH_WSREP } else if (trx && wsrep_on(trx->mysql_thd) && - (!parent || (que_node_get_type(parent) != QUE_NODE_UPDATE) || - ((upd_node_t*)parent)->cascade_upd_nodes->empty())) { + wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); From d186b99251da03617abab6aaa71c89b3c7e92bea Mon Sep 17 00:00:00 2001 From: sjaakola Date: Fri, 25 Aug 2017 11:33:13 +0300 Subject: [PATCH 28/30] MW-402 cascading FK issue (5.7 version) Optimizing cascade node list traversal, which could turn out as performance bottleneck Even this current cascade node check could be skipped, but a dedicated mtr test is needed to confirm that --- storage/innobase/row/row0upd.cc | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 59626326687..7b3283dfaf5 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -2430,6 +2430,10 @@ row_upd_sec_index_entry( if (!rec_get_deleted_flag( rec, dict_table_is_comp(index->table))) { +#ifdef WITH_WSREP + que_node_t *parent = que_node_get_parent(node); +#endif /* WITH_WSREP */ + err = btr_cur_del_mark_set_sec_rec( flags, btr_cur, TRUE, thr, &mtr); @@ -2437,10 +2441,9 @@ row_upd_sec_index_entry( break; } #ifdef WITH_WSREP - if (!referenced && foreign && - wsrep_on(trx->mysql_thd) && - !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && - wsrep_must_process_fk(node, trx)) { + if (!referenced && foreign + && wsrep_must_process_fk(node, trx) + && !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { ulint* offsets = rec_get_offsets( rec, index, NULL, ULINT_UNDEFINED, &heap); @@ -2659,6 +2662,9 @@ row_upd_clust_rec_by_insert( rec_t* rec; ulint* offsets = NULL; +#ifdef WITH_WSREP + que_node_t *parent = que_node_get_parent(node); +#endif /* WITH_WSREP */ ut_ad(node); ut_ad(dict_index_is_clust(index)); @@ -2747,11 +2753,12 @@ check_fk: } #ifdef WITH_WSREP } else if (foreign && wsrep_on(trx->mysql_thd) && - wsrep_must_process_fk(node, trx)) { + wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: @@ -2964,6 +2971,10 @@ row_upd_del_mark_clust_rec( ut_ad(dict_index_is_clust(index)); ut_ad(node->is_delete); +#ifdef WITH_WSREP + que_node_t *parent = que_node_get_parent(node); +#endif /* WITH_WSREP */ + pcur = node->pcur; btr_cur = btr_pcur_get_btr_cur(pcur); @@ -2992,6 +3003,7 @@ row_upd_del_mark_clust_rec( } else if (trx && wsrep_on(trx->mysql_thd) && wsrep_must_process_fk(node, trx)) { + err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); From 3894fdd47ad1bb0973d76e969b5cf785e95532cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 13 Sep 2017 19:51:30 +0300 Subject: [PATCH 29/30] MDEV-13678: DELETE with CASCADE takes a long time when Galera is enabled Ported fix from mysql-wsrep-bugs with some refactoring. Test case is MW-402 where MariaDB needs record as there is extra connection lines. --- mysql-test/suite/galera/r/MW-402.result | 50 +++++++++++++++++++++++++ storage/innobase/row/row0upd.cc | 31 ++++++--------- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/mysql-test/suite/galera/r/MW-402.result b/mysql-test/suite/galera/r/MW-402.result index 4261bc6bac3..81d6f1a555e 100644 --- a/mysql-test/suite/galera/r/MW-402.result +++ b/mysql-test/suite/galera/r/MW-402.result @@ -1,26 +1,37 @@ +connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; +connection node_1; CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON DELETE CASCADE); INSERT INTO p VALUES (1, 0); INSERT INTO p VALUES (2, 0); INSERT INTO c VALUES (1, 1, 0); +connection node_1a; +connection node_1; SET AUTOCOMMIT=ON; START TRANSACTION; UPDATE c SET f2=1 where f1=1; +connection node_1a; SET SESSION wsrep_sync_wait = 0; SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +connection node_2; DELETE FROM p WHERE f1 = 1; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'dbug='; SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +connection node_1; COMMIT; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; SET GLOBAL wsrep_provider_options = 'dbug='; +connection node_1; ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +connection node_2; SELECT * FROM p; f1 f2 2 0 @@ -28,29 +39,39 @@ SELECT * FROM c; f1 p_id f2 DROP TABLE c; DROP TABLE p; +connection node_1; CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON UPDATE CASCADE); INSERT INTO p VALUES (1, 0); INSERT INTO p VALUES (2, 0); INSERT INTO c VALUES (1, 1, 0); +connection node_1a; +connection node_1; SET AUTOCOMMIT=ON; START TRANSACTION; UPDATE c SET f2=2 where f1=1; +connection node_1a; SET SESSION wsrep_sync_wait = 0; SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +connection node_2; UPDATE p set f1=11 WHERE f1 = 1; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'dbug='; SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +connection node_1; COMMIT; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; SET GLOBAL wsrep_provider_options = 'dbug='; +connection node_1; ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +connection node_2; SELECT * FROM p; f1 f2 2 0 @@ -60,29 +81,39 @@ f1 p_id f2 1 11 0 DROP TABLE c; DROP TABLE p; +connection node_1; CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON UPDATE CASCADE); INSERT INTO p VALUES (1, 0); INSERT INTO p VALUES (2, 0); INSERT INTO c VALUES (1, 1, 0); +connection node_1a; +connection node_1; SET AUTOCOMMIT=ON; START TRANSACTION; UPDATE c SET p_id=2 where f1=1; +connection node_1a; SET SESSION wsrep_sync_wait = 0; SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +connection node_2; UPDATE p set f1=11 WHERE f1 = 1; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'dbug='; SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +connection node_1; COMMIT; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; SET GLOBAL wsrep_provider_options = 'dbug='; +connection node_1; ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +connection node_2; SELECT * FROM p; f1 f2 2 0 @@ -90,23 +121,32 @@ f1 f2 SELECT * FROM c; f1 p_id f2 1 11 0 +connection node_1a; +connection node_1; SET AUTOCOMMIT=ON; START TRANSACTION; UPDATE p set f1=21 WHERE f1 = 11; +connection node_1a; SET SESSION wsrep_sync_wait = 0; SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +connection node_2; UPDATE c SET p_id=2 where f1=1; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'dbug='; SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +connection node_1; COMMIT; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; SET GLOBAL wsrep_provider_options = 'dbug='; +connection node_1; ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +connection node_2; SELECT * FROM p; f1 f2 2 0 @@ -116,6 +156,7 @@ f1 p_id f2 1 2 0 DROP TABLE c; DROP TABLE p; +connection node_1; CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; CREATE TABLE p2 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER, f2 INTEGER, @@ -124,22 +165,31 @@ CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)); INSERT INTO p1 VALUES (1, 0); INSERT INTO p2 VALUES (1, 0); INSERT INTO c VALUES (1, 1, 1, 0); +connection node_1a; +connection node_1; SET AUTOCOMMIT=ON; START TRANSACTION; UPDATE p2 SET f2=2 where f1=1; +connection node_1a; SET SESSION wsrep_sync_wait = 0; SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; +connection node_2; DELETE FROM p1 WHERE f1 = 1; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'dbug='; SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync'; +connection node_1; COMMIT; +connection node_1a; SET SESSION wsrep_on = 0; SET SESSION wsrep_on = 1; SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync'; SET GLOBAL wsrep_provider_options = 'dbug='; +connection node_1; +connection node_2; SELECT * FROM p1; f1 f2 SELECT * FROM p2; diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 7b3283dfaf5..a7e84d7ef64 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -460,14 +460,18 @@ func_exit: @param[in] node query node @param[in] trx transaction @return whether the node cannot be ignored */ -static +inline bool wsrep_must_process_fk(const upd_node_t* node, const trx_t* trx) { + if (que_node_get_type(node->common.parent) != QUE_NODE_UPDATE || + !wsrep_on(trx->mysql_thd)) { + return false; + } + const upd_node_t* parent = static_cast(node->common.parent); - return (!parent || (que_node_get_type(parent) != QUE_NODE_UPDATE) || - parent->cascade_upd_nodes->empty()); + return parent->cascade_upd_nodes->empty(); } #endif /* WITH_WSREP */ @@ -2430,10 +2434,6 @@ row_upd_sec_index_entry( if (!rec_get_deleted_flag( rec, dict_table_is_comp(index->table))) { -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ - err = btr_cur_del_mark_set_sec_rec( flags, btr_cur, TRUE, thr, &mtr); @@ -2442,8 +2442,9 @@ row_upd_sec_index_entry( } #ifdef WITH_WSREP if (!referenced && foreign - && wsrep_must_process_fk(node, trx) - && !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + wsrep_must_process_fk(node, trx) && + !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + ulint* offsets = rec_get_offsets( rec, index, NULL, ULINT_UNDEFINED, &heap); @@ -2662,9 +2663,6 @@ row_upd_clust_rec_by_insert( rec_t* rec; ulint* offsets = NULL; -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ ut_ad(node); ut_ad(dict_index_is_clust(index)); @@ -2752,7 +2750,7 @@ check_fk: goto err_exit; } #ifdef WITH_WSREP - } else if (foreign && wsrep_on(trx->mysql_thd) && + } else if (foreign && wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( @@ -2971,10 +2969,6 @@ row_upd_del_mark_clust_rec( ut_ad(dict_index_is_clust(index)); ut_ad(node->is_delete); -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ - pcur = node->pcur; btr_cur = btr_pcur_get_btr_cur(pcur); @@ -3000,10 +2994,9 @@ row_upd_del_mark_clust_rec( err = row_upd_check_references_constraints( node, pcur, index->table, index, offsets, thr, mtr); #ifdef WITH_WSREP - } else if (trx && wsrep_on(trx->mysql_thd) && + } else if (trx && wsrep_must_process_fk(node, trx)) { - err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); From 4bf087986f7cdbb8164acbed77c644afda1d353b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 18 Sep 2017 11:10:06 +0300 Subject: [PATCH 30/30] Fix the WSREP build --- mysql-test/suite/galera/r/MW-402.result | 4 +--- mysql-test/suite/galera/t/MW-402.test | 6 +----- storage/innobase/row/row0upd.cc | 22 +++++----------------- 3 files changed, 7 insertions(+), 25 deletions(-) diff --git a/mysql-test/suite/galera/r/MW-402.result b/mysql-test/suite/galera/r/MW-402.result index 81d6f1a555e..9be98d629fb 100644 --- a/mysql-test/suite/galera/r/MW-402.result +++ b/mysql-test/suite/galera/r/MW-402.result @@ -197,6 +197,4 @@ f1 f2 1 2 SELECT * FROM c; f1 p1_id p2_id f2 -DROP TABLE c; -DROP TABLE p1; -DROP TABLE p2; +DROP TABLE c,p1,p2; diff --git a/mysql-test/suite/galera/t/MW-402.test b/mysql-test/suite/galera/t/MW-402.test index 5713cda5282..36b691c6295 100644 --- a/mysql-test/suite/galera/t/MW-402.test +++ b/mysql-test/suite/galera/t/MW-402.test @@ -1,6 +1,5 @@ --source include/galera_cluster.inc --source include/have_innodb.inc ---source include/have_debug_sync.inc --source suite/galera/include/galera_have_debug_sync.inc # @@ -19,7 +18,6 @@ CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB; CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER, f2 INTEGER, CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1) ON DELETE CASCADE); - INSERT INTO p VALUES (1, 0); INSERT INTO p VALUES (2, 0); @@ -179,6 +177,4 @@ SELECT * FROM p1; SELECT * FROM p2; SELECT * FROM c; -DROP TABLE c; -DROP TABLE p1; -DROP TABLE p2; \ No newline at end of file +DROP TABLE c,p1,p2; diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index a7e84d7ef64..8ba3da255a4 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -2433,17 +2433,15 @@ row_upd_sec_index_entry( row_ins_sec_index_entry() below */ if (!rec_get_deleted_flag( rec, dict_table_is_comp(index->table))) { - err = btr_cur_del_mark_set_sec_rec( flags, btr_cur, TRUE, thr, &mtr); - if (err != DB_SUCCESS) { break; } #ifdef WITH_WSREP if (!referenced && foreign - wsrep_must_process_fk(node, trx) && - !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + && wsrep_must_process_fk(node, trx) + && !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { ulint* offsets = rec_get_offsets( rec, index, NULL, ULINT_UNDEFINED, @@ -2750,13 +2748,10 @@ check_fk: goto err_exit; } #ifdef WITH_WSREP - } else if (foreign && - wsrep_must_process_fk(node, trx)) { - + } else if (foreign && wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, table, index, offsets, thr, mtr); - switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: @@ -2768,16 +2763,11 @@ check_fk: << " index " << index->name << " table " << index->table->name; } - break; + goto err_exit; default: ib::error() << "WSREP: referenced FK check fail: " << ut_strerr(err) << " index " << index->name << " table " << index->table->name; - - break; - } - - if (err != DB_SUCCESS) { goto err_exit; } #endif /* WITH_WSREP */ @@ -2994,9 +2984,7 @@ row_upd_del_mark_clust_rec( err = row_upd_check_references_constraints( node, pcur, index->table, index, offsets, thr, mtr); #ifdef WITH_WSREP - } else if (trx && - wsrep_must_process_fk(node, trx)) { - + } else if (foreign && wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr);