From 04f92dde67a3d4f6fcbdbf65fdb63a54ff6761d8 Mon Sep 17 00:00:00 2001 From: Hartmut Holzgraefe Date: Wed, 21 Sep 2016 10:51:37 +0200 Subject: [PATCH 001/276] MDEV-10853 netcat help output in error log when running xtrabackup SST --- scripts/wsrep_sst_xtrabackup-v2.sh | 2 +- scripts/wsrep_sst_xtrabackup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 962c170dc1b..327c92fb6cf 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -175,7 +175,7 @@ get_transfer() fi wsrep_log_info "Using netcat as streamer" if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - if nc -h | grep -q ncat;then + if nc -h 2>&1 | grep -q ncat;then tcmd="nc -l ${TSST_PORT}" else tcmd="nc -dl ${TSST_PORT}" diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh index 7279c623288..7ba421840ab 100644 --- a/scripts/wsrep_sst_xtrabackup.sh +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -149,7 +149,7 @@ get_transfer() fi wsrep_log_info "Using netcat as streamer" if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - if nc -h | grep -q ncat;then + if nc -h 2>&1 | grep -q ncat;then tcmd="nc -l ${TSST_PORT}" else tcmd="nc -dl ${TSST_PORT}" From fc1798785f4daf878e2c9f4c28ed35ca1af6dcba Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 17 Oct 2016 12:10:12 -0400 Subject: [PATCH 002/276] Adjust test to adapt to a recent change in mysqltest. --- mysql-test/suite/galera/t/galera_concurrent_ctas.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_concurrent_ctas.test b/mysql-test/suite/galera/t/galera_concurrent_ctas.test index f0dcf8e4900..8b9b461d0fd 100644 --- a/mysql-test/suite/galera/t/galera_concurrent_ctas.test +++ b/mysql-test/suite/galera/t/galera_concurrent_ctas.test @@ -43,10 +43,10 @@ let $run=10; while($run) { --error 0,1 - exec $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_1 test - < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql & - $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_2 test - < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql; + exec $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_1 test \ + < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql & \ + $MYSQL --user=root --host=127.0.0.1 --port=$NODE_MYPORT_2 test \ + < $MYSQLTEST_VARDIR/tmp/galera_concurrent.sql; dec $run; } From 1bba40f0dfae5b2c634a50a5de8a44ca63c55510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 9 Nov 2016 08:49:33 +0200 Subject: [PATCH 003/276] MDEV-10544: Galera: Failing assertion: (lock->trx)->wait_lock == lock In Galera case this assertion is too strict as conflicting lock could be same as lock here. --- storage/innobase/lock/lock0lock.c | 1 - storage/xtradb/lock/lock0lock.c | 1 - 2 files changed, 2 deletions(-) diff --git a/storage/innobase/lock/lock0lock.c b/storage/innobase/lock/lock0lock.c index cf77a9ebb8c..3890a2c44ff 100644 --- a/storage/innobase/lock/lock0lock.c +++ b/storage/innobase/lock/lock0lock.c @@ -798,7 +798,6 @@ lock_reset_lock_and_trx_wait( /* Reset the back pointer in trx to this waiting lock request */ if (!(lock->type_mode & LOCK_CONV_BY_OTHER)) { - ut_ad((lock->trx)->wait_lock == lock); (lock->trx)->wait_lock = NULL; } else { ut_ad(lock_get_type_low(lock) == LOCK_REC); diff --git a/storage/xtradb/lock/lock0lock.c b/storage/xtradb/lock/lock0lock.c index c5f8d6770df..b6cee714dec 100644 --- a/storage/xtradb/lock/lock0lock.c +++ b/storage/xtradb/lock/lock0lock.c @@ -799,7 +799,6 @@ lock_reset_lock_and_trx_wait( /* Reset the back pointer in trx to this waiting lock request */ if (!(lock->type_mode & LOCK_CONV_BY_OTHER)) { - ut_ad((lock->trx)->wait_lock == lock); (lock->trx)->wait_lock = NULL; } else { ut_ad(lock_get_type_low(lock) == LOCK_REC); From 72fd15f7c31aa3e3705ae1b005a3247a985c5bb3 Mon Sep 17 00:00:00 2001 From: SachinSetiya Date: Thu, 1 Dec 2016 12:16:13 +0530 Subject: [PATCH 004/276] MDEV-11016 wsrep_node_is_ready() check is too strict Problem:- The condition that checks for node readiness is too strict as it does not allow SELECTs even if these selects do not access any tables. For example,if we run SELECT 1; OR SELECT @@max_allowed_packet; Solution:- We need not to report this error when all_tables(lex->query_tables) is NULL: --- .../suite/galera/r/galera_var_dirty_reads.result | 13 +++++++++++++ .../suite/galera/t/galera_var_dirty_reads.test | 7 +++++++ sql/sql_parse.cc | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 6d703c8cf95..6a2aa1eb5e7 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,6 +18,19 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 +SET @@session.wsrep_dirty_reads=OFF; +SELECT 2; +2 +2 +SELECT @@max_allowed_packet; +@@max_allowed_packet +4194304 +SELECT 2+2 from DUAL; +2+2 +4 +SELECT sysdate() from DUAL; +sysdate() +2016-10-28 23:13:06 SELECT * FROM t1; i 1 diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index 9eea8efdaf3..dd7bc88cb1c 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -39,6 +39,13 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; +#Select query which does not access table should be allowed MDEV-11016 +SET @@session.wsrep_dirty_reads=OFF; +SELECT 2; +SELECT @@max_allowed_packet; +SELECT 2+2 from DUAL; +SELECT sysdate() from DUAL; + --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' --enable_query_log diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 48addaab143..eb4f7141918 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2388,11 +2388,15 @@ mysql_execute_command(THD *thd) /* Bail out if DB snapshot has not been installed. We however, allow SET and SHOW queries. + SHOW and SELECT queries (only if wsrep_dirty_reads is set or when it + does not access any table). */ if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && lex->sql_command != SQLCOM_SET_OPTION && !(thd->variables.wsrep_dirty_reads && lex->sql_command == SQLCOM_SELECT) && + !(lex->sql_command == SQLCOM_SELECT && + !all_tables) && !wsrep_is_show_query(lex->sql_command)) { #if DIRTY_HACK From 313a14f79e6de43acd5c2ceffbcc01af2506d800 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Fri, 9 Dec 2016 12:24:08 -0500 Subject: [PATCH 005/276] Fix test failures. --- .../suite/galera/r/galera_var_dirty_reads.result | 15 +++++++++------ mysql-test/suite/galera/suite.pm | 1 + .../suite/galera/t/galera_var_dirty_reads.test | 12 +++++++++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 6a2aa1eb5e7..c0ffb2d4860 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,19 +18,22 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 +# +# MDEV-11016: wsrep_node_is_ready() check is too strict +# SET @@session.wsrep_dirty_reads=OFF; SELECT 2; 2 2 -SELECT @@max_allowed_packet; -@@max_allowed_packet -4194304 SELECT 2+2 from DUAL; 2+2 4 -SELECT sysdate() from DUAL; -sysdate() -2016-10-28 23:13:06 +SET @VAR=1; +SELECT @VAR; +@VAR +1 +SELECT @@max_allowed_packet; +SELECT SYSDATE() from DUAL; SELECT * FROM t1; i 1 diff --git a/mysql-test/suite/galera/suite.pm b/mysql-test/suite/galera/suite.pm index fdaf64580c7..a790e951762 100644 --- a/mysql-test/suite/galera/suite.pm +++ b/mysql-test/suite/galera/suite.pm @@ -37,6 +37,7 @@ push @::global_suppressions, qr|WSREP: access file\(.*gvwstate.dat\) failed\(No such file or directory\)|, qr|WSREP: no nodes coming from prim view, prim not possible|, qr(WSREP: SYNC message from member .* in non-primary configuration. Ignored.), + qr(WSREP: discarding established .*), ); diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index dd7bc88cb1c..48fbe37139c 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -39,12 +39,18 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; -#Select query which does not access table should be allowed MDEV-11016 +--echo # +--echo # MDEV-11016: wsrep_node_is_ready() check is too strict +--echo # SET @@session.wsrep_dirty_reads=OFF; SELECT 2; -SELECT @@max_allowed_packet; SELECT 2+2 from DUAL; -SELECT sysdate() from DUAL; +SET @VAR=1; +SELECT @VAR; +--disable_result_log +SELECT @@max_allowed_packet; +SELECT SYSDATE() from DUAL; +--enable_result_log --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' From 95422c445db7db60fca688711d98955e59e009e2 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 14 Dec 2016 15:58:14 +0530 Subject: [PATCH 006/276] Revert " MDEV-11016 wsrep_node_is_ready() check is too strict" This reverts commit 72fd15f7c31aa3e3705ae1b005a3247a985c5bb3. # Conflicts: # mysql-test/suite/galera/r/galera_var_dirty_reads.result # mysql-test/suite/galera/t/galera_var_dirty_reads.test --- .../suite/galera/r/galera_var_dirty_reads.result | 16 ---------------- .../suite/galera/t/galera_var_dirty_reads.test | 13 ------------- sql/sql_parse.cc | 4 ---- 3 files changed, 33 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index c0ffb2d4860..6d703c8cf95 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,22 +18,6 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 -# -# MDEV-11016: wsrep_node_is_ready() check is too strict -# -SET @@session.wsrep_dirty_reads=OFF; -SELECT 2; -2 -2 -SELECT 2+2 from DUAL; -2+2 -4 -SET @VAR=1; -SELECT @VAR; -@VAR -1 -SELECT @@max_allowed_packet; -SELECT SYSDATE() from DUAL; SELECT * FROM t1; i 1 diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index 48fbe37139c..9eea8efdaf3 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -39,19 +39,6 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; ---echo # ---echo # MDEV-11016: wsrep_node_is_ready() check is too strict ---echo # -SET @@session.wsrep_dirty_reads=OFF; -SELECT 2; -SELECT 2+2 from DUAL; -SET @VAR=1; -SELECT @VAR; ---disable_result_log -SELECT @@max_allowed_packet; -SELECT SYSDATE() from DUAL; ---enable_result_log - --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' --enable_query_log diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index eb4f7141918..48addaab143 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2388,15 +2388,11 @@ mysql_execute_command(THD *thd) /* Bail out if DB snapshot has not been installed. We however, allow SET and SHOW queries. - SHOW and SELECT queries (only if wsrep_dirty_reads is set or when it - does not access any table). */ if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && lex->sql_command != SQLCOM_SET_OPTION && !(thd->variables.wsrep_dirty_reads && lex->sql_command == SQLCOM_SELECT) && - !(lex->sql_command == SQLCOM_SELECT && - !all_tables) && !wsrep_is_show_query(lex->sql_command)) { #if DIRTY_HACK From 5ddd8149e44122feb0989b9944113d2b92d6f84a Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 14 Dec 2016 17:14:42 +0530 Subject: [PATCH 007/276] MDEV-11479 Improved wsrep_dirty_reads Tasks:- Changes in wsrep_dirty_reads variable 1.) Global + Session scope (Current: session-only) 2.) Can be set using command line. 3.) Allow all commands that do not change data (besides SELECT) 4.) Allow prepared Statements that do not change data 5.) Works with wsrep_sync_wait enabled --- mysql-test/r/mysqld--help.result | 3 + .../galera/r/galera_var_dirty_reads.result | 70 +++++++++++++++++++ .../galera/t/galera_var_dirty_reads.test | 68 ++++++++++++++++++ .../sys_vars/r/wsrep_dirty_reads_basic.result | 23 +++++- .../sys_vars/t/wsrep_dirty_reads_basic.test | 17 ++++- sql/sql_parse.cc | 9 ++- sql/sys_vars.cc | 5 +- sql/wsrep_mysqld.cc | 4 ++ sql/wsrep_mysqld.h | 1 + 9 files changed, 191 insertions(+), 9 deletions(-) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 61243e02363..34d4bae0f28 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -879,6 +879,8 @@ The following options may be given as the first argument: DBUG options to provider library --wsrep-debug To enable debug level logging --wsrep-desync To desynchronize the node from the cluster + --wsrep-dirty-reads Allow reads even when the node is not in the primary + component. --wsrep-drupal-282555-workaround To use a workaround forbad autoincrement value --wsrep-forced-binlog-format=name @@ -1201,6 +1203,7 @@ wsrep-convert-LOCK-to-trx FALSE wsrep-dbug-option wsrep-debug FALSE wsrep-desync FALSE +wsrep-dirty-reads FALSE wsrep-drupal-282555-workaround FALSE wsrep-forced-binlog-format NONE wsrep-load-data-splitting TRUE diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 6d703c8cf95..8a3175912c7 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -3,6 +3,10 @@ INSERT INTO t1 VALUES(1); SELECT * FROM t1; i 1 +create user user1; +grant all privileges on *.* to user1; +create user user2; +grant all privileges on *.* to user2; SET @@global.wsrep_cluster_address = ''; SET @@session.wsrep_dirty_reads=OFF; SET SESSION wsrep_sync_wait=0; @@ -18,8 +22,74 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i 1 +connect con1, localhost, user1,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2; +SET SESSION wsrep_sync_wait=0; +set session wsrep_dirty_reads=1; +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; +set session wsrep_dirty_reads=0; +execute stmt_show; +ERROR 08S01: WSREP has not yet prepared node for application use +execute stmt_select; +ERROR 08S01: WSREP has not yet prepared node for application use +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET wsrep_dirty_reads=ON; +select @@session.wsrep_dirty_reads; +@@session.wsrep_dirty_reads +1 +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET @@global.wsrep_dirty_reads=ON; +connect con2, localhost, user2,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2; +select @@session.wsrep_dirty_reads; +@@session.wsrep_dirty_reads +1 +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET SESSION wsrep_sync_wait=1; +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +SET SESSION wsrep_sync_wait=7; +execute stmt_show; +1 +1 +execute stmt_select; +i +1 +execute stmt_insert; +ERROR 08S01: WSREP has not yet prepared node for application use +connection node_2; +SET @@global.wsrep_dirty_reads=OFF; +connection node_1; SELECT * FROM t1; i 1 DROP TABLE t1; +drop user user1; +drop user user2; +disconnect node_2; +disconnect node_1; # End of test diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index 9eea8efdaf3..8fd3b1d22f2 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -20,6 +20,11 @@ CREATE TABLE t1(i INT) ENGINE=INNODB; INSERT INTO t1 VALUES(1); SELECT * FROM t1; +create user user1; +grant all privileges on *.* to user1; +create user user2; +grant all privileges on *.* to user2; + SET @@global.wsrep_cluster_address = ''; SET @@session.wsrep_dirty_reads=OFF; @@ -39,6 +44,67 @@ SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; +--enable_connect_log +--connect (con1, localhost, user1,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2) +#Just test the session behavior +SET SESSION wsrep_sync_wait=0; + +set session wsrep_dirty_reads=1; +#Prepared statement creation should be allowed MDEV-11479 +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; +set session wsrep_dirty_reads=0; + +#No Preapare stmt/proceure will be allowed +--error ER_UNKNOWN_COM_ERROR +execute stmt_show; +--error ER_UNKNOWN_COM_ERROR +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +SET wsrep_dirty_reads=ON; +select @@session.wsrep_dirty_reads; +#Only prepare statement which does not change data should be allowed +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; +SET @@global.wsrep_dirty_reads=ON; + +--connect (con2, localhost, user2,,test,$NODE_MYPORT_2,$NODE_MYSOCK_2) +#Just test the session behavior +select @@session.wsrep_dirty_reads; + +prepare stmt_show from 'select 1'; +prepare stmt_select from 'select * from t1'; +prepare stmt_insert from 'insert into t1 values(1)'; + +#Only prepare statement which does not change data should be allowed +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +#wsrep_dirty_read should work when wsrep_sync_wait is 1 or non zero +#because we already are disconnected , So It does not make any sense +#to wait for other nodes +SET SESSION wsrep_sync_wait=1; +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +SET SESSION wsrep_sync_wait=7; +execute stmt_show; +execute stmt_select; +--error ER_UNKNOWN_COM_ERROR +execute stmt_insert; + +--connection node_2 +SET @@global.wsrep_dirty_reads=OFF; + --disable_query_log --eval SET @@global.wsrep_cluster_address = '$wsrep_cluster_address_saved' --enable_query_log @@ -48,6 +114,8 @@ SELECT * FROM t1; SELECT * FROM t1; # Cleanup DROP TABLE t1; +drop user user1; +drop user user2; --disable_query_log # Restore original auto_increment_offset values. diff --git a/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result b/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result index d2a62d6136f..1968103873a 100644 --- a/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result +++ b/mysql-test/suite/sys_vars/r/wsrep_dirty_reads_basic.result @@ -5,12 +5,13 @@ SET @wsrep_dirty_reads_session_saved = @@session.wsrep_dirty_reads; # default SELECT @@global.wsrep_dirty_reads; -ERROR HY000: Variable 'wsrep_dirty_reads' is a SESSION variable +@@global.wsrep_dirty_reads +0 SELECT @@session.wsrep_dirty_reads; @@session.wsrep_dirty_reads 0 -# scope and valid values +# valid values for session SET @@session.wsrep_dirty_reads=OFF; SELECT @@session.wsrep_dirty_reads; @@session.wsrep_dirty_reads @@ -24,11 +25,29 @@ SELECT @@session.wsrep_dirty_reads; @@session.wsrep_dirty_reads 0 +# valid values for global +SET @@global.wsrep_dirty_reads=OFF; +SELECT @@global.wsrep_dirty_reads; +@@global.wsrep_dirty_reads +0 +SET @@global.wsrep_dirty_reads=ON; +SELECT @@global.wsrep_dirty_reads; +@@global.wsrep_dirty_reads +1 +SET @@global.wsrep_dirty_reads=default; +SELECT @@global.wsrep_dirty_reads; +@@global.wsrep_dirty_reads +0 + # invalid values SET @@session.wsrep_dirty_reads=NULL; ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'NULL' SET @@session.wsrep_dirty_reads='junk'; ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'junk' +SET @@global.wsrep_dirty_reads=NULL; +ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'NULL' +SET @@global.wsrep_dirty_reads='junk'; +ERROR 42000: Variable 'wsrep_dirty_reads' can't be set to the value of 'junk' # restore the initial values SET @@session.wsrep_dirty_reads = @wsrep_dirty_reads_session_saved; diff --git a/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test b/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test index a47524fcfe3..ffe767a051b 100644 --- a/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test +++ b/mysql-test/suite/sys_vars/t/wsrep_dirty_reads_basic.test @@ -8,12 +8,12 @@ SET @wsrep_dirty_reads_session_saved = @@session.wsrep_dirty_reads; --echo # default ---error ER_INCORRECT_GLOBAL_LOCAL_VAR + SELECT @@global.wsrep_dirty_reads; SELECT @@session.wsrep_dirty_reads; --echo ---echo # scope and valid values +--echo # valid values for session SET @@session.wsrep_dirty_reads=OFF; SELECT @@session.wsrep_dirty_reads; SET @@session.wsrep_dirty_reads=ON; @@ -21,12 +21,25 @@ SELECT @@session.wsrep_dirty_reads; SET @@session.wsrep_dirty_reads=default; SELECT @@session.wsrep_dirty_reads; +--echo +--echo # valid values for global +SET @@global.wsrep_dirty_reads=OFF; +SELECT @@global.wsrep_dirty_reads; +SET @@global.wsrep_dirty_reads=ON; +SELECT @@global.wsrep_dirty_reads; +SET @@global.wsrep_dirty_reads=default; +SELECT @@global.wsrep_dirty_reads; + --echo --echo # invalid values --error ER_WRONG_VALUE_FOR_VAR SET @@session.wsrep_dirty_reads=NULL; --error ER_WRONG_VALUE_FOR_VAR SET @@session.wsrep_dirty_reads='junk'; +--error ER_WRONG_VALUE_FOR_VAR +SET @@global.wsrep_dirty_reads=NULL; +--error ER_WRONG_VALUE_FOR_VAR +SET @@global.wsrep_dirty_reads='junk'; --echo --echo # restore the initial values diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 48addaab143..ec6eb0e6f87 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2386,13 +2386,16 @@ mysql_execute_command(THD *thd) } /* - Bail out if DB snapshot has not been installed. We however, - allow SET and SHOW queries. + Bail out if DB snapshot has not been installed. SET and SHOW commands, + however, are always allowed. + + We additionally allow all other commands that do not change data in + case wsrep_dirty_reads is enabled. */ if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && lex->sql_command != SQLCOM_SET_OPTION && !(thd->variables.wsrep_dirty_reads && - lex->sql_command == SQLCOM_SELECT) && + !is_update_query(lex->sql_command)) && !wsrep_is_show_query(lex->sql_command)) { #if DIRTY_HACK diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index bdcedfff5a3..e0de7960ec9 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3966,8 +3966,9 @@ static Sys_var_mybool Sys_wsrep_restart_slave( GLOBAL_VAR(wsrep_restart_slave), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); static Sys_var_mybool Sys_wsrep_dirty_reads( - "wsrep_dirty_reads", "Do not reject SELECT queries even when the node " - "is not ready.", SESSION_ONLY(wsrep_dirty_reads), NO_CMD_LINE, + "wsrep_dirty_reads", + "Allow reads even when the node is not in the primary component.", + SESSION_VAR(wsrep_dirty_reads), CMD_LINE(OPT_ARG), DEFAULT(FALSE), NO_MUTEX_GUARD, NOT_IN_BINLOG); #endif /* WITH_WSREP */ diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 48114fcba34..21c7e94096d 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -65,6 +65,8 @@ my_bool wsrep_restart_slave_activated = 0; // node has dropped, and slave // restart will be needed my_bool wsrep_slave_UK_checks = 0; // slave thread does UK checks my_bool wsrep_slave_FK_checks = 0; // slave thread does FK checks +// Allow reads even if the node is not in the primary component. +bool wsrep_dirty_reads = false; /* Set during the creation of first wsrep applier and rollback threads. @@ -894,6 +896,8 @@ bool wsrep_must_sync_wait (THD* thd, uint mask) { return (thd->variables.wsrep_sync_wait & mask) && thd->variables.wsrep_on && + !(thd->variables.wsrep_dirty_reads && + !is_update_query(thd->lex->sql_command)) && !thd->in_active_multi_stmt_transaction() && thd->wsrep_conflict_state != REPLAYING && thd->wsrep_sync_wait_gtid.seqno == WSREP_SEQNO_UNDEFINED; diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 3fb5189937e..85d4cea241f 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -80,6 +80,7 @@ extern ulong wsrep_retry_autocommit; extern my_bool wsrep_auto_increment_control; extern my_bool wsrep_drupal_282555_workaround; extern my_bool wsrep_incremental_data_collection; +extern bool wsrep_dirty_reads; extern const char* wsrep_start_position; extern ulong wsrep_max_ws_size; extern ulong wsrep_max_ws_rows; From ec9a48112b3caac39fb133d524f499172942d43a Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Sun, 15 Jan 2017 22:32:22 -0500 Subject: [PATCH 008/276] Fix bad merge --- scripts/mysqld_safe.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 9ab51ded8a8..5bd3b5b06ac 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -988,7 +988,6 @@ do chmod "$fmode" "$err_log" # wrong owner next time we log, so set fi # it up correctly while we can! - eval_log_error "$cmd" end_time=`date +%M%S` if test ! -f "$pid_file" # This is removed if normal shutdown From aa820f9471ccd68a48ed81c1c8b5ad98f9cf20ba Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 10 Mar 2017 11:49:14 +0100 Subject: [PATCH 009/276] MDEV-12217 misssing full license clarify the license --- policy/selinux/mariadb-server.fc | 2 +- policy/selinux/mariadb-server.te | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/policy/selinux/mariadb-server.fc b/policy/selinux/mariadb-server.fc index 1a69ecc2c40..409f72923aa 100644 --- a/policy/selinux/mariadb-server.fc +++ b/policy/selinux/mariadb-server.fc @@ -1,4 +1,4 @@ -# This SELinux file contexts (.fc) file has been copied under BSD License from +# This SELinux file contexts (.fc) file has been copied under New BSD License from # Percona XtraDB Cluster. /etc/init\.d/rc\.d/mysql -- gen_context(system_u:object_r:mysqld_initrc_exec_t,s0) diff --git a/policy/selinux/mariadb-server.te b/policy/selinux/mariadb-server.te index 34d79326b10..45ef40f4153 100644 --- a/policy/selinux/mariadb-server.te +++ b/policy/selinux/mariadb-server.te @@ -1,4 +1,4 @@ -# This SELinux type enforcement (.te) file has been copied under BSD License +# This SELinux type enforcement (.te) file has been copied under New BSD License # from Percona XtraDB Cluster, along with some additions. module mariadb-server 1.0; From ea2695ccbc4ef80cb8b21a2ca4bf0eedb195adae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 5 Apr 2017 08:54:20 +0300 Subject: [PATCH 010/276] fix warning "ignoring return value" of fwrite. Merge pull request https://github.com/MariaDB/server/pull/343 contributed by Eric Herman. --- sql/wsrep_binlog.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/wsrep_binlog.cc b/sql/wsrep_binlog.cc index af4ec84db85..7feebc63cf4 100644 --- a/sql/wsrep_binlog.cc +++ b/sql/wsrep_binlog.cc @@ -308,9 +308,13 @@ void wsrep_dump_rbr_buf(THD *thd, const void* rbr_buf, size_t buf_len) } FILE *of= fopen(filename, "wb"); + if (of) { - fwrite (rbr_buf, buf_len, 1, of); + if (fwrite(rbr_buf, buf_len, 1, of) == 0) + WSREP_ERROR("Failed to write buffer of length %llu to '%s'", + (unsigned long long)buf_len, filename); + fclose(of); } else From 19b9fe07f5356b1d5965c16b322a0ca7d5fbdfe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 5 Apr 2017 10:50:12 +0300 Subject: [PATCH 011/276] Fix compiler error on gcc 6.x and most of the compiler warnings. --- sql/sql_class.cc | 4 ++-- sql/wsrep_sst.cc | 4 ++-- storage/innobase/btr/btr0btr.c | 1 - storage/innobase/btr/btr0cur.c | 8 +------- storage/innobase/buf/buf0buddy.c | 1 - storage/innobase/dict/dict0crea.c | 10 +++++----- storage/innobase/dict/dict0dict.c | 1 - storage/innobase/dict/dict0load.c | 10 +++++----- storage/innobase/dict/dict0mem.c | 2 +- storage/innobase/dyn/dyn0dyn.c | 1 - storage/innobase/fil/fil0fil.c | 2 +- storage/innobase/fsp/fsp0fsp.c | 5 ----- storage/innobase/handler/ha_innodb.h | 4 ++-- storage/innobase/include/dict0dict.ic | 7 ------- storage/innobase/include/dict0mem.h | 2 +- storage/innobase/include/dyn0dyn.ic | 10 ---------- storage/innobase/include/fsp0fsp.h | 2 +- storage/innobase/include/mach0data.ic | 13 ------------- storage/innobase/include/mtr0mtr.ic | 1 - storage/innobase/include/page0page.ic | 1 - storage/innobase/include/rem0rec.ic | 1 - storage/innobase/include/ut0lst.h | 3 --- storage/innobase/mtr/mtr0mtr.c | 1 - storage/innobase/page/page0page.c | 2 -- storage/innobase/page/page0zip.c | 1 - storage/innobase/rem/rem0rec.c | 3 +-- storage/innobase/row/row0merge.c | 12 ------------ storage/innobase/row/row0purge.c | 3 --- storage/innobase/row/row0upd.c | 5 ----- storage/innobase/trx/trx0trx.c | 1 - storage/xtradb/btr/btr0btr.c | 1 - storage/xtradb/btr/btr0cur.c | 10 ++-------- storage/xtradb/buf/buf0buddy.c | 1 - storage/xtradb/dict/dict0crea.c | 10 +++++----- storage/xtradb/dict/dict0dict.c | 1 - storage/xtradb/dict/dict0load.c | 10 +++++----- storage/xtradb/dict/dict0mem.c | 2 +- storage/xtradb/dyn/dyn0dyn.c | 1 - storage/xtradb/fil/fil0fil.c | 2 +- storage/xtradb/fsp/fsp0fsp.c | 5 ----- storage/xtradb/handler/ha_innodb.h | 4 ++-- storage/xtradb/include/btr0sea.ic | 4 ---- storage/xtradb/include/dict0dict.ic | 7 ------- storage/xtradb/include/dict0mem.h | 2 +- storage/xtradb/include/dyn0dyn.ic | 10 ---------- storage/xtradb/include/fsp0fsp.h | 2 +- storage/xtradb/include/mach0data.ic | 13 ------------- storage/xtradb/include/mtr0mtr.ic | 1 - storage/xtradb/include/page0page.ic | 1 - storage/xtradb/include/rem0rec.ic | 1 - storage/xtradb/include/ut0lst.h | 3 --- storage/xtradb/mtr/mtr0mtr.c | 1 - storage/xtradb/page/page0page.c | 2 -- storage/xtradb/page/page0zip.c | 1 - storage/xtradb/rem/rem0rec.c | 3 +-- storage/xtradb/row/row0merge.c | 12 ------------ storage/xtradb/row/row0purge.c | 3 --- storage/xtradb/row/row0upd.c | 5 ----- storage/xtradb/trx/trx0trx.c | 1 - 59 files changed, 41 insertions(+), 199 deletions(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 7905daa235d..fd2dec5a146 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -853,11 +853,11 @@ extern "C" wsrep_ws_handle_t* wsrep_thd_ws_handle(THD *thd) return &thd->wsrep_ws_handle; } -extern "C"void wsrep_thd_LOCK(THD *thd) +extern "C" void wsrep_thd_LOCK(THD *thd) { mysql_mutex_lock(&thd->LOCK_wsrep_thd); } -extern "C"void wsrep_thd_UNLOCK(THD *thd) +extern "C" void wsrep_thd_UNLOCK(THD *thd) { mysql_mutex_unlock(&thd->LOCK_wsrep_thd); } diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index 8008a977a84..a5fe8a1e2d8 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -748,7 +748,7 @@ static int sst_donate_mysqldump (const char* addr, "%s", host, port, mysqld_port, mysqld_unix_port, mysql_real_data_home, wsrep_defaults_file, uuid_str, - (long long)seqno, bypass ? " "WSREP_SST_OPT_BYPASS : ""); + (long long)seqno, bypass ? " " WSREP_SST_OPT_BYPASS : ""); if (ret < 0 || ret >= cmd_len) { @@ -1023,7 +1023,7 @@ static int sst_donate_other (const char* method, method, addr, mysqld_unix_port, mysql_real_data_home, wsrep_defaults_file, uuid, (long long) seqno, - bypass ? " "WSREP_SST_OPT_BYPASS : ""); + bypass ? " " WSREP_SST_OPT_BYPASS : ""); if (ret < 0 || ret >= cmd_len) { diff --git a/storage/innobase/btr/btr0btr.c b/storage/innobase/btr/btr0btr.c index ede72ba57bb..58d816559d4 100644 --- a/storage/innobase/btr/btr0btr.c +++ b/storage/innobase/btr/btr0btr.c @@ -2932,7 +2932,6 @@ btr_level_list_remove_func( ulint prev_page_no; ulint next_page_no; - ut_ad(page && mtr); ut_ad(mtr_memo_contains_page(mtr, page, MTR_MEMO_PAGE_X_FIX)); ut_ad(space == page_get_space_id(page)); /* Get the previous and next page numbers of page */ diff --git a/storage/innobase/btr/btr0cur.c b/storage/innobase/btr/btr0cur.c index 3b533909e67..2c1ab4bffae 100644 --- a/storage/innobase/btr/btr0cur.c +++ b/storage/innobase/btr/btr0cur.c @@ -1877,7 +1877,6 @@ btr_cur_update_alloc_zip( mtr_t* mtr) /*!< in: mini-transaction */ { ut_a(page_zip == buf_block_get_page_zip(block)); - ut_ad(page_zip); ut_ad(!dict_index_is_ibuf(index)); if (page_zip_available(page_zip, dict_index_is_clust(index), @@ -2837,7 +2836,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(page_is_leaf(page_align(rec))); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && thr) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr), index, "del mark "); rec_print_new(stderr, rec, offsets); } @@ -4128,7 +4127,6 @@ btr_cur_disown_inherited_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(!rec_offs_comp(offsets) || !rec_get_node_ptr_flag(rec)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(mtr); for (i = 0; i < rec_offs_n_fields(offsets); i++) { if (rec_offs_nth_extern(offsets, i) @@ -4191,9 +4189,6 @@ btr_push_update_extern_fields( ulint n; const upd_field_t* uf; - ut_ad(tuple); - ut_ad(update); - uf = update->fields; n = upd_get_n_fields(update); @@ -4366,7 +4361,6 @@ btr_store_big_rec_extern_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(btr_mtr); ut_ad(mtr_memo_contains(btr_mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK)); ut_ad(mtr_memo_contains(btr_mtr, rec_block, MTR_MEMO_PAGE_X_FIX)); diff --git a/storage/innobase/buf/buf0buddy.c b/storage/innobase/buf/buf0buddy.c index 9277a89ce66..fa2515eddc2 100644 --- a/storage/innobase/buf/buf0buddy.c +++ b/storage/innobase/buf/buf0buddy.c @@ -276,7 +276,6 @@ buf_buddy_alloc_low( { buf_block_t* block; - ut_ad(lru); ut_ad(buf_pool_mutex_own(buf_pool)); ut_ad(!mutex_own(&buf_pool->zip_mutex)); ut_ad(i >= buf_buddy_get_slot(PAGE_ZIP_MIN_SIZE)); diff --git a/storage/innobase/dict/dict0crea.c b/storage/innobase/dict/dict0crea.c index f8bcc6f2d5f..4e173fb905d 100644 --- a/storage/innobase/dict/dict0crea.c +++ b/storage/innobase/dict/dict0crea.c @@ -112,13 +112,13 @@ dict_create_sys_tables_tuple( dfield = dtuple_get_nth_field(entry, 3/*TYPE*/); ptr = mem_heap_alloc(heap, 4); - if (table->flags & (~DICT_TF_COMPACT & ~(~0 << DICT_TF_BITS))) { + if (table->flags & (~DICT_TF_COMPACT & ~(~0U << DICT_TF_BITS))) { ut_a(table->flags & DICT_TF_COMPACT); ut_a(dict_table_get_format(table) >= DICT_TF_FORMAT_ZIP); ut_a((table->flags & DICT_TF_ZSSIZE_MASK) <= (DICT_TF_ZSSIZE_MAX << DICT_TF_ZSSIZE_SHIFT)); - ut_a(!(table->flags & (~0 << DICT_TF2_BITS))); - mach_write_to_4(ptr, table->flags & ~(~0 << DICT_TF_BITS)); + ut_a(!(table->flags & (~0U << DICT_TF2_BITS))); + mach_write_to_4(ptr, table->flags & ~(~0U << DICT_TF_BITS)); } else { mach_write_to_4(ptr, DICT_TABLE_ORDINARY); } @@ -306,7 +306,7 @@ dict_build_table_def_step( ut_ad(!dict_table_zip_size(table) || dict_table_get_format(table) >= DICT_TF_FORMAT_ZIP); - flags = table->flags & ~(~0 << DICT_TF_BITS); + flags = table->flags & ~(~0U << DICT_TF_BITS); error = fil_create_new_single_table_tablespace( space, path_or_name, is_path, flags == DICT_TF_COMPACT ? 0 : flags, @@ -325,7 +325,7 @@ dict_build_table_def_step( mtr_commit(&mtr); } else { /* Create in the system tablespace: disallow new features */ - table->flags &= (~0 << DICT_TF_BITS) | DICT_TF_COMPACT; + table->flags &= (~0U << DICT_TF_BITS) | DICT_TF_COMPACT; } row = dict_create_sys_tables_tuple(table, node->heap); diff --git a/storage/innobase/dict/dict0dict.c b/storage/innobase/dict/dict0dict.c index 2a59347bc78..17cf10412a8 100644 --- a/storage/innobase/dict/dict0dict.c +++ b/storage/innobase/dict/dict0dict.c @@ -5868,7 +5868,6 @@ dict_set_corrupted( const char* status; btr_cur_t cursor; - ut_ad(index); ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(!dict_table_is_comp(dict_sys->sys_tables)); ut_ad(!dict_table_is_comp(dict_sys->sys_indexes)); diff --git a/storage/innobase/dict/dict0load.c b/storage/innobase/dict/dict0load.c index 22de23af7ee..351a42f110d 100644 --- a/storage/innobase/dict/dict0load.c +++ b/storage/innobase/dict/dict0load.c @@ -643,7 +643,7 @@ dict_sys_tables_get_flags( return(ULINT_UNDEFINED); } - if (UNIV_UNLIKELY(flags & (~0 << DICT_TF_BITS))) { + if (UNIV_UNLIKELY(flags & (~0U << DICT_TF_BITS))) { /* Some unused bits are set. */ return(ULINT_UNDEFINED); } @@ -1332,7 +1332,7 @@ err_len: goto err_len; } type = mach_read_from_4(field); - if (UNIV_UNLIKELY(type & (~0 << DICT_IT_BITS))) { + if (UNIV_UNLIKELY(type & (~0U << DICT_IT_BITS))) { return("unknown SYS_INDEXES.TYPE bits"); } @@ -1696,7 +1696,7 @@ err_len: flags2 = mach_read_from_4(field); - if (flags2 & (~0 << (DICT_TF2_BITS - DICT_TF2_SHIFT))) { + if (flags2 & (~0U << (DICT_TF2_BITS - DICT_TF2_SHIFT))) { ut_print_timestamp(stderr); fputs(" InnoDB: Warning: table ", stderr); ut_print_filename(stderr, name); @@ -1705,7 +1705,7 @@ err_len: " has unknown flags %lx.\n", (ulong) flags2); - flags2 &= ~(~0 << (DICT_TF2_BITS - DICT_TF2_SHIFT)); + flags2 &= ~(~0U << (DICT_TF2_BITS - DICT_TF2_SHIFT)); } flags |= flags2 << DICT_TF2_SHIFT; @@ -1838,7 +1838,7 @@ err_exit: if (!fil_open_single_table_tablespace( TRUE, table->space, table->flags == DICT_TF_COMPACT ? 0 : - table->flags & ~(~0 << DICT_TF_BITS), name)) { + table->flags & ~(~0U << DICT_TF_BITS), name)) { /* We failed to find a sensible tablespace file */ diff --git a/storage/innobase/dict/dict0mem.c b/storage/innobase/dict/dict0mem.c index cea7a253ba0..87d03eff3c2 100644 --- a/storage/innobase/dict/dict0mem.c +++ b/storage/innobase/dict/dict0mem.c @@ -69,7 +69,7 @@ dict_mem_table_create( DBUG_ENTER("dict_mem_table_create"); ut_ad(name); - ut_a(!(flags & (~0 << DICT_TF2_BITS))); + ut_a(!(flags & (~0U << DICT_TF2_BITS))); heap = mem_heap_create(DICT_HEAP_SIZE); diff --git a/storage/innobase/dyn/dyn0dyn.c b/storage/innobase/dyn/dyn0dyn.c index d0f50ad0c32..b723877d648 100644 --- a/storage/innobase/dyn/dyn0dyn.c +++ b/storage/innobase/dyn/dyn0dyn.c @@ -40,7 +40,6 @@ dyn_array_add_block( mem_heap_t* heap; dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/innobase/fil/fil0fil.c b/storage/innobase/fil/fil0fil.c index 7aa1a42358e..7b29eda02c6 100644 --- a/storage/innobase/fil/fil0fil.c +++ b/storage/innobase/fil/fil0fil.c @@ -3308,7 +3308,7 @@ fil_open_single_table_tablespace( } if (space_id != id - || space_flags != (flags & ~(~0 << DICT_TF_BITS))) { + || space_flags != (flags & ~(~0U << DICT_TF_BITS))) { ut_print_timestamp(stderr); fputs(" InnoDB: Error: tablespace id and flags in file ", diff --git a/storage/innobase/fsp/fsp0fsp.c b/storage/innobase/fsp/fsp0fsp.c index 6d00a1f1d4a..e2e27608e5d 100644 --- a/storage/innobase/fsp/fsp0fsp.c +++ b/storage/innobase/fsp/fsp0fsp.c @@ -1285,7 +1285,6 @@ fsp_fill_free_list( ulint i; mtr_t ibuf_mtr; - ut_ad(header && mtr); ut_ad(page_offset(header) == FSP_HEADER_OFFSET); /* Check if we can fill free list from above the free list limit */ @@ -1584,9 +1583,6 @@ fsp_alloc_free_page( ulint page_no; ulint space_size; - ut_ad(mtr); - ut_ad(init_mtr); - header = fsp_get_space_header(space, zip_size, mtr); /* Get the hinted descriptor */ @@ -2602,7 +2598,6 @@ fseg_alloc_free_page_low( ibool success; ulint n; - ut_ad(mtr); ut_ad((direction >= FSP_UP) && (direction <= FSP_NO_DIR)); ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 8034f4dd535..f9b6d6ff582 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -319,8 +319,8 @@ extern "C" void wsrep_thd_set_conflict_state( extern "C" void wsrep_thd_set_trx_to_replay(THD *thd, uint64 trx_id); -extern "C"void wsrep_thd_LOCK(THD *thd); -extern "C"void wsrep_thd_UNLOCK(THD *thd); +extern "C" void wsrep_thd_LOCK(THD *thd); +extern "C" void wsrep_thd_UNLOCK(THD *thd); extern "C" uint32 wsrep_thd_wsrep_rand(THD *thd); extern "C" time_t wsrep_thd_query_start(THD *thd); extern "C" my_thread_id wsrep_thd_thread_id(THD *thd); diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index b65cae2a1d8..dbc3ce99ab0 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -247,7 +247,6 @@ dict_index_is_clust( /*================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_CLUSTERED)); @@ -261,7 +260,6 @@ dict_index_is_unique( /*=================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_UNIQUE)); @@ -276,7 +274,6 @@ dict_index_is_ibuf( /*===============*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_IBUF)); @@ -293,7 +290,6 @@ dict_index_is_sec_or_ibuf( { ulint type; - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); type = index->type; @@ -311,7 +307,6 @@ dict_table_get_n_user_cols( /*=======================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols - DATA_N_SYS_COLS); @@ -343,7 +338,6 @@ dict_table_get_n_cols( /*==================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols); @@ -937,7 +931,6 @@ dict_index_is_corrupted( /*====================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY((index->type & DICT_CORRUPT) diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index daa3c35ebfb..09834308bb6 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -87,7 +87,7 @@ combination of types */ /* @{ */ #define DICT_TF_FORMAT_SHIFT 5 /* file format */ #define DICT_TF_FORMAT_MASK \ -((~(~0 << (DICT_TF_BITS - DICT_TF_FORMAT_SHIFT))) << DICT_TF_FORMAT_SHIFT) +((~(~0U << (DICT_TF_BITS - DICT_TF_FORMAT_SHIFT))) << DICT_TF_FORMAT_SHIFT) #define DICT_TF_FORMAT_51 0 /*!< InnoDB/MySQL up to 5.1 */ #define DICT_TF_FORMAT_ZIP 1 /*!< InnoDB plugin for 5.1: compressed tables, diff --git a/storage/innobase/include/dyn0dyn.ic b/storage/innobase/include/dyn0dyn.ic index 177877ed1fd..2565a249271 100644 --- a/storage/innobase/include/dyn0dyn.ic +++ b/storage/innobase/include/dyn0dyn.ic @@ -47,8 +47,6 @@ dyn_block_get_used( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((block->used) & ~DYN_BLOCK_FULL_FLAG); } @@ -61,8 +59,6 @@ dyn_block_get_data( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((byte*) block->data); } @@ -76,7 +72,6 @@ dyn_array_create( dyn_array_t* arr) /*!< in/out: memory buffer of size sizeof(dyn_array_t) */ { - ut_ad(arr); #if DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG # error "DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG" #endif @@ -119,7 +114,6 @@ dyn_array_push( dyn_block_t* block; ulint used; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -159,7 +153,6 @@ dyn_array_open( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -195,7 +188,6 @@ dyn_array_close( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); block = dyn_array_get_last_block(arr); @@ -222,7 +214,6 @@ dyn_array_get_element( { const dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); /* Get the first array block */ @@ -260,7 +251,6 @@ dyn_array_get_data_size( const dyn_block_t* block; ulint sum = 0; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index f07e3decc66..0b813182947 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -42,7 +42,7 @@ Created 12/18/1995 Heikki Tuuri #define FSP_FLAGS_POS_PAGE_SSIZE 6 /** Bit mask of the PAGE_SSIZE field */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ + ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) /** Return the value of the PAGE_SSIZE field */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index 238a56577af..fdcb07ca29b 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -50,7 +50,6 @@ mach_read_from_1( /*=============*/ const byte* b) /*!< in: pointer to byte */ { - ut_ad(b); return((ulint)(b[0])); } @@ -143,7 +142,6 @@ mach_read_from_3( /*=============*/ const byte* b) /*!< in: pointer to 3 bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 16) | ((ulint)(b[1]) << 8) | (ulint)(b[2]) @@ -178,7 +176,6 @@ mach_read_from_4( /*=============*/ const byte* b) /*!< in: pointer to four bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 24) | ((ulint)(b[1]) << 16) | ((ulint)(b[2]) << 8) @@ -255,8 +252,6 @@ mach_read_compressed( { ulint flag; - ut_ad(b); - flag = mach_read_from_1(b); if (flag < 0x80UL) { @@ -333,8 +328,6 @@ mach_read_from_7( /*=============*/ const byte* b) /*!< in: pointer to 7 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_3(b), mach_read_from_4(b + 3))); } @@ -364,8 +357,6 @@ mach_read_from_6( /*=============*/ const byte* b) /*!< in: pointer to 6 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_2(b), mach_read_from_4(b + 2))); } @@ -413,8 +404,6 @@ mach_ull_read_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - n = (ib_uint64_t) mach_read_compressed(b); size = mach_get_compressed_size((ulint) n); @@ -480,8 +469,6 @@ mach_ull_read_much_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - if (*b != (byte)0xFF) { n = 0; size = 0; diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic index a03a0271535..c9e2f2e39de 100644 --- a/storage/innobase/include/mtr0mtr.ic +++ b/storage/innobase/include/mtr0mtr.ic @@ -149,7 +149,6 @@ mtr_memo_contains( dyn_array_t* memo; ulint offset; - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE || mtr->state == MTR_COMMITTING); diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 781ad029e87..1689c0ade6b 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -154,7 +154,6 @@ page_header_get_offs( { ulint offs; - ut_ad(page); ut_ad((field == PAGE_FREE) || (field == PAGE_LAST_INSERT) || (field == PAGE_HEAP_TOP)); diff --git a/storage/innobase/include/rem0rec.ic b/storage/innobase/include/rem0rec.ic index 564d2d1b31c..c81388391d7 100644 --- a/storage/innobase/include/rem0rec.ic +++ b/storage/innobase/include/rem0rec.ic @@ -1545,7 +1545,6 @@ rec_copy( ulint extra_len; ulint data_len; - ut_ad(rec && buf); ut_ad(rec_offs_validate((rec_t*) rec, NULL, offsets)); ut_ad(rec_validate(rec, offsets)); diff --git a/storage/innobase/include/ut0lst.h b/storage/innobase/include/ut0lst.h index a010f464570..1fb43fd4805 100644 --- a/storage/innobase/include/ut0lst.h +++ b/storage/innobase/include/ut0lst.h @@ -88,7 +88,6 @@ Adds the node as the first element in a two-way linked list. */ #define UT_LIST_ADD_FIRST(NAME, BASE, N)\ {\ - ut_ad(N);\ ((BASE).count)++;\ ((N)->NAME).next = (BASE).start;\ ((N)->NAME).prev = NULL;\ @@ -134,7 +133,6 @@ Inserts a NODE2 after NODE1 in a list. #define UT_LIST_INSERT_AFTER(NAME, BASE, NODE1, NODE2)\ {\ ut_ad(NODE1);\ - ut_ad(NODE2);\ ut_ad((NODE1) != (NODE2));\ ((BASE).count)++;\ ((NODE2)->NAME).prev = (NODE1);\ @@ -169,7 +167,6 @@ Removes a node from a two-way linked list. */ #define UT_LIST_REMOVE(NAME, BASE, N) \ do { \ - ut_ad(N); \ ut_a((BASE).count > 0); \ ((BASE).count)--; \ if (((N)->NAME).next != NULL) { \ diff --git a/storage/innobase/mtr/mtr0mtr.c b/storage/innobase/mtr/mtr0mtr.c index a5c98761523..ebad6e7bfa1 100644 --- a/storage/innobase/mtr/mtr0mtr.c +++ b/storage/innobase/mtr/mtr0mtr.c @@ -257,7 +257,6 @@ mtr_commit( /*=======*/ mtr_t* mtr) /*!< in: mini-transaction */ { - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE); ut_ad(!mtr->inside_ibuf); diff --git a/storage/innobase/page/page0page.c b/storage/innobase/page/page0page.c index 81051f8f4fe..9af4b1c8faa 100644 --- a/storage/innobase/page/page0page.c +++ b/storage/innobase/page/page0page.c @@ -1345,7 +1345,6 @@ page_dir_split_slot( ulint i; ulint n_owned; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); @@ -1407,7 +1406,6 @@ page_dir_balance_slot( rec_t* old_rec; rec_t* new_rec; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); diff --git a/storage/innobase/page/page0zip.c b/storage/innobase/page/page0zip.c index 7c04e0b9183..faa004be730 100644 --- a/storage/innobase/page/page0zip.c +++ b/storage/innobase/page/page0zip.c @@ -4739,7 +4739,6 @@ page_zip_parse_compress( ulint size; ulint trailer_size; - ut_ad(ptr && end_ptr); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(ptr + (2 + 2) > end_ptr)) { diff --git a/storage/innobase/rem/rem0rec.c b/storage/innobase/rem/rem0rec.c index 5351690dcef..84f04beb61d 100644 --- a/storage/innobase/rem/rem0rec.c +++ b/storage/innobase/rem/rem0rec.c @@ -794,8 +794,7 @@ rec_get_converted_size_comp_prefix_low( ulint extra_size; ulint data_size; ulint i; - ut_ad(index); - ut_ad(fields); + ut_ad(n_fields > 0); ut_ad(n_fields <= dict_index_get_n_fields(index)); ut_ad(!temp || extra); diff --git a/storage/innobase/row/row0merge.c b/storage/innobase/row/row0merge.c index 7d87d1f9c8f..a393254d145 100644 --- a/storage/innobase/row/row0merge.c +++ b/storage/innobase/row/row0merge.c @@ -807,14 +807,8 @@ row_merge_read_rec( ulint data_size; ulint avail_size; - ut_ad(block); - ut_ad(buf); ut_ad(b >= block[0]); ut_ad(b < block[1]); - ut_ad(index); - ut_ad(foffs); - ut_ad(mrec); - ut_ad(offsets); ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); @@ -1175,12 +1169,6 @@ row_merge_read_clustered_index( trx->op_info = "reading clustered index"; - ut_ad(trx); - ut_ad(old_table); - ut_ad(new_table); - ut_ad(index); - ut_ad(files); - /* Create and initialize memory for record buffers */ merge_buf = mem_alloc(n_index * sizeof *merge_buf); diff --git a/storage/innobase/row/row0purge.c b/storage/innobase/row/row0purge.c index 9018582f5d6..5f3e4175544 100644 --- a/storage/innobase/row/row0purge.c +++ b/storage/innobase/row/row0purge.c @@ -751,9 +751,6 @@ row_purge( { ibool updated_extern; - ut_ad(node); - ut_ad(thr); - node->undo_rec = trx_purge_fetch_next_rec(&node->roll_ptr, &node->reservation, node->heap); diff --git a/storage/innobase/row/row0upd.c b/storage/innobase/row/row0upd.c index 05924958edd..3560a8bff1a 100644 --- a/storage/innobase/row/row0upd.c +++ b/storage/innobase/row/row0upd.c @@ -1235,8 +1235,6 @@ row_upd_index_replace_new_col_vals_index_pos( ulint n_fields; const ulint zip_size = dict_table_zip_size(index->table); - ut_ad(index); - dtuple_set_info_bits(entry, update->info_bits); if (order_only) { @@ -1419,9 +1417,6 @@ row_upd_changes_ord_field_binary_func( ulint i; const dict_index_t* clust_index; - ut_ad(index); - ut_ad(update); - ut_ad(thr); ut_ad(thr->graph); ut_ad(thr->graph->trx); diff --git a/storage/innobase/trx/trx0trx.c b/storage/innobase/trx/trx0trx.c index ce43c80e826..558cd74cfca 100644 --- a/storage/innobase/trx/trx0trx.c +++ b/storage/innobase/trx/trx0trx.c @@ -97,7 +97,6 @@ trx_create( trx_t* trx; ut_ad(mutex_own(&kernel_mutex)); - ut_ad(sess); trx = mem_alloc(sizeof(trx_t)); diff --git a/storage/xtradb/btr/btr0btr.c b/storage/xtradb/btr/btr0btr.c index 0c429363789..fb670df2329 100644 --- a/storage/xtradb/btr/btr0btr.c +++ b/storage/xtradb/btr/btr0btr.c @@ -2974,7 +2974,6 @@ btr_level_list_remove_func( ulint prev_page_no; ulint next_page_no; - ut_ad(page && mtr); ut_ad(mtr_memo_contains_page(mtr, page, MTR_MEMO_PAGE_X_FIX)); ut_ad(space == page_get_space_id(page)); /* Get the previous and next page numbers of page */ diff --git a/storage/xtradb/btr/btr0cur.c b/storage/xtradb/btr/btr0cur.c index 0ec9367b27c..d9c1a2ef7a8 100644 --- a/storage/xtradb/btr/btr0cur.c +++ b/storage/xtradb/btr/btr0cur.c @@ -2008,7 +2008,6 @@ btr_cur_update_alloc_zip( trx_t* trx) /*!< in: NULL or transaction */ { ut_a(page_zip == buf_block_get_page_zip(block)); - ut_ad(page_zip); ut_ad(!dict_index_is_ibuf(index)); if (page_zip_available(page_zip, dict_index_is_clust(index), @@ -3008,7 +3007,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(page_is_leaf(page_align(rec))); #ifdef UNIV_DEBUG - if (btr_cur_print_record_ops && thr) { + if (btr_cur_print_record_ops) { btr_cur_trx_report(thr_get_trx(thr), index, "del mark "); rec_print_new(stderr, rec, offsets); } @@ -3017,7 +3016,7 @@ btr_cur_del_mark_set_clust_rec( ut_ad(dict_index_is_clust(index)); ut_ad(!rec_get_deleted_flag(rec, rec_offs_comp(offsets))); - if (UNIV_UNLIKELY(thr && thr_get_trx(thr)->fake_changes)) { + if (UNIV_UNLIKELY(thr_get_trx(thr)->fake_changes)) { /* skip LOCK, UNDO, CHANGE, LOG */ return(DB_SUCCESS); } @@ -4323,7 +4322,6 @@ btr_cur_disown_inherited_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(!rec_offs_comp(offsets) || !rec_get_node_ptr_flag(rec)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(mtr); for (i = 0; i < rec_offs_n_fields(offsets); i++) { if (rec_offs_nth_extern(offsets, i) @@ -4386,9 +4384,6 @@ btr_push_update_extern_fields( ulint n; const upd_field_t* uf; - ut_ad(tuple); - ut_ad(update); - uf = update->fields; n = upd_get_n_fields(update); @@ -4571,7 +4566,6 @@ btr_store_big_rec_extern_fields( ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(rec_offs_any_extern(offsets)); - ut_ad(btr_mtr); ut_ad(mtr_memo_contains(btr_mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK)); ut_ad(mtr_memo_contains(btr_mtr, rec_block, MTR_MEMO_PAGE_X_FIX)); diff --git a/storage/xtradb/buf/buf0buddy.c b/storage/xtradb/buf/buf0buddy.c index 439be08b01f..493d0d2d41c 100644 --- a/storage/xtradb/buf/buf0buddy.c +++ b/storage/xtradb/buf/buf0buddy.c @@ -288,7 +288,6 @@ buf_buddy_alloc_low( { buf_block_t* block; - ut_ad(lru); //ut_ad(buf_pool_mutex_own(buf_pool)); ut_ad(mutex_own(&buf_pool->LRU_list_mutex)); ut_ad(!mutex_own(&buf_pool->zip_mutex)); diff --git a/storage/xtradb/dict/dict0crea.c b/storage/xtradb/dict/dict0crea.c index b44fdc1d2e4..1529c22bbb9 100644 --- a/storage/xtradb/dict/dict0crea.c +++ b/storage/xtradb/dict/dict0crea.c @@ -112,13 +112,13 @@ dict_create_sys_tables_tuple( dfield = dtuple_get_nth_field(entry, 3/*TYPE*/); ptr = mem_heap_alloc(heap, 4); - if (table->flags & (~DICT_TF_COMPACT & ~(~0 << DICT_TF_BITS))) { + if (table->flags & (~DICT_TF_COMPACT & ~(~0U << DICT_TF_BITS))) { ut_a(table->flags & DICT_TF_COMPACT); ut_a(dict_table_get_format(table) >= DICT_TF_FORMAT_ZIP); ut_a(((ulonglong) table->flags & DICT_TF_ZSSIZE_MASK) <= (ulonglong) (DICT_TF_ZSSIZE_MAX << DICT_TF_ZSSIZE_SHIFT)); - ut_a(!(table->flags & (~0 << DICT_TF2_BITS))); - mach_write_to_4(ptr, table->flags & ~(~0 << DICT_TF_BITS)); + ut_a(!(table->flags & (~0U << DICT_TF2_BITS))); + mach_write_to_4(ptr, table->flags & ~(~0U << DICT_TF_BITS)); } else { mach_write_to_4(ptr, DICT_TABLE_ORDINARY); } @@ -306,7 +306,7 @@ dict_build_table_def_step( ut_ad(!dict_table_zip_size(table) || dict_table_get_format(table) >= DICT_TF_FORMAT_ZIP); - flags = table->flags & ~(~0 << DICT_TF_BITS); + flags = table->flags & ~(~0U << DICT_TF_BITS); error = fil_create_new_single_table_tablespace( space, path_or_name, is_path, flags == DICT_TF_COMPACT ? 0 : flags, @@ -325,7 +325,7 @@ dict_build_table_def_step( mtr_commit(&mtr); } else { /* Create in the system tablespace: disallow new features */ - table->flags &= (~0 << DICT_TF_BITS) | DICT_TF_COMPACT; + table->flags &= (~0U << DICT_TF_BITS) | DICT_TF_COMPACT; } row = dict_create_sys_tables_tuple(table, node->heap); diff --git a/storage/xtradb/dict/dict0dict.c b/storage/xtradb/dict/dict0dict.c index 6f16fc0bce0..a6970c20831 100644 --- a/storage/xtradb/dict/dict0dict.c +++ b/storage/xtradb/dict/dict0dict.c @@ -6414,7 +6414,6 @@ dict_set_corrupted( const char* status; btr_cur_t cursor; - ut_ad(index); ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(!dict_table_is_comp(dict_sys->sys_tables)); ut_ad(!dict_table_is_comp(dict_sys->sys_indexes)); diff --git a/storage/xtradb/dict/dict0load.c b/storage/xtradb/dict/dict0load.c index 1fdbffee3c8..5184746e48e 100644 --- a/storage/xtradb/dict/dict0load.c +++ b/storage/xtradb/dict/dict0load.c @@ -716,7 +716,7 @@ dict_sys_tables_get_flags( return(ULINT_UNDEFINED); } - if (UNIV_UNLIKELY(flags & (~0 << DICT_TF_BITS))) { + if (UNIV_UNLIKELY(flags & (~0U << DICT_TF_BITS))) { /* Some unused bits are set. */ return(ULINT_UNDEFINED); } @@ -1405,7 +1405,7 @@ err_len: goto err_len; } type = mach_read_from_4(field); - if (UNIV_UNLIKELY(type & (~0 << DICT_IT_BITS))) { + if (UNIV_UNLIKELY(type & (~0U << DICT_IT_BITS))) { return("unknown SYS_INDEXES.TYPE bits"); } @@ -1770,7 +1770,7 @@ err_len: flags2 = mach_read_from_4(field); - if (flags2 & (~0 << (DICT_TF2_BITS - DICT_TF2_SHIFT))) { + if (flags2 & (~0U << (DICT_TF2_BITS - DICT_TF2_SHIFT))) { ut_print_timestamp(stderr); fputs(" InnoDB: Warning: table ", stderr); ut_print_filename(stderr, name); @@ -1779,7 +1779,7 @@ err_len: " has unknown flags %lx.\n", (ulong) flags2); - flags2 &= ~(~0 << (DICT_TF2_BITS - DICT_TF2_SHIFT)); + flags2 &= ~(~0U << (DICT_TF2_BITS - DICT_TF2_SHIFT)); } flags |= flags2 << DICT_TF2_SHIFT; @@ -1912,7 +1912,7 @@ err_exit: if (!fil_open_single_table_tablespace( TRUE, table->space, table->flags == DICT_TF_COMPACT ? 0 : - table->flags & ~(~0 << DICT_TF_BITS), name, NULL)) { + table->flags & ~(~0U << DICT_TF_BITS), name, NULL)) { /* We failed to find a sensible tablespace file */ diff --git a/storage/xtradb/dict/dict0mem.c b/storage/xtradb/dict/dict0mem.c index 40099c06823..18917a30ff6 100644 --- a/storage/xtradb/dict/dict0mem.c +++ b/storage/xtradb/dict/dict0mem.c @@ -69,7 +69,7 @@ dict_mem_table_create( DBUG_ENTER("dict_mem_table_create"); ut_ad(name); - ut_a(!(flags & (~0 << DICT_TF2_BITS))); + ut_a(!(flags & (~0U << DICT_TF2_BITS))); heap = mem_heap_create(DICT_HEAP_SIZE); diff --git a/storage/xtradb/dyn/dyn0dyn.c b/storage/xtradb/dyn/dyn0dyn.c index d0f50ad0c32..b723877d648 100644 --- a/storage/xtradb/dyn/dyn0dyn.c +++ b/storage/xtradb/dyn/dyn0dyn.c @@ -40,7 +40,6 @@ dyn_array_add_block( mem_heap_t* heap; dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/xtradb/fil/fil0fil.c b/storage/xtradb/fil/fil0fil.c index 7682b49e755..86e00dc22e4 100644 --- a/storage/xtradb/fil/fil0fil.c +++ b/storage/xtradb/fil/fil0fil.c @@ -3883,7 +3883,7 @@ skip_write: } if (space_id != id - || space_flags != (flags & ~(~0 << DICT_TF_BITS))) { + || space_flags != (flags & ~(~0U << DICT_TF_BITS))) { ut_print_timestamp(stderr); fputs(" InnoDB: Error: tablespace id and flags in file ", diff --git a/storage/xtradb/fsp/fsp0fsp.c b/storage/xtradb/fsp/fsp0fsp.c index 772e224f6f7..4477b34f7b4 100644 --- a/storage/xtradb/fsp/fsp0fsp.c +++ b/storage/xtradb/fsp/fsp0fsp.c @@ -1290,7 +1290,6 @@ fsp_fill_free_list( ulint i; mtr_t ibuf_mtr; - ut_ad(header && mtr); ut_ad(page_offset(header) == FSP_HEADER_OFFSET); /* Check if we can fill free list from above the free list limit */ @@ -1589,9 +1588,6 @@ fsp_alloc_free_page( ulint page_no; ulint space_size; - ut_ad(mtr); - ut_ad(init_mtr); - header = fsp_get_space_header(space, zip_size, mtr); /* Get the hinted descriptor */ @@ -2613,7 +2609,6 @@ fseg_alloc_free_page_low( ibool success; ulint n; - ut_ad(mtr); ut_ad((direction >= FSP_UP) && (direction <= FSP_NO_DIR)); ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index 521a9f2664c..69a28905f22 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -398,8 +398,8 @@ extern "C" void wsrep_thd_set_conflict_state( extern "C" void wsrep_thd_set_trx_to_replay(THD *thd, uint64 trx_id); -extern "C"void wsrep_thd_LOCK(THD *thd); -extern "C"void wsrep_thd_UNLOCK(THD *thd); +extern "C" void wsrep_thd_LOCK(THD *thd); +extern "C" void wsrep_thd_UNLOCK(THD *thd); extern "C" uint32 wsrep_thd_wsrep_rand(THD *thd); extern "C" time_t wsrep_thd_query_start(THD *thd); extern "C" my_thread_id wsrep_thd_thread_id(THD *thd); diff --git a/storage/xtradb/include/btr0sea.ic b/storage/xtradb/include/btr0sea.ic index 3f0dfdaa511..677cb52f458 100644 --- a/storage/xtradb/include/btr0sea.ic +++ b/storage/xtradb/include/btr0sea.ic @@ -91,7 +91,6 @@ btr_search_get_hash_table( /*======================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->search_table); return(index->search_table); @@ -103,7 +102,6 @@ btr_search_get_latch( /*=================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->search_latch >= btr_search_latch_arr && index->search_latch < btr_search_latch_arr + btr_search_index_num); @@ -130,8 +128,6 @@ btr_search_index_init( /*===============*/ dict_index_t* index) /*!< in: index */ { - ut_ad(index); - index->search_latch = &btr_search_latch_arr[btr_search_get_key(index->id)]; index->search_table = diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index 6836928ff49..3410c945a80 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -247,7 +247,6 @@ dict_index_is_clust( /*================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_CLUSTERED)); @@ -261,7 +260,6 @@ dict_index_is_unique( /*=================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_UNIQUE)); @@ -276,7 +274,6 @@ dict_index_is_ibuf( /*===============*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY(index->type & DICT_IBUF)); @@ -293,7 +290,6 @@ dict_index_is_sec_or_ibuf( { ulint type; - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); type = index->type; @@ -311,7 +307,6 @@ dict_table_get_n_user_cols( /*=======================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols - DATA_N_SYS_COLS); @@ -343,7 +338,6 @@ dict_table_get_n_cols( /*==================*/ const dict_table_t* table) /*!< in: table */ { - ut_ad(table); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); return(table->n_cols); @@ -950,7 +944,6 @@ dict_index_is_corrupted( /*====================*/ const dict_index_t* index) /*!< in: index */ { - ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return(UNIV_UNLIKELY((index->type & DICT_CORRUPT) diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index f174103c3fe..489a30592d1 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -87,7 +87,7 @@ combination of types */ /* @{ */ #define DICT_TF_FORMAT_SHIFT 5 /* file format */ #define DICT_TF_FORMAT_MASK \ -((~(~0 << (DICT_TF_BITS - DICT_TF_FORMAT_SHIFT))) << DICT_TF_FORMAT_SHIFT) +((~(~0U << (DICT_TF_BITS - DICT_TF_FORMAT_SHIFT))) << DICT_TF_FORMAT_SHIFT) #define DICT_TF_FORMAT_51 0 /*!< InnoDB/MySQL up to 5.1 */ #define DICT_TF_FORMAT_ZIP 1 /*!< InnoDB plugin for 5.1: compressed tables, diff --git a/storage/xtradb/include/dyn0dyn.ic b/storage/xtradb/include/dyn0dyn.ic index 177877ed1fd..2565a249271 100644 --- a/storage/xtradb/include/dyn0dyn.ic +++ b/storage/xtradb/include/dyn0dyn.ic @@ -47,8 +47,6 @@ dyn_block_get_used( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((block->used) & ~DYN_BLOCK_FULL_FLAG); } @@ -61,8 +59,6 @@ dyn_block_get_data( /*===============*/ const dyn_block_t* block) /*!< in: dyn array block */ { - ut_ad(block); - return((byte*) block->data); } @@ -76,7 +72,6 @@ dyn_array_create( dyn_array_t* arr) /*!< in/out: memory buffer of size sizeof(dyn_array_t) */ { - ut_ad(arr); #if DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG # error "DYN_ARRAY_DATA_SIZE >= DYN_BLOCK_FULL_FLAG" #endif @@ -119,7 +114,6 @@ dyn_array_push( dyn_block_t* block; ulint used; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -159,7 +153,6 @@ dyn_array_open( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); ut_ad(size <= DYN_ARRAY_DATA_SIZE); ut_ad(size); @@ -195,7 +188,6 @@ dyn_array_close( { dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); block = dyn_array_get_last_block(arr); @@ -222,7 +214,6 @@ dyn_array_get_element( { const dyn_block_t* block; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); /* Get the first array block */ @@ -260,7 +251,6 @@ dyn_array_get_data_size( const dyn_block_t* block; ulint sum = 0; - ut_ad(arr); ut_ad(arr->magic_n == DYN_BLOCK_MAGIC_N); if (arr->heap == NULL) { diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index f07e3decc66..0b813182947 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -42,7 +42,7 @@ Created 12/18/1995 Heikki Tuuri #define FSP_FLAGS_POS_PAGE_SSIZE 6 /** Bit mask of the PAGE_SSIZE field */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ + ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) /** Return the value of the PAGE_SSIZE field */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ diff --git a/storage/xtradb/include/mach0data.ic b/storage/xtradb/include/mach0data.ic index 238a56577af..fdcb07ca29b 100644 --- a/storage/xtradb/include/mach0data.ic +++ b/storage/xtradb/include/mach0data.ic @@ -50,7 +50,6 @@ mach_read_from_1( /*=============*/ const byte* b) /*!< in: pointer to byte */ { - ut_ad(b); return((ulint)(b[0])); } @@ -143,7 +142,6 @@ mach_read_from_3( /*=============*/ const byte* b) /*!< in: pointer to 3 bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 16) | ((ulint)(b[1]) << 8) | (ulint)(b[2]) @@ -178,7 +176,6 @@ mach_read_from_4( /*=============*/ const byte* b) /*!< in: pointer to four bytes */ { - ut_ad(b); return( ((ulint)(b[0]) << 24) | ((ulint)(b[1]) << 16) | ((ulint)(b[2]) << 8) @@ -255,8 +252,6 @@ mach_read_compressed( { ulint flag; - ut_ad(b); - flag = mach_read_from_1(b); if (flag < 0x80UL) { @@ -333,8 +328,6 @@ mach_read_from_7( /*=============*/ const byte* b) /*!< in: pointer to 7 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_3(b), mach_read_from_4(b + 3))); } @@ -364,8 +357,6 @@ mach_read_from_6( /*=============*/ const byte* b) /*!< in: pointer to 6 bytes */ { - ut_ad(b); - return(ut_ull_create(mach_read_from_2(b), mach_read_from_4(b + 2))); } @@ -413,8 +404,6 @@ mach_ull_read_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - n = (ib_uint64_t) mach_read_compressed(b); size = mach_get_compressed_size((ulint) n); @@ -480,8 +469,6 @@ mach_ull_read_much_compressed( ib_uint64_t n; ulint size; - ut_ad(b); - if (*b != (byte)0xFF) { n = 0; size = 0; diff --git a/storage/xtradb/include/mtr0mtr.ic b/storage/xtradb/include/mtr0mtr.ic index 7b5d268b70f..bebe202e6ac 100644 --- a/storage/xtradb/include/mtr0mtr.ic +++ b/storage/xtradb/include/mtr0mtr.ic @@ -170,7 +170,6 @@ mtr_memo_contains( dyn_array_t* memo; ulint offset; - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE || mtr->state == MTR_COMMITTING); diff --git a/storage/xtradb/include/page0page.ic b/storage/xtradb/include/page0page.ic index 4fe93345ce5..da9fbc5f01d 100644 --- a/storage/xtradb/include/page0page.ic +++ b/storage/xtradb/include/page0page.ic @@ -154,7 +154,6 @@ page_header_get_offs( { ulint offs; - ut_ad(page); ut_ad((field == PAGE_FREE) || (field == PAGE_LAST_INSERT) || (field == PAGE_HEAP_TOP)); diff --git a/storage/xtradb/include/rem0rec.ic b/storage/xtradb/include/rem0rec.ic index b14366312e0..b99d076b500 100644 --- a/storage/xtradb/include/rem0rec.ic +++ b/storage/xtradb/include/rem0rec.ic @@ -1545,7 +1545,6 @@ rec_copy( ulint extra_len; ulint data_len; - ut_ad(rec && buf); ut_ad(rec_offs_validate((rec_t*) rec, NULL, offsets)); ut_ad(rec_validate(rec, offsets)); diff --git a/storage/xtradb/include/ut0lst.h b/storage/xtradb/include/ut0lst.h index 9bb4bc7723f..2b5f5c7a705 100644 --- a/storage/xtradb/include/ut0lst.h +++ b/storage/xtradb/include/ut0lst.h @@ -88,7 +88,6 @@ Adds the node as the first element in a two-way linked list. */ #define UT_LIST_ADD_FIRST(NAME, BASE, N)\ {\ - ut_ad(N);\ ((BASE).count)++;\ ((N)->NAME).next = (BASE).start;\ ((N)->NAME).prev = NULL;\ @@ -134,7 +133,6 @@ Inserts a NODE2 after NODE1 in a list. #define UT_LIST_INSERT_AFTER(NAME, BASE, NODE1, NODE2)\ {\ ut_ad(NODE1);\ - ut_ad(NODE2);\ ut_ad((NODE1) != (NODE2));\ ((BASE).count)++;\ ((NODE2)->NAME).prev = (NODE1);\ @@ -169,7 +167,6 @@ Removes a node from a two-way linked list. */ #define UT_LIST_REMOVE(NAME, BASE, N) \ do { \ - ut_ad(N); \ ut_a((BASE).count > 0); \ ((BASE).count)--; \ if (((N)->NAME).next != NULL) { \ diff --git a/storage/xtradb/mtr/mtr0mtr.c b/storage/xtradb/mtr/mtr0mtr.c index 083692efd77..d6aed7ee723 100644 --- a/storage/xtradb/mtr/mtr0mtr.c +++ b/storage/xtradb/mtr/mtr0mtr.c @@ -289,7 +289,6 @@ mtr_commit( /*=======*/ mtr_t* mtr) /*!< in: mini-transaction */ { - ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE); ut_ad(!mtr->inside_ibuf); diff --git a/storage/xtradb/page/page0page.c b/storage/xtradb/page/page0page.c index f2ce6c9fe16..2dd5a4ba472 100644 --- a/storage/xtradb/page/page0page.c +++ b/storage/xtradb/page/page0page.c @@ -1345,7 +1345,6 @@ page_dir_split_slot( ulint i; ulint n_owned; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); @@ -1407,7 +1406,6 @@ page_dir_balance_slot( rec_t* old_rec; rec_t* new_rec; - ut_ad(page); ut_ad(!page_zip || page_is_comp(page)); ut_ad(slot_no > 0); diff --git a/storage/xtradb/page/page0zip.c b/storage/xtradb/page/page0zip.c index 3d3605d6362..7482323c9b5 100644 --- a/storage/xtradb/page/page0zip.c +++ b/storage/xtradb/page/page0zip.c @@ -4743,7 +4743,6 @@ page_zip_parse_compress( ulint size; ulint trailer_size; - ut_ad(ptr && end_ptr); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(ptr + (2 + 2) > end_ptr)) { diff --git a/storage/xtradb/rem/rem0rec.c b/storage/xtradb/rem/rem0rec.c index a0e289e2163..cc181c7d333 100644 --- a/storage/xtradb/rem/rem0rec.c +++ b/storage/xtradb/rem/rem0rec.c @@ -794,8 +794,7 @@ rec_get_converted_size_comp_prefix_low( ulint extra_size; ulint data_size; ulint i; - ut_ad(index); - ut_ad(fields); + ut_ad(n_fields > 0); ut_ad(n_fields <= dict_index_get_n_fields(index)); ut_ad(!temp || extra); diff --git a/storage/xtradb/row/row0merge.c b/storage/xtradb/row/row0merge.c index 3a03c7c8578..7409d5a5e1a 100644 --- a/storage/xtradb/row/row0merge.c +++ b/storage/xtradb/row/row0merge.c @@ -823,14 +823,8 @@ row_merge_read_rec( ulint data_size; ulint avail_size; - ut_ad(block); - ut_ad(buf); ut_ad(b >= block[0]); ut_ad(b < block[1]); - ut_ad(index); - ut_ad(foffs); - ut_ad(mrec); - ut_ad(offsets); ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); @@ -1202,12 +1196,6 @@ row_merge_read_clustered_index( trx->op_info = "reading clustered index"; - ut_ad(trx); - ut_ad(old_table); - ut_ad(new_table); - ut_ad(index); - ut_ad(files); - /* Create and initialize memory for record buffers */ merge_buf = mem_alloc(n_index * sizeof *merge_buf); diff --git a/storage/xtradb/row/row0purge.c b/storage/xtradb/row/row0purge.c index 1e87016bc5e..8ada79a2bbc 100644 --- a/storage/xtradb/row/row0purge.c +++ b/storage/xtradb/row/row0purge.c @@ -751,9 +751,6 @@ row_purge( { ibool updated_extern; - ut_ad(node); - ut_ad(thr); - node->undo_rec = trx_purge_fetch_next_rec(&node->roll_ptr, &node->reservation, node->heap); diff --git a/storage/xtradb/row/row0upd.c b/storage/xtradb/row/row0upd.c index 33a7e45765e..191d3e5815a 100644 --- a/storage/xtradb/row/row0upd.c +++ b/storage/xtradb/row/row0upd.c @@ -1253,8 +1253,6 @@ row_upd_index_replace_new_col_vals_index_pos( ulint n_fields; const ulint zip_size = dict_table_zip_size(index->table); - ut_ad(index); - dtuple_set_info_bits(entry, update->info_bits); if (order_only) { @@ -1437,9 +1435,6 @@ row_upd_changes_ord_field_binary_func( ulint i; const dict_index_t* clust_index; - ut_ad(index); - ut_ad(update); - ut_ad(thr); ut_ad(thr->graph); ut_ad(thr->graph->trx); diff --git a/storage/xtradb/trx/trx0trx.c b/storage/xtradb/trx/trx0trx.c index d6ce4767b39..fbdf11f378d 100644 --- a/storage/xtradb/trx/trx0trx.c +++ b/storage/xtradb/trx/trx0trx.c @@ -218,7 +218,6 @@ trx_create( trx_t* trx; ut_ad(mutex_own(&kernel_mutex)); - ut_ad(sess); trx = mem_alloc(sizeof(trx_t)); From b8afa783bcd32c9a0e5b26d780f4debf859f1b8f Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 18 Apr 2017 11:50:43 +0530 Subject: [PATCH 012/276] MW-267 Fix Galera crash at startup when compiled with gcc 6 Signed-off-by: Sachin Setiya --- sql/wsrep_mysqld.cc | 2 +- sql/wsrep_mysqld.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 21c7e94096d..a0938bdfb43 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -83,7 +83,7 @@ my_bool wsrep_creating_startup_threads = 0; */ my_bool wsrep_inited = 0; // initialized ? -static const wsrep_uuid_t cluster_uuid = WSREP_UUID_UNDEFINED; +static wsrep_uuid_t cluster_uuid = WSREP_UUID_UNDEFINED; const wsrep_uuid_t* wsrep_cluster_uuid() { return &cluster_uuid; diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 85d4cea241f..bd45399a948 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -314,7 +314,6 @@ int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len); int wsrep_create_event_query(THD *thd, uchar** buf, size_t* buf_len); int wsrep_alter_event_query(THD *thd, uchar** buf, size_t* buf_len); -const wsrep_uuid_t* wsrep_cluster_uuid(); struct xid_t; void wsrep_set_SE_checkpoint(xid_t*); void wsrep_get_SE_checkpoint(wsrep_uuid_t&, wsrep_seqno_t&); From 1d4cc423886201d9d373e17bb29d667ab8a5f503 Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko Date: Fri, 12 Aug 2016 14:21:01 +0300 Subject: [PATCH 013/276] MW-267: followup to the original pull request, removed unnecessary cast. --- sql/wsrep_mysqld.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index a0938bdfb43..53d53c2c404 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -249,8 +249,7 @@ wsrep_view_handler_cb (void* app_ctx, if (memcmp(&cluster_uuid, &view->state_id.uuid, sizeof(wsrep_uuid_t))) { - memcpy((wsrep_uuid_t*)&cluster_uuid, &view->state_id.uuid, - sizeof(cluster_uuid)); + memcpy(&cluster_uuid, &view->state_id.uuid, sizeof(cluster_uuid)); wsrep_uuid_print (&cluster_uuid, cluster_uuid_str, sizeof(cluster_uuid_str)); From 33aaee8ee997dc76a9148d96ebe0d40b2830ddd9 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 18 May 2016 11:07:58 +0200 Subject: [PATCH 014/276] MW-175 Fix definitively lost memory in wsrep_get_params --- sql/wsrep_var.cc | 5 ++--- wsrep/wsrep_dummy.c | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 66b33e7697c..6385fa926a2 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -185,9 +185,8 @@ static bool refresh_provider_options() char* opts= wsrep->options_get(wsrep); if (opts) { - if (wsrep_provider_options) my_free((void *)wsrep_provider_options); - wsrep_provider_options = (char*)my_memdup(opts, strlen(opts) + 1, - MYF(MY_WME)); + wsrep_provider_options_init(opts); + free(opts); } else { diff --git a/wsrep/wsrep_dummy.c b/wsrep/wsrep_dummy.c index bab5329dc02..5f1ea63cc40 100644 --- a/wsrep/wsrep_dummy.c +++ b/wsrep/wsrep_dummy.c @@ -86,7 +86,7 @@ static wsrep_status_t dummy_options_set( static char* dummy_options_get (wsrep_t* w) { WSREP_DBUG_ENTER(w); - return WSREP_DUMMY(w)->options; + return strdup(WSREP_DUMMY(w)->options); } static wsrep_status_t dummy_connect( From d1313d605e34a0e805022b1fd207d8df849eee4c Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 18 Apr 2017 11:53:03 +0530 Subject: [PATCH 015/276] GCF-837 Check wsrep interface version before loading provider Signed-off-by: Sachin Setiya --- wsrep/wsrep_loader.c | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/wsrep/wsrep_loader.c b/wsrep/wsrep_loader.c index 0825d7e9ecf..5f98b0ace6a 100644 --- a/wsrep/wsrep_loader.c +++ b/wsrep/wsrep_loader.c @@ -37,6 +37,22 @@ static wsrep_log_cb_t logger = default_logger; * Library loader **************************************************************************/ +static int wsrep_check_iface_version(const char* found, const char* iface_ver) +{ + const size_t msg_len = 128; + char msg[128]; + + if (strcmp(found, iface_ver)) { + snprintf (msg, msg_len, + "provider interface version mismatch: need '%s', found '%s'", + iface_ver, found); + logger (WSREP_LOG_ERROR, msg); + return EINVAL; + } + + return 0; +} + static int verify(const wsrep_t *wh, const char *iface_ver) { char msg[128]; @@ -50,13 +66,8 @@ static int verify(const wsrep_t *wh, const char *iface_ver) VERIFY(wh); VERIFY(wh->version); - if (strcmp(wh->version, iface_ver)) { - snprintf (msg, sizeof(msg), - "provider interface version mismatch: need '%s', found '%s'", - iface_ver, wh->version); - logger (WSREP_LOG_ERROR, msg); + if (wsrep_check_iface_version(wh->version, iface_ver)) return EINVAL; - } VERIFY(wh->init); VERIFY(wh->options_set); @@ -107,6 +118,15 @@ static wsrep_loader_fun wsrep_dlf(void *dlh, const char *sym) return alias.dlfun; } +static int wsrep_check_version_symbol(void *dlh) +{ + char** dlversion = NULL; + dlversion = (char**) dlsym(dlh, "wsrep_interface_version"); + if (dlversion == NULL) + return 0; + return wsrep_check_iface_version(*dlversion, WSREP_INTERFACE_VERSION); +} + extern int wsrep_dummy_loader(wsrep_t *w); int wsrep_load(const char *spec, wsrep_t **hptr, wsrep_log_cb_t log_cb) @@ -151,6 +171,11 @@ int wsrep_load(const char *spec, wsrep_t **hptr, wsrep_log_cb_t log_cb) goto out; } + if (wsrep_check_version_symbol(dlh) != 0) { + ret = EINVAL; + goto out; + } + if ((ret = (*dlfun)(*hptr)) != 0) { snprintf(msg, sizeof(msg), "wsrep_load(): loader failed: %s", strerror(ret)); From fce9a0c46ab9a8a95345a8d661d2b047e64ad7d6 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 18 Apr 2017 11:55:04 +0530 Subject: [PATCH 016/276] Bump WSREP_PATCH_VERSION to 20 Signed-off-by: Sachin Setiya --- cmake/wsrep.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index 6d92537e40f..b9e79539ac6 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -18,7 +18,7 @@ # so WSREP_VERSION is produced regardless # Set the patch version -SET(WSREP_PATCH_VERSION "14") +SET(WSREP_PATCH_VERSION "20") # MariaDB addition: Revision number of the last revision merged from # codership branch visible in @@visible_comment. From 1d821bad8cc4a64a27092fa1f476fef6a5deea21 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 18 Apr 2017 14:00:21 +0530 Subject: [PATCH 017/276] Fix sys_vars.wsrep_provider_options_basic test failure. --- wsrep/wsrep_dummy.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wsrep/wsrep_dummy.c b/wsrep/wsrep_dummy.c index 5f1ea63cc40..7c3e3d7f9e2 100644 --- a/wsrep/wsrep_dummy.c +++ b/wsrep/wsrep_dummy.c @@ -86,7 +86,8 @@ static wsrep_status_t dummy_options_set( static char* dummy_options_get (wsrep_t* w) { WSREP_DBUG_ENTER(w); - return strdup(WSREP_DUMMY(w)->options); + char * opt= WSREP_DUMMY(w)->options; + return opt ? strdup(opt) : NULL; } static wsrep_status_t dummy_connect( From b0bae66b8ef95844b5d94a72c35d2a5fe6460275 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Thu, 20 Apr 2017 18:33:20 +0530 Subject: [PATCH 018/276] FIX test failures --- mysql-test/suite/galera/disabled.def | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 mysql-test/suite/galera/disabled.def diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def new file mode 100644 index 00000000000..c91654c2023 --- /dev/null +++ b/mysql-test/suite/galera/disabled.def @@ -0,0 +1,2 @@ +galera_var_dirty_reads : MDEV-12539 +query_cache : MDEV-12539 From 85f53e46f049725431ded4718abae07d4d779e66 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Fri, 21 Apr 2017 09:27:23 +0530 Subject: [PATCH 019/276] Fix BuildBot Failure. In some slave like p8-xenial-bintar-debug we are using C90 standard which does not allow declaration and code to be mixed. --- wsrep/wsrep_dummy.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wsrep/wsrep_dummy.c b/wsrep/wsrep_dummy.c index 7c3e3d7f9e2..ff9b0b0ea92 100644 --- a/wsrep/wsrep_dummy.c +++ b/wsrep/wsrep_dummy.c @@ -85,8 +85,9 @@ static wsrep_status_t dummy_options_set( static char* dummy_options_get (wsrep_t* w) { + char * opt; WSREP_DBUG_ENTER(w); - char * opt= WSREP_DUMMY(w)->options; + opt= WSREP_DUMMY(w)->options; return opt ? strdup(opt) : NULL; } From 2cd585ad34a64b250a17e29301ffd72d8ddb4d9c Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Sat, 22 Apr 2017 11:19:48 -0400 Subject: [PATCH 020/276] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index bc0b19220b6..ade2e129c54 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=55 +MYSQL_VERSION_PATCH=56 MYSQL_VERSION_EXTRA= From 92316dcdeb01c1da208f92445d68411ab3ecefa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Dor=C5=88=C3=A1k?= Date: Sun, 30 Apr 2017 14:40:38 +1000 Subject: [PATCH 021/276] wsrep.cnf - wsrep_on=1 by default From: https://github.com/devexp-db/mariadb/blob/f27/mariadb-galera.cnf.patch --- support-files/wsrep.cnf.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/support-files/wsrep.cnf.sh b/support-files/wsrep.cnf.sh index a5390855ca1..51ce3dca2dd 100644 --- a/support-files/wsrep.cnf.sh +++ b/support-files/wsrep.cnf.sh @@ -30,6 +30,9 @@ bind-address=0.0.0.0 ## WSREP options ## +# Enable wsrep +wsrep_on=1 + # Full path to wsrep provider library or 'none' wsrep_provider=none From 4f1a3dd115ec2050265f0a4cf0e8f3a3791c3e96 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 3 May 2017 11:11:33 +0530 Subject: [PATCH 022/276] Add tokudb_bugs test to disabled. --- storage/tokudb/mysql-test/tokudb_bugs/disabled.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/tokudb/mysql-test/tokudb_bugs/disabled.def b/storage/tokudb/mysql-test/tokudb_bugs/disabled.def index b6fa575522b..81f9c69f9bc 100644 --- a/storage/tokudb/mysql-test/tokudb_bugs/disabled.def +++ b/storage/tokudb/mysql-test/tokudb_bugs/disabled.def @@ -12,4 +12,4 @@ xa-2: Not for 5.5 6053: N/A checkpoint_lock_2: test can not work when the checkpoint_safe_lock is a fair rwlock - +xa-3 : failed in 5.5.56-galera From 9df17790f2b0199864443d8aa99e30cb10e71b20 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 3 May 2017 13:10:17 -0400 Subject: [PATCH 023/276] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ade2e129c54..68358acf754 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=56 +MYSQL_VERSION_PATCH=57 MYSQL_VERSION_EXTRA= From e8a2a751212a04e835b2b03408132ecbbab52410 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 20 Jun 2017 14:29:25 +0530 Subject: [PATCH 024/276] MDEV-11036 Add link wsrep_sst_rsync_wan -> wsrep_sst_rsync Currently galera has capability of using delta transfer algorithm (rsync) for SST. But for using delta transfer we have change/copy wsrep_sst_rsync wsrep_sst_rsync_wan. This patch creates a symbolic link of wsrep_sst_rsync_wan to wsrep_sst_rsync. --- .gitignore | 1 + .../Ubuntu/mariadb-galera-server-5.5.files.in | 1 + scripts/CMakeLists.txt | 16 ++++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/.gitignore b/.gitignore index 4e3d2a1e66e..b00f7dd5643 100644 --- a/.gitignore +++ b/.gitignore @@ -116,6 +116,7 @@ scripts/mytop scripts/wsrep_sst_common scripts/wsrep_sst_mysqldump scripts/wsrep_sst_rsync +scripts/wsrep_sst_rsync_wan scripts/wsrep_sst_xtrabackup scripts/wsrep_sst_xtrabackup-v2 sql-bench/bench-count-distinct diff --git a/debian/dist/Ubuntu/mariadb-galera-server-5.5.files.in b/debian/dist/Ubuntu/mariadb-galera-server-5.5.files.in index f921759fa84..ccfc7350ed0 100644 --- a/debian/dist/Ubuntu/mariadb-galera-server-5.5.files.in +++ b/debian/dist/Ubuntu/mariadb-galera-server-5.5.files.in @@ -44,6 +44,7 @@ usr/bin/resolveip usr/bin/wsrep_sst_common usr/bin/wsrep_sst_mysqldump usr/bin/wsrep_sst_rsync +usr/bin/wsrep_sst_rsync_wan usr/bin/wsrep_sst_xtrabackup usr/bin/wsrep_sst_xtrabackup-v2 usr/share/doc/mariadb-galera-server-5.5/mysqld.sym.gz diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index eacb516d618..da4f0d9b3bb 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -374,6 +374,22 @@ ELSE() COMPONENT ${${file}_COMPONENT} ) ENDFOREACH() + SET (wsrep_sst_rsync_wan ${CMAKE_CURRENT_BINARY_DIR}/wsrep_sst_rsync_wan) + ADD_CUSTOM_COMMAND( + OUTPUT ${wsrep_sst_rsync_wan} + COMMAND ${CMAKE_COMMAND} ARGS -E create_symlink + wsrep_sst_rsync + wsrep_sst_rsync_wan + ) + ADD_CUSTOM_TARGET(symlink_wsrep_sst_rsync + ALL + DEPENDS ${wsrep_sst_rsync_wan} + ) + INSTALL( + FILES ${wsrep_sst_rsync_wan} + DESTINATION ${INSTALL_BINDIR} + COMPONENT Server + ) ENDIF() # Install libgcc as mylibgcc.a From a4bc8db216b4dd61ca0b1cb5a8b7806437416dc7 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Thu, 27 Apr 2017 19:51:18 +0300 Subject: [PATCH 025/276] MW-322 - CTAS fix merged to 5.5-v25 branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Lindström --- sql/handler.cc | 5 +++ sql/sql_base.cc | 13 +++++- sql/sql_insert.cc | 29 +++++++++++++ sql/wsrep_mysqld.cc | 24 +++-------- sql/wsrep_mysqld.h | 11 +++++ storage/innobase/handler/ha_innodb.cc | 62 +++++++++++++++------------ 6 files changed, 99 insertions(+), 45 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index 5917ba505a3..5cff15821c7 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -5426,6 +5426,11 @@ void ha_wsrep_fake_trx_id(THD *thd) DBUG_VOID_RETURN; } + if (thd->wsrep_ws_handle.trx_id != WSREP_UNDEFINED_TRX_ID) + { + WSREP_DEBUG("fake trx id skipped: %lu", thd->wsrep_ws_handle.trx_id); + DBUG_VOID_RETURN; + } handlerton *hton= installed_htons[DB_TYPE_INNODB]; if (hton && hton->wsrep_fake_trx_id) { diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 345c7acb508..3832a4ab471 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -9506,11 +9506,22 @@ void tdc_remove_table(THD *thd, enum_tdc_remove_table_type remove_type, I_P_List_iterator it(share->free_tables); #ifndef DBUG_OFF if (remove_type == TDC_RT_REMOVE_ALL) +#ifdef WITH_WSREP { - DBUG_ASSERT(share->used_tables.is_empty()); + /* following assert may cause false posivive fire for CTAS */ + if (thd->lex->sql_command != SQLCOM_CREATE_TABLE) + { +#endif /* WITH_WSREP */ + { + DBUG_ASSERT(share->used_tables.is_empty()); + } +#ifdef WITH_WSREP + } } else if (remove_type == TDC_RT_REMOVE_NOT_OWN || remove_type == TDC_RT_REMOVE_NOT_OWN_AND_MARK_NOT_USABLE) +#endif /* WITH_WSREP */ + { I_P_List_iterator it2(share->used_tables); while ((table= it2++)) diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 4f0129c6e70..e8f00b76d5a 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -4313,9 +4313,38 @@ bool select_create::send_eof() */ if (!table->s->tmp_table) { + #ifdef WITH_WSREP + /* + append table level exclusive key for CTAS + */ + wsrep_key_arr_t key_arr= {0, 0}; + wsrep_prepare_keys_for_isolation(thd, + create_table->db, + create_table->table_name, + table_list, + &key_arr); + int rcode = wsrep->append_key( + wsrep, + &thd->wsrep_ws_handle, + key_arr.keys, //&wkey, + key_arr.keys_len, + WSREP_KEY_EXCLUSIVE, + false); + wsrep_keys_free(&key_arr); + if (rcode) { + DBUG_PRINT("wsrep", ("row key failed: %d", rcode)); + WSREP_ERROR("Appending table key for CTAS failed: %s, %d", + (wsrep_thd_query(thd)) ? + wsrep_thd_query(thd) : "void", rcode); + return true; + } + /* If commit fails, we should be able to reset the OK status. */ + thd->stmt_da->can_overwrite_status= TRUE; +#endif /* WITH_WSREP */ trans_commit_stmt(thd); trans_commit_implicit(thd); #ifdef WITH_WSREP + thd->stmt_da->can_overwrite_status= FALSE; mysql_mutex_lock(&thd->LOCK_wsrep_thd); if (thd->wsrep_conflict_state != NO_CONFLICT) { diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 53d53c2c404..4ac346ca0a3 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -944,17 +944,7 @@ bool wsrep_sync_wait (THD* thd, uint mask) return false; } -/* - * Helpers to deal with TOI key arrays - */ -typedef struct wsrep_key_arr -{ - wsrep_key_t* keys; - size_t keys_len; -} wsrep_key_arr_t; - - -static void wsrep_keys_free(wsrep_key_arr_t* key_arr) +void wsrep_keys_free(wsrep_key_arr_t* key_arr) { for (size_t i= 0; i < key_arr->keys_len; ++i) { @@ -1019,11 +1009,11 @@ static bool wsrep_prepare_key_for_isolation(const char* db, } /* Prepare key list from db/table and table_list */ -static bool wsrep_prepare_keys_for_isolation(THD* thd, - const char* db, - const char* table, - const TABLE_LIST* table_list, - wsrep_key_arr_t* ka) +bool wsrep_prepare_keys_for_isolation(THD* thd, + const char* db, + const char* table, + const TABLE_LIST* table_list, + wsrep_key_arr_t* ka) { ka->keys= 0; ka->keys_len= 0; @@ -1554,7 +1544,7 @@ wsrep_grant_mdl_exception(MDL_context *requestor_ctx, } else if (request_thd->lex->sql_command == SQLCOM_DROP_TABLE) { - WSREP_DEBUG("DROP caused BF abort"); + WSREP_DEBUG("DROP caused BF abort, conf %d", granted_thd->wsrep_conflict_state); ticket->wsrep_report(wsrep_debug); mysql_mutex_unlock(&granted_thd->LOCK_wsrep_thd); wsrep_abort_thd((void*)request_thd, (void*)granted_thd, 1); diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index bd45399a948..23e9718e533 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -322,4 +322,15 @@ const wsrep_uuid_t* wsrep_xid_uuid(const xid_t*); wsrep_seqno_t wsrep_xid_seqno(const xid_t*); extern "C" int wsrep_is_wsrep_xid(const void* xid); +typedef struct wsrep_key_arr +{ + wsrep_key_t* keys; + size_t keys_len; +} wsrep_key_arr_t; +bool wsrep_prepare_keys_for_isolation(THD* thd, + const char* db, + const char* table, + const TABLE_LIST* table_list, + wsrep_key_arr_t* ka); +void wsrep_keys_free(wsrep_key_arr_t* key_arr); #endif /* WSREP_MYSQLD_H */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5261a50fdd6..ad31598384c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6000,10 +6000,14 @@ report_error: prebuilt->table->flags, user_thd); #ifdef WITH_WSREP - if (!error_result && - wsrep_thd_exec_mode(user_thd) == LOCAL_STATE && - wsrep_on(user_thd) && - !wsrep_consistency_check(user_thd)) + if (!error_result + && wsrep_on(user_thd) + && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE + && !wsrep_consistency_check(user_thd)) + && (sql_command != SQLCOM_CREATE_TABLE) + && (sql_command != SQLCOM_LOAD || + thd_binlog_format(user_thd) == BINLOG_FORMAT_ROW)) { + { if (wsrep_append_keys(user_thd, false, record, NULL)) { @@ -12788,8 +12792,10 @@ wsrep_innobase_kill_one_trx( wsrep_thd_thread_id(thd), victim_trx->id); - WSREP_DEBUG("Aborting query: %s", - (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void"); + WSREP_DEBUG("Aborting query: %s conf %d trx: %lu", + (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void", + wsrep_thd_conflict_state(thd), + wsrep_thd_ws_handle(thd)->trx_id); wsrep_thd_LOCK(thd); @@ -12841,9 +12847,8 @@ wsrep_innobase_kill_one_trx( } else { rcode = wsrep->abort_pre_commit( wsrep, bf_seqno, - (wsrep_trx_id_t)victim_trx->id + (wsrep_trx_id_t)wsrep_thd_ws_handle(thd)->trx_id ); - switch (rcode) { case WSREP_WARNING: WSREP_DEBUG("cancel commit warning: %llu", @@ -12973,15 +12978,17 @@ wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd, my_bool signal) { DBUG_ENTER("wsrep_innobase_abort_thd"); - trx_t* victim_trx = thd_to_trx(victim_thd); - trx_t* bf_trx = (bf_thd) ? thd_to_trx(bf_thd) : NULL; + + trx_t* victim_trx = thd_to_trx(victim_thd); + trx_t* bf_trx = (bf_thd) ? thd_to_trx(bf_thd) : NULL; + + WSREP_DEBUG("abort transaction: BF: %s victim: %s victim conf: %d", + wsrep_thd_query(bf_thd), + wsrep_thd_query(victim_thd), + wsrep_thd_conflict_state(victim_thd)); ut_ad(!mutex_own(&kernel_mutex)); - WSREP_DEBUG("abort transaction: BF: %s victim: %s", - wsrep_thd_query(bf_thd), - wsrep_thd_query(victim_thd)); - if (victim_trx) { int rcode = wsrep_innobase_kill_one_trx( @@ -13001,24 +13008,24 @@ wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd, static int innobase_wsrep_set_checkpoint(handlerton* hton, const XID* xid) { DBUG_ASSERT(hton == innodb_hton_ptr); - if (wsrep_is_wsrep_xid(xid)) { - mtr_t mtr; - mtr_start(&mtr); - trx_sysf_t* sys_header = trx_sysf_get(&mtr); - trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr); - mtr_commit(&mtr); - innobase_flush_logs(hton); - return 0; - } else { - return 1; - } + if (wsrep_is_wsrep_xid(xid)) { + mtr_t mtr; + mtr_start(&mtr); + trx_sysf_t* sys_header = trx_sysf_get(&mtr); + trx_sys_update_wsrep_checkpoint(xid, sys_header, &mtr); + mtr_commit(&mtr); + innobase_flush_logs(hton); + return 0; + } else { + return 1; + } } static int innobase_wsrep_get_checkpoint(handlerton* hton, XID* xid) { DBUG_ASSERT(hton == innodb_hton_ptr); - trx_sys_read_wsrep_checkpoint(xid); - return 0; + trx_sys_read_wsrep_checkpoint(xid); + return 0; } static void @@ -13030,6 +13037,7 @@ wsrep_fake_trx_id( mutex_enter(&kernel_mutex); trx_id_t trx_id = trx_sys_get_new_trx_id(); mutex_exit(&kernel_mutex); + WSREP_DEBUG("innodb fake trx id: %lu thd: %s", trx_id, wsrep_thd_query(thd)); (void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); } From 48b7245bf285b757aa46bd37c86b9c202705e3bb Mon Sep 17 00:00:00 2001 From: sjaakola Date: Thu, 27 Apr 2017 20:28:22 +0300 Subject: [PATCH 026/276] MW-369 - merged fix for FK issue from 5.6-v25 branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Lindström --- storage/innobase/row/row0ins.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/storage/innobase/row/row0ins.c b/storage/innobase/row/row0ins.c index c380390d62a..3726c517ff0 100644 --- a/storage/innobase/row/row0ins.c +++ b/storage/innobase/row/row0ins.c @@ -1082,11 +1082,12 @@ row_ins_foreign_check_on_constraint( #ifdef WITH_WSREP err = wsrep_append_foreign_key( - thr_get_trx(thr), - foreign, - clust_rec, - clust_index, - FALSE, FALSE); + thr_get_trx(thr), + foreign, + clust_rec, + clust_index, + FALSE, + (node) ? TRUE : FALSE); if (err != DB_SUCCESS) { fprintf(stderr, "WSREP: foreign key append failed: %lu\n", err); @@ -1247,6 +1248,9 @@ row_ins_check_foreign_constraint( ulint* offsets = offsets_; rec_offs_init(offsets_); +#ifdef WITH_WSREP + upd_node= NULL; +#endif /* WITH_WSREP */ run_again: #ifdef UNIV_SYNC_DEBUG ut_ad(rw_lock_own(&dict_operation_lock, RW_LOCK_SHARED)); @@ -1429,9 +1433,10 @@ run_again: err = wsrep_append_foreign_key( thr_get_trx(thr), foreign, - rec, - check_index, - check_ref, TRUE); + rec, + check_index, + check_ref, + (upd_node) ? TRUE : FALSE); #endif /* WITH_WSREP */ goto end_scan; } else if (foreign->type != 0) { From dbda504275af5eded33f62f509bff0b46e1560e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 20 Jul 2017 13:52:22 +0300 Subject: [PATCH 027/276] Fix merge error and compiler warning. --- storage/innobase/handler/ha_innodb.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index ad31598384c..2f1e3b41c47 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6003,14 +6003,12 @@ report_error: if (!error_result && wsrep_on(user_thd) && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE - && !wsrep_consistency_check(user_thd)) + && !wsrep_consistency_check(user_thd) && (sql_command != SQLCOM_CREATE_TABLE) && (sql_command != SQLCOM_LOAD || thd_binlog_format(user_thd) == BINLOG_FORMAT_ROW)) { - { - if (wsrep_append_keys(user_thd, false, record, NULL)) - { + if (wsrep_append_keys(user_thd, false, record, NULL)) { DBUG_PRINT("wsrep", ("row key failed")); error_result = HA_ERR_INTERNAL_ERROR; goto wsrep_error; @@ -13028,16 +13026,15 @@ static int innobase_wsrep_get_checkpoint(handlerton* hton, XID* xid) return 0; } -static void -wsrep_fake_trx_id( -/*==================*/ +static void wsrep_fake_trx_id( handlerton *hton, THD *thd) /*!< in: user thread handle */ { mutex_enter(&kernel_mutex); trx_id_t trx_id = trx_sys_get_new_trx_id(); mutex_exit(&kernel_mutex); - WSREP_DEBUG("innodb fake trx id: %lu thd: %s", trx_id, wsrep_thd_query(thd)); + WSREP_DEBUG("innodb fake trx id: %llu thd: %s", + trx_id, wsrep_thd_query(thd)); (void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); } From 20ab1665af07080441ecea0734eb78b3aa95e58e Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Fri, 5 May 2017 10:55:45 +0300 Subject: [PATCH 028/276] MW-322 Fix compilation error with debug build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Lindström --- storage/innobase/handler/ha_innodb.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 2f1e3b41c47..97c6cf84910 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -13033,8 +13033,7 @@ static void wsrep_fake_trx_id( mutex_enter(&kernel_mutex); trx_id_t trx_id = trx_sys_get_new_trx_id(); mutex_exit(&kernel_mutex); - WSREP_DEBUG("innodb fake trx id: %llu thd: %s", - trx_id, wsrep_thd_query(thd)); + WSREP_DEBUG("innodb fake trx id: %llu thd: %s", trx_id, wsrep_thd_query(thd)); (void *)wsrep_ws_handle_for_trx(wsrep_thd_ws_handle(thd), trx_id); } From 6326f0eac6f5a7b8132676af1b37bd7ec453009e Mon Sep 17 00:00:00 2001 From: sjaakola Date: Fri, 5 May 2017 11:09:01 +0300 Subject: [PATCH 029/276] MW-322 CTAS fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pushed fix for a typo Signed-off-by: Jan Lindström --- sql/sql_insert.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index e8f00b76d5a..09e3d2c3c61 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -4313,7 +4313,7 @@ bool select_create::send_eof() */ if (!table->s->tmp_table) { - #ifdef WITH_WSREP +#ifdef WITH_WSREP /* append table level exclusive key for CTAS */ From 519e4322e1d5ae928d6939d88d0395d60f224da2 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Mon, 8 May 2017 23:12:51 +0300 Subject: [PATCH 030/276] MW-369 FK fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skipping wsrep extra FK check for applier and replayer Signed-off-by: Jan Lindström --- storage/innobase/row/row0upd.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/innobase/row/row0upd.c b/storage/innobase/row/row0upd.c index 40e9fbfde23..dd9c7e8defd 100644 --- a/storage/innobase/row/row0upd.c +++ b/storage/innobase/row/row0upd.c @@ -1822,7 +1822,9 @@ row_upd_sec_index_entry( index, offsets, thr, &mtr); } #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && + if (wsrep_on(trx->mysql_thd) && + !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && + err == DB_SUCCESS && !referenced && !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && ((upd_node_t*)parent)->cascade_node == node) && From a82611771b58fcfee024ce59c13adc2a928c5428 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 8 Mar 2017 13:08:21 +0100 Subject: [PATCH 031/276] MW-86 Add separate wsrep_sync_wait bitmask value for SHOW commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, setting `wsrep_sync_wait = 1` would have an effect on both SELECT and SHOW statements. This patch changes wsrep_sync_wait so that bitmask value 1 is used for SELECT statements, while bitmask value 8 is reserved for SHOW statements. It is still possible to achieve sync wait on both SELECT and SHOW statements by setting `wsrep_sync_wait = 9`. Signed-off-by: Jan Lindström --- sql/sql_parse.cc | 72 ++++++++++++++++------------------------------ sql/sql_parse.h | 5 ++++ sql/wsrep_mysqld.h | 5 ++-- 3 files changed, 32 insertions(+), 50 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 745ea171030..3862de0c135 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2473,9 +2473,7 @@ mysql_execute_command(THD *thd) #endif case SQLCOM_SHOW_STATUS_PROC: case SQLCOM_SHOW_STATUS_FUNC: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if ((res= check_table_access(thd, SELECT_ACL, all_tables, FALSE, UINT_MAX, FALSE))) goto error; @@ -2483,7 +2481,9 @@ mysql_execute_command(THD *thd) break; case SQLCOM_SHOW_STATUS: { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); execute_show_status(thd, all_tables); + #ifdef WITH_WSREP if (lex->sql_command == SQLCOM_SHOW_STATUS) wsrep_free_status(thd); #endif /* WITH_WSREP */ @@ -2497,27 +2497,22 @@ mysql_execute_command(THD *thd) case SQLCOM_SHOW_PLUGINS: case SQLCOM_SHOW_FIELDS: case SQLCOM_SHOW_KEYS: -#ifndef WITH_WSREP case SQLCOM_SHOW_VARIABLES: case SQLCOM_SHOW_CHARSETS: case SQLCOM_SHOW_COLLATIONS: case SQLCOM_SHOW_STORAGE_ENGINES: case SQLCOM_SHOW_PROFILE: -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); case SQLCOM_SHOW_CLIENT_STATS: case SQLCOM_SHOW_USER_STATS: case SQLCOM_SHOW_TABLE_STATS: case SQLCOM_SHOW_INDEX_STATS: case SQLCOM_SELECT: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; - case SQLCOM_SHOW_VARIABLES: - case SQLCOM_SHOW_CHARSETS: - case SQLCOM_SHOW_COLLATIONS: - case SQLCOM_SHOW_STORAGE_ENGINES: - case SQLCOM_SHOW_PROFILE: -#endif /* WITH_WSREP */ { +#ifdef WITH_WSREP + if (lex->sql_command == SQLCOM_SELECT) + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_READ); +#endif /* WITH_WSREP */ thd->status_var.last_query_cost= 0.0; /* @@ -3079,9 +3074,7 @@ end_with_restore_list: goto error; #else { -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); /* Access check: @@ -3138,9 +3131,7 @@ end_with_restore_list: case SQLCOM_CHECKSUM: { DBUG_ASSERT(first_table == all_tables && first_table != 0); -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && wsrep_sync_wait(thd)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_READ); if (check_table_access(thd, SELECT_ACL, all_tables, FALSE, UINT_MAX, FALSE)) @@ -3150,11 +3141,8 @@ end_with_restore_list: break; } case SQLCOM_UPDATE: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE)) goto error; -#endif /* WITH_WSREP */ { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE); ha_rows found= 0, updated= 0; DBUG_ASSERT(first_table == all_tables && first_table != 0); if (update_precheck(thd, all_tables)) @@ -3193,10 +3181,7 @@ end_with_restore_list: /* if we switched from normal update, rights are checked */ if (up_result != 2) { -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE); if ((res= multi_update_precheck(thd, all_tables))) break; } @@ -3266,10 +3251,7 @@ end_with_restore_list: break; } case SQLCOM_REPLACE: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE)) goto error; -#endif /* WITH_WSREP */ + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); #ifndef DBUG_OFF if (mysql_bin_log.is_open()) { @@ -3305,11 +3287,8 @@ end_with_restore_list: } #endif case SQLCOM_INSERT: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE)) goto error; -#endif /* WITH_WSREP */ { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); DBUG_ASSERT(first_table == all_tables && first_table != 0); if ((res= insert_precheck(thd, all_tables))) break; @@ -3350,12 +3329,9 @@ end_with_restore_list: } case SQLCOM_REPLACE_SELECT: case SQLCOM_INSERT_SELECT: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE)) goto error; -#endif /* WITH_WSREP */ { - select_result *sel_result; + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); + select_insert *sel_result; DBUG_ASSERT(first_table == all_tables && first_table != 0); if ((res= insert_precheck(thd, all_tables))) break; @@ -3450,11 +3426,8 @@ end_with_restore_list: break; } case SQLCOM_DELETE: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE)) goto error; -#endif /* WITH_WSREP */ { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE); DBUG_ASSERT(first_table == all_tables && first_table != 0); if ((res= delete_precheck(thd, all_tables))) break; @@ -3469,11 +3442,8 @@ end_with_restore_list: break; } case SQLCOM_DELETE_MULTI: -#ifdef WITH_WSREP - if (WSREP_CLIENT(thd) && - wsrep_sync_wait(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE)) goto error; -#endif /* WITH_WSREP */ { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE); DBUG_ASSERT(first_table == all_tables && first_table != 0); TABLE_LIST *aux_tables= thd->lex->auxiliary_table_list.first; multi_delete *del_result; @@ -3840,6 +3810,7 @@ end_with_restore_list: my_error(ER_WRONG_DB_NAME, MYF(0), lex->name.str); break; } + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); res= mysqld_show_create_db(thd, lex->name.str, &lex->create_info); break; } @@ -3891,6 +3862,7 @@ end_with_restore_list: /* lex->unit.cleanup() is called outside, no need to call it here */ break; case SQLCOM_SHOW_CREATE_EVENT: + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); res= Events::show_create_event(thd, lex->spname->m_db, lex->spname->m_name); break; @@ -4751,12 +4723,14 @@ create_sp_error: } case SQLCOM_SHOW_CREATE_PROC: { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (sp_show_create_routine(thd, TYPE_ENUM_PROCEDURE, lex->spname)) goto error; break; } case SQLCOM_SHOW_CREATE_FUNC: { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (sp_show_create_routine(thd, TYPE_ENUM_FUNCTION, lex->spname)) goto error; break; @@ -4769,6 +4743,7 @@ create_sp_error: stored_procedure_type type= (lex->sql_command == SQLCOM_SHOW_PROC_CODE ? TYPE_ENUM_PROCEDURE : TYPE_ENUM_FUNCTION); + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (sp_cache_routine(thd, type, lex->spname, FALSE, &sp)) goto error; if (!sp || sp->show_routine_code(thd)) @@ -4790,6 +4765,7 @@ create_sp_error: if (check_ident_length(&lex->spname->m_name)) goto error; + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (show_create_trigger(thd, lex->spname)) goto error; /* Error has been already logged. */ diff --git a/sql/sql_parse.h b/sql/sql_parse.h index 58885594a5d..e176e4eadf8 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -218,10 +218,15 @@ inline bool is_supported_parser_charset(CHARSET_INFO *cs) if (WSREP(thd) || (thd && thd->wsrep_exec_mode==TOTAL_ORDER)) \ wsrep_to_isolation_end(thd); +#define WSREP_SYNC_WAIT(thd_, before_) \ + if (WSREP_CLIENT(thd_) && \ + wsrep_sync_wait(thd_, before_)) goto error; + #else #define WSREP_TO_ISOLATION_BEGIN(db_, table_, table_list_) #define WSREP_TO_ISOLATION_END +#define WSREP_SYNC_WAIT(thd_, before_) #endif /* WITH_WSREP */ diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 23e9718e533..bb33d2f1069 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -105,11 +105,12 @@ extern my_bool wsrep_creating_startup_threads; enum enum_wsrep_OSU_method { WSREP_OSU_TOI, WSREP_OSU_RSU }; enum enum_wsrep_sync_wait { WSREP_SYNC_WAIT_NONE = 0x0, - // show, select, begin + // select, begin WSREP_SYNC_WAIT_BEFORE_READ = 0x1, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE = 0x2, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE = 0x4, - WSREP_SYNC_WAIT_MAX = 0x7 + WSREP_SYNC_WAIT_BEFORE_SHOW = 0x8, + WSREP_SYNC_WAIT_MAX = 0xF }; // MySQL status variables From 34853fa7937b8081516eddb0226e8031f06ba96f Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Mon, 13 Mar 2017 15:35:04 +0100 Subject: [PATCH 032/276] MW-86 Additional wsrep_sync_wait coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following commands are now subject to wsrep_sync_wait with bitmask value 8: SHOW BINARY LOGS SHOW BINLOG EVENTS SHOW GRANTS Signed-off-by: Jan Lindström --- sql/sql_parse.cc | 10 +++++++--- sql/sql_parse.h | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 3862de0c135..c222adbbac2 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2502,7 +2502,6 @@ mysql_execute_command(THD *thd) case SQLCOM_SHOW_COLLATIONS: case SQLCOM_SHOW_STORAGE_ENGINES: case SQLCOM_SHOW_PROFILE: - WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); case SQLCOM_SHOW_CLIENT_STATS: case SQLCOM_SHOW_USER_STATS: case SQLCOM_SHOW_TABLE_STATS: @@ -2510,8 +2509,10 @@ mysql_execute_command(THD *thd) case SQLCOM_SELECT: { #ifdef WITH_WSREP - if (lex->sql_command == SQLCOM_SELECT) - WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_READ); + if (lex->sql_command == SQLCOM_SELECT) + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_READ) + else + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW) #endif /* WITH_WSREP */ thd->status_var.last_query_cost= 0.0; @@ -2635,6 +2636,7 @@ case SQLCOM_PREPARE: case SQLCOM_SHOW_RELAYLOG_EVENTS: /* fall through */ case SQLCOM_SHOW_BINLOG_EVENTS: { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); if (check_global_access(thd, REPL_SLAVE_ACL)) goto error; res = mysql_show_binlog_events(thd); @@ -3061,6 +3063,7 @@ end_with_restore_list: { if (check_global_access(thd, SUPER_ACL | REPL_CLIENT_ACL)) goto error; + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); res = show_binlogs(thd); break; } @@ -4163,6 +4166,7 @@ end_with_restore_list: !strcmp(thd->security_ctx->priv_user, grant_user->user.str)) || !check_access(thd, SELECT_ACL, "mysql", NULL, NULL, 1, 0)) { + WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_SHOW); res = mysql_show_grants(thd, grant_user); } break; diff --git a/sql/sql_parse.h b/sql/sql_parse.h index e176e4eadf8..2093ecc85ee 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -219,8 +219,8 @@ inline bool is_supported_parser_charset(CHARSET_INFO *cs) wsrep_to_isolation_end(thd); #define WSREP_SYNC_WAIT(thd_, before_) \ - if (WSREP_CLIENT(thd_) && \ - wsrep_sync_wait(thd_, before_)) goto error; + { if (WSREP_CLIENT(thd_) && \ + wsrep_sync_wait(thd_, before_)) goto error; } #else From be416cfa3b6ec6faad3e0ac273fc8698de61094d Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Mon, 13 Mar 2017 22:45:42 +0100 Subject: [PATCH 033/276] MW-86 Removed unnecessary wsrep_sync_wait before processing SQLCOM_REPLACE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync waiting before processing SQLCOM_REPLACE was not necessary given that this case falls through to processing of SQLCOM_INSERT. In case of SQLCOM_REPLACE, wsrep_sync_wait would be called twice. Signed-off-by: Jan Lindström --- sql/sql_parse.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index c222adbbac2..52b18318849 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -3254,7 +3254,6 @@ end_with_restore_list: break; } case SQLCOM_REPLACE: - WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); #ifndef DBUG_OFF if (mysql_bin_log.is_open()) { From 970859a599714d469c15b5fd49703d22b35317ad Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Wed, 24 May 2017 14:46:25 +0300 Subject: [PATCH 034/276] MW-383 Bumped wsrep patch version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Lindström --- cmake/wsrep.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index b9e79539ac6..6a9dd720f77 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -18,7 +18,7 @@ # so WSREP_VERSION is produced regardless # Set the patch version -SET(WSREP_PATCH_VERSION "20") +SET(WSREP_PATCH_VERSION "21") # MariaDB addition: Revision number of the last revision merged from # codership branch visible in @@visible_comment. From e5c488a49ba54aee596f4ff0080f3a855461b911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 21 Jul 2017 08:29:52 +0300 Subject: [PATCH 035/276] Fix failing test case. --- mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result b/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result index 1e7b9364570..0df3dff8990 100644 --- a/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result +++ b/mysql-test/suite/sys_vars/r/wsrep_sync_wait_basic.result @@ -34,11 +34,9 @@ SELECT @@session.wsrep_sync_wait; @@session.wsrep_sync_wait 7 SET @@session.wsrep_sync_wait=8; -Warnings: -Warning 1292 Truncated incorrect wsrep_sync_wait value: '8' SELECT @@session.wsrep_sync_wait; @@session.wsrep_sync_wait -7 +8 # invalid values SET @@global.wsrep_sync_wait=NULL; From 07f8360f173c824df592c3ff130bfc256aba2bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Jul 2017 11:06:42 +0300 Subject: [PATCH 036/276] MDEV-10379: Failing assertion: xid_seqno > trx_sys_cur_xid_seqno Remove too strict debug assertion. --- storage/innobase/trx/trx0sys.c | 20 -------------------- storage/xtradb/trx/trx0sys.c | 20 -------------------- 2 files changed, 40 deletions(-) diff --git a/storage/innobase/trx/trx0sys.c b/storage/innobase/trx/trx0sys.c index a0fcc2b2370..f8bd87034e8 100644 --- a/storage/innobase/trx/trx0sys.c +++ b/storage/innobase/trx/trx0sys.c @@ -825,26 +825,6 @@ trx_sys_update_wsrep_checkpoint( trx_sysf_t* sys_header, /*!< in: sys_header */ mtr_t* mtr) /*!< in: mtr */ { - -#ifdef UNIV_DEBUG - { - /* Check that seqno is monotonically increasing */ - unsigned char xid_uuid[16]; - long long xid_seqno = read_wsrep_xid_seqno(xid); - read_wsrep_xid_uuid(xid, xid_uuid); - if (!memcmp(xid_uuid, trx_sys_cur_xid_uuid, 8)) - { - ut_ad(xid_seqno > trx_sys_cur_xid_seqno); - trx_sys_cur_xid_seqno = xid_seqno; - } - else - { - memcpy(trx_sys_cur_xid_uuid, xid_uuid, 16); - } - trx_sys_cur_xid_seqno = xid_seqno; - } -#endif /* UNIV_DEBUG */ - ut_ad(xid && mtr && sys_header); ut_a(xid->formatID == -1 || wsrep_is_wsrep_xid(xid)); diff --git a/storage/xtradb/trx/trx0sys.c b/storage/xtradb/trx/trx0sys.c index 7d9397b8e2c..198430f9a01 100644 --- a/storage/xtradb/trx/trx0sys.c +++ b/storage/xtradb/trx/trx0sys.c @@ -990,26 +990,6 @@ trx_sys_update_wsrep_checkpoint( trx_sysf_t* sys_header, /*!< in: sys_header */ mtr_t* mtr) /*!< in: mtr */ { - -#ifdef UNIV_DEBUG - { - /* Check that seqno is monotonically increasing */ - unsigned char xid_uuid[16]; - long long xid_seqno = read_wsrep_xid_seqno(xid); - read_wsrep_xid_uuid(xid, xid_uuid); - if (!memcmp(xid_uuid, trx_sys_cur_xid_uuid, 8)) - { - ut_ad(xid_seqno > trx_sys_cur_xid_seqno); - trx_sys_cur_xid_seqno = xid_seqno; - } - else - { - memcpy(trx_sys_cur_xid_uuid, xid_uuid, 16); - } - trx_sys_cur_xid_seqno = xid_seqno; - } -#endif /* UNIV_DEBUG */ - ut_ad(xid && mtr && sys_header); ut_a(xid->formatID == -1 || wsrep_is_wsrep_xid(xid)); From 2aaed4489f92ede9fd1b28a848a2d8434dea17f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Jul 2017 15:43:45 +0300 Subject: [PATCH 037/276] Fix regression on galera.partition test case by commenting the problematic condition. --- storage/innobase/handler/ha_innodb.cc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 97c6cf84910..094d6f76d17 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6000,13 +6000,22 @@ report_error: prebuilt->table->flags, user_thd); #ifdef WITH_WSREP + fprintf(stderr, "JAN: split %d load %d\n", wsrep_load_data_splitting, + sql_command == SQLCOM_LOAD); if (!error_result && wsrep_on(user_thd) && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE && !wsrep_consistency_check(user_thd) - && (sql_command != SQLCOM_CREATE_TABLE) - && (sql_command != SQLCOM_LOAD || - thd_binlog_format(user_thd) == BINLOG_FORMAT_ROW)) { + && (sql_command != SQLCOM_CREATE_TABLE)) { + /* This change part of commit + a4bc8db216b4dd61ca0b1cb5a8b7806437416dc7 + MW-322 - CTAS fix will cause regression on galera.partition + test case. Commented until galeracluster developers have + looked the issue. + && (sql_command != SQLCOM_LOAD || + thd_binlog_format(user_thd) == + BINLOG_FORMAT_ROW)) { + */ if (wsrep_append_keys(user_thd, false, record, NULL)) { DBUG_PRINT("wsrep", ("row key failed")); From eec6417e05f6b5f76758e619a44fbdb00860258a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Jul 2017 16:14:27 +0300 Subject: [PATCH 038/276] Apply galera patches to XtraDB storage engine and remove one debug output. --- storage/innobase/handler/ha_innodb.cc | 2 -- storage/xtradb/handler/ha_innodb.cc | 22 +++++++++++++++------- storage/xtradb/row/row0ins.c | 25 +++++++++++++++---------- storage/xtradb/row/row0upd.c | 4 +++- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 094d6f76d17..1991850e6cd 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6000,8 +6000,6 @@ report_error: prebuilt->table->flags, user_thd); #ifdef WITH_WSREP - fprintf(stderr, "JAN: split %d load %d\n", wsrep_load_data_splitting, - sql_command == SQLCOM_LOAD); if (!error_result && wsrep_on(user_thd) && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 805b28d9d21..d1f40905184 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -7025,13 +7025,21 @@ report_error: prebuilt->table->flags, user_thd); #ifdef WITH_WSREP - if (!error_result && - wsrep_thd_exec_mode(user_thd) == LOCAL_STATE && - wsrep_on(user_thd) && - !wsrep_consistency_check(user_thd)) - { - if (wsrep_append_keys(user_thd, false, record, NULL)) - { + if (!error_result + && wsrep_on(user_thd) + && wsrep_thd_exec_mode(user_thd) == LOCAL_STATE + && !wsrep_consistency_check(user_thd) + && (sql_command != SQLCOM_CREATE_TABLE)) { + /* This change part of commit + a4bc8db216b4dd61ca0b1cb5a8b7806437416dc7 + MW-322 - CTAS fix will cause regression on galera.partition + test case. Commented until galeracluster developers have + looked the issue. + && (sql_command != SQLCOM_LOAD || + thd_binlog_format(user_thd) == + BINLOG_FORMAT_ROW)) { + */ + if (wsrep_append_keys(user_thd, false, record, NULL)) { DBUG_PRINT("wsrep", ("row key failed")); error_result = HA_ERR_INTERNAL_ERROR; goto wsrep_error; diff --git a/storage/xtradb/row/row0ins.c b/storage/xtradb/row/row0ins.c index a4309612377..52495c7ec60 100644 --- a/storage/xtradb/row/row0ins.c +++ b/storage/xtradb/row/row0ins.c @@ -761,7 +761,7 @@ row_ins_invalidate_query_cache( mem_free(buf); } #ifdef WITH_WSREP -ulint wsrep_append_foreign_key(trx_t *trx, +ulint wsrep_append_foreign_key(trx_t *trx, dict_foreign_t* foreign, const rec_t* clust_rec, dict_index_t* clust_index, @@ -1083,13 +1083,14 @@ row_ins_foreign_check_on_constraint( #ifdef WITH_WSREP err = wsrep_append_foreign_key( - thr_get_trx(thr), - foreign, - clust_rec, - clust_index, - FALSE, FALSE); + thr_get_trx(thr), + foreign, + clust_rec, + clust_index, + FALSE, + (node) ? TRUE : FALSE); if (err != DB_SUCCESS) { - fprintf(stderr, + fprintf(stderr, "WSREP: foreign key append failed: %lu\n", err); } else #endif @@ -1248,6 +1249,9 @@ row_ins_check_foreign_constraint( ulint* offsets = offsets_; rec_offs_init(offsets_); +#ifdef WITH_WSREP + upd_node= NULL; +#endif /* WITH_WSREP */ run_again: #ifdef UNIV_SYNC_DEBUG ut_ad(rw_lock_own(&dict_operation_lock, RW_LOCK_SHARED)); @@ -1436,9 +1440,10 @@ run_again: err = wsrep_append_foreign_key( thr_get_trx(thr), foreign, - rec, - check_index, - check_ref, TRUE); + rec, + check_index, + check_ref, + (upd_node) ? TRUE : FALSE); #endif /* WITH_WSREP */ goto end_scan; } else if (foreign->type != 0) { diff --git a/storage/xtradb/row/row0upd.c b/storage/xtradb/row/row0upd.c index a2eabc347d6..d75100c91e8 100644 --- a/storage/xtradb/row/row0upd.c +++ b/storage/xtradb/row/row0upd.c @@ -1842,7 +1842,9 @@ row_upd_sec_index_entry( index, offsets, thr, &mtr); } #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && + if (wsrep_on(trx->mysql_thd) && + !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && + err == DB_SUCCESS && !referenced && !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && ((upd_node_t*)parent)->cascade_node == node) && From d9675a10d5b1494458730367e1b790efc3b03bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 24 Jul 2017 20:24:03 +0300 Subject: [PATCH 039/276] Remove unneeded code. --- storage/innobase/trx/trx0sys.c | 18 ------------------ storage/xtradb/trx/trx0sys.c | 18 ------------------ 2 files changed, 36 deletions(-) diff --git a/storage/innobase/trx/trx0sys.c b/storage/innobase/trx/trx0sys.c index f8bd87034e8..51729e05c9d 100644 --- a/storage/innobase/trx/trx0sys.c +++ b/storage/innobase/trx/trx0sys.c @@ -801,24 +801,6 @@ trx_sys_print_mysql_binlog_offset(void) #ifdef WITH_WSREP -#ifdef UNIV_DEBUG -static long long trx_sys_cur_xid_seqno = -1; -static unsigned char trx_sys_cur_xid_uuid[16]; - -long long read_wsrep_xid_seqno(const XID* xid) -{ - long long seqno; - memcpy(&seqno, xid->data + 24, sizeof(long long)); - return seqno; -} - -void read_wsrep_xid_uuid(const XID* xid, unsigned char* buf) -{ - memcpy(buf, xid->data + 8, 16); -} - -#endif /* UNIV_DEBUG */ - void trx_sys_update_wsrep_checkpoint( const XID* xid, /*!< in: transaction XID */ diff --git a/storage/xtradb/trx/trx0sys.c b/storage/xtradb/trx/trx0sys.c index 198430f9a01..0d13de23e0e 100644 --- a/storage/xtradb/trx/trx0sys.c +++ b/storage/xtradb/trx/trx0sys.c @@ -966,24 +966,6 @@ trx_sys_print_mysql_binlog_offset(void) #ifdef WITH_WSREP -#ifdef UNIV_DEBUG -static long long trx_sys_cur_xid_seqno = -1; -static unsigned char trx_sys_cur_xid_uuid[16]; - -long long read_wsrep_xid_seqno(const XID* xid) -{ - long long seqno; - memcpy(&seqno, xid->data + 24, sizeof(long long)); - return seqno; -} - -void read_wsrep_xid_uuid(const XID* xid, unsigned char* buf) -{ - memcpy(buf, xid->data + 8, 16); -} - -#endif /* UNIV_DEBUG */ - void trx_sys_update_wsrep_checkpoint( const XID* xid, /*!< in: transaction XID */ From 2b811f0624c46469185ff4d93bd8f184e1f7ac92 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 26 Jul 2017 11:36:57 -0400 Subject: [PATCH 040/276] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 68358acf754..2905c37cc09 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=57 +MYSQL_VERSION_PATCH=58 MYSQL_VERSION_EXTRA= From 86d31ce9f141d34fffcf7f355d4c0e6e54077eb3 Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko Date: Mon, 19 Jun 2017 17:23:02 +0700 Subject: [PATCH 041/276] MW-384 protect access to wsrep_ready variable with mutex --- scripts/mysql_config.pl | 6 +++--- sql/mysqld.cc | 2 +- sql/slave.cc | 2 +- sql/sql_parse.cc | 4 ++-- sql/wsrep_mysqld.cc | 36 +++++++++++++++++++++++------------- sql/wsrep_mysqld.h | 3 ++- sql/wsrep_priv.h | 2 +- sql/wsrep_sst.cc | 1 - 8 files changed, 33 insertions(+), 23 deletions(-) diff --git a/scripts/mysql_config.pl b/scripts/mysql_config.pl index 5490853c091..a9254bb966e 100644 --- a/scripts/mysql_config.pl +++ b/scripts/mysql_config.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl # -*- cperl -*- # -# Copyright (c) 2007, 2014, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -53,7 +53,7 @@ my $cwd = cwd(); my $basedir; my $socket = '/tmp/mysql.sock'; -my $version = '5.5.38'; +my $version = '5.5.58'; sub which { @@ -204,7 +204,7 @@ $flags->{embedded_libs} = [@ldflags,@lib_e_opts,'','','','','']; $flags->{include} = ["-I$pkgincludedir"]; -$flags->{cflags} = [@{$flags->{include}},split(" ",'-Wall -DWITH_WSREP -DWSREP_PROC_INFO -DMYSQL_MAX_VARIABLE_VALUE_LEN=2048 -DWITH_INNODB_DISALLOW_WRITES -O2 -g -DNDEBUG -DDBUG_OFF')]; +$flags->{cflags} = [@{$flags->{include}},split(" ",'')]; # ---------------------------------------------------------------------- # Remove some options that a client doesn't have to care about diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 9198f616d15..f76cbe5e5b2 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -7929,7 +7929,7 @@ SHOW_VAR status_vars[]= { #endif #ifdef WITH_WSREP {"wsrep_connected", (char*) &wsrep_connected, SHOW_BOOL}, - {"wsrep_ready", (char*) &wsrep_ready, SHOW_BOOL}, + {"wsrep_ready", (char*) &wsrep_show_ready, SHOW_FUNC}, {"wsrep_cluster_state_uuid", (char*) &wsrep_cluster_state_uuid,SHOW_CHAR_PTR}, {"wsrep_cluster_conf_id", (char*) &wsrep_cluster_conf_id, SHOW_LONGLONG}, {"wsrep_cluster_status", (char*) &wsrep_cluster_status, SHOW_CHAR_PTR}, diff --git a/sql/slave.cc b/sql/slave.cc index 5f7c5ffdb84..f370e3dd27f 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3794,7 +3794,7 @@ err_during_init: */ if (wsrep_node_dropped && wsrep_restart_slave) { - if (wsrep_ready) + if (wsrep_ready_get()) { WSREP_INFO("Slave error due to node temporarily non-primary" "SQL slave will continue"); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index f0f021d4047..d102c6162d0 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -851,7 +851,7 @@ bool do_command(THD *thd) * bail out if DB snapshot has not been installed. We however, * allow queries "SET" and "SHOW", they are trapped later in execute_command */ - if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && + if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready_get() && command != COM_QUERY && command != COM_PING && command != COM_QUIT && @@ -2382,7 +2382,7 @@ mysql_execute_command(THD *thd) We additionally allow all other commands that do not change data in case wsrep_dirty_reads is enabled. */ - if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready && + if (thd->variables.wsrep_on && !thd->wsrep_applier && !wsrep_ready_get() && lex->sql_command != SQLCOM_SET_OPTION && !(thd->variables.wsrep_dirty_reads && !is_update_query(lex->sql_command)) && diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 4ac346ca0a3..54fdf430f86 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -13,6 +13,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include // SHOW_MY_BOOL #include #include #include @@ -295,7 +296,7 @@ wsrep_view_handler_cb (void* app_ctx, // version change if (view->proto_ver != wsrep_protocol_version) { - my_bool wsrep_ready_saved= wsrep_ready; + my_bool wsrep_ready_saved= wsrep_ready_get(); wsrep_ready_set(FALSE); WSREP_INFO("closing client connections for " "protocol change %ld -> %d", @@ -409,16 +410,34 @@ out: return WSREP_CB_SUCCESS; } -void wsrep_ready_set (my_bool x) +my_bool wsrep_ready_set (my_bool x) { WSREP_DEBUG("Setting wsrep_ready to %d", x); if (mysql_mutex_lock (&LOCK_wsrep_ready)) abort(); - if (wsrep_ready != x) + my_bool ret= (wsrep_ready != x); + if (ret) { wsrep_ready= x; mysql_cond_signal (&COND_wsrep_ready); } mysql_mutex_unlock (&LOCK_wsrep_ready); + return ret; +} + +my_bool wsrep_ready_get (void) +{ + if (mysql_mutex_lock (&LOCK_wsrep_ready)) abort(); + my_bool ret= wsrep_ready; + mysql_mutex_unlock (&LOCK_wsrep_ready); + return ret; +} + +int wsrep_show_ready(THD *thd, SHOW_VAR *var, char *buff) +{ + var->type= SHOW_MY_BOOL; + var->value= buff; + *((my_bool *)buff)= wsrep_ready_get(); + return 0; } // Wait until wsrep has reached ready state @@ -437,17 +456,8 @@ void wsrep_ready_wait () static void wsrep_synced_cb(void* app_ctx) { WSREP_INFO("Synchronized with group, ready for connections"); - bool signal_main= false; - if (mysql_mutex_lock (&LOCK_wsrep_ready)) abort(); - if (!wsrep_ready) - { - wsrep_ready= TRUE; - mysql_cond_signal (&COND_wsrep_ready); - signal_main= true; - - } + my_bool signal_main= wsrep_ready_set(TRUE); local_status.set(WSREP_MEMBER_SYNCED); - mysql_mutex_unlock (&LOCK_wsrep_ready); if (signal_main) { diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index bb33d2f1069..56e3baae7cc 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -115,7 +115,6 @@ enum enum_wsrep_sync_wait { // MySQL status variables extern my_bool wsrep_connected; -extern my_bool wsrep_ready; extern const char* wsrep_cluster_state_uuid; extern long long wsrep_cluster_conf_id; extern const char* wsrep_cluster_status; @@ -130,6 +129,7 @@ extern const char* wsrep_provider_vendor; extern my_bool wsrep_inited; // whether wsrep is initialized ? int wsrep_show_status(THD *thd, SHOW_VAR *var, char *buff); +int wsrep_show_ready(THD *thd, SHOW_VAR *var, char *buff); void wsrep_free_status(THD *thd); /* Filters out --wsrep-new-cluster oprtion from argv[] @@ -245,6 +245,7 @@ extern wsrep_seqno_t wsrep_locked_seqno; #define WSREP_PROVIDER_EXISTS \ (wsrep_provider && strncasecmp(wsrep_provider, WSREP_NONE, FN_REFLEN)) +extern my_bool wsrep_ready_get(); extern void wsrep_ready_wait(); enum wsrep_trx_status { diff --git a/sql/wsrep_priv.h b/sql/wsrep_priv.h index 93640fbcc03..ce97ac65182 100644 --- a/sql/wsrep_priv.h +++ b/sql/wsrep_priv.h @@ -26,7 +26,7 @@ #include #include -void wsrep_ready_set (my_bool x); +my_bool wsrep_ready_set (my_bool x); ssize_t wsrep_sst_prepare (void** msg); wsrep_cb_status wsrep_sst_donate_cb (void* app_ctx, diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index a5fe8a1e2d8..d870824ed9d 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -1057,7 +1057,6 @@ wsrep_cb_status_t wsrep_sst_donate_cb (void* app_ctx, void* recv_ctx, { /* This will be reset when sync callback is called. * Should we set wsrep_ready to FALSE here too? */ -// wsrep_notify_status(WSREP_MEMBER_DONOR); local_status.set(WSREP_MEMBER_DONOR); const char* method = (char*)msg; From 0332acc77cc7e22ed2f98537477d3e23fb3efa36 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Fri, 4 Aug 2017 11:22:35 +0300 Subject: [PATCH 042/276] MW-394 Merged fix for LOAD DATA splitting with log_bin==OFF, from mysql-wsrep-bugs/MW-394 --- sql/sql_class.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 43d7ffd94d6..d540abb276c 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -4297,14 +4297,16 @@ extern "C" int thd_non_transactional_update(const MYSQL_THD thd) extern "C" int thd_binlog_format(const MYSQL_THD thd) { #ifdef WITH_WSREP - if (((WSREP(thd) && wsrep_emulate_bin_log) || mysql_bin_log.is_open()) && - (thd->variables.option_bits & OPTION_BIN_LOG)) -#else + if (WSREP(thd)) + { + /* for wsrep binlog format is meaningful also when binlogging is off */ + return (int) WSREP_BINLOG_FORMAT(thd->variables.binlog_format); + } +#endif /* WITH_WSREP */ if (mysql_bin_log.is_open() && (thd->variables.option_bits & OPTION_BIN_LOG)) -#endif - return (int) WSREP_BINLOG_FORMAT(thd->variables.binlog_format); + return (int) thd->variables.binlog_format; else - return BINLOG_FORMAT_UNSPEC; + return BINLOG_FORMAT_UNSPEC; } extern "C" void thd_mark_transaction_to_rollback(MYSQL_THD thd, bool all) From e90249bf472eda519a1485e5f3381ef2a80ce59d Mon Sep 17 00:00:00 2001 From: sjaakola Date: Fri, 25 Aug 2017 12:41:56 +0300 Subject: [PATCH 043/276] MW-402 cascading FK issues (5.5 version) copied the fix for MW-402 from mysql-wsrep-bugs/MW-402-5.6 (note, this is not git merge) --- storage/innobase/row/row0ins.c | 3 +-- storage/innobase/row/row0upd.c | 10 +++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/storage/innobase/row/row0ins.c b/storage/innobase/row/row0ins.c index 3726c517ff0..1cd3e9e7c74 100644 --- a/storage/innobase/row/row0ins.c +++ b/storage/innobase/row/row0ins.c @@ -1086,8 +1086,7 @@ row_ins_foreign_check_on_constraint( foreign, clust_rec, clust_index, - FALSE, - (node) ? TRUE : FALSE); + FALSE, FALSE); if (err != DB_SUCCESS) { fprintf(stderr, "WSREP: foreign key append failed: %lu\n", err); diff --git a/storage/innobase/row/row0upd.c b/storage/innobase/row/row0upd.c index dd9c7e8defd..0a503f01f40 100644 --- a/storage/innobase/row/row0upd.c +++ b/storage/innobase/row/row0upd.c @@ -2107,7 +2107,7 @@ err_exit: } } #ifdef WITH_WSREP - if (!referenced && + if (wsrep_on(trx->mysql_thd) && !referenced && !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && ((upd_node_t*)parent)->cascade_node == node) && foreign @@ -2313,6 +2313,7 @@ row_upd_del_mark_clust_rec( #ifdef WITH_WSREP rec_t* rec; que_node_t *parent = que_node_get_parent(node); + trx_t* trx = thr_get_trx(thr); #endif /* WITH_WSREP */ ut_ad(node); @@ -2348,11 +2349,10 @@ row_upd_del_mark_clust_rec( node, pcur, index->table, index, offsets, thr, mtr); } #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && + if (trx && wsrep_on(trx->mysql_thd) && + err == DB_SUCCESS && !referenced && foreign && !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - thr_get_trx(thr) && - foreign + ((upd_node_t*)parent)->cascade_node == node) ) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); From 2864c37d6c5d4c13472438863d513d0ea8b9063c Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Mon, 18 Sep 2017 16:12:13 +0300 Subject: [PATCH 044/276] MW-406 Bumped up the wsrep patch version (5.5.57-25.22) --- cmake/wsrep.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index 6a9dd720f77..afab9105d8d 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -18,7 +18,7 @@ # so WSREP_VERSION is produced regardless # Set the patch version -SET(WSREP_PATCH_VERSION "21") +SET(WSREP_PATCH_VERSION "22") # MariaDB addition: Revision number of the last revision merged from # codership branch visible in @@visible_comment. From 38530c86aa36e2cbd315447a5362bcc751fb9205 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Thu, 5 Oct 2017 11:41:02 +0200 Subject: [PATCH 045/276] MW-415 THD::COND_wsrep_thd is never destroyed THD::COND_wsrep_thd is never destroyed. This patch adds missing call to mysql_cond_destroy(&COND_wsrep_thd) in THD::release_resources(). --- sql/sql_class.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index d540abb276c..1398fdff94f 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1693,6 +1693,7 @@ THD::~THD() mysql_mutex_lock(&LOCK_wsrep_thd); mysql_mutex_unlock(&LOCK_wsrep_thd); mysql_mutex_destroy(&LOCK_wsrep_thd); + mysql_cond_destroy(&COND_wsrep_thd); if (wsrep_rli) delete wsrep_rli; if (wsrep_status_vars) wsrep->stats_free(wsrep, wsrep_status_vars); #endif From 8822b30f1e258a5f0efc124043f42970e600c5d4 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Tue, 10 Oct 2017 23:39:48 +0300 Subject: [PATCH 046/276] MW-416 Replicating DDL after ACL check, 5.6 version Re-implemented the fix for MW-416 according to 5.7 version --- mysql-test/suite/galera/r/MW-416.result | 114 ++++++++++++++++++++ mysql-test/suite/galera/t/MW-416.test | 132 ++++++++++++++++++++++++ sql/events.cc | 15 +++ sql/sql_class.h | 1 + sql/sql_parse.cc | 7 -- sql/sql_plugin.cc | 19 +++- sql/sql_trigger.cc | 5 + sql/sql_view.cc | 5 + 8 files changed, 287 insertions(+), 11 deletions(-) create mode 100644 mysql-test/suite/galera/r/MW-416.result create mode 100644 mysql-test/suite/galera/t/MW-416.test diff --git a/mysql-test/suite/galera/r/MW-416.result b/mysql-test/suite/galera/r/MW-416.result new file mode 100644 index 00000000000..05399b213a8 --- /dev/null +++ b/mysql-test/suite/galera/r/MW-416.result @@ -0,0 +1,114 @@ +CREATE USER 'userMW416'@'localhost'; +GRANT SELECT, INSERT, UPDATE ON test.* TO 'userMW416'@'localhost'; +SHOW GLOBAL STATUS LIKE 'wsrep_replicated'; +Variable_name Value +wsrep_replicated 2 +ALTER DATABASE db CHARACTER SET = utf8; +ERROR 42000: Access denied for user 'userMW416'@'localhost' to database 'db' +ALTER EVENT ev1 RENAME TO ev2; +ERROR 42000: Access denied for user 'userMW416'@'localhost' to database 'test' +ALTER FUNCTION fun1 COMMENT 'foo'; +ERROR 42000: alter routine command denied to user 'userMW416'@'localhost' for routine 'test.fun1' +ALTER LOGFILE GROUP lfg ADD UNDOFILE 'file' ENGINE=InnoDB; +Got one of the listed errors +ALTER PROCEDURE proc1 COMMENT 'foo'; +Got one of the listed errors +ALTER SERVER srv OPTIONS (USER 'sally'); +Got one of the listed errors +ALTER TABLE tbl DROP COLUMN col; +Got one of the listed errors +ALTER TABLESPACE tblspc DROP DATAFILE 'file' ENGINE=innodb; +Got one of the listed errors +ALTER VIEW vw AS SELECT 1; +Got one of the listed errors +CREATE DATABASE db; +Got one of the listed errors +CREATE EVENT ev1 ON SCHEDULE AT CURRENT_TIMESTAMP DO SELECT 1; +Got one of the listed errors +CREATE FUNCTION fun1() RETURNS int RETURN(1); +Got one of the listed errors +CREATE FUNCTION fun1 RETURNS STRING SONAME 'funlib.so'; +Got one of the listed errors +CREATE PROCEDURE proc1() BEGIN END; +Got one of the listed errors +CREATE INDEX idx ON tbl(id); +Got one of the listed errors +CREATE LOGFILE GROUP lfg ADD UNDOFILE 'undofile' ENGINE innodb; +Got one of the listed errors +CREATE SERVER srv FOREIGN DATA WRAPPER 'fdw' OPTIONS (USER 'user'); +Got one of the listed errors +CREATE TABLE t (i int); +Got one of the listed errors +CREATE TABLESPACE tblspc ADD DATAFILE 'file' ENGINE=innodb; +Got one of the listed errors +CREATE TRIGGER trg BEFORE UPDATE ON t FOR EACH ROW BEGIN END; +Got one of the listed errors +CREATE VIEW vw AS SELECT 1; +Got one of the listed errors +DROP DATABASE db; +Got one of the listed errors +DROP EVENT ev; +Got one of the listed errors +DROP FUNCTION fun1; +Got one of the listed errors +DROP INDEX idx ON t0; +Got one of the listed errors +DROP LOGFILE GROUP lfg; +Got one of the listed errors +DROP PROCEDURE proc1; +Got one of the listed errors +DROP SERVEr srv; +Got one of the listed errors +DROP TABLE t0; +Got one of the listed errors +DROP TABLESPACE tblspc; +Got one of the listed errors +DROP TRIGGER trg; +Got one of the listed errors +DROP VIEW vw; +Got one of the listed errors +RENAME TABLE t0 TO t1; +Got one of the listed errors +TRUNCATE TABLE t0; +Got one of the listed errors +ALTER USER myuser PASSWORD EXPIRE; +Got one of the listed errors +CREATE USER myuser IDENTIFIED BY 'pass'; +Got one of the listed errors +DROP USER myuser; +Got one of the listed errors +GRANT ALL ON *.* TO 'myuser'; +Got one of the listed errors +RENAME USER myuser TO mariauser; +Got one of the listed errors +REVOKE SELECT ON test FROM myuser; +Got one of the listed errors +REVOKE ALL, GRANT OPTION FROM myuser; +Got one of the listed errors +REVOKE PROXY ON myuser FROM myuser; +Got one of the listed errors +ANALYZE TABLE db.tbl; +Got one of the listed errors +CHECK TABLE db.tbl; +Got one of the listed errors +CHECKSUM TABLE db.tbl; +Got one of the listed errors +OPTIMIZE TABLE db.tbl; +Got one of the listed errors +REPAIR TABLE db.tbl; +Got one of the listed errors +INSTALL PLUGIN plg SONAME 'plg.so'; +Got one of the listed errors +UNINSTALL PLUGIN plg; +Got one of the listed errors +DROP USER 'userMW416'@'localhost'; +SHOW DATABASES; +Database +information_schema +mtr +mysql +performance_schema +test +SHOW GLOBAL STATUS LIKE 'wsrep_replicated'; +Variable_name Value +wsrep_replicated 3 diff --git a/mysql-test/suite/galera/t/MW-416.test b/mysql-test/suite/galera/t/MW-416.test new file mode 100644 index 00000000000..ebc15438adc --- /dev/null +++ b/mysql-test/suite/galera/t/MW-416.test @@ -0,0 +1,132 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +CREATE USER 'userMW416'@'localhost'; +GRANT SELECT, INSERT, UPDATE ON test.* TO 'userMW416'@'localhost'; + +SHOW GLOBAL STATUS LIKE 'wsrep_replicated'; + +--connect userMW416, localhost, userMW416,, test, $NODE_MYPORT_1 +--connection userMW416 + +# DDL + +--error 1044 +ALTER DATABASE db CHARACTER SET = utf8; +--error 1044 +ALTER EVENT ev1 RENAME TO ev2; +--error 1370 +ALTER FUNCTION fun1 COMMENT 'foo'; +#--error 1044,1227 +#ALTER INSTANCE ROTATE INNODB MASTER KEY; +--error 1044,1227 +ALTER LOGFILE GROUP lfg ADD UNDOFILE 'file' ENGINE=InnoDB; +--error 1044,1227,1370 +ALTER PROCEDURE proc1 COMMENT 'foo'; +--error 1044,1227,1370 +ALTER SERVER srv OPTIONS (USER 'sally'); +--error 1044,1142,1227,1370 +ALTER TABLE tbl DROP COLUMN col; +--error 1044,1227,1370 +ALTER TABLESPACE tblspc DROP DATAFILE 'file' ENGINE=innodb; +--error 1044,1142,1227,1370 +ALTER VIEW vw AS SELECT 1; + +--error 1044,1227,1370 +CREATE DATABASE db; +--error 1044,1227,1370 +CREATE EVENT ev1 ON SCHEDULE AT CURRENT_TIMESTAMP DO SELECT 1; +--error 1044,1227,1370 +CREATE FUNCTION fun1() RETURNS int RETURN(1); +--error 1044,1227,1370 +CREATE FUNCTION fun1 RETURNS STRING SONAME 'funlib.so'; +--error 1044,1227,1370 +CREATE PROCEDURE proc1() BEGIN END; +--error 1044,1142,1227,1370 +CREATE INDEX idx ON tbl(id); +--error 1044,1142,1227,1370 +CREATE LOGFILE GROUP lfg ADD UNDOFILE 'undofile' ENGINE innodb; +--error 1044,1142,1227,1370 +CREATE SERVER srv FOREIGN DATA WRAPPER 'fdw' OPTIONS (USER 'user'); +--error 1044,1142,1227,1370 +CREATE TABLE t (i int); +--error 1044,1142,1227,1370 +CREATE TABLESPACE tblspc ADD DATAFILE 'file' ENGINE=innodb; +--error 1044,1142,1227,1370 +CREATE TRIGGER trg BEFORE UPDATE ON t FOR EACH ROW BEGIN END; +--error 1044,1142,1227,1370 +CREATE VIEW vw AS SELECT 1; + + + +--error 1044,1142,1227,1370 +DROP DATABASE db; +--error 1044,1142,1227,1370 +DROP EVENT ev; +--error 1044,1142,1227,1370 +DROP FUNCTION fun1; +--error 1044,1142,1227,1370 +DROP INDEX idx ON t0; +--error 1044,1142,1227,1370 +DROP LOGFILE GROUP lfg; +--error 1044,1142,1227,1370 +DROP PROCEDURE proc1; +--error 1044,1142,1227,1370 +DROP SERVEr srv; +--error 1044,1142,1227,1370 +DROP TABLE t0; +--error 1044,1142,1227,1370 +DROP TABLESPACE tblspc; +--error 1044,1142,1227,1360,1370 +DROP TRIGGER trg; +--error 1044,1142,1227,1370 +DROP VIEW vw; + +--error 1044,1142,1227,1370 +RENAME TABLE t0 TO t1; + +--error 1044,1142,1227,1370 +TRUNCATE TABLE t0; + +# DCL + +# account management +--error 1044,1142,1227,1370 +ALTER USER myuser PASSWORD EXPIRE; +--error 1044,1142,1227,1370 +CREATE USER myuser IDENTIFIED BY 'pass'; +--error 1044,1142,1227,1370 +DROP USER myuser; +--error 1044,1045,1142,1227,1370 +GRANT ALL ON *.* TO 'myuser'; +--error 1044,1142,1227,1370 +RENAME USER myuser TO mariauser; +--error 1044,1142,1227,1370 +REVOKE SELECT ON test FROM myuser; +--error 1044,1142,1227,1370,1698 +REVOKE ALL, GRANT OPTION FROM myuser; +--error 1044,1142,1227,1370,1698 +REVOKE PROXY ON myuser FROM myuser; + +# table maintenance +--error 1044,1142,1227,1370 +ANALYZE TABLE db.tbl; +--error 1044,1142,1227,1370 +CHECK TABLE db.tbl; +--error 1044,1142,1227,1370 +CHECKSUM TABLE db.tbl; +--error 1044,1142,1227,1370 +OPTIMIZE TABLE db.tbl; +--error 1044,1142,1227,1370 +REPAIR TABLE db.tbl; + +# plugin and user defined functions +--error 1044,1142,1227,1370 +INSTALL PLUGIN plg SONAME 'plg.so'; +--error 1044,1142,1227,1370 +UNINSTALL PLUGIN plg; + +--connection node_1 +DROP USER 'userMW416'@'localhost'; +SHOW DATABASES; +SHOW GLOBAL STATUS LIKE 'wsrep_replicated'; diff --git a/sql/events.cc b/sql/events.cc index 7aa7dae4d7f..a6379ec5a46 100644 --- a/sql/events.cc +++ b/sql/events.cc @@ -327,6 +327,7 @@ Events::create_event(THD *thd, Event_parse_data *parse_data, if (check_access(thd, EVENT_ACL, parse_data->dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); if (check_db_dir_existence(parse_data->dbname.str)) { @@ -406,6 +407,10 @@ Events::create_event(THD *thd, Event_parse_data *parse_data, thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(ret); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -446,6 +451,7 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, if (check_access(thd, EVENT_ACL, parse_data->dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); if (new_dbname) /* It's a rename */ { @@ -521,6 +527,10 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(ret); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -560,6 +570,7 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) if (check_access(thd, EVENT_ACL, dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); /* Turn off row binlogging of this statement and use statement-based so @@ -585,6 +596,10 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) if (save_binlog_row_based) thd->set_current_stmt_binlog_format_row(); DBUG_RETURN(ret); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } diff --git a/sql/sql_class.h b/sql/sql_class.h index 8cfb3ced4b6..70bf6f7783d 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -2383,6 +2383,7 @@ public: enum wsrep_query_state wsrep_query_state; enum wsrep_conflict_state wsrep_conflict_state; mysql_mutex_t LOCK_wsrep_thd; + mysql_cond_t COND_wsrep_thd; // changed from wsrep_seqno_t to wsrep_trx_meta_t in wsrep API rev 75 // wsrep_seqno_t wsrep_trx_seqno; wsrep_trx_meta_t wsrep_trx_meta; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index d102c6162d0..553a6e7539d 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -3825,7 +3825,6 @@ end_with_restore_list: if (res) break; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) switch (lex->sql_command) { case SQLCOM_CREATE_EVENT: { @@ -3861,7 +3860,6 @@ end_with_restore_list: lex->spname->m_name); break; case SQLCOM_DROP_EVENT: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (!(res= Events::drop_event(thd, lex->spname->m_db, lex->spname->m_name, lex->drop_if_exists))) @@ -4773,7 +4771,6 @@ create_sp_error: Note: SQLCOM_CREATE_VIEW also handles 'ALTER VIEW' commands as specified through the thd->lex->create_view_mode flag. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_view(thd, first_table, thd->lex->create_view_mode); break; } @@ -4789,7 +4786,6 @@ create_sp_error: case SQLCOM_CREATE_TRIGGER: { /* Conditionally writes to binlog. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_or_drop_trigger(thd, all_tables, 1); break; @@ -4797,7 +4793,6 @@ create_sp_error: case SQLCOM_DROP_TRIGGER: { /* Conditionally writes to binlog. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_or_drop_trigger(thd, all_tables, 0); break; } @@ -4860,13 +4855,11 @@ create_sp_error: my_ok(thd); break; case SQLCOM_INSTALL_PLUGIN: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (! (res= mysql_install_plugin(thd, &thd->lex->comment, &thd->lex->ident))) my_ok(thd); break; case SQLCOM_UNINSTALL_PLUGIN: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (! (res= mysql_uninstall_plugin(thd, &thd->lex->comment, &thd->lex->ident))) my_ok(thd); diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index d4f497743b7..7e5c7c9c852 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2079,6 +2079,8 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, bool error; int argc=orig_argc; char **argv=orig_argv; + unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = + { MYSQL_AUDIT_GENERAL_CLASSMASK }; DBUG_ENTER("mysql_install_plugin"); if (opt_noacl) @@ -2090,6 +2092,7 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, tables.init_one_table("mysql", 5, "plugin", 6, "plugin", TL_WRITE); if (check_table_access(thd, INSERT_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table = open_ltable(thd, &tables, TL_WRITE, @@ -2123,8 +2126,7 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, See also mysql_uninstall_plugin() and initialize_audit_plugin() */ - unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = - { MYSQL_AUDIT_GENERAL_CLASSMASK }; + mysql_audit_acquire_plugins(thd, event_class_mask); mysql_mutex_lock(&LOCK_plugin); @@ -2155,6 +2157,10 @@ err: if (argv) free_defaults(argv); DBUG_RETURN(error); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -2221,6 +2227,8 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, TABLE_LIST tables; LEX_STRING dl= *dl_arg; bool error= false; + unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = + { MYSQL_AUDIT_GENERAL_CLASSMASK }; DBUG_ENTER("mysql_uninstall_plugin"); if (opt_noacl) @@ -2233,6 +2241,7 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, if (check_table_access(thd, DELETE_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table= open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT))) @@ -2259,8 +2268,6 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, See also mysql_install_plugin() and initialize_audit_plugin() */ - unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = - { MYSQL_AUDIT_GENERAL_CLASSMASK }; mysql_audit_acquire_plugins(thd, event_class_mask); mysql_mutex_lock(&LOCK_plugin); @@ -2290,6 +2297,10 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, mysql_mutex_unlock(&LOCK_plugin); DBUG_RETURN(error); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index c562ee9762c..70ed6f0e600 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -501,6 +501,7 @@ bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create) if (err_status) goto end; } + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); /* We should have only one table in table list. */ DBUG_ASSERT(tables->next_global == 0); @@ -605,6 +606,10 @@ end: my_ok(thd); DBUG_RETURN(result); +#ifdef WITH_WSREP + error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 5bd82fdd842..f6948a76f6a 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -429,6 +429,7 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, if ((res= create_view_precheck(thd, tables, view, mode))) goto err; + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; @@ -721,6 +722,10 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, lex->link_first_table_back(view, link_to_local); DBUG_RETURN(0); +#ifdef WITH_WSREP + error: + res= TRUE; +#endif /* WITH_WSREP */ err: thd_proc_info(thd, "end"); lex->link_first_table_back(view, link_to_local); From 12d7ee03efa36e9eb293fefe8313d3848fc41113 Mon Sep 17 00:00:00 2001 From: sjaakola Date: Tue, 10 Oct 2017 23:39:48 +0300 Subject: [PATCH 047/276] MW-416 Replicating DDL after ACL check --- sql/event_data_objects.cc | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/sql/event_data_objects.cc b/sql/event_data_objects.cc index 656881b9977..0cb123451df 100644 --- a/sql/event_data_objects.cc +++ b/sql/event_data_objects.cc @@ -1468,23 +1468,35 @@ end: thd->security_ctx->master_access |= SUPER_ACL; #ifdef WITH_WSREP - if (WSREP(thd)) { - // sql_print_information("sizeof(LEX) = %d", sizeof(struct LEX)); - // sizeof(LEX) = 4512, so it's relatively safe to allocate it on stack. - LEX lex; - lex.sql_command = SQLCOM_DROP_EVENT; - LEX* saved = thd->lex; - thd->lex = &lex; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); - thd->lex = saved; + /* + This code is processing event execution and does not have client + connection. Here, event execution will now execute a prepared + DROP EVENT statement, but thd->lex->sql_command is set to + SQLCOM_CREATE_PROCEDURE + DROP EVENT will be logged in binlog, and we have to + replicate it to make all nodes have consistent event definitions + Wsrep DDL replication is triggered inside Events::drop_event(), + and here we need to prepare the THD so that DDL replication is + possible, essentially it requires setting sql_command to + SQLCOMM_DROP_EVENT, we will switch sql_command for the duration + of DDL replication only. + */ + const enum_sql_command sql_command_save= thd->lex->sql_command; + const bool sql_command_set= WSREP(thd); + if (WSREP(thd)) + { + thd->lex->sql_command = SQLCOM_DROP_EVENT; } #endif ret= Events::drop_event(thd, dbname, name, FALSE); #ifdef WITH_WSREP - WSREP_TO_ISOLATION_END; - error: + if (sql_command_set) + { + WSREP_TO_ISOLATION_END; + thd->lex->sql_command = sql_command_save; + } #endif thd->security_ctx->master_access= saved_master_access; } From 241a2687d7ce07572adb2fb4629bbd319a732e5a Mon Sep 17 00:00:00 2001 From: sjaakola Date: Wed, 11 Oct 2017 15:35:17 +0300 Subject: [PATCH 048/276] MW-416 Replicate DDL after ACL check CREATE VIEW handling had an issue, because CREATE VIEW replication happened when view was unlinked from table list. This caused debug assert in wsrep_mysqld.cc:wsrep_can_run_in_toi() MySQL 5.5 version does not have this assert, but it is still better to run replication when table list is not manipulated. --- sql/sql_view.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_view.cc b/sql/sql_view.cc index f6948a76f6a..bbc5f002461 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -429,10 +429,10 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, if ((res= create_view_precheck(thd, tables, view, mode))) goto err; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); if (open_and_lock_tables(thd, lex->query_tables, TRUE, 0)) { From 181f3015bf62bccc108aacd4df55065e0158b060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 19 Oct 2017 11:06:55 +0300 Subject: [PATCH 049/276] MariaDB adjustments. Add missing COND variable and disable test that fail. --- mysql-test/suite/galera/disabled.def | 2 ++ mysql-test/suite/galera/t/MW-416.test | 8 +++++--- sql/mysqld.cc | 3 ++- sql/mysqld.h | 1 + sql/sql_class.cc | 1 + 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index c91654c2023..ad966ebab0d 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -1,2 +1,4 @@ galera_var_dirty_reads : MDEV-12539 query_cache : MDEV-12539 +MW-421 : MDEV-12539 +galera_concurrent_ctas : MDEV-12539 diff --git a/mysql-test/suite/galera/t/MW-416.test b/mysql-test/suite/galera/t/MW-416.test index ebc15438adc..df4fa35abc7 100644 --- a/mysql-test/suite/galera/t/MW-416.test +++ b/mysql-test/suite/galera/t/MW-416.test @@ -1,6 +1,8 @@ --source include/galera_cluster.inc --source include/have_innodb.inc +--source include/wait_until_ready.inc + CREATE USER 'userMW416'@'localhost'; GRANT SELECT, INSERT, UPDATE ON test.* TO 'userMW416'@'localhost'; @@ -67,7 +69,7 @@ DROP EVENT ev; DROP FUNCTION fun1; --error 1044,1142,1227,1370 DROP INDEX idx ON t0; ---error 1044,1142,1227,1370 +--error 1044,1142,1227,1370,1064 DROP LOGFILE GROUP lfg; --error 1044,1142,1227,1370 DROP PROCEDURE proc1; @@ -75,7 +77,7 @@ DROP PROCEDURE proc1; DROP SERVEr srv; --error 1044,1142,1227,1370 DROP TABLE t0; ---error 1044,1142,1227,1370 +--error 1044,1142,1227,1370,1064 DROP TABLESPACE tblspc; --error 1044,1142,1227,1360,1370 DROP TRIGGER trg; @@ -91,7 +93,7 @@ TRUNCATE TABLE t0; # DCL # account management ---error 1044,1142,1227,1370 +--error 1044,1142,1227,1370,1064 ALTER USER myuser PASSWORD EXPIRE; --error 1044,1142,1227,1370 CREATE USER myuser IDENTIFIED BY 'pass'; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index f76cbe5e5b2..3edbeafe224 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -911,7 +911,7 @@ PSI_cond_key key_BINLOG_COND_prep_xids, key_BINLOG_update_cond, key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache, key_BINLOG_COND_queue_busy; #ifdef WITH_WSREP -PSI_cond_key key_COND_wsrep_rollback, +PSI_cond_key key_COND_wsrep_rollback, key_COND_wsrep_thd, key_COND_wsrep_replaying, key_COND_wsrep_ready, key_COND_wsrep_sst, key_COND_wsrep_sst_init, key_COND_wsrep_sst_thread; #endif /* WITH_WSREP */ @@ -962,6 +962,7 @@ static PSI_cond_info all_server_conds[]= { &key_COND_wsrep_sst_init, "COND_wsrep_sst_init", PSI_FLAG_GLOBAL}, { &key_COND_wsrep_sst_thread, "wsrep_sst_thread", 0}, { &key_COND_wsrep_rollback, "COND_wsrep_rollback", PSI_FLAG_GLOBAL}, + { &key_COND_wsrep_thd, "THD::COND_wsrep_thd", 0}, { &key_COND_wsrep_replaying, "COND_wsrep_replaying", PSI_FLAG_GLOBAL}, #endif { &key_COND_flush_thread_cache, "COND_flush_thread_cache", PSI_FLAG_GLOBAL} diff --git a/sql/mysqld.h b/sql/mysqld.h index 14997455f2f..91fa2eda7fd 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -223,6 +223,7 @@ extern PSI_mutex_key key_PAGE_lock, key_LOCK_sync, key_LOCK_active, #endif /* HAVE_MMAP */ #ifdef WITH_WSREP extern PSI_mutex_key key_LOCK_wsrep_thd; +extern PSI_cond_key key_COND_wsrep_thd; #endif /* HAVE_WSREP */ #ifdef HAVE_OPENSSL diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 1398fdff94f..fcba6661410 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1105,6 +1105,7 @@ THD::THD() #ifdef WITH_WSREP mysql_mutex_init(key_LOCK_wsrep_thd, &LOCK_wsrep_thd, MY_MUTEX_INIT_FAST); + mysql_cond_init(key_COND_wsrep_thd, &COND_wsrep_thd, NULL); wsrep_ws_handle.trx_id = WSREP_UNDEFINED_TRX_ID; wsrep_ws_handle.opaque = NULL; wsrep_retry_counter = 0; From 016785f6aa9c56b731b612363e7d52488735ee98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 19 Oct 2017 15:02:14 +0300 Subject: [PATCH 050/276] Fix test failure on perfschema.all_instances. --- mysql-test/suite/perfschema/r/all_instances.result | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/perfschema/r/all_instances.result b/mysql-test/suite/perfschema/r/all_instances.result index 3b0b4db4541..bfe50b881da 100644 --- a/mysql-test/suite/perfschema/r/all_instances.result +++ b/mysql-test/suite/perfschema/r/all_instances.result @@ -152,6 +152,7 @@ wait/synch/cond/sql/Relay_log_info::sleep_cond wait/synch/cond/sql/Relay_log_info::start_cond wait/synch/cond/sql/Relay_log_info::stop_cond wait/synch/cond/sql/THD::COND_wakeup_ready +wait/synch/cond/sql/THD::COND_wsrep_thd select event_name from file_instances group by event_name; event_name wait/io/file/aria/control From 22936df631633be2b136e6bf9a88b66e94fa1402 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 25 Oct 2017 11:42:04 -0400 Subject: [PATCH 051/276] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2905c37cc09..87b72051a84 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=58 +MYSQL_VERSION_PATCH=59 MYSQL_VERSION_EXTRA= From e6e026ae51a77969749de201d491a176483bbc69 Mon Sep 17 00:00:00 2001 From: Ian Gilfillan Date: Thu, 23 Nov 2017 14:02:36 +0200 Subject: [PATCH 052/276] Update wsrep_sync_wait documentation as per MW-86 --- mysql-test/r/mysqld--help.result | 4 ++-- sql/sys_vars.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 34d4bae0f28..91131870752 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -942,8 +942,8 @@ The following options may be given as the first argument: --wsrep-sync-wait[=#] Ensure "synchronous" read view before executing an operation of the type specified by bitmask: 1 - - READ(includes SELECT, SHOW and BEGIN/START TRANSACTION); - 2 - UPDATE and DELETE; 4 - INSERT and REPLACE + READ(includes SELECT and BEGIN/START TRANSACTION); 2 - + UPDATE and DELETE; 4 - INSERT and REPLACE; 8 - SHOW Variables (--variable-name=value) allow-suspicious-udfs FALSE diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index a0475c136ae..53b606c0336 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3897,8 +3897,8 @@ static Sys_var_mybool Sys_wsrep_causal_reads( static Sys_var_uint Sys_wsrep_sync_wait( "wsrep_sync_wait", "Ensure \"synchronous\" read view before executing " "an operation of the type specified by bitmask: 1 - READ(includes " - "SELECT, SHOW and BEGIN/START TRANSACTION); 2 - UPDATE and DELETE; 4 - " - "INSERT and REPLACE", + "SELECT and BEGIN/START TRANSACTION); 2 - UPDATE and DELETE; 4 - " + "INSERT and REPLACE; 8 - SHOW", SESSION_VAR(wsrep_sync_wait), CMD_LINE(OPT_ARG), VALID_RANGE(WSREP_SYNC_WAIT_NONE, WSREP_SYNC_WAIT_MAX), DEFAULT(WSREP_SYNC_WAIT_NONE), BLOCK_SIZE(1), From 9007ca68738cb11b24854aa1bfa12dcd204a459d Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Wed, 20 Dec 2017 00:06:02 +0530 Subject: [PATCH 053/276] MDEV-13478 Full SST sync fails because of the error in the cleaning part Problem: The command was: find $paths -mindepth 1 -regex $cpat -prune -o -exec rm -rf {} \+ Which was supposed to work as * skipping $paths directories themselves (-mindepth 1) * see if the dir/file name matches $cpat (-regex) * if yes - don't dive into the directory, skip it (-prune) * otherwise (-o) * remove it and everything inside (-exec) Now -exec ... \+ works like this: every new found path is appended to the end of the command line. when accumulated command line length reaches `getconf ARG_MAX` (~2Gb) it's executed, and find continues, appending to a new command line. What happens here, find appends some directory to the command line, then dives into it, and starts appending files from that directory. At some point command line overflows, rm -rf gets executed and removes the whole directory. Now find tries to continue scanning the directory that was already removed. Fix: don't dive into directories that will be recursively removed anyway, use -prune for them. Basically, we should be pruning both paths that have matched $cpat and paths that have not matched it. This is achived by pruning unconditionally, before the regex is tested: find $paths -mindepth 1 -prune -regex $cpat -o -exec rm -rf {} \+ Patch Credit:- Serg --- scripts/wsrep_sst_xtrabackup-v2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 327c92fb6cf..00d8fe21113 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -863,7 +863,7 @@ then wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" - find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ tempdir=$(parse_cnf mysqld log-bin "") if [[ -n ${tempdir:-} ]];then From 42ccfd82110292a4693b760295699cd6c57bbd94 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 16 Jan 2018 22:45:48 +1100 Subject: [PATCH 054/276] wsrep_sst_mysqldump: safer test of version != 5 --- scripts/wsrep_sst_mysqldump.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index f6f2187006d..56174e914dc 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -124,7 +124,7 @@ SET_GTID_BINLOG_STATE="" SQL_LOG_BIN_OFF="" # Safety check -if echo $SERVER_VERSION | grep '^10.0' > /dev/null +if [ ${SERVER_VERSION%%.*} != '5' ] then # If binary logging is enabled on the joiner node, we need to copy donor's # gtid_binlog_state to joiner. In order to do that, a RESET MASTER must be From 843503e90f902a0c4691467b294a5edc34cf9f21 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 24 Oct 2017 16:48:08 +0300 Subject: [PATCH 055/276] Set wsrep_rli to NULL after deleting it --- sql/sql_class.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 883f0c9cc88..7aae3ed91ed 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1695,7 +1695,10 @@ THD::~THD() mysql_mutex_unlock(&LOCK_wsrep_thd); mysql_mutex_destroy(&LOCK_wsrep_thd); mysql_cond_destroy(&COND_wsrep_thd); - if (wsrep_rli) delete wsrep_rli; + if (wsrep_rli != NULL) { + delete wsrep_rli; + wsrep_rli = NULL; + } if (wsrep_status_vars) wsrep->stats_free(wsrep, wsrep_status_vars); #endif /* Close connection */ From 2400b769c641adaaed3badc1f8eeae1de4065c05 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 21 Nov 2016 16:20:10 -0500 Subject: [PATCH 056/276] MDEV-10442: "Address already in use" on restart SST processes should inherit mysqld's process group. --- sql/wsrep_utils.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/wsrep_utils.cc b/sql/wsrep_utils.cc index 951007c2660..14bf4f29794 100644 --- a/sql/wsrep_utils.cc +++ b/sql/wsrep_utils.cc @@ -264,7 +264,6 @@ process::process (const char* cmd, const char* type, char** env) err_ = posix_spawnattr_setflags (&attr, POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK | - /* start a new process group */ POSIX_SPAWN_SETPGROUP | POSIX_SPAWN_USEVFORK); if (err_) { From 4132b1785a51234b4204682419308da0c967e66e Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 23 Jan 2018 12:05:10 -0500 Subject: [PATCH 057/276] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 87b72051a84..44f719ca097 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=59 +MYSQL_VERSION_PATCH=60 MYSQL_VERSION_EXTRA= From 7cec685758c9ae222e30eb799a673b819c91fa0f Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Tue, 30 Jan 2018 05:37:22 -0800 Subject: [PATCH 058/276] Bump wsrep patch version to 25.23 --- cmake/wsrep.cmake | 2 +- scripts/mysql_config.pl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake index afab9105d8d..5020a2669b4 100644 --- a/cmake/wsrep.cmake +++ b/cmake/wsrep.cmake @@ -18,7 +18,7 @@ # so WSREP_VERSION is produced regardless # Set the patch version -SET(WSREP_PATCH_VERSION "22") +SET(WSREP_PATCH_VERSION "23") # MariaDB addition: Revision number of the last revision merged from # codership branch visible in @@visible_comment. diff --git a/scripts/mysql_config.pl b/scripts/mysql_config.pl index a9254bb966e..0a022034a6d 100644 --- a/scripts/mysql_config.pl +++ b/scripts/mysql_config.pl @@ -53,7 +53,7 @@ my $cwd = cwd(); my $basedir; my $socket = '/tmp/mysql.sock'; -my $version = '5.5.58'; +my $version = '5.5.59'; sub which { From 09b25f85966f44aae933e86b84b4ebe59ded47c3 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 1 Mar 2018 16:35:46 +0100 Subject: [PATCH 059/276] only allow SUPER user to modify wsrep_on --- mysql-test/suite/wsrep/r/variables.result | 7 +++++++ mysql-test/suite/wsrep/t/variables.test | 14 ++++++++++++++ sql/sys_vars.cc | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/wsrep/r/variables.result b/mysql-test/suite/wsrep/r/variables.result index 5117dd1b048..72ea8eacfbf 100644 --- a/mysql-test/suite/wsrep/r/variables.result +++ b/mysql-test/suite/wsrep/r/variables.result @@ -73,6 +73,13 @@ Threads_connected 1 SHOW STATUS LIKE 'wsrep_thread_count'; Variable_name Value wsrep_thread_count 11 +set wsrep_on=0; +set wsrep_on=1; +create user test@localhost; +set auto_increment_increment=10; +set wsrep_on=0; +ERROR 42000: Access denied; you need (at least one of) the SUPER privilege(s) for this operation +drop user test@localhost; # # MDEV#6411: Setting set @@global.wsrep_sst_auth=NULL causes crash # diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test index 3093d8ba942..e148152fc22 100644 --- a/mysql-test/suite/wsrep/t/variables.test +++ b/mysql-test/suite/wsrep/t/variables.test @@ -68,6 +68,20 @@ sleep 3; SHOW STATUS LIKE 'threads_connected'; SHOW STATUS LIKE 'wsrep_thread_count'; +# +# privileges for wsrep_on +# +set wsrep_on=0; +set wsrep_on=1; +create user test@localhost; +connect con1,localhost,test; +set auto_increment_increment=10; +--error ER_SPECIFIC_ACCESS_DENIED_ERROR +set wsrep_on=0; +disconnect con1; +connection default; +drop user test@localhost; + --echo # --echo # MDEV#6411: Setting set @@global.wsrep_sst_auth=NULL causes crash --echo # diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 53b606c0336..8a9120abdd0 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3854,7 +3854,7 @@ static Sys_var_mybool Sys_wsrep_on ( "wsrep_on", "To enable wsrep replication ", SESSION_VAR(wsrep_on), CMD_LINE(OPT_ARG), DEFAULT(TRUE), - NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(check_has_super), ON_UPDATE(wsrep_on_update)); static Sys_var_charptr Sys_wsrep_start_position ( From 9629bca1f0be1bf160c2e8ba205e7bf7fa4ba780 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 10:54:00 +1100 Subject: [PATCH 060/276] MDEV-8743: use O_CLOEXEC (innodb/xtradb) --- storage/innobase/os/os0file.c | 6 +++--- storage/xtradb/os/os0file.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/storage/innobase/os/os0file.c b/storage/innobase/os/os0file.c index a6156e555b5..a410d4644ac 100644 --- a/storage/innobase/os/os0file.c +++ b/storage/innobase/os/os0file.c @@ -1184,10 +1184,10 @@ try_again: } if (create_mode == OS_FILE_CREATE) { - file = open(name, create_flag, S_IRUSR | S_IWUSR + file = open(name, create_flag | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); } else { - file = open(name, create_flag); + file = open(name, create_flag | O_CLOEXEC); } if (file == -1) { @@ -1631,7 +1631,7 @@ try_again: } #endif /* O_SYNC */ - file = open(name, create_flag, os_innodb_umask); + file = open(name, create_flag | O_CLOEXEC, os_innodb_umask); if (file == -1) { *success = FALSE; diff --git a/storage/xtradb/os/os0file.c b/storage/xtradb/os/os0file.c index 8e5cc9a6ba6..d0f014e8f08 100644 --- a/storage/xtradb/os/os0file.c +++ b/storage/xtradb/os/os0file.c @@ -1286,10 +1286,10 @@ try_again: } if (create_mode == OS_FILE_CREATE) { - file = open(name, create_flag, S_IRUSR | S_IWUSR + file = open(name, create_flag | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); } else { - file = open(name, create_flag); + file = open(name, create_flag | O_CLOEXEC); } if (file == -1) { @@ -1752,7 +1752,7 @@ try_again: } #endif /* O_SYNC */ - file = open(name, create_flag, os_innodb_umask); + file = open(name, create_flag | O_CLOEXEC, os_innodb_umask); if (file == -1) { *success = FALSE; From 4ec7b840776e498456b9a22a51178099e4d66aa3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 10:54:34 +1100 Subject: [PATCH 061/276] MDEV-8743: use O_CLOEXEC MYSQL_LOG::open / TC_LOG_MMAP::open For galera compatibility, the main thing is to ensure the FD 1, 2 are not opened with O_CLOEXEC otherwise galera sst errors don't appear in the error.log Files without O_CLOEXEC from the test below: 0 -> /dev/pts/9 1 -> /tmp/error.log (intended) 2 -> /tmp/error.log (intended) 5 -> /tmp/datadir 6 -> /tmp/datadir/aria_log.00000001 (Innodb temp files) 8 -> /tmp/ibIIrhFL (deleted) 9 -> /tmp/ibfx1vai (deleted) 10 -> /tmp/ibAQUKFO (deleted) 11 -> /tmp/ibWBQSHR (deleted) 15 -> /tmp/ibEXEcfo (deleted) 20 -> /tmp/datadir/mysql/host.MYD 22 -> /tmp/datadir/mysql/user.MYD ... (rest of MYD files) Test for this and the previous commit. sql/mysqld --skip-networking --datadir=/tmp/datadir --log-bin=/tmp/datadir/mysqlbin --socket /tmp/s.sock --lc-messages-dir=${PWD}/sql/share --verbose --log-error=/tmp/error.log --general-log-file=/tmp/general.log --general-log=1 --slow-query-log-file=/tmp/slow.log --slow-query-log=1 180302 10:56:41 [Note] sql/mysqld (mysqld 5.5.60-MariaDB-wsrep) starting as process 26056 ... $ cd /proc/26056 $ ls -la --sort=none fd total 0 dr-x------. 2 dan dan 0 Mar 2 10:57 . dr-xr-xr-x. 9 dan dan 0 Mar 2 10:56 .. lrwx------. 1 dan dan 64 Mar 2 10:57 0 -> /dev/pts/9 l-wx------. 1 dan dan 64 Mar 2 10:57 1 -> /tmp/error.log l-wx------. 1 dan dan 64 Mar 2 10:57 2 -> /tmp/error.log lrwx------. 1 dan dan 64 Mar 2 10:57 3 -> /tmp/datadir/mysqlbin.index lrwx------. 1 dan dan 64 Mar 2 10:57 4 -> /tmp/datadir/aria_log_control lr-x------. 1 dan dan 64 Mar 2 10:57 5 -> /tmp/datadir lrwx------. 1 dan dan 64 Mar 2 10:57 6 -> /tmp/datadir/aria_log.00000001 lrwx------. 1 dan dan 64 Mar 2 10:57 7 -> /tmp/datadir/ibdata1 lrwx------. 1 dan dan 64 Mar 2 10:57 8 -> /tmp/ibIIrhFL (deleted) lrwx------. 1 dan dan 64 Mar 2 10:57 9 -> /tmp/ibfx1vai (deleted) lrwx------. 1 dan dan 64 Mar 2 10:57 10 -> /tmp/ibAQUKFO (deleted) lrwx------. 1 dan dan 64 Mar 2 10:57 11 -> /tmp/ibWBQSHR (deleted) lrwx------. 1 dan dan 64 Mar 2 10:57 12 -> /tmp/datadir/ib_logfile0 lrwx------. 1 dan dan 64 Mar 2 10:57 13 -> /tmp/datadir/ib_logfile1 l-wx------. 1 dan dan 64 Mar 2 10:57 14 -> /tmp/slow.log lrwx------. 1 dan dan 64 Mar 2 10:57 15 -> /tmp/ibEXEcfo (deleted) l-wx------. 1 dan dan 64 Mar 2 10:57 16 -> /tmp/general.log lrwx------. 1 dan dan 64 Mar 2 10:57 17 -> socket:[1897356] lrwx------. 1 dan dan 64 Mar 2 10:57 18 -> socket:[45335] l-wx------. 1 dan dan 64 Mar 2 10:57 19 -> /tmp/datadir/mysqlbin.000004 lrwx------. 1 dan dan 64 Mar 2 10:57 20 -> /tmp/datadir/mysql/host.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 21 -> /tmp/datadir/mysql/host.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 22 -> /tmp/datadir/mysql/user.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 23 -> /tmp/datadir/mysql/user.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 24 -> /tmp/datadir/mysql/db.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 25 -> /tmp/datadir/mysql/db.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 26 -> /tmp/datadir/mysql/proxies_priv.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 27 -> /tmp/datadir/mysql/proxies_priv.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 28 -> /tmp/datadir/mysql/tables_priv.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 29 -> /tmp/datadir/mysql/tables_priv.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 30 -> /tmp/datadir/mysql/columns_priv.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 31 -> /tmp/datadir/mysql/columns_priv.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 32 -> /tmp/datadir/mysql/procs_priv.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 33 -> /tmp/datadir/mysql/procs_priv.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 34 -> /tmp/datadir/mysql/servers.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 35 -> /tmp/datadir/mysql/servers.MYI lrwx------. 1 dan dan 64 Mar 2 10:57 36 -> /tmp/datadir/mysql/event.MYD lrwx------. 1 dan dan 64 Mar 2 10:57 37 -> /tmp/datadir/mysql/event.MYI O_CLOEXEC files are those with flags 02000000 /usr/include/bits/fcntl-linux.h:# define __O_CLOEXEC 02000000 /usr/include/bits/fcntl-linux.h:# define O_CLOEXEC __O_CLOEXEC /* Set close_on_exec. */ $ find fdinfo/ -type f -ls -exec cat {} \; | more 1924720 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/0 pos: 0 flags: 0100002 mnt_id: 25 1924721 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/1 pos: 9954 flags: 0102001 mnt_id: 82 1924722 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/2 pos: 10951 flags: 0102001 mnt_id: 82 1924723 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/3 pos: 116 flags: 02100002 mnt_id: 82 1924724 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/4 pos: 52 flags: 0100002 mnt_id: 82 lock: 1: POSIX ADVISORY WRITE 26056 00:2c:1866365 0 EOF 1924725 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/5 pos: 0 flags: 0100000 mnt_id: 82 1924726 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/6 pos: 16384 flags: 0100002 mnt_id: 82 1924727 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/7 pos: 0 flags: 02100002 mnt_id: 82 lock: 1: POSIX ADVISORY WRITE 26056 00:2c:1866491 0 EOF 1924728 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/8 pos: 0 flags: 0100002 mnt_id: 82 1924729 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/9 pos: 0 flags: 0100002 mnt_id: 82 1924730 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/10 pos: 0 flags: 0100002 mnt_id: 82 1924731 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/11 pos: 0 flags: 0100002 mnt_id: 82 1924732 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/12 pos: 0 flags: 02100002 mnt_id: 82 lock: 1: POSIX ADVISORY WRITE 26056 00:2c:1866492 0 EOF 1924733 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/13 pos: 0 flags: 02100002 mnt_id: 82 lock: 1: POSIX ADVISORY WRITE 26056 00:2c:1866493 0 EOF 1924734 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/14 pos: 763 flags: 02102001 mnt_id: 82 1924735 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/15 pos: 0 flags: 0100002 mnt_id: 82 1924736 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/16 pos: 473 flags: 02102001 mnt_id: 82 1924737 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/17 pos: 0 flags: 02000002 mnt_id: 9 1924738 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/18 pos: 0 flags: 02 mnt_id: 9 1924739 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/19 pos: 245 flags: 02100001 mnt_id: 82 1924740 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/20 pos: 0 flags: 0100002 mnt_id: 82 1924741 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/21 pos: 503 flags: 0500002 mnt_id: 82 1924742 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/22 pos: 324 flags: 0100002 mnt_id: 82 1924743 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/23 pos: 642 flags: 0500002 mnt_id: 82 1924744 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/24 pos: 880 flags: 0100002 mnt_id: 82 1924745 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/25 pos: 581 flags: 0500002 mnt_id: 82 1924746 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/26 pos: 1386 flags: 0100002 mnt_id: 82 1924747 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/27 pos: 498 flags: 0500002 mnt_id: 82 1924748 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/28 pos: 0 flags: 0100002 mnt_id: 82 1924749 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/29 pos: 513 flags: 0500002 mnt_id: 82 1924750 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/30 pos: 0 flags: 0100002 mnt_id: 82 1924751 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/31 pos: 494 flags: 0500002 mnt_id: 82 1924752 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/32 pos: 0 flags: 0100002 mnt_id: 82 1924753 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/33 pos: 535 flags: 0500002 mnt_id: 82 1924754 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/34 pos: 0 flags: 0100002 mnt_id: 82 1924755 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/35 pos: 396 flags: 0500002 mnt_id: 82 1924756 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/36 pos: 0 flags: 0100002 mnt_id: 82 1924757 0 -r-------- 1 dan dan 0 Mar 2 10:57 fdinfo/37 pos: 517 flags: 0500002 mnt_id: 82 --- mysys/my_symlink.c | 2 +- sql/log.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mysys/my_symlink.c b/mysys/my_symlink.c index 12959f1f24a..f2dd2e596ff 100644 --- a/mysys/my_symlink.c +++ b/mysys/my_symlink.c @@ -236,7 +236,7 @@ const char *my_open_parent_dir_nosymlinks(const char *pathname, int *pdfd) return pathname + (s - buf); } - fd = openat(dfd, s, O_NOFOLLOW | O_PATH); + fd = openat(dfd, s, O_NOFOLLOW | O_PATH | O_CLOEXEC); if (fd < 0) goto err; diff --git a/sql/log.cc b/sql/log.cc index 4760e668aeb..ca7833a0460 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -2466,7 +2466,7 @@ bool MYSQL_LOG::open( File file= -1; my_off_t seek_offset; bool is_fifo = false; - int open_flags= O_CREAT | O_BINARY; + int open_flags= O_CREAT | O_BINARY | O_CLOEXEC; DBUG_ENTER("MYSQL_LOG::open"); DBUG_PRINT("enter", ("log_type: %d", (int) log_type_arg)); @@ -3088,7 +3088,7 @@ bool MYSQL_BIN_LOG::open_index_file(const char *index_file_name_arg, ".index", opt); if ((index_file_nr= mysql_file_open(m_key_file_log_index, index_file_name, - O_RDWR | O_CREAT | O_BINARY, + O_RDWR | O_CREAT | O_BINARY | O_CLOEXEC, MYF(MY_WME))) < 0 || mysql_file_sync(index_file_nr, MYF(MY_WME)) || init_io_cache(&index_file, index_file_nr, @@ -7045,14 +7045,14 @@ int TC_LOG_MMAP::open(const char *opt_name) DBUG_ASSERT(TC_LOG_PAGE_SIZE % tc_log_page_size == 0); fn_format(logname,opt_name,mysql_data_home,"",MY_UNPACK_FILENAME); - if ((fd= mysql_file_open(key_file_tclog, logname, O_RDWR, MYF(0))) < 0) + if ((fd= mysql_file_open(key_file_tclog, logname, O_RDWR | O_CLOEXEC, MYF(0))) < 0) { if (my_errno != ENOENT) goto err; if (using_heuristic_recover()) return 1; if ((fd= mysql_file_create(key_file_tclog, logname, CREATE_MODE, - O_RDWR, MYF(MY_WME))) < 0) + O_RDWR | O_CLOEXEC, MYF(MY_WME))) < 0) goto err; inited=1; file_length= opt_tc_log_size; From c54c490c59e2d6aed21934cfd5948974ee0ee774 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Fri, 2 Mar 2018 11:17:35 +1100 Subject: [PATCH 062/276] MDEV-8743: O_CLOEXEC/SOCK_CLOEXEC defines for non-unix compatibility From 0f4478105f5027ce589a0f08c6019d906d632029 --- include/my_global.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/my_global.h b/include/my_global.h index cf140cf54ce..6e8fb33b137 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -578,6 +578,12 @@ typedef SOCKET_SIZE_TYPE size_socket; #ifndef O_NOFOLLOW #define O_NOFOLLOW 0 #endif +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif +#ifndef SOCK_CLOEXEC +#define SOCK_CLOEXEC 0 +#endif /* additional file share flags for win32 */ #ifdef __WIN__ From 88fb8b2e367158dce7a4c3e14f2517c5deb9b51d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 11:52:20 +1100 Subject: [PATCH 063/276] MDEV-8743: protect myisam/aria MYD files and aria log files --- storage/maria/ma_control_file.c | 2 +- storage/maria/ma_loghandler.c | 6 +++--- storage/maria/ma_open.c | 4 ++-- storage/myisam/mi_open.c | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/storage/maria/ma_control_file.c b/storage/maria/ma_control_file.c index 2fa3e6ed18e..11cb4c5fd24 100644 --- a/storage/maria/ma_control_file.c +++ b/storage/maria/ma_control_file.c @@ -273,7 +273,7 @@ CONTROL_FILE_ERROR ma_control_file_open(my_bool create_if_missing, " file is probably in use by another process"; uint new_cf_create_time_size, new_cf_changeable_size, new_block_size; my_off_t file_size; - int open_flags= O_BINARY | /*O_DIRECT |*/ O_RDWR; + int open_flags= O_BINARY | /*O_DIRECT |*/ O_RDWR | O_CLOEXEC; int error= CONTROL_FILE_UNKNOWN_ERROR; DBUG_ENTER("ma_control_file_open"); diff --git a/storage/maria/ma_loghandler.c b/storage/maria/ma_loghandler.c index 2118b3b6ce6..f31662ce003 100644 --- a/storage/maria/ma_loghandler.c +++ b/storage/maria/ma_loghandler.c @@ -942,7 +942,7 @@ static File create_logfile_by_number_no_cache(uint32 file_no) /* TODO: add O_DIRECT to open flags (when buffer is aligned) */ if ((file= mysql_file_create(key_file_translog, translog_filename_by_fileno(file_no, path), - 0, O_BINARY | O_RDWR, MYF(MY_WME))) < 0) + 0, O_BINARY | O_RDWR | O_CLOEXEC, MYF(MY_WME))) < 0) { DBUG_PRINT("error", ("Error %d during creating file '%s'", errno, path)); translog_stop_writing(); @@ -979,7 +979,7 @@ static File open_logfile_by_number_no_cache(uint32 file_no) /* TODO: use mysql_file_create() */ if ((file= mysql_file_open(key_file_translog, translog_filename_by_fileno(file_no, path), - log_descriptor.open_flags, + log_descriptor.open_flags | O_CLOEXEC, MYF(MY_WME))) < 0) { DBUG_PRINT("error", ("Error %d during opening file '%s'", errno, path)); @@ -3263,7 +3263,7 @@ static my_bool translog_get_last_page_addr(TRANSLOG_ADDRESS *addr, File fd; if ((fd= mysql_file_open(key_file_translog, translog_filename_by_fileno(file_no, path), - O_RDONLY, (no_errors ? MYF(0) : MYF(MY_WME)))) < 0) + O_RDONLY | O_CLOEXEC, (no_errors ? MYF(0) : MYF(MY_WME)))) < 0) { my_errno= errno; DBUG_PRINT("error", ("Error %d during opening file #%d", diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c index a382c30e025..d20fa06e069 100644 --- a/storage/maria/ma_open.c +++ b/storage/maria/ma_open.c @@ -1885,7 +1885,7 @@ int _ma_open_datafile(MARIA_HA *info, MARIA_SHARE *share) DEBUG_SYNC_C("mi_open_datafile"); info->dfile.file= share->bitmap.file.file= mysql_file_open(key_file_dfile, share->data_file_name.str, - share->mode | O_SHARE, MYF(flags)); + share->mode | O_SHARE | O_CLOEXEC, MYF(flags)); return info->dfile.file >= 0 ? 0 : 1; } @@ -1899,7 +1899,7 @@ int _ma_open_keyfile(MARIA_SHARE *share) mysql_mutex_lock(&share->intern_lock); share->kfile.file= mysql_file_open(key_file_kfile, share->unique_file_name.str, - share->mode | O_SHARE | O_NOFOLLOW, + share->mode | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_WME | MY_NOSYMLINKS)); mysql_mutex_unlock(&share->intern_lock); return (share->kfile.file < 0); diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c index 86b70129203..8e739109c0d 100644 --- a/storage/myisam/mi_open.c +++ b/storage/myisam/mi_open.c @@ -1249,7 +1249,7 @@ int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share) myf flags= MY_WME | (share->mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0); DEBUG_SYNC_C("mi_open_datafile"); info->dfile= mysql_file_open(mi_key_file_dfile, share->data_file_name, - share->mode | O_SHARE, MYF(flags)); + share->mode | O_SHARE | O_CLOEXEC, MYF(flags)); return info->dfile >= 0 ? 0 : 1; } @@ -1258,7 +1258,7 @@ int mi_open_keyfile(MYISAM_SHARE *share) { if ((share->kfile= mysql_file_open(mi_key_file_kfile, share->unique_file_name, - share->mode | O_SHARE | O_NOFOLLOW, + share->mode | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS | MY_WME))) < 0) return 1; return 0; From bbee025370d885103cba55716c9c2ab44e533283 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 12:45:36 +1100 Subject: [PATCH 064/276] MDEV-8743: O_CLOEXEC on innodb/xtradb temp files Thread 1 "mysqld" hit Breakpoint 1, innobase_mysql_tmpfile () at /home/dan/repos/mariadb-server-5.5/storage/xtradb/handler/ha_innodb.cc:1639 1639 os_event_wait(srv_allow_writes_event); (gdb) p fd $2 = 7 (gdb) n 1682 fd2 = fcntl(fd, F_DUPFD_CLOEXEC, 0); (gdb) 1687 if (fd2 < 0) { (gdb) p fd2 $3 = 8 cat /proc/20448/fdinfo/{7,8} 2051972 0 -r-------- 1 dan dan 0 Mar 2 12:04 fdinfo/7 pos: 0 flags: 0100002 mnt_id: 82 2051973 0 -r-------- 1 dan dan 0 Mar 2 12:04 fdinfo/8 pos: 0 flags: 02100002 mnt_id: 82 So fd 8 has 02000000 hence CLOEXEC --- storage/innobase/handler/ha_innodb.cc | 4 ++++ storage/xtradb/handler/ha_innodb.cc | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3a96763d6c2..c9eee5b4dcd 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1410,8 +1410,12 @@ innobase_mysql_tmpfile(void) fd2 = -1; } } +#else +#ifdef F_DUPFD_CLOEXEC + fd2 = fcntl(fd, F_DUPFD_CLOEXEC, 0); #else fd2 = dup(fd); +#endif #endif if (fd2 < 0) { DBUG_PRINT("error",("Got error %d on dup",fd2)); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index f99ae7c7de9..04fbcd1a9fd 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1677,8 +1677,12 @@ innobase_mysql_tmpfile(void) fd2 = -1; } } +#else +#ifdef F_DUPFD_CLOEXEC + fd2 = fcntl(fd, F_DUPFD_CLOEXEC, 0); #else fd2 = dup(fd); +#endif #endif if (fd2 < 0) { DBUG_PRINT("error",("Got error %d on dup",fd2)); From d3c0e34bdc99ee57fd5d63237d29531d1afdef2b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 2 Mar 2018 16:19:14 +1100 Subject: [PATCH 065/276] MDEV-8743: protect myisam/aria MYI with O_CLOEXEC --- storage/maria/ma_open.c | 4 ++-- storage/myisam/mi_open.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c index d20fa06e069..d9ff2179691 100644 --- a/storage/maria/ma_open.c +++ b/storage/maria/ma_open.c @@ -314,13 +314,13 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) }); DEBUG_SYNC_C("mi_open_kfile"); if ((kfile=mysql_file_open(key_file_kfile, name_buff, - (open_mode=O_RDWR) | O_SHARE | O_NOFOLLOW, + (open_mode=O_RDWR) | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) { if ((errno != EROFS && errno != EACCES) || mode != O_RDONLY || (kfile=mysql_file_open(key_file_kfile, name_buff, - (open_mode=O_RDONLY) | O_SHARE | O_NOFOLLOW, + (open_mode=O_RDONLY) | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) goto err; } diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c index 8e739109c0d..9775aade9c1 100644 --- a/storage/myisam/mi_open.c +++ b/storage/myisam/mi_open.c @@ -117,13 +117,13 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) DEBUG_SYNC_C("mi_open_kfile"); if ((kfile= mysql_file_open(mi_key_file_kfile, name_buff, - (open_mode= O_RDWR) | O_SHARE | O_NOFOLLOW, + (open_mode= O_RDWR) | O_SHARE | O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) { if ((errno != EROFS && errno != EACCES) || mode != O_RDONLY || (kfile= mysql_file_open(mi_key_file_kfile, name_buff, - (open_mode= O_RDONLY) | O_SHARE| O_NOFOLLOW, + (open_mode= O_RDONLY) | O_SHARE| O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) goto err; } From ccd566af20dc47a5708e2e707c9624796ea79bef Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 19 Apr 2018 07:38:57 +1000 Subject: [PATCH 066/276] MDEV-8743: mysqld port/socket - FD_CLOEXEC if no SOCK_CLOEXEC In MDEV-8743, the port/socket of mysqld was changed to set FD_CLOEXEC. The existing mysql_socket_socket function already set that with SOCK_CLOEXEC when the socket was created. So here we move the fcntl functionality to the mysql_socket_socket as port/socket are the only callers. Preprocessor checks of SOCK_CLOEXEC cannot be done as its a 0 if not there and SOCK_CLOEXEC (being the value of the enum in bits/socket_type.h) Preprocesssor logic for arithmetic and non-arithmetic defines are hard/nonportable/ugly to read. As such we just check in my_global.h and define HAVE_SOCK_CLOEXEC if we have it. There was a disparity in behaviour between defined(WITH_WSREP) and not depending on the OS, so the WITH_WSREP condition was removed from setting calling fcntl. All sockets are now maked SOCK_CLOEXEC/FD_CLOEXEC. strace of mysqld with SOCK_CLOEXEC: socket(AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 10 write(2, "180419 14:52:40 [Note] Server socket created on IP: '127.0.0.1'.\n", 65180419 14:52:40 [Note] Server socket created on IP: '127.0.0.1'. ) = 65 setsockopt(10, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 bind(10, {sa_family=AF_INET, sin_port=htons(16020), sin_addr=inet_addr("127.0.0.1")}, 16) = 0 listen(10, 150) = 0 socket(AF_INET, SOCK_STREAM|SOCK_CLOEXEC, IPPROTO_TCP) = 11 write(2, "180419 14:52:40 [Note] Server socket created on IP: '127.0.0.1'.\n", 65180419 14:52:40 [Note] Server socket created on IP: '127.0.0.1'. ) = 65 setsockopt(11, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 bind(11, {sa_family=AF_INET, sin_port=htons(16021), sin_addr=inet_addr("127.0.0.1")}, 16) = 0 listen(11, 150) = 0 socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0) = 12 unlink("/home/dan/repos/build-mariadb-server-10.0/mysql-test/var/tmp/mysqld.1.sock") = -1 ENOENT (No such file or directory) setsockopt(12, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0 umask(000) = 006 bind(12, {sa_family=AF_UNIX, sun_path="/home/dan/repos/build-mariadb-server-10.0/mysql-test/var/tmp/mysqld.1.sock"}, 110) = 0 umask(006) = 000 listen(12, 150) = 0 --- include/my_global.h | 2 ++ include/mysql/psi/mysql_socket.h | 6 ++++++ sql/mysqld.cc | 7 ------- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/include/my_global.h b/include/my_global.h index ad1527d7b37..057c613d64c 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -608,6 +608,8 @@ typedef SOCKET_SIZE_TYPE size_socket; #endif #ifndef SOCK_CLOEXEC #define SOCK_CLOEXEC 0 +#else +#define HAVE_SOCK_CLOEXEC #endif /* additional file share flags for win32 */ diff --git a/include/mysql/psi/mysql_socket.h b/include/mysql/psi/mysql_socket.h index bd05ef9b8a9..70700a733d8 100644 --- a/include/mysql/psi/mysql_socket.h +++ b/include/mysql/psi/mysql_socket.h @@ -562,6 +562,12 @@ inline_mysql_socket_socket (key, (const my_socket*)&mysql_socket.fd, NULL, 0); } #endif + + /* SOCK_CLOEXEC isn't always a number - can't preprocessor compare */ +#if defined(HAVE_FCNTL) && defined(FD_CLOEXEC) && !defined(HAVE_SOCK_CLOEXEC) + (void) fcntl(mysql_socket.fd, F_SETFD, FD_CLOEXEC); +#endif + return mysql_socket; } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 8575709203c..15f4af5edbb 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2593,10 +2593,6 @@ static MYSQL_SOCKET activate_tcp_port(uint port) socket_errno); unireg_abort(1); } -#if defined(WITH_WSREP) && defined(HAVE_FCNTL) && defined(FD_CLOEXEC) - (void) fcntl(mysql_socket_getfd(ip_sock), F_SETFD, FD_CLOEXEC); -#endif /* WITH_WSREP */ - DBUG_RETURN(ip_sock); } @@ -2714,9 +2710,6 @@ static void network_init(void) if (mysql_socket_listen(unix_sock,(int) back_log) < 0) sql_print_warning("listen() on Unix socket failed with error %d", socket_errno); -#if defined(WITH_WSREP) && defined(HAVE_FCNTL) - (void) fcntl(mysql_socket_getfd(unix_sock), F_SETFD, FD_CLOEXEC); -#endif /* WITH_WSREP */ } #endif DBUG_PRINT("info",("server started")); From c2c61bbccea3525e12d8897c009898138a335392 Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Sun, 17 Dec 2017 14:41:55 +0200 Subject: [PATCH 067/276] Provider rollback for ineffective trx codership/mysql-wsrep#318 Adapt MTR tests to new Galera status variables and fix exposed leaks In certain cases it is possible that transaction has populated keys in the provider but during commit time the IO cache is empty, so the provider commit does not happen (for example early ROLLBACK TO SAVEPOINT followed by COMMIT). Run provider post_rollback() to clean up the transaction object. --- sql/wsrep_hton.cc | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/sql/wsrep_hton.cc b/sql/wsrep_hton.cc index 0572230b18b..c18c179a4bd 100644 --- a/sql/wsrep_hton.cc +++ b/sql/wsrep_hton.cc @@ -99,16 +99,45 @@ void wsrep_register_hton(THD* thd, bool all) */ void wsrep_post_commit(THD* thd, bool all) { - if (thd->wsrep_exec_mode == LOCAL_COMMIT) + if (!WSREP(thd)) return; + + switch (thd->wsrep_exec_mode) { - DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED); - if (wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) + case LOCAL_COMMIT: { + DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED); + if (wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) + { DBUG_PRINT("wsrep", ("set committed fail")); - WSREP_WARN("set committed fail: %llu %d", + WSREP_WARN("set committed fail: %llu %d", (long long)thd->real_id, thd->stmt_da->status()); + } + wsrep_cleanup_transaction(thd); + break; } - wsrep_cleanup_transaction(thd); + case LOCAL_STATE: + { + /* non-InnoDB statements may have populated events in stmt cache + => cleanup + */ + WSREP_DEBUG("cleanup transaction for LOCAL_STATE"); + /* + Run post-rollback hook to clean up in the case if + some keys were populated for the transaction in provider + but during commit time there was no write set to replicate. + This may happen when client sets the SAVEPOINT and immediately + rolls back to savepoint after first operation. + */ + if (all && thd->wsrep_conflict_state != MUST_REPLAY && + wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) + { + WSREP_WARN("post_rollback fail: %llu %d", + (long long)thd->thread_id, thd->stmt_da->status()); + } + wsrep_cleanup_transaction(thd); + break; + } + default: break; } } From 231c02f7b9fc05ffa07dacd5f31bbd2ff14e7cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 24 Apr 2018 13:58:42 +0300 Subject: [PATCH 068/276] MariaDB adjustments. --- scripts/mysql_config.pl | 2 +- sql/wsrep_hton.cc | 48 ++++++++++++++++++++--------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/scripts/mysql_config.pl b/scripts/mysql_config.pl index 0a022034a6d..113d8fc10be 100644 --- a/scripts/mysql_config.pl +++ b/scripts/mysql_config.pl @@ -53,7 +53,7 @@ my $cwd = cwd(); my $basedir; my $socket = '/tmp/mysql.sock'; -my $version = '5.5.59'; +my $version = '5.5.60'; sub which { diff --git a/sql/wsrep_hton.cc b/sql/wsrep_hton.cc index c18c179a4bd..1676daab5fe 100644 --- a/sql/wsrep_hton.cc +++ b/sql/wsrep_hton.cc @@ -103,10 +103,10 @@ void wsrep_post_commit(THD* thd, bool all) switch (thd->wsrep_exec_mode) { - case LOCAL_COMMIT: + case LOCAL_COMMIT: { DBUG_ASSERT(thd->wsrep_trx_meta.gtid.seqno != WSREP_SEQNO_UNDEFINED); - if (wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) + if (wsrep && wsrep->post_commit(wsrep, &thd->wsrep_ws_handle)) { DBUG_PRINT("wsrep", ("set committed fail")); WSREP_WARN("set committed fail: %llu %d", @@ -115,29 +115,29 @@ void wsrep_post_commit(THD* thd, bool all) wsrep_cleanup_transaction(thd); break; } - case LOCAL_STATE: - { - /* non-InnoDB statements may have populated events in stmt cache - => cleanup - */ - WSREP_DEBUG("cleanup transaction for LOCAL_STATE"); - /* - Run post-rollback hook to clean up in the case if - some keys were populated for the transaction in provider - but during commit time there was no write set to replicate. - This may happen when client sets the SAVEPOINT and immediately - rolls back to savepoint after first operation. + case LOCAL_STATE: + { + /* non-InnoDB statements may have populated events in stmt cache + => cleanup */ - if (all && thd->wsrep_conflict_state != MUST_REPLAY && - wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) - { - WSREP_WARN("post_rollback fail: %llu %d", - (long long)thd->thread_id, thd->stmt_da->status()); - } - wsrep_cleanup_transaction(thd); - break; - } - default: break; + WSREP_DEBUG("cleanup transaction for LOCAL_STATE"); + /* + Run post-rollback hook to clean up in the case if + some keys were populated for the transaction in provider + but during commit time there was no write set to replicate. + This may happen when client sets the SAVEPOINT and immediately + rolls back to savepoint after first operation. + */ + if (all && thd->wsrep_conflict_state != MUST_REPLAY && + wsrep && wsrep->post_rollback(wsrep, &thd->wsrep_ws_handle)) + { + WSREP_WARN("post_rollback fail: %llu %d", + (long long)thd->thread_id, thd->stmt_da->status()); + } + wsrep_cleanup_transaction(thd); + break; + } + default: break; } } From 75dec85c2ef7383cb452fcd4e5b0fe06c0016bc0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 27 Apr 2018 11:21:55 +0200 Subject: [PATCH 069/276] Bug#25471090: MYSQL USE AFTER FREE fix another similar line followup for 7828ba0df488 --- sql-common/client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql-common/client.c b/sql-common/client.c index bb7bdb1ff7d..9cb3311d2e1 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -1636,7 +1636,7 @@ MYSQL_DATA *cli_read_rows(MYSQL *mysql,MYSQL_FIELD *mysql_fields, else { cur->data[field] = to; - if (len > (ulong) (end_to - to)) + if (to + len > end_to) { free_rows(result); set_mysql_error(mysql, CR_MALFORMED_PACKET, unknown_sqlstate); From ccad629d7e453b06b16193deb4685fc10908f92f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 30 Apr 2018 13:50:59 +0200 Subject: [PATCH 070/276] Bug#25471090: MYSQL USE AFTER FREE a better fix --- sql-common/client.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql-common/client.c b/sql-common/client.c index 9cb3311d2e1..b485ebf4f60 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -1636,7 +1636,7 @@ MYSQL_DATA *cli_read_rows(MYSQL *mysql,MYSQL_FIELD *mysql_fields, else { cur->data[field] = to; - if (to + len > end_to) + if (unlikely(len > (ulong)(end_to-to) || to > end_to)) { free_rows(result); set_mysql_error(mysql, CR_MALFORMED_PACKET, unknown_sqlstate); @@ -1708,7 +1708,7 @@ read_one_row(MYSQL *mysql,uint fields,MYSQL_ROW row, ulong *lengths) } else { - if (pos + len > end_pos) + if (unlikely(len > (ulong)(end_pos - pos) || pos > end_pos)) { set_mysql_error(mysql, CR_UNKNOWN_ERROR, unknown_sqlstate); return -1; From 1ecd68d867ced1d00ebffdcedbf6bc97493f5067 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 30 Apr 2018 23:06:09 +0200 Subject: [PATCH 071/276] Use after free in authentication --- mysql-test/r/connect_debug.result | 5 +++++ mysql-test/t/connect_debug.test | 12 ++++++++++++ sql-common/client.c | 4 ++-- sql/sql_acl.cc | 1 + 4 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 mysql-test/r/connect_debug.result create mode 100644 mysql-test/t/connect_debug.test diff --git a/mysql-test/r/connect_debug.result b/mysql-test/r/connect_debug.result new file mode 100644 index 00000000000..0452b238db9 --- /dev/null +++ b/mysql-test/r/connect_debug.result @@ -0,0 +1,5 @@ +set @old_dbug=@@global.debug_dbug; +set global debug_dbug='+d,auth_disconnect'; +create user 'bad' identified by 'worse'; +set global debug_dbug=@old_dbug; +drop user bad; diff --git a/mysql-test/t/connect_debug.test b/mysql-test/t/connect_debug.test new file mode 100644 index 00000000000..299b605b2cd --- /dev/null +++ b/mysql-test/t/connect_debug.test @@ -0,0 +1,12 @@ +source include/have_debug.inc; +set @old_dbug=@@global.debug_dbug; + +# +# use after free if need plugin change and auth aborted +# +set global debug_dbug='+d,auth_disconnect'; +create user 'bad' identified by 'worse'; +--error 1 +--exec $MYSQL --default-auth=mysql_old_password --user=bad --password=worse +set global debug_dbug=@old_dbug; +drop user bad; diff --git a/sql-common/client.c b/sql-common/client.c index b485ebf4f60..00e2877bedb 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -2742,7 +2742,7 @@ static int client_mpvio_read_packet(struct st_plugin_vio *mpv, uchar **buf) *buf= mysql->net.read_pos; /* was it a request to change plugins ? */ - if (**buf == 254) + if (pkt_len == packet_error || **buf == 254) return (int)packet_error; /* if yes, this plugin shan't continue */ /* @@ -2927,7 +2927,7 @@ int run_plugin_auth(MYSQL *mysql, char *data, uint data_len, compile_time_assert(CR_OK == -1); compile_time_assert(CR_ERROR == 0); - if (res > CR_OK && mysql->net.read_pos[0] != 254) + if (res > CR_OK && (mysql->net.last_errno || mysql->net.read_pos[0] != 254)) { /* the plugin returned an error. write it down in mysql, diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index c3740f8ab29..15a238193dd 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -8253,6 +8253,7 @@ static bool send_plugin_request_packet(MPVIO_EXT *mpvio, const char *client_auth_plugin= ((st_mysql_auth *) (plugin_decl(mpvio->plugin)->info))->client_auth_plugin; + DBUG_EXECUTE_IF("auth_disconnect", { vio_close(net->vio); DBUG_RETURN(1); }); DBUG_ASSERT(client_auth_plugin); /* From 0088fb91f36574ea79a575282951e956ab7f15f8 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Fri, 2 Feb 2018 14:18:39 +0100 Subject: [PATCH 072/276] Fix sporadic failure of MTR test galera.galera_many_tables_pk The test relies on the false assumption that node_1 is always "assigned" autoincrement value 1 for a given key. However that is no necessarily the case when wsrep autoincrement control is enabled. This patch changed the test so that it does not make use of a autoincrement key, which is not fundamental for the test itself. codership/mysql-wsrep#322 --- mysql-test/suite/galera/t/galera_many_tables_pk.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_many_tables_pk.test b/mysql-test/suite/galera/t/galera_many_tables_pk.test index 72c7e08bcc3..0af7160d4d8 100644 --- a/mysql-test/suite/galera/t/galera_many_tables_pk.test +++ b/mysql-test/suite/galera/t/galera_many_tables_pk.test @@ -16,7 +16,7 @@ while ($count) { --disable_query_log - --let $ddl_var = `SELECT CONCAT("CREATE TABLE t", $count, " (f1 INTEGER AUTO_INCREMENT PRIMARY KEY) ENGINE=InnoDB")` + --let $ddl_var = `SELECT CONCAT("CREATE TABLE t", $count, " (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB")` --eval $ddl_var --enable_query_log --dec $count @@ -37,7 +37,7 @@ START TRANSACTION; while ($count) { --disable_query_log - --let $ddl_var = `SELECT CONCAT("INSERT INTO t", $count, " VALUES (DEFAULT)")` + --let $ddl_var = `SELECT CONCAT("INSERT INTO t", $count, " VALUES (1)")` --eval $ddl_var --enable_query_log --dec $count From b1fabb2c0aecc4518a04341325abdbf26e341754 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 7 Feb 2018 14:14:21 +0100 Subject: [PATCH 073/276] Galera MTR Tests: missing wsrep_sync_wait in test galera_evs_suspect_timeout --- .../suite/galera_3nodes/r/galera_evs_suspect_timeout.result | 1 + mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result b/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result index 1464222a079..7e0d282ec7f 100644 --- a/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result +++ b/mysql-test/suite/galera_3nodes/r/galera_evs_suspect_timeout.result @@ -11,6 +11,7 @@ SET SESSION wsrep_sync_wait = 0; SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; VARIABLE_VALUE = 2 1 +SET SESSION wsrep_sync_wait = DEFAULT; SELECT COUNT(*) = 1 FROM t1; COUNT(*) = 1 1 diff --git a/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test b/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test index a87f19ac94e..03236a3cb93 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test +++ b/mysql-test/suite/galera_3nodes/t/galera_evs_suspect_timeout.test @@ -50,6 +50,7 @@ SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_N --enable_query_log --source include/wait_until_connected_again.inc +SET SESSION wsrep_sync_wait = DEFAULT; SELECT COUNT(*) = 1 FROM t1; DROP TABLE t1; From ba07581c811a3991116605f51143a5f3e3dd8b7f Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 7 Feb 2018 14:16:50 +0100 Subject: [PATCH 074/276] Galera MTR tests: remove unused config files in galera suite --- .../galera/galera_2nodes_as_master_slave.cnf | 83 ------------------ ...lera_2nodes_as_master_with_repl_filter.cnf | 87 ------------------- 2 files changed, 170 deletions(-) delete mode 100644 mysql-test/suite/galera/galera_2nodes_as_master_slave.cnf delete mode 100644 mysql-test/suite/galera/galera_2nodes_as_master_with_repl_filter.cnf diff --git a/mysql-test/suite/galera/galera_2nodes_as_master_slave.cnf b/mysql-test/suite/galera/galera_2nodes_as_master_slave.cnf deleted file mode 100644 index f68fe524904..00000000000 --- a/mysql-test/suite/galera/galera_2nodes_as_master_slave.cnf +++ /dev/null @@ -1,83 +0,0 @@ - -# -# Let's understand the topology. -# * Independent Master with server-id = 1 -# * Galera cluster with 2 nodes: node#1 and node#2 with server-id = 2, 3 -# node#1 act as slave to Independent Master with server-id = 1 -# * Independent Slave with server-id = 4 replicating from galera node#2 -# - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf - -[mysqld] -log-slave-updates -log-bin=mysqld-bin -binlog-format=row -gtid-mode=on -enforce-gtid-consistency=true - -[mysqld.1] -server-id=1 - -[mysqld.2] -server-id=2 - -wsrep_provider=@ENV.WSREP_PROVIDER -wsrep_cluster_address='gcomm://' -wsrep_provider_options='base_port=@mysqld.2.#galera_port;evs.install_timeout = PT15S; evs.max_install_timeouts=1;' - -# enforce read-committed characteristics across the cluster -wsrep_causal_reads=ON -wsrep_sync_wait = 15 - -wsrep_node_address=127.0.0.1 -wsrep_sst_receive_address=127.0.0.2:@mysqld.2.#sst_port -wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port - -# Required for Galera -innodb_autoinc_lock_mode=2 - -innodb_flush_log_at_trx_commit=2 - -[mysqld.3] -server-id=3 - -wsrep_provider=@ENV.WSREP_PROVIDER -wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.2.#galera_port' -wsrep_provider_options='base_port=@mysqld.3.#galera_port;evs.install_timeout = PT15S; evs.max_install_timeouts = 1;' - -# enforce read-committed characteristics across the cluster -wsrep_causal_reads=ON -wsrep_sync_wait = 15 - -wsrep_node_address=127.0.0.1 -wsrep_sst_receive_address=127.0.0.2:@mysqld.3.#sst_port -wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port - -# Required for Galera -innodb_autoinc_lock_mode=2 - -innodb_flush_log_at_trx_commit=2 - -[mysqld.4] -server-id=4 - -[ENV] -NODE_MYPORT_1= @mysqld.1.port -NODE_MYSOCK_1= @mysqld.1.socket - -NODE_MYPORT_2= @mysqld.2.port -NODE_MYSOCK_2= @mysqld.2.socket - -NODE_MYPORT_3= @mysqld.3.port -NODE_MYSOCK_3= @mysqld.3.socket - -NODE_MYPORT_4= @mysqld.4.port -NODE_MYSOCK_4= @mysqld.4.socket - -NODE_GALERAPORT_2= @mysqld.2.#galera_port -NODE_GALERAPORT_3= @mysqld.3.#galera_port - -NODE_SSTPORT_2= @mysqld.2.#sst_port -NODE_SSTPORT_3= @mysqld.3.#sst_port diff --git a/mysql-test/suite/galera/galera_2nodes_as_master_with_repl_filter.cnf b/mysql-test/suite/galera/galera_2nodes_as_master_with_repl_filter.cnf deleted file mode 100644 index d5490280ab2..00000000000 --- a/mysql-test/suite/galera/galera_2nodes_as_master_with_repl_filter.cnf +++ /dev/null @@ -1,87 +0,0 @@ -# -# This .cnf file creates a setup with a 2-node Galera cluster and one stand-alone MySQL server, to be used as a slave -# - -# Use default setting for mysqld processes -!include include/default_mysqld.cnf - -[mysqld] -default-storage-engine=InnoDB - -[mysqld.1] -server-id=1 -binlog-format=row -log-bin=mysqld-bin -log_slave_updates -gtid-mode=on -enforce-gtid-consistency=true -event-scheduler=1 - -wsrep_provider=@ENV.WSREP_PROVIDER -wsrep_cluster_address='gcomm://' -wsrep_provider_options='base_port=@mysqld.1.#galera_port' - -# enforce read-committed characteristics across the cluster -wsrep_causal_reads=ON -wsrep_sync_wait = 15 - -wsrep_node_address=127.0.0.1 -wsrep_sst_receive_address=127.0.0.2:@mysqld.1.#sst_port -wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port - -# Required for Galera -innodb_autoinc_lock_mode=2 - -innodb_flush_log_at_trx_commit=2 - -[mysqld.2] -server-id=2 -binlog-format=row -log-bin=mysqld-bin -log_slave_updates -gtid-mode=on -enforce-gtid-consistency=true -event-scheduler=1 - -wsrep_provider=@ENV.WSREP_PROVIDER -wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port' -wsrep_provider_options='base_port=@mysqld.2.#galera_port' - -# enforce read-committed characteristics across the cluster -wsrep_causal_reads=ON -wsrep_sync_wait = 15 - -wsrep_node_address=127.0.0.1 -wsrep_sst_receive_address=127.0.0.2:@mysqld.2.#sst_port -wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port - -# Required for Galera -innodb_autoinc_lock_mode=2 - -innodb_flush_log_at_trx_commit=2 - -[mysqld.3] -server-id=3 -replicate-ignore-db=test -replicate-wild-ignore-table=test.% -log-bin=mysqld-bin -log_slave_updates -gtid-mode=on -enforce-gtid-consistency=true -event-scheduler=1 - -[ENV] -NODE_MYPORT_1= @mysqld.1.port -NODE_MYSOCK_1= @mysqld.1.socket - -NODE_MYPORT_2= @mysqld.2.port -NODE_MYSOCK_2= @mysqld.2.socket - -NODE_MYPORT_3= @mysqld.3.port -NODE_MYSOCK_3= @mysqld.3.socket - -NODE_GALERAPORT_1= @mysqld.1.#galera_port -NODE_GALERAPORT_2= @mysqld.2.#galera_port - -NODE_SSTPORT_1= @mysqld.1.#sst_port -NODE_SSTPORT_2= @mysqld.2.#sst_port From 7274bed257420d85fe07c986b57f864fc6052e24 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Thu, 22 Feb 2018 11:14:16 +0100 Subject: [PATCH 075/276] Fix MTR test galera.galera_gcache_recover_manytrx The kills the two nodes that compose the cluster and then recovers them. The first node that is recovered is expected to recover its gcache, so that when the second node is recovered, it is expected that it rejoins the cluster via IST. However, it is possible that if the second node is killed while it is applying in TOI mode, its local state is marked unsafe. In which case, SST is the only option to rejoin the cluster. codership/mysql-wsrep#323 --- .../galera/t/galera_gcache_recover_manytrx.test | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test b/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test index 3bfcdc9f117..08165f30f7d 100644 --- a/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test +++ b/mysql-test/suite/galera/t/galera_gcache_recover_manytrx.test @@ -5,6 +5,7 @@ --source include/galera_cluster.inc --source include/big_test.inc +--source include/have_log_bin.inc SET SESSION wsrep_sync_wait = 0; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 LONGBLOB) ENGINE=InnoDB; @@ -93,6 +94,8 @@ END| DELIMITER ;| +--let $wsrep_last_committed_before = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'` + --connect node_1_insert_simple, 127.0.0.1, root, , test, $NODE_MYPORT_1 --connect node_1_insert_multi, 127.0.0.1, root, , test, $NODE_MYPORT_1 --connect node_1_insert_transaction, 127.0.0.1, root, , test, $NODE_MYPORT_1 @@ -124,6 +127,13 @@ DELIMITER ;| --connection node_2 SET SESSION wsrep_sync_wait = 0; + +# Make sure that node_2 is not killed while TOIs are applied. +# Otherwhise we risk that grastate file is marked unsafe, and +# as a consequence the node cannot rejoin with IST. +--let $wait_condition = SELECT VARIABLE_VALUE > $wsrep_last_committed_before FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed' +--source include/wait_condition.inc + --source include/kill_galera.inc --sleep 10 @@ -172,9 +182,8 @@ SET SESSION wsrep_sync_wait = 0; --source include/start_mysqld.inc --connection node_1 ---source include/wait_until_connected_again.inc ---source include/galera_wait_ready.inc --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc --let $diff_servers = 1 2 --source include/diff_servers.inc From f8ea96b80c3eb79c45b646b61ff4472e1326ec18 Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Tue, 27 Feb 2018 04:52:55 +0200 Subject: [PATCH 076/276] codership/galera#500 MTR test for proper Galera provider tear down Added a test which verifies that if the gcomm background thread is forced to quit via exception, the error is propagated all the way up the stack and wsrep_ready becomes 0. --- mysql-test/suite/galera/r/galera#500.result | 10 ++++++ mysql-test/suite/galera/t/galera#500.test | 37 +++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 mysql-test/suite/galera/r/galera#500.result create mode 100644 mysql-test/suite/galera/t/galera#500.test diff --git a/mysql-test/suite/galera/r/galera#500.result b/mysql-test/suite/galera/r/galera#500.result new file mode 100644 index 00000000000..6a07d0359a4 --- /dev/null +++ b/mysql-test/suite/galera/r/galera#500.result @@ -0,0 +1,10 @@ +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options="gmcast.isolate=2"; +SET SESSION wsrep_sync_wait = 0; +SHOW STATUS LIKE 'wsrep_cluster_status'; +Variable_name Value +wsrep_cluster_status non-Primary +SET SESSION wsrep_sync_wait = default; +SET GLOBAL wsrep_provider_options="pc.bootstrap=1"; +SET SESSION wsrep_on=0; +CALL mtr.add_suppression("WSREP: exception from gcomm, backend must be restarted: Gcomm backend termination was requested by setting gmcast.isolate=2."); diff --git a/mysql-test/suite/galera/t/galera#500.test b/mysql-test/suite/galera/t/galera#500.test new file mode 100644 index 00000000000..a313d5778d3 --- /dev/null +++ b/mysql-test/suite/galera/t/galera#500.test @@ -0,0 +1,37 @@ +# +# The purpose of this test is to verify that if an exception is +# thrown from gcomm background thread, the provider terminates properly +# and wsrep_ready becomes 0. +# + +--source include/have_innodb.inc +--source include/galera_cluster.inc + +# Force node_2 gcomm background thread to terminate via exception. +--connection node_2 +--let $wsrep_cluster_address = `SELECT @@wsrep_cluster_address` +# Setting gmcast.isolate=2 will force gcomm background thread to +# throw exception. +SET SESSION wsrep_sync_wait = 0; +SET GLOBAL wsrep_provider_options="gmcast.isolate=2"; + +# Wait until wsrep_ready becomes 0. +--let $wait_condition = SELECT VARIABLE_VALUE = 0 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME ='wsrep_ready' +--source include/wait_condition.inc + +# Wait until node_1 ends up in non-prim and rebootstrap the cluster. +--connection node_1 +SET SESSION wsrep_sync_wait = 0; +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME ='wsrep_cluster_size' +--source include/wait_condition.inc +SHOW STATUS LIKE 'wsrep_cluster_status'; +SET SESSION wsrep_sync_wait = default; +SET GLOBAL wsrep_provider_options="pc.bootstrap=1"; + +# Restart node_2 +--connection node_2 +SET SESSION wsrep_on=0; +--source include/restart_mysqld.inc + +--connection node_2 +CALL mtr.add_suppression("WSREP: exception from gcomm, backend must be restarted: Gcomm backend termination was requested by setting gmcast.isolate=2."); From 2e5681a450627709512138a7d15b8ee035a29c6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 8 May 2018 09:33:48 +0300 Subject: [PATCH 077/276] Test requires galera debug library. --- mysql-test/suite/galera/t/galera#500.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/galera/t/galera#500.test b/mysql-test/suite/galera/t/galera#500.test index a313d5778d3..3c8490b6907 100644 --- a/mysql-test/suite/galera/t/galera#500.test +++ b/mysql-test/suite/galera/t/galera#500.test @@ -6,6 +6,7 @@ --source include/have_innodb.inc --source include/galera_cluster.inc +--source include/galera_have_debug_sync.inc # Force node_2 gcomm background thread to terminate via exception. --connection node_2 From 0f80c014c1dd1523293aa4420f7e7be9d69afa72 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 9 May 2018 11:22:26 -0400 Subject: [PATCH 078/276] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 75daccdd9d4..59a439d4152 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=0 -MYSQL_VERSION_PATCH=35 +MYSQL_VERSION_PATCH=36 From 2b35db5ac4e9f8e1ba20a1f733f094028d92b969 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 7 Mar 2018 13:10:01 +1100 Subject: [PATCH 079/276] MDEV-15496: wsrep_sst_common - parse IPv6 correct Fix to e78e308e818a4a763ebb981302851f58f4c42bba, IPv6 got split at : Also got rid of awk (yuck!) to parse port (commit: b4fbb4183b35f921efe079a312b0c1be87866b40) Tested with dash to avoid bashisms: $ dash -vx './wsrep_sst_common.sh' --address '128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more + readonly WSREP_SST_OPT_ADDR=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_HOST=128.0.0.5 + readonly WSREP_SST_OPT_HOST_UNESCAPED=128.0.0.5 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver $ dash -vx './wsrep_sst_common.sh' --address '[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more + readonly WSREP_SST_OPT_ADDR=[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + addr_no_bracket=2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_HOST_UNESCAPED=2001:db8::1 + readonly WSREP_SST_OPT_HOST=[2001:db8::1] + remain=:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver And bash as well: $ bash -vx './wsrep_sst_common.sh' --address '127.0.0.3:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more + readonly WSREP_SST_OPT_ADDR=127.0.0.3:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + WSREP_SST_OPT_ADDR=127.0.0.3:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + case "${WSREP_SST_OPT_ADDR}" in + readonly WSREP_SST_OPT_HOST=127.0.0.3 + WSREP_SST_OPT_HOST=127.0.0.3 + readonly WSREP_SST_OPT_HOST_UNESCAPED=127.0.0.3 + WSREP_SST_OPT_HOST_UNESCAPED=127.0.0.3 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_PORT=3333 + WSREP_SST_OPT_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver + WSREP_SST_OPT_SST_VER=sst_ver $ bash -vx './wsrep_sst_common.sh' --address '[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more + readonly 'WSREP_SST_OPT_ADDR=[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' + WSREP_SST_OPT_ADDR='[2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' + case "${WSREP_SST_OPT_ADDR}" in + addr_no_bracket='2001:db8::1]:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' + readonly WSREP_SST_OPT_HOST_UNESCAPED=2001:db8::1 + WSREP_SST_OPT_HOST_UNESCAPED=2001:db8::1 + readonly 'WSREP_SST_OPT_HOST=[2001:db8::1]' + WSREP_SST_OPT_HOST='[2001:db8::1]' + remain=:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_PORT=3333 + WSREP_SST_OPT_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver + WSREP_SST_OPT_SST_VER=sst_ver --- scripts/wsrep_sst_common.sh | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index b9d9d04fef6..9926b4ac9d9 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -35,18 +35,25 @@ case "$1" in # # Break address string into host:port/path parts # - readonly WSREP_SST_OPT_HOST=${WSREP_SST_OPT_ADDR%%[:/]*} - if [ ${WSREP_SST_OPT_HOST:0:1} = '[' ] - then - # IPv6 notation - readonly WSREP_SST_OPT_HOST_UNESCAPED=${WSREP_SST_OPT_HOST:1:-1} - else - readonly WSREP_SST_OPT_HOST_UNESCAPED=${WSREP_SST_OPT_HOST} - fi - readonly WSREP_SST_OPT_PORT=$(echo $WSREP_SST_OPT_ADDR | \ - cut -d ']' -f 2 | cut -s -d ':' -f 2 | cut -d '/' -f 1) + case "${WSREP_SST_OPT_ADDR}" in + \[*) + # IPv6 + addr_no_bracket=${WSREP_SST_OPT_ADDR#\[} + readonly WSREP_SST_OPT_HOST_UNESCAPED=${addr_no_bracket%%\]*} + readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" + remain=${WSREP_SST_OPT_ADDR#*\]} + remain=${remain#*:} + ;; + *) + readonly WSREP_SST_OPT_HOST=${WSREP_SST_OPT_ADDR%%[:/]*} + readonly WSREP_SST_OPT_HOST_UNESCAPED=$WSREP_SST_OPT_HOST + remain=${WSREP_SST_OPT_ADDR#*:} + ;; + esac + readonly WSREP_SST_OPT_PORT=${remain%%/*} + remain=${remain#*/} + readonly WSREP_SST_OPT_MODULE=${remain%%/*} readonly WSREP_SST_OPT_PATH=${WSREP_SST_OPT_ADDR#*/} - readonly WSREP_SST_OPT_MODULE=${WSREP_SST_OPT_PATH%%/*} remain=${WSREP_SST_OPT_PATH#*/} readonly WSREP_SST_OPT_LSN=${remain%%/*} remain=${remain#*/} From 24ab82e67560f840f0dddda066c2612e7fb3a20e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 12 Mar 2018 15:14:15 +1100 Subject: [PATCH 080/276] MDEV-15541: wsrep_sst_common - WSREP_SST_OPT_PORT set twice (--address and --port) --- scripts/wsrep_sst_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 9926b4ac9d9..c69dbf92448 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -50,7 +50,7 @@ case "$1" in remain=${WSREP_SST_OPT_ADDR#*:} ;; esac - readonly WSREP_SST_OPT_PORT=${remain%%/*} + readonly WSREP_SST_OPT_ADDR_PORT=${remain%%/*} remain=${remain#*/} readonly WSREP_SST_OPT_MODULE=${remain%%/*} readonly WSREP_SST_OPT_PATH=${WSREP_SST_OPT_ADDR#*/} From e6b31df6dfce1a412d20f32092da676768cff8ad Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 16 Oct 2017 17:49:52 +0200 Subject: [PATCH 081/276] MDEV-13968 sst fails with "WSREP_SST_OPT_PORT: readonly variable" --- scripts/wsrep_sst_common.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index c69dbf92448..bb91eeaeba2 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -129,6 +129,17 @@ done readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG +if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then + if [ -n "$WSREP_SST_OPT_PORT" ]; then + if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then + wsrep_log_error "port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" + exit 2 + fi + else + readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" + fi +fi + # try to use my_print_defaults, mysql and mysqldump that come with the sources # (for MTR suite) SCRIPTS_DIR="$(cd $(dirname "$0"); pwd -P)" From fe3c4a41829582f41f2db00cd410229dca2b6c38 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 24 Oct 2017 20:59:54 +0200 Subject: [PATCH 082/276] MDEV-13968 sst fails with "WSREP_SST_OPT_PORT: readonly variable" followup for 990c4318a779645ee fix broken script (set -u) --- scripts/wsrep_sst_common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index bb91eeaeba2..c421bbfdc64 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -129,8 +129,8 @@ done readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG -if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then - if [ -n "$WSREP_SST_OPT_PORT" ]; then +if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then + if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then wsrep_log_error "port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" exit 2 From 82f26dafcb72e74a11916e1d22a2fabac4f51ce8 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 12 Mar 2018 15:23:58 +1100 Subject: [PATCH 083/276] MDEV-13968: wsrep_log_error not defined until later in wsrep_sst_common Fix for 990c4318a779645ee dash -vx './wsrep_sst_common.sh' --address '128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' --port 1111 2>&1 | more .... + readonly WSREP_SST_OPT_ADDR=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_HOST=128.0.0.5 + readonly WSREP_SST_OPT_HOST_UNESCAPED=128.0.0.5 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_ADDR_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver + shift + shift + [ 2 -gt 0 ] + readonly WSREP_SST_OPT_PORT=1111 + shift + shift + [ 0 -gt 0 ] readonly WSREP_SST_OPT_BYPASS + readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG + readonly WSREP_SST_OPT_BINLOG if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 exit 2 fi else readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" fi fi + [ -n 3333 ] + [ -n 1111 ] + [ 1111 != 3333 ] + echo WSREP_SST: [ERROR] port in --port=1111 differs from port in --address=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 WSREP_SST: [ERROR] port in --port=1111 differs from port in --address=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + exit 2 dash -vx './wsrep_sst_common.sh' --address '128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2' 2>&1 | more ... + [ 2 -gt 0 ] + readonly WSREP_SST_OPT_ADDR=128.0.0.5:3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_HOST=128.0.0.5 + readonly WSREP_SST_OPT_HOST_UNESCAPED=128.0.0.5 + remain=3333/module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_ADDR_PORT=3333 + remain=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_MODULE=module + readonly WSREP_SST_OPT_PATH=module/lsn_version/sst_ver/some_future_option1/some_future_option2 + remain=lsn_version/sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_LSN=lsn_version + remain=sst_ver/some_future_option1/some_future_option2 + readonly WSREP_SST_OPT_SST_VER=sst_ver + shift + shift + [ 0 -gt 0 ] readonly WSREP_SST_OPT_BYPASS + readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG + readonly WSREP_SST_OPT_BINLOG if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 exit 2 fi else readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" fi fi + [ -n 3333 ] + [ -n ] + readonly WSREP_SST_OPT_PORT=3333 --- scripts/wsrep_sst_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index c421bbfdc64..d20e3bc36c9 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -132,7 +132,7 @@ readonly WSREP_SST_OPT_BINLOG if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then - wsrep_log_error "port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" + echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 exit 2 fi else From 215d652c66474c4578b0476385d055e7feae2d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 9 May 2018 09:16:20 +0300 Subject: [PATCH 084/276] MDEV-15351: wsrep_sst_xtrabackup is broken in 10.1.31 Remove the setup_ports function call. This is related to https://github.com/MariaDB/server/pull/717 Thanks to Daniel Black and Bart S. --- scripts/wsrep_sst_xtrabackup.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh index 75aedb7ae63..eb059f8bb9d 100644 --- a/scripts/wsrep_sst_xtrabackup.sh +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -394,7 +394,6 @@ if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' fi read_cnf -setup_ports get_stream get_transfer From 6a04c2a1aa60e67c07d4e6ada63f65cc3b26a5a9 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Tue, 22 May 2018 12:09:05 -0700 Subject: [PATCH 085/276] MDEV-16235 Server crashes in my_utf8_uni or in my_strtod_int upon SELECT .. LIMIT 0 The code must differentiate between a SELECT with contradictory WHERE/HAVING and one with LIMIT 0. Also for the latter printed 'Zero limit' instead of 'Impossible where' in the EXPLAIN output. --- mysql-test/r/limit.result | 16 ++++++++++++++++ mysql-test/r/subselect4.result | 2 +- mysql-test/r/subselect_mat_cost_bugs.result | 2 +- mysql-test/t/limit.test | 14 ++++++++++++++ mysql-test/t/subselect4.test | 2 +- sql/sql_select.cc | 17 +++++++++++++---- 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/mysql-test/r/limit.result b/mysql-test/r/limit.result index 176a93c7a46..deb4ca2ab95 100644 --- a/mysql-test/r/limit.result +++ b/mysql-test/r/limit.result @@ -146,3 +146,19 @@ a 16 DROP TABLE t1; End of 5.1 tests +# +# mdev-16235: SELECT over a table with LIMIT 0 +# +EXPLAIN +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Zero limit +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; +start_time user_host query_time lock_time rows_sent rows_examined db last_insert_id insert_id server_id sql_text +EXPLAIN +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Zero limit +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; +help_topic_id name help_category_id description example url +End of 5.5 tests diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 6accc23d18a..8973330c7da 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -2512,7 +2512,7 @@ SELECT 2 IN (SELECT 2 from DUAL WHERE 1 != 1); SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; # -# mfrv-14515: Wrong results for tableless query with subquery in WHERE +# mdev-14515: Wrong results for tableless query with subquery in WHERE # and implicit aggregation # create table t1 (i1 int, i2 int); diff --git a/mysql-test/r/subselect_mat_cost_bugs.result b/mysql-test/r/subselect_mat_cost_bugs.result index df6b543bab8..d33f1488e4d 100644 --- a/mysql-test/r/subselect_mat_cost_bugs.result +++ b/mysql-test/r/subselect_mat_cost_bugs.result @@ -334,7 +334,7 @@ SELECT * FROM t1 WHERE (f1) IN (SELECT f1 FROM t2) LIMIT 0; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE +1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Zero limit 2 DEPENDENT SUBQUERY t2 ALL NULL NULL NULL NULL 2 Using where SELECT * FROM t1 WHERE (f1) IN (SELECT f1 FROM t2) diff --git a/mysql-test/t/limit.test b/mysql-test/t/limit.test index 4dbe13096d4..668d3b74518 100644 --- a/mysql-test/t/limit.test +++ b/mysql-test/t/limit.test @@ -115,3 +115,17 @@ SELECT a FROM t1 ORDER BY a LIMIT 2 OFFSET 14; DROP TABLE t1; --echo End of 5.1 tests + +--echo # +--echo # mdev-16235: SELECT over a table with LIMIT 0 +--echo # + +EXPLAIN +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; +SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; + +EXPLAIN +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; +SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; + +--echo End of 5.5 tests diff --git a/mysql-test/t/subselect4.test b/mysql-test/t/subselect4.test index 2b53b55b735..2727fd9daed 100644 --- a/mysql-test/t/subselect4.test +++ b/mysql-test/t/subselect4.test @@ -2047,7 +2047,7 @@ SET optimizer_switch= @@global.optimizer_switch; set @@tmp_table_size= @@global.tmp_table_size; --echo # ---echo # mfrv-14515: Wrong results for tableless query with subquery in WHERE +--echo # mdev-14515: Wrong results for tableless query with subquery in WHERE --echo # and implicit aggregation --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1e9f1c0848b..d6d269a700f 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1136,10 +1136,19 @@ JOIN::optimize() if (cond_value == Item::COND_FALSE || having_value == Item::COND_FALSE || (!unit->select_limit_cnt && !(select_options & OPTION_FOUND_ROWS))) { /* Impossible cond */ - DBUG_PRINT("info", (having_value == Item::COND_FALSE ? - "Impossible HAVING" : "Impossible WHERE")); - zero_result_cause= having_value == Item::COND_FALSE ? - "Impossible HAVING" : "Impossible WHERE"; + if (unit->select_limit_cnt) + { + DBUG_PRINT("info", (having_value == Item::COND_FALSE ? + "Impossible HAVING" : "Impossible WHERE")); + zero_result_cause= having_value == Item::COND_FALSE ? + "Impossible HAVING" : "Impossible WHERE"; + } + else + { + DBUG_PRINT("info", ("Zero limit")); + zero_result_cause= "Zero limit"; + conds= 0; + } table_count= top_join_tab_count= 0; error= 0; goto setup_subq_exit; From 1ada4afb0a51f7283b6187a95019ec2cb80c8a0b Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Sun, 29 Apr 2018 19:47:48 +0300 Subject: [PATCH 086/276] mtr: use process launch -- args to start mysqld in lldb gdb's "set args" equivalent for lldb would be "settings set target.run-args", however it doesn't play well with double dashed args (--). --- mysql-test/mysql-test-run.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index de1871cbf03..ba49d88a85f 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -6051,7 +6051,7 @@ sub lldb_arguments { $input = $input ? "< $input" : ""; # write init file for mysqld or client - mtr_tofile($lldb_init_file, "set args $str $input\n"); + mtr_tofile($lldb_init_file, "process launch --stop-at-entry -- $str $input\n"); print "\nTo start lldb for $type, type in another window:\n"; print "cd $glob_mysql_test_dir && lldb -s $lldb_init_file $$exe\n"; From c1698e8dc50f0c342c1a6886426ba1d43396cd1e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 29 May 2018 16:47:47 +0100 Subject: [PATCH 087/276] MDEV-16303 - do not install wsrep.ini on Windows --- support-files/CMakeLists.txt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/support-files/CMakeLists.txt b/support-files/CMakeLists.txt index 3b6e86030ab..dff4610d490 100644 --- a/support-files/CMakeLists.txt +++ b/support-files/CMakeLists.txt @@ -41,13 +41,20 @@ ELSE() SET(inst_location ${INSTALL_SUPPORTFILESDIR}) ENDIF() -FOREACH(inifile my-huge my-innodb-heavy-4G my-large my-medium my-small wsrep) - CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${inifile}.cnf.sh +FOREACH(inifile my-huge my-innodb-heavy-4G my-large my-medium my-small) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${inifile}.cnf.sh ${CMAKE_CURRENT_BINARY_DIR}/${inifile}.${ini_file_extension} @ONLY) INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/${inifile}.${ini_file_extension} DESTINATION ${inst_location} COMPONENT IniFiles) ENDFOREACH() +IF(WITH_WSREP) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/wsrep.cnf.sh + ${CMAKE_CURRENT_BINARY_DIR}/wsrep.${ini_file_extension} @ONLY) + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/wsrep.${ini_file_extension} + DESTINATION ${inst_location} COMPONENT IniFiles) +ENDIF() + IF(UNIX) SET(prefix ${CMAKE_INSTALL_PREFIX}) FOREACH(script mysqld_multi.server mysql-log-rotate binary-configure wsrep_notify) From 5fb2c586f225361deae0f04dc6f72a3f0d168ac2 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 2 Jun 2018 11:52:48 +0530 Subject: [PATCH 088/276] MDEV-16225: wrong resultset from query with semijoin=on For non-semi-join subquery optimization we do a cost based decision between Materialisation and IN -> EXIST transformation. The issue in this case is that for IN->EXIST transformation we run JOIN::reoptimize with the IN->EXISt conditions and we come up with a new query plan. But when we compare the cost with Materialization, we make the decision to chose Materialization so we need to restore the query plan for Materilization. The saving and restoring for keyuse array and join_tab keyuse is only done when we have atleast one element in the keyuse_array , we are now changing to do it even for 0 elements to main the generality. --- mysql-test/r/subselect_sj2_mat.result | 57 +++++++++++++++++++++++++++ mysql-test/t/subselect_sj2_mat.test | 42 ++++++++++++++++++++ sql/sql_select.cc | 46 ++++++++++----------- 3 files changed, 119 insertions(+), 26 deletions(-) diff --git a/mysql-test/r/subselect_sj2_mat.result b/mysql-test/r/subselect_sj2_mat.result index 835742a3ff4..19aa3bd3f02 100644 --- a/mysql-test/r/subselect_sj2_mat.result +++ b/mysql-test/r/subselect_sj2_mat.result @@ -1601,3 +1601,60 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY eq_ref distinct_key distinct_key 11 func 1 2 MATERIALIZED t2 ALL NULL NULL NULL NULL 3 Using where DROP TABLE t1,t2; +# +# MDEV-16225: wrong resultset from query with semijoin=on +# +CREATE TABLE t1 ( +`id` int(10) NOT NULL AUTO_INCREMENT, +`local_name` varchar(64) NOT NULL, +PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=latin1; +insert into t1(`id`,`local_name`) values +(1,'Cash Advance'), +(2,'Cash Advance'), +(3,'Rollover'), +(4,'AL Installment'), +(5,'AL Installment'), +(6,'AL Installment'), +(7,'AL Installment'), +(8,'AL Installment'), +(9,'AL Installment'), +(10,'Internet Payday'), +(11,'Rollover - Internet Payday'), +(12,'AL Monthly Installment'), +(13,'AL Semi-Monthly Installment'); +explain +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( +t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) +OR +(t.id IN (0,4,12,13,1,10,3,11)) +); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t index PRIMARY PRIMARY 4 NULL 13 Using where; Using index +2 MATERIALIZED ALL distinct_key NULL NULL NULL 8 +2 MATERIALIZED A ALL PRIMARY NULL NULL NULL 13 Using where; Using join buffer (flat, BNL join) +3 MATERIALIZED B ALL PRIMARY NULL NULL NULL 13 Using where +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( +t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) +OR +(t.id IN (0,4,12,13,1,10,3,11)) +); +id +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +drop table t1; diff --git a/mysql-test/t/subselect_sj2_mat.test b/mysql-test/t/subselect_sj2_mat.test index cfb6c8c2819..0665cdf68fe 100644 --- a/mysql-test/t/subselect_sj2_mat.test +++ b/mysql-test/t/subselect_sj2_mat.test @@ -303,3 +303,45 @@ eval $q; eval explain $q; DROP TABLE t1,t2; + +--echo # +--echo # MDEV-16225: wrong resultset from query with semijoin=on +--echo # + +CREATE TABLE t1 ( + `id` int(10) NOT NULL AUTO_INCREMENT, + `local_name` varchar(64) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=latin1; + +insert into t1(`id`,`local_name`) values +(1,'Cash Advance'), +(2,'Cash Advance'), +(3,'Rollover'), +(4,'AL Installment'), +(5,'AL Installment'), +(6,'AL Installment'), +(7,'AL Installment'), +(8,'AL Installment'), +(9,'AL Installment'), +(10,'Internet Payday'), +(11,'Rollover - Internet Payday'), +(12,'AL Monthly Installment'), +(13,'AL Semi-Monthly Installment'); + +explain +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( + t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) + OR + (t.id IN (0,4,12,13,1,10,3,11)) +); +SELECT SQL_NO_CACHE t.id +FROM t1 t +WHERE ( + t.id IN (SELECT A.id FROM t1 AS A WHERE A.local_name IN (SELECT B.local_name FROM t1 AS B WHERE B.id IN (0,4,12,13,1,10,3,11))) + OR + (t.id IN (0,4,12,13,1,10,3,11)) +); +drop table t1; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index d6d269a700f..016dd594e08 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -23739,21 +23739,18 @@ void JOIN::set_allowed_join_cache_types() void JOIN::save_query_plan(Join_plan_state *save_to) { - if (keyuse.elements) - { - DYNAMIC_ARRAY tmp_keyuse; - /* Swap the current and the backup keyuse internal arrays. */ - tmp_keyuse= keyuse; - keyuse= save_to->keyuse; /* keyuse is reset to an empty array. */ - save_to->keyuse= tmp_keyuse; + DYNAMIC_ARRAY tmp_keyuse; + /* Swap the current and the backup keyuse internal arrays. */ + tmp_keyuse= keyuse; + keyuse= save_to->keyuse; /* keyuse is reset to an empty array. */ + save_to->keyuse= tmp_keyuse; - for (uint i= 0; i < table_count; i++) - { - save_to->join_tab_keyuse[i]= join_tab[i].keyuse; - join_tab[i].keyuse= NULL; - save_to->join_tab_checked_keys[i]= join_tab[i].checked_keys; - join_tab[i].checked_keys.clear_all(); - } + for (uint i= 0; i < table_count; i++) + { + save_to->join_tab_keyuse[i]= join_tab[i].keyuse; + join_tab[i].keyuse= NULL; + save_to->join_tab_checked_keys[i]= join_tab[i].checked_keys; + join_tab[i].checked_keys.clear_all(); } memcpy((uchar*) save_to->best_positions, (uchar*) best_positions, sizeof(POSITION) * (table_count + 1)); @@ -23791,20 +23788,17 @@ void JOIN::reset_query_plan() void JOIN::restore_query_plan(Join_plan_state *restore_from) { - if (restore_from->keyuse.elements) + DYNAMIC_ARRAY tmp_keyuse; + tmp_keyuse= keyuse; + keyuse= restore_from->keyuse; + restore_from->keyuse= tmp_keyuse; + + for (uint i= 0; i < table_count; i++) { - DYNAMIC_ARRAY tmp_keyuse; - tmp_keyuse= keyuse; - keyuse= restore_from->keyuse; - restore_from->keyuse= tmp_keyuse; - - for (uint i= 0; i < table_count; i++) - { - join_tab[i].keyuse= restore_from->join_tab_keyuse[i]; - join_tab[i].checked_keys= restore_from->join_tab_checked_keys[i]; - } - + join_tab[i].keyuse= restore_from->join_tab_keyuse[i]; + join_tab[i].checked_keys= restore_from->join_tab_checked_keys[i]; } + memcpy((uchar*) best_positions, (uchar*) restore_from->best_positions, sizeof(POSITION) * (table_count + 1)); /* Restore SJM nests */ From 41cbe92bf116cf35a935496fa8b6c8a9f80a5edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Jun 2018 08:42:24 +0300 Subject: [PATCH 089/276] Remove dead code that was added in MDEV-5834 ha_innobase::set_partition_owner_stats(): Remove (unused function). ha_innobase::ha_partition_stats: Remove (the variable is never read). Remove unused ut_timer functions. --- storage/innobase/btr/btr0defragment.cc | 51 --------------------- storage/innobase/handler/ha_innodb.cc | 10 +---- storage/innobase/handler/ha_innodb.h | 5 +-- storage/innobase/include/ut0timer.h | 39 +--------------- storage/innobase/include/ut0timer.ic | 61 +------------------------- storage/xtradb/btr/btr0defragment.cc | 50 --------------------- storage/xtradb/handler/ha_innodb.cc | 10 +---- storage/xtradb/handler/ha_innodb.h | 5 +-- storage/xtradb/include/ut0timer.h | 39 +--------------- storage/xtradb/include/ut0timer.ic | 61 +------------------------- 10 files changed, 12 insertions(+), 319 deletions(-) diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index ca4c90eef41..9f17815cd43 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -39,57 +39,6 @@ Modified 30/07/2014 Jan Lindström jan.lindstrom@mariadb.com #include -/**************************************************//** -Custom nullptr implementation for under g++ 4.6 -*******************************************************/ -// #pragma once -/* -namespace std -{ - // based on SC22/WG21/N2431 = J16/07-0301 - struct nullptr_t - { - template operator any * () const - { - return 0; - } - template operator T any:: * () const - { - return 0; - } - -#ifdef _MSC_VER - struct pad {}; - pad __[sizeof(void*)/sizeof(pad)]; -#else - char __[sizeof(void*)]; -#endif -private: - // nullptr_t();// {} - // nullptr_t(const nullptr_t&); - // void operator = (const nullptr_t&); - void operator &() const; - template void operator +(any) const - { - // I Love MSVC 2005! - } - template void operator -(any) const - { - // I Love MSVC 2005! - } - }; -static const nullptr_t __nullptr = {}; -} - -#ifndef nullptr -#define nullptr std::__nullptr -#endif -*/ - -/**************************************************//** -End of Custom nullptr implementation for under g++ 4.6 -*******************************************************/ - /* When there's no work, either because defragment is disabled, or because no query is submitted, thread checks state every BTR_DEFRAGMENT_SLEEP_IN_USECS.*/ #define BTR_DEFRAGMENT_SLEEP_IN_USECS 1000000 diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 961166fed9a..f2c241d0869 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2849,8 +2849,7 @@ ha_innobase::ha_innobase( (srv_force_primary_key ? HA_REQUIRE_PRIMARY_KEY : 0 ) | HA_CAN_FULLTEXT_EXT | HA_CAN_EXPORT), start_of_scan(0), - num_write_row(0), - ha_partition_stats(NULL) + num_write_row(0) {} /*********************************************************************//** @@ -16749,13 +16748,6 @@ innodb_max_dirty_pages_pct_lwm_update( srv_max_dirty_pages_pct_lwm = in_val; } -UNIV_INTERN -void -ha_innobase::set_partition_owner_stats(ha_statistics *stats) -{ - ha_partition_stats= stats; -} - /************************************************************//** Validate the file format name and return its corresponding id. @return valid file format id */ diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 300f49c93f9..9c838910434 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -100,8 +100,6 @@ class ha_innobase: public handler or undefined */ uint num_write_row; /*!< number of write_row() calls */ - ha_statistics* ha_partition_stats; /*!< stats of the partition owner - handler (if there is one) */ uint store_key_val_for_row(uint keynr, char* buff, uint buff_len, const uchar* record); inline void update_thd(THD* thd); @@ -316,7 +314,6 @@ class ha_innobase: public handler Alter_inplace_info* ha_alter_info, bool commit); /** @} */ - void set_partition_owner_stats(ha_statistics *stats); bool check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes); private: diff --git a/storage/innobase/include/ut0timer.h b/storage/innobase/include/ut0timer.h index f361ae79bf5..3015771b434 100644 --- a/storage/innobase/include/ut0timer.h +++ b/storage/innobase/include/ut0timer.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2014, Facebook, Inc. All Rights Reserved. -Copyright (c) 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -19,7 +19,7 @@ this program; if not, write to the Free Software Foundation, Inc., /********************************************************************//** @file include/ut0timer.h -Timer rountines +Timer routines Created 30/07/2014 Jan Lindström jan.lindstrom@skysql.com modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11d7269bdfea06f96d6b6 @@ -28,8 +28,6 @@ modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11 #define ut0timer_h #include "univ.i" -#include "data0type.h" -#include /* Current timer stats */ extern struct my_timer_unit_info ut_timer; @@ -47,39 +45,6 @@ Initializes my_timer struct to contain the info for selected timer.*/ UNIV_INTERN void ut_init_timer(void); -/**************************************************************//** -Return time passed since time then, automatically adjusted -for the estimated timer overhead. -@return time passed since "then" */ -UNIV_INLINE -ulonglong -ut_timer_since( -/*===========*/ - ulonglong then); /*!< in: time where to calculate */ -/**************************************************************//** -Get time passed since "then", and update then to now -@return time passed sinche "then" */ -UNIV_INLINE -ulonglong -ut_timer_since_and_update( -/*======================*/ - ulonglong *then); /*!< in: time where to calculate */ -/**************************************************************//** -Convert native timer units in a ulonglong into seconds in a double -@return time in a seconds */ -UNIV_INLINE -double -ut_timer_to_seconds( -/*=================*/ - ulonglong when); /*!< in: time where to calculate */ -/**************************************************************//** -Convert native timer units in a ulonglong into milliseconds in a double -@return time in milliseconds */ -UNIV_INLINE -double -ut_timer_to_milliseconds( -/*=====================*/ - ulonglong when); /*!< in: time where to calculate */ /**************************************************************//** Convert native timer units in a ulonglong into microseconds in a double @return time in microseconds */ diff --git a/storage/innobase/include/ut0timer.ic b/storage/innobase/include/ut0timer.ic index 46dcd0cb718..34886481efa 100644 --- a/storage/innobase/include/ut0timer.ic +++ b/storage/innobase/include/ut0timer.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2014, Facebook, Inc. All Rights Reserved. -Copyright (c) 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -19,69 +19,12 @@ this program; if not, write to the Free Software Foundation, Inc., /********************************************************************//** @file include/ut0timer.ic -Timer rountines +Timer routines Created 30/07/2014 Jan Lindström jan.lindstrom@skysql.com modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11d7269bdfea06f96d6b6 *************************************************************************/ -/**************************************************************//** -Return time passed since time then, automatically adjusted -for the estimated timer overhead. -@return time passed since "then" */ -UNIV_INLINE -ulonglong -ut_timer_since( -/*===========*/ - ulonglong then) /*!< in: time where to calculate */ -{ - return (ut_timer_now() - then) - ut_timer.overhead; -} - -/**************************************************************//** -Get time passed since "then", and update then to now -@return time passed sinche "then" */ -UNIV_INLINE -ulonglong -ut_timer_since_and_update( -/*======================*/ - ulonglong *then) /*!< in: time where to calculate */ -{ - ulonglong now = ut_timer_now(); - ulonglong ret = (now - (*then)) - ut_timer.overhead; - *then = now; - return ret; -} - -/**************************************************************//** -Convert native timer units in a ulonglong into seconds in a double -@return time in a seconds */ -UNIV_INLINE -double -ut_timer_to_seconds( -/*=================*/ - ulonglong when) /*!< in: time where to calculate */ -{ - double ret = (double)(when); - ret /= (double)(ut_timer.frequency); - return ret; -} - -/**************************************************************//** -Convert native timer units in a ulonglong into milliseconds in a double -@return time in milliseconds */ -UNIV_INLINE -double -ut_timer_to_milliseconds( -/*=====================*/ - ulonglong when) /*!< in: time where to calculate */ -{ - double ret = (double)(when); - ret *= 1000.0; - ret /= (double)(ut_timer.frequency); - return ret; -} - /**************************************************************//** Convert native timer units in a ulonglong into microseconds in a double @return time in microseconds */ diff --git a/storage/xtradb/btr/btr0defragment.cc b/storage/xtradb/btr/btr0defragment.cc index 44acd9118d7..3124db650c6 100644 --- a/storage/xtradb/btr/btr0defragment.cc +++ b/storage/xtradb/btr/btr0defragment.cc @@ -40,56 +40,6 @@ Modified 30/07/2014 Jan Lindström jan.lindstrom@mariadb.com #include -/**************************************************//** -Custom nullptr implementation for under g++ 4.6 -*******************************************************/ -/* -// #pragma once -namespace std -{ - // based on SC22/WG21/N2431 = J16/07-0301 - struct nullptr_t - { - template operator any * () const - { - return 0; - } - template operator T any:: * () const - { - return 0; - } - -#ifdef _MSC_VER - struct pad {}; - pad __[sizeof(void*)/sizeof(pad)]; -#else - char __[sizeof(void*)]; -#endif -private: - // nullptr_t();// {} - // nullptr_t(const nullptr_t&); - // void operator = (const nullptr_t&); - void operator &() const; - template void operator +(any) const - { - // I Love MSVC 2005! - } - template void operator -(any) const - { - // I Love MSVC 2005! - } - }; -static const nullptr_t __nullptr = {}; -} - -#ifndef nullptr -#define nullptr std::__nullptr -#endif -*/ -/**************************************************//** -End of Custom nullptr implementation for under g++ 4.6 -*******************************************************/ - /* When there's no work, either because defragment is disabled, or because no query is submitted, thread checks state every BTR_DEFRAGMENT_SLEEP_IN_USECS.*/ #define BTR_DEFRAGMENT_SLEEP_IN_USECS 1000000 diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 238627ce003..a673dd78c70 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -3259,8 +3259,7 @@ ha_innobase::ha_innobase( (srv_force_primary_key ? HA_REQUIRE_PRIMARY_KEY : 0 ) | HA_CAN_FULLTEXT_EXT | HA_CAN_EXPORT), start_of_scan(0), - num_write_row(0), - ha_partition_stats(NULL) + num_write_row(0) {} /*********************************************************************//** @@ -17527,13 +17526,6 @@ innodb_max_dirty_pages_pct_lwm_update( srv_max_dirty_pages_pct_lwm = in_val; } -UNIV_INTERN -void -ha_innobase::set_partition_owner_stats(ha_statistics *stats) -{ - ha_partition_stats= stats; -} - /************************************************************//** Validate the file format name and return its corresponding id. @return valid file format id */ diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index c5b0e723702..3dbbc53a0e3 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -100,8 +100,6 @@ class ha_innobase: public handler or undefined */ uint num_write_row; /*!< number of write_row() calls */ - ha_statistics* ha_partition_stats; /*!< stats of the partition owner - handler (if there is one) */ uint store_key_val_for_row(uint keynr, char* buff, uint buff_len, const uchar* record); inline void update_thd(THD* thd); @@ -318,7 +316,6 @@ class ha_innobase: public handler Alter_inplace_info* ha_alter_info, bool commit); /** @} */ - void set_partition_owner_stats(ha_statistics *stats); bool check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes); diff --git a/storage/xtradb/include/ut0timer.h b/storage/xtradb/include/ut0timer.h index f361ae79bf5..3015771b434 100644 --- a/storage/xtradb/include/ut0timer.h +++ b/storage/xtradb/include/ut0timer.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2014, Facebook, Inc. All Rights Reserved. -Copyright (c) 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -19,7 +19,7 @@ this program; if not, write to the Free Software Foundation, Inc., /********************************************************************//** @file include/ut0timer.h -Timer rountines +Timer routines Created 30/07/2014 Jan Lindström jan.lindstrom@skysql.com modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11d7269bdfea06f96d6b6 @@ -28,8 +28,6 @@ modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11 #define ut0timer_h #include "univ.i" -#include "data0type.h" -#include /* Current timer stats */ extern struct my_timer_unit_info ut_timer; @@ -47,39 +45,6 @@ Initializes my_timer struct to contain the info for selected timer.*/ UNIV_INTERN void ut_init_timer(void); -/**************************************************************//** -Return time passed since time then, automatically adjusted -for the estimated timer overhead. -@return time passed since "then" */ -UNIV_INLINE -ulonglong -ut_timer_since( -/*===========*/ - ulonglong then); /*!< in: time where to calculate */ -/**************************************************************//** -Get time passed since "then", and update then to now -@return time passed sinche "then" */ -UNIV_INLINE -ulonglong -ut_timer_since_and_update( -/*======================*/ - ulonglong *then); /*!< in: time where to calculate */ -/**************************************************************//** -Convert native timer units in a ulonglong into seconds in a double -@return time in a seconds */ -UNIV_INLINE -double -ut_timer_to_seconds( -/*=================*/ - ulonglong when); /*!< in: time where to calculate */ -/**************************************************************//** -Convert native timer units in a ulonglong into milliseconds in a double -@return time in milliseconds */ -UNIV_INLINE -double -ut_timer_to_milliseconds( -/*=====================*/ - ulonglong when); /*!< in: time where to calculate */ /**************************************************************//** Convert native timer units in a ulonglong into microseconds in a double @return time in microseconds */ diff --git a/storage/xtradb/include/ut0timer.ic b/storage/xtradb/include/ut0timer.ic index 815726e9d0a..02af23cdc6d 100644 --- a/storage/xtradb/include/ut0timer.ic +++ b/storage/xtradb/include/ut0timer.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2014, Facebook, Inc. All Rights Reserved. -Copyright (c) 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -19,69 +19,12 @@ this program; if not, write to the Free Software Foundation, Inc., /********************************************************************//** @file include/ut0timer.ic -Timer rountines +Timer routines Created 30/07/2014 Jan Lindström jan.lindstrom@skysql.com modified from https://github.com/facebook/mysql-5.6/commit/c75a413edeb96eb99bf11d7269bdfea06f96d6b6 *************************************************************************/ -/**************************************************************//** -Return time passed since time then, automatically adjusted -for the estimated timer overhead. -@return time passed since "then" */ -UNIV_INLINE -ulonglong -ut_timer_since( -/*===========*/ - ulonglong then) /*!< in: time where to calculate */ -{ - return (ut_timer_now() - then) - ut_timer.overhead; -} - -/**************************************************************//** -Get time passed since "then", and update then to now -@return time passed sinche "then" */ -UNIV_INLINE -ulonglong -ut_timer_since_and_update( -/*======================*/ - ulonglong *then) /*!< in: time where to calculate */ -{ - ulonglong now = ut_timer_now(); - ulonglong ret = (now - (*then)) - ut_timer.overhead; - *then = now; - return ret; -} - -/**************************************************************//** -Convert native timer units in a ulonglong into seconds in a double -@return time in a seconds */ -UNIV_INLINE -double -ut_timer_to_seconds( -/*=================*/ - ulonglong when) /*!< in: time where to calculate */ -{ - double ret = (double)(when); - ret /= (double)(ut_timer.frequency); - return ret; -} - -/**************************************************************//** -Convert native timer units in a ulonglong into milliseconds in a double -@return time in milliseconds */ -UNIV_INLINE -double -ut_timer_to_milliseconds( -/*=====================*/ - ulonglong when) /*!< in: time where to calculate */ -{ - double ret = (double)(when); - ret *= 1000.0; - ret /= (double)(ut_timer.frequency); - return ret; -} - /**************************************************************//** Convert native timer units in a ulonglong into microseconds in a double @return time in microseconds */ From f6376bfd1c25997283cbaf4eb835cc1e09e49def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Jun 2018 09:25:34 +0300 Subject: [PATCH 090/276] Add FLUSH TABLES to avoid corruption of MyISAM system tables --- mysql-test/suite/innodb/r/innodb-blob.result | 1 + mysql-test/suite/innodb/t/innodb-blob.test | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/innodb/r/innodb-blob.result b/mysql-test/suite/innodb/r/innodb-blob.result index eca77972400..5d2101d8eeb 100644 --- a/mysql-test/suite/innodb/r/innodb-blob.result +++ b/mysql-test/suite/innodb/r/innodb-blob.result @@ -1,3 +1,4 @@ +FLUSH TABLES; CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=InnoDB; CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE t3 (a INT PRIMARY KEY, b TEXT, c TEXT) ENGINE=InnoDB; diff --git a/mysql-test/suite/innodb/t/innodb-blob.test b/mysql-test/suite/innodb/t/innodb-blob.test index 775d52c9ce8..17fd194dbeb 100644 --- a/mysql-test/suite/innodb/t/innodb-blob.test +++ b/mysql-test/suite/innodb/t/innodb-blob.test @@ -5,12 +5,14 @@ --source include/have_innodb.inc --source include/have_innodb_16k.inc -# DEBUG_SYNC must be compiled in. +--source include/have_debug.inc --source include/have_debug_sync.inc # Embedded server does not support restarting --source include/not_embedded.inc +FLUSH TABLES; + CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=InnoDB; CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE t3 (a INT PRIMARY KEY, b TEXT, c TEXT) ENGINE=InnoDB; From 3b7da8a44c8a0ff4b40b37e4db01f7e397aefab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Jun 2018 10:36:25 +0300 Subject: [PATCH 091/276] MDEV-15824 innodb_defragment=ON trumps innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE ha_innobase::optimize(): If both innodb_defragment and innodb_optimize_fulltext_only are at their default settings (OFF), fall back to ALTER TABLE. Else process one or both options. --- .../innodb/r/innodb_defragment_small.result | 21 +++++++++++++---- .../innodb/t/innodb_defragment_small.test | 23 +++++++++++++------ storage/innobase/handler/ha_innodb.cc | 14 +++++------ storage/xtradb/handler/ha_innodb.cc | 14 +++++------ 4 files changed, 44 insertions(+), 28 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_defragment_small.result b/mysql-test/suite/innodb/r/innodb_defragment_small.result index 3f594f79e70..07e58f442d9 100644 --- a/mysql-test/suite/innodb/r/innodb_defragment_small.result +++ b/mysql-test/suite/innodb/r/innodb_defragment_small.result @@ -1,6 +1,13 @@ SET @innodb_defragment_orig=@@GLOBAL.innodb_defragment; +SET @innodb_optimize_fulltext_orig=@@GLOBAL.innodb_optimize_fulltext_only; SET GLOBAL innodb_defragment = 1; -CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), KEY(a, b)) ENGINE=INNODB; +SET GLOBAL innodb_optimize_fulltext_only = 0; +# +# MDEV-12198 innodb_defragment=1 crashes server on +# OPTIMIZE TABLE when FULLTEXT index exists +# +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), +KEY(a, b), FULLTEXT KEY(b)) ENGINE=INNODB; OPTIMIZE TABLE t1; Table Op Msg_type Msg_text test.t1 optimize status OK @@ -11,14 +18,18 @@ INSERT INTO t1 VALUES (400000, REPEAT('A', 256)); OPTIMIZE TABLE t1; Table Op Msg_type Msg_text test.t1 optimize status OK -DROP TABLE t1; # -# MDEV-12198 innodb_defragment=1 crashes server on -# OPTIMIZE TABLE when FULLTEXT index exists +# MDEV-15824 innodb_defragment=ON trumps +# innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE # -CREATE TABLE t1 (c TEXT, FULLTEXT KEY (c)) ENGINE=InnoDB; +SET GLOBAL innodb_optimize_fulltext_only = 1; +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +SET GLOBAL innodb_defragment = 0; OPTIMIZE TABLE t1; Table Op Msg_type Msg_text test.t1 optimize status OK DROP TABLE t1; SET GLOBAL innodb_defragment = @innodb_defragment_orig; +SET GLOBAL innodb_optimize_fulltext_only = @innodb_optimize_fulltext_orig; diff --git a/mysql-test/suite/innodb/t/innodb_defragment_small.test b/mysql-test/suite/innodb/t/innodb_defragment_small.test index 454333d6986..c52a58fc5a7 100644 --- a/mysql-test/suite/innodb/t/innodb_defragment_small.test +++ b/mysql-test/suite/innodb/t/innodb_defragment_small.test @@ -1,10 +1,17 @@ --source include/have_innodb.inc SET @innodb_defragment_orig=@@GLOBAL.innodb_defragment; +SET @innodb_optimize_fulltext_orig=@@GLOBAL.innodb_optimize_fulltext_only; SET GLOBAL innodb_defragment = 1; +SET GLOBAL innodb_optimize_fulltext_only = 0; -# Small tests copied from innodb.innodb_defragment -CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), KEY(a, b)) ENGINE=INNODB; +--echo # +--echo # MDEV-12198 innodb_defragment=1 crashes server on +--echo # OPTIMIZE TABLE when FULLTEXT index exists +--echo # + +CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), + KEY(a, b), FULLTEXT KEY(b)) ENGINE=INNODB; OPTIMIZE TABLE t1; INSERT INTO t1 VALUES (100000, REPEAT('A', 256)); @@ -13,16 +20,18 @@ INSERT INTO t1 VALUES (300000, REPEAT('A', 256)); INSERT INTO t1 VALUES (400000, REPEAT('A', 256)); OPTIMIZE TABLE t1; -DROP TABLE t1; --echo # ---echo # MDEV-12198 innodb_defragment=1 crashes server on ---echo # OPTIMIZE TABLE when FULLTEXT index exists +--echo # MDEV-15824 innodb_defragment=ON trumps +--echo # innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE --echo # -CREATE TABLE t1 (c TEXT, FULLTEXT KEY (c)) ENGINE=InnoDB; - +SET GLOBAL innodb_optimize_fulltext_only = 1; OPTIMIZE TABLE t1; +SET GLOBAL innodb_defragment = 0; +OPTIMIZE TABLE t1; + DROP TABLE t1; SET GLOBAL innodb_defragment = @innodb_defragment_orig; +SET GLOBAL innodb_optimize_fulltext_only = @innodb_optimize_fulltext_orig; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index f2c241d0869..c525c2f0efa 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -13904,6 +13904,7 @@ ha_innobase::optimize( This works OK otherwise, but MySQL locks the entire table during calls to OPTIMIZE, which is undesirable. */ + bool try_alter = true; if (srv_defragment) { int err; @@ -13911,7 +13912,7 @@ ha_innobase::optimize( err = defragment_table(prebuilt->table->name, NULL, false); if (err == 0) { - return (HA_ADMIN_OK); + try_alter = false; } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, err, @@ -13919,9 +13920,7 @@ ha_innobase::optimize( prebuilt->table->name, err); if(err == ER_SP_ALREADY_EXISTS) { - return (HA_ADMIN_OK); - } else { - return (HA_ADMIN_TRY_ALTER); + try_alter = false; } } } @@ -13932,11 +13931,10 @@ ha_innobase::optimize( fts_sync_table(prebuilt->table, false, true, false); fts_optimize_table(prebuilt->table); } - return(HA_ADMIN_OK); - } else { - - return(HA_ADMIN_TRY_ALTER); + try_alter = false; } + + return try_alter ? HA_ADMIN_TRY_ALTER : HA_ADMIN_OK; } /*******************************************************************//** diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index a673dd78c70..d642f9c0a06 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -14522,6 +14522,7 @@ ha_innobase::optimize( This works OK otherwise, but MySQL locks the entire table during calls to OPTIMIZE, which is undesirable. */ + bool try_alter = true; if (srv_defragment) { int err; @@ -14529,7 +14530,7 @@ ha_innobase::optimize( err = defragment_table(prebuilt->table->name, NULL, false); if (err == 0) { - return (HA_ADMIN_OK); + try_alter = false; } else { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, err, @@ -14537,9 +14538,7 @@ ha_innobase::optimize( prebuilt->table->name, err); if(err == ER_SP_ALREADY_EXISTS) { - return (HA_ADMIN_OK); - } else { - return (HA_ADMIN_TRY_ALTER); + try_alter = false; } } } @@ -14550,11 +14549,10 @@ ha_innobase::optimize( fts_sync_table(prebuilt->table, false, true, false); fts_optimize_table(prebuilt->table); } - return(HA_ADMIN_OK); - } else { - - return(HA_ADMIN_TRY_ALTER); + try_alter = false; } + + return try_alter ? HA_ADMIN_TRY_ALTER : HA_ADMIN_OK; } /*******************************************************************//** From c10bed17ddaaa2635fe758a47d89a12c4169e31e Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 29 May 2018 14:31:53 +1000 Subject: [PATCH 092/276] mysqld_safe: use sh not bash c902d5a4ded2a492eb47379910111717dbded2d3 changed this to bash. This isn't approprate as mysqld_safe is used by FreeBSD where there is no bash by default. The original reason to use bash seems related to MDEV-3279 which was a dash bug fixed in 2009 - https://git.kernel.org/pub/scm/utils/dash/dash.git/commit/?id=3800d4934391b144fd261a7957aea72ced7d47ea Working around 9 year old fixed bugs shouldn't be done here. --- scripts/mysqld_safe.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 696ac552fad..2abdfd7faf5 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Copyright Abandoned 1996 TCX DataKonsult AB & Monty Program KB & Detron HB # This file is public domain and comes with NO WARRANTY of any kind # From 88a263eabb7a7b993c135ba6b91ef969a7c41bd3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sun, 3 Jun 2018 20:31:10 +1000 Subject: [PATCH 093/276] MDEV-16318: mysqld_safe - partial revert on 64094e1 Revert part of 64094e1 because mysqld_safe isn't just used for Debian. As such references to specific packaging files like /etc/mysql/debian.cnf shouldn't have passed a review. There's also no history of why 64094e1 was needed based on the history of debian/patches/38_scripts__mysqld_safe.sh__signals.dpatch Other distos have survived without mysqld_safe handling signals. --- scripts/mysqld_safe.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 2abdfd7faf5..5797bdc68d7 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -36,6 +36,7 @@ skip_err_log=0 syslog_tag_mysqld=mysqld syslog_tag_mysqld_safe=mysqld_safe +trap '' 1 2 3 15 # we shouldn't let anyone kill us # MySQL-specific environment variable. First off, it's not really a umask, # it's the desired mode. Second, it follows umask(2), not umask(3) in that @@ -159,7 +160,7 @@ eval_log_error () { # sed buffers output (only GNU sed supports a -u (unbuffered) option) # which means that messages may not get sent to syslog until the # mysqld process quits. - cmd="$cmd 2>&1 | logger -t '$syslog_tag_mysqld' -p daemon.error & wait" + cmd="$cmd 2>&1 | logger -t '$syslog_tag_mysqld' -p daemon.error" ;; *) echo "Internal program error (non-fatal):" \ @@ -876,13 +877,6 @@ then exit 1 fi -# -# From now on, we catch signals to do a proper shutdown of mysqld -# when signalled to do so. -# -trap '/usr/bin/mysqladmin --defaults-extra-file=/etc/mysql/debian.cnf refresh & wait' 1 # HUP -trap '/usr/bin/mysqladmin --defaults-extra-file=/etc/mysql/debian.cnf shutdown' 2 3 15 # INT QUIT and TERM - # # Uncomment the following lines if you want all tables to be automatically # checked and repaired during startup. You should add sensible key_buffer From 738c5c84245f27c839a0713f52452a45f141b3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 13:06:14 +0000 Subject: [PATCH 094/276] MDEV-12642: Build deb source packages on buildbot, just not on Travis-CI --- debian/autobake-deb.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index c26fcaa4e85..d4de8045788 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -107,12 +107,20 @@ dch -b -D ${CODENAME} -v "${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME}" "Automat echo "Creating package version ${EPOCH}${UPSTREAM}${PATCHLEVEL}~${CODENAME} ... " +# On Travis CI, use -b to build binary only packages as there is no need to +# waste time on generating the source package. +if [[ $TRAVIS ]] +then + BUILDPACKAGE_FLAGS="-b" +fi + # Build the package # Pass -I so that .git and other unnecessary temporary and source control files # will be ignored by dpkg-source when creating the tar.gz source package. -# Use -b to build binary only packages as there is no need to waste time on -# generating the source package. -fakeroot dpkg-buildpackage -us -uc -I -b +fakeroot dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS + +# If the step above fails due to missing dependencies, you can manually run +# sudo mk-build-deps debian/control -r -i # Don't log package contents on Travis-CI to save time and log size if [[ ! $TRAVIS ]] From 55abcfa7b70968246a1a26a8839013ebb8f5c506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 5 Jun 2018 18:16:12 +0300 Subject: [PATCH 095/276] MDEV-16124 fil_rename_tablespace() times out and crashes server during table-rebuilding ALTER TABLE InnoDB insisted on closing the file handle before renaming a file. Renaming a file should never be a problem on POSIX systems. Also on Windows it should work if the file was opened in FILE_SHARE_DELETE mode. fil_space_t::stop_ios: Remove. We no longer need to stop file access during rename operations. fil_mutex_enter_and_prepare_for_io(): Remove the wait for stop_ios. fil_rename_tablespace(): Remove the retry logic; do not close the file handle. Remove the unused fault injection that was added along with the DATA DIRECTORY functionality (MySQL WL#5980). os_file_create_simple_func(), os_file_create_func(), os_file_create_simple_no_error_handling_func(): Include FILE_SHARE_DELETE in the share_mode. (We will still prevent multiple InnoDB instances from using the same files by not setting FILE_SHARE_WRITE.) --- storage/innobase/fil/fil0fil.cc | 110 ----------------------------- storage/innobase/include/fil0fil.h | 4 -- storage/innobase/os/os0file.cc | 9 +-- storage/xtradb/fil/fil0fil.cc | 110 ----------------------------- storage/xtradb/include/fil0fil.h | 4 -- storage/xtradb/os/os0file.cc | 9 +-- 6 files changed, 10 insertions(+), 236 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7167b99a750..f852a64e2e9 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -845,7 +845,6 @@ fil_mutex_enter_and_prepare_for_io( ibool success; ibool print_info = FALSE; ulint count = 0; - ulint count2 = 0; retry: mutex_enter(&fil_system->mutex); @@ -862,46 +861,6 @@ retry: space = fil_space_get_by_id(space_id); - if (space != NULL && space->stop_ios) { - /* We are going to do a rename file and want to stop new i/o's - for a while */ - - if (count2 > 20000) { - fputs("InnoDB: Warning: tablespace ", stderr); - ut_print_filename(stderr, space->name); - fprintf(stderr, - " has i/o ops stopped for a long time %lu\n", - (ulong) count2); - } - - mutex_exit(&fil_system->mutex); - -#ifndef UNIV_HOTBACKUP - - /* Wake the i/o-handler threads to make sure pending - i/o's are performed */ - os_aio_simulated_wake_handler_threads(); - - /* The sleep here is just to give IO helper threads a - bit of time to do some work. It is not required that - all IO related to the tablespace being renamed must - be flushed here as we do fil_flush() in - fil_rename_tablespace() as well. */ - os_thread_sleep(20000); - -#endif /* UNIV_HOTBACKUP */ - - /* Flush tablespaces so that we can close modified - files in the LRU list */ - fil_flush_file_spaces(FIL_TABLESPACE); - - os_thread_sleep(20000); - - count2++; - - goto retry; - } - if (fil_system->n_open < fil_system->max_n_open) { return; @@ -2898,7 +2857,6 @@ fil_rename_tablespace( ibool success; fil_space_t* space; fil_node_t* node; - ulint count = 0; char* new_path; char* old_name; char* old_path; @@ -2906,25 +2864,10 @@ fil_rename_tablespace( ut_a(id != 0); -retry: - count++; - - if (!(count % 1000)) { - ut_print_timestamp(stderr); - fputs(" InnoDB: Warning: problems renaming ", stderr); - ut_print_filename(stderr, - old_name_in ? old_name_in : not_given); - fputs(" to ", stderr); - ut_print_filename(stderr, new_name); - fprintf(stderr, ", %lu iterations\n", (ulong) count); - } - mutex_enter(&fil_system->mutex); space = fil_space_get_by_id(id); - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_1", space = NULL; ); - if (space == NULL) { ib_logf(IB_LOG_LEVEL_ERROR, "Cannot find space id %lu in the tablespace " @@ -2936,54 +2879,11 @@ retry: return(FALSE); } - if (count > 25000) { - space->stop_ios = FALSE; - mutex_exit(&fil_system->mutex); - - return(FALSE); - } - - /* We temporarily close the .ibd file because we do not trust that - operating systems can rename an open file. For the closing we have to - wait until there are no pending i/o's or flushes on the file. */ - - space->stop_ios = TRUE; - /* The following code must change when InnoDB supports multiple datafiles per tablespace. */ ut_a(UT_LIST_GET_LEN(space->chain) == 1); node = UT_LIST_GET_FIRST(space->chain); - if (node->n_pending > 0 - || node->n_pending_flushes > 0 - || node->being_extended) { - /* There are pending i/o's or flushes or the file is - currently being extended, sleep for a while and - retry */ - - mutex_exit(&fil_system->mutex); - - os_thread_sleep(20000); - - goto retry; - - } else if (node->modification_counter > node->flush_counter) { - /* Flush the space */ - - mutex_exit(&fil_system->mutex); - - os_thread_sleep(20000); - - fil_flush(id); - - goto retry; - - } else if (node->open) { - /* Close the file */ - - fil_node_close_file(node, fil_system); - } - /* Check that the old name in the space is right */ if (old_name_in) { @@ -3002,17 +2902,9 @@ retry: space, node, new_name, new_path); if (success) { - - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", - goto skip_second_rename; ); - success = os_file_rename( innodb_file_data_key, old_path, new_path); - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", -skip_second_rename: - success = FALSE; ); - if (!success) { /* We have to revert the changes we made to the tablespace memory cache */ @@ -3022,8 +2914,6 @@ skip_second_rename: } } - space->stop_ios = FALSE; - mutex_exit(&fil_system->mutex); #ifndef UNIV_HOTBACKUP diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9374bb88bb7..6e772e31772 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -256,10 +256,6 @@ struct fil_space_t { the space corresponds to a table in the InnoDB data dictionary; so we can print a warning of orphaned tablespaces */ - ibool stop_ios;/*!< TRUE if we want to rename the - .ibd file of tablespace and want to - stop temporarily posting of new i/o - requests on the file */ ibool stop_new_ops; /*!< we set this TRUE when we start deleting a single-table tablespace. diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 55d2c15f906..8f3f3716fc2 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -1186,7 +1186,8 @@ os_file_create_simple_func( /* Use default security attributes and no template file. */ file = CreateFile( - (LPCTSTR) name, access, FILE_SHARE_READ, NULL, + (LPCTSTR) name, access, + FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, create_flag, attributes, NULL); if (file == INVALID_HANDLE_VALUE) { @@ -1314,7 +1315,7 @@ os_file_create_simple_no_error_handling_func( DWORD access; DWORD create_flag; DWORD attributes = 0; - DWORD share_mode = FILE_SHARE_READ; + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE; ut_a(name); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); @@ -1554,7 +1555,7 @@ os_file_create_func( #ifdef __WIN__ DWORD create_flag; - DWORD share_mode = FILE_SHARE_READ; + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE; on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT ? TRUE : FALSE; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 3807e6b3395..49c5f4b090b 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -862,7 +862,6 @@ fil_mutex_enter_and_prepare_for_io( ibool success; ibool print_info = FALSE; ulint count = 0; - ulint count2 = 0; retry: mutex_enter(&fil_system->mutex); @@ -879,46 +878,6 @@ retry: space = fil_space_get_by_id(space_id); - if (space != NULL && space->stop_ios) { - /* We are going to do a rename file and want to stop new i/o's - for a while */ - - if (count2 > 20000) { - fputs("InnoDB: Warning: tablespace ", stderr); - ut_print_filename(stderr, space->name); - fprintf(stderr, - " has i/o ops stopped for a long time %lu\n", - (ulong) count2); - } - - mutex_exit(&fil_system->mutex); - -#ifndef UNIV_HOTBACKUP - - /* Wake the i/o-handler threads to make sure pending - i/o's are performed */ - os_aio_simulated_wake_handler_threads(); - - /* The sleep here is just to give IO helper threads a - bit of time to do some work. It is not required that - all IO related to the tablespace being renamed must - be flushed here as we do fil_flush() in - fil_rename_tablespace() as well. */ - os_thread_sleep(20000); - -#endif /* UNIV_HOTBACKUP */ - - /* Flush tablespaces so that we can close modified - files in the LRU list */ - fil_flush_file_spaces(FIL_TABLESPACE); - - os_thread_sleep(20000); - - count2++; - - goto retry; - } - if (fil_system->n_open < fil_system->max_n_open) { return; @@ -2950,7 +2909,6 @@ fil_rename_tablespace( ibool success; fil_space_t* space; fil_node_t* node; - ulint count = 0; char* new_path; char* old_name; char* old_path; @@ -2958,25 +2916,10 @@ fil_rename_tablespace( ut_a(id != 0); -retry: - count++; - - if (!(count % 1000)) { - ut_print_timestamp(stderr); - fputs(" InnoDB: Warning: problems renaming ", stderr); - ut_print_filename(stderr, - old_name_in ? old_name_in : not_given); - fputs(" to ", stderr); - ut_print_filename(stderr, new_name); - fprintf(stderr, ", %lu iterations\n", (ulong) count); - } - mutex_enter(&fil_system->mutex); space = fil_space_get_by_id(id); - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_1", space = NULL; ); - if (space == NULL) { ib_logf(IB_LOG_LEVEL_ERROR, "Cannot find space id %lu in the tablespace " @@ -2988,54 +2931,11 @@ retry: return(FALSE); } - if (count > 25000) { - space->stop_ios = FALSE; - mutex_exit(&fil_system->mutex); - - return(FALSE); - } - - /* We temporarily close the .ibd file because we do not trust that - operating systems can rename an open file. For the closing we have to - wait until there are no pending i/o's or flushes on the file. */ - - space->stop_ios = TRUE; - /* The following code must change when InnoDB supports multiple datafiles per tablespace. */ ut_a(UT_LIST_GET_LEN(space->chain) == 1); node = UT_LIST_GET_FIRST(space->chain); - if (node->n_pending > 0 - || node->n_pending_flushes > 0 - || node->being_extended) { - /* There are pending i/o's or flushes or the file is - currently being extended, sleep for a while and - retry */ - - mutex_exit(&fil_system->mutex); - - os_thread_sleep(20000); - - goto retry; - - } else if (node->modification_counter > node->flush_counter) { - /* Flush the space */ - - mutex_exit(&fil_system->mutex); - - os_thread_sleep(20000); - - fil_flush(id); - - goto retry; - - } else if (node->open) { - /* Close the file */ - - fil_node_close_file(node, fil_system); - } - /* Check that the old name in the space is right */ if (old_name_in) { @@ -3054,17 +2954,9 @@ retry: space, node, new_name, new_path); if (success) { - - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", - goto skip_second_rename; ); - success = os_file_rename( innodb_file_data_key, old_path, new_path); - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", -skip_second_rename: - success = FALSE; ); - if (!success) { /* We have to revert the changes we made to the tablespace memory cache */ @@ -3074,8 +2966,6 @@ skip_second_rename: } } - space->stop_ios = FALSE; - mutex_exit(&fil_system->mutex); #ifndef UNIV_HOTBACKUP diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 6230bd7a4d4..86b1c561349 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -249,10 +249,6 @@ struct fil_space_t { the space corresponds to a table in the InnoDB data dictionary; so we can print a warning of orphaned tablespaces */ - ibool stop_ios;/*!< TRUE if we want to rename the - .ibd file of tablespace and want to - stop temporarily posting of new i/o - requests on the file */ ibool stop_new_ops; /*!< we set this TRUE when we start deleting a single-table tablespace. diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 5ed7cbbc9b3..c1ddc83d852 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -1301,7 +1301,8 @@ os_file_create_simple_func( /* Use default security attributes and no template file. */ file = CreateFile( - (LPCTSTR) name, access, FILE_SHARE_READ, NULL, + (LPCTSTR) name, access, + FILE_SHARE_READ | FILE_SHARE_DELETE, NULL, create_flag, attributes, NULL); if (file == INVALID_HANDLE_VALUE) { @@ -1460,7 +1461,7 @@ os_file_create_simple_no_error_handling_func( DWORD access; DWORD create_flag; DWORD attributes = 0; - DWORD share_mode = FILE_SHARE_READ; + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE; ut_a(name); ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); @@ -1737,7 +1738,7 @@ os_file_create_func( #ifdef __WIN__ DWORD create_flag; - DWORD share_mode = FILE_SHARE_READ; + DWORD share_mode = FILE_SHARE_READ | FILE_SHARE_DELETE; on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT ? TRUE : FALSE; From 72b6d01848e56a75349d663bc61bbe71f97a280b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 5 Jun 2018 22:13:19 +0100 Subject: [PATCH 096/276] MDEV-10246 ssl-* config file options have no effect without mysql_ssl_set() Partially revert 4ef74979969ac9339d0d42c11a6f26632e6776f1 that caused regression. Any ssl- option must imply use_ssl=1, even if mysql_set_ssl() was not used. --- sql-common/client.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sql-common/client.c b/sql-common/client.c index 6321e64bc75..ec992f80e8d 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -2541,6 +2541,10 @@ static int send_client_reply_packet(MCPVIO_EXT *mpvio, mysql->client_flag|= CLIENT_MULTI_RESULTS; #ifdef HAVE_OPENSSL + if (mysql->options.ssl_key || mysql->options.ssl_cert || + mysql->options.ssl_ca || mysql->options.ssl_capath || + mysql->options.ssl_cipher) + mysql->options.use_ssl = 1; if (mysql->options.use_ssl) mysql->client_flag|= CLIENT_SSL; #endif /* HAVE_OPENSSL */ From 75b4eb5cc969a1f5fc221d03c62987b8e57d6332 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 6 Jun 2018 15:27:57 +0200 Subject: [PATCH 097/276] Catch of OOM situation. --- sql/sql_base.cc | 3 +++ sql/table.cc | 2 ++ 2 files changed, 5 insertions(+) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index f5864256440..71bbb538b5c 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -6426,6 +6426,9 @@ find_field_in_natural_join(THD *thd, TABLE_LIST *table_ref, const char *name, column reference. See create_view_field() for details. */ item= nj_col->create_item(thd); + if (!item) + DBUG_RETURN(NULL); + /* *ref != NULL means that *ref contains the item that we need to replace. If the item was aliased by the user, set the alias to diff --git a/sql/table.cc b/sql/table.cc index bb4eae9b1e2..552f514283d 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -5252,6 +5252,8 @@ Item *create_view_field(THD *thd, TABLE_LIST *view, Item **field_ref, Item *item= new Item_direct_view_ref(&view->view->select_lex.context, field_ref, view->alias, name, view); + if (!item) + return NULL; /* Force creation of nullable item for the result tmp table for outer joined views/derived tables. From 3627dd7f6a1cf5bd7151ff23290d64cb6dffea90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 7 Jun 2018 10:31:39 +0300 Subject: [PATCH 098/276] MDEV-16416 Crash on IMPORT TABLESPACE of a ROW_FORMAT=COMPRESSED table fil_iterate(): Invoke fil_encrypt_buf() correctly when a ROW_FORMAT=COMPRESSED table with a physical page size of innodb_page_size is being imported. Also, validate the page checksum before decryption, and reduce the scope of some variables. AbstractCallback::operator()(): Remove the parameter 'offset'. The check for it in FetchIndexRootPages::operator() was basically redundant and dead code since the previous refactoring. --- .../encryption/r/innodb-discard-import.result | 6 + .../encryption/t/innodb-discard-import.test | 32 ++-- storage/innobase/row/row0import.cc | 145 ++++++++---------- storage/xtradb/row/row0import.cc | 145 ++++++++---------- 4 files changed, 147 insertions(+), 181 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb-discard-import.result b/mysql-test/suite/encryption/r/innodb-discard-import.result index 06f4abab9f4..406460d0a23 100644 --- a/mysql-test/suite/encryption/r/innodb-discard-import.result +++ b/mysql-test/suite/encryption/r/innodb-discard-import.result @@ -1,5 +1,8 @@ call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); +SET @innodb_file_format_orig = @@GLOBAL.innodb_file_format; +SET @innodb_file_per_table_orig = @@GLOBAL.innodb_file_per_table; +SET @innodb_compression_algo = @@GLOBAL.innodb_compression_algorithm; SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; @@ -134,3 +137,6 @@ NOT FOUND /tmpres/ in t3.ibd NOT FOUND /mysql/ in t4.ibd DROP PROCEDURE innodb_insert_proc; DROP TABLE t1,t2,t3,t4; +SET GLOBAL innodb_file_format = @innodb_file_format_orig; +SET GLOBAL innodb_file_per_table = @innodb_file_per_table_orig; +SET GLOBAL innodb_compression_algorithm = @innodb_compression_algo; diff --git a/mysql-test/suite/encryption/t/innodb-discard-import.test b/mysql-test/suite/encryption/t/innodb-discard-import.test index 9feaacc41e5..54471ae3bae 100644 --- a/mysql-test/suite/encryption/t/innodb-discard-import.test +++ b/mysql-test/suite/encryption/t/innodb-discard-import.test @@ -1,10 +1,6 @@ -- source include/have_innodb.inc +-- source include/innodb_page_size_small.inc -- source include/have_file_key_management_plugin.inc -# embedded does not support restart --- source include/not_embedded.inc --- source include/not_valgrind.inc -# Avoid CrashReporter popup on Mac --- source include/not_crashrep.inc # # MDEV-8770: Incorrect error message when importing page compressed tablespace @@ -13,17 +9,13 @@ call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); ---disable_query_log -let $innodb_file_format_orig = `SELECT @@innodb_file_format`; -let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; -let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`; ---enable_query_log +SET @innodb_file_format_orig = @@GLOBAL.innodb_file_format; +SET @innodb_file_per_table_orig = @@GLOBAL.innodb_file_per_table; +SET @innodb_compression_algo = @@GLOBAL.innodb_compression_algorithm; ---disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; ---enable_warnings --let $MYSQLD_TMPDIR = `SELECT @@tmpdir` --let $MYSQLD_DATADIR = `SELECT @@datadir` @@ -37,7 +29,9 @@ create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes en show warnings; create table t2(c1 bigint not null, b char(200)) engine=innodb page_compressed=1 encrypted=yes encryption_key_id=4; show warnings; -create table t3(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4; +let $kbs= `select floor(@@global.innodb_page_size/1024)`; +--replace_regex / key_block_size=\d+//i +eval create table t3(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4 key_block_size=$kbs; show warnings; create table t4(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; show warnings; @@ -98,6 +92,7 @@ ALTER TABLE t2 IMPORT TABLESPACE; SHOW CREATE TABLE t2; SELECT COUNT(*) FROM t2; ALTER TABLE t3 IMPORT TABLESPACE; +--replace_regex / key_block_size=\d+//i SHOW CREATE TABLE t3; SELECT COUNT(*) FROM t3; ALTER TABLE t4 IMPORT TABLESPACE; @@ -125,11 +120,6 @@ SELECT COUNT(*) FROM t4; DROP PROCEDURE innodb_insert_proc; DROP TABLE t1,t2,t3,t4; -# reset system ---disable_warnings ---disable_query_log -EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; -EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; -EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo; ---enable_query_log ---enable_warnings +SET GLOBAL innodb_file_format = @innodb_file_format_orig; +SET GLOBAL innodb_file_per_table = @innodb_file_per_table_orig; +SET GLOBAL innodb_compression_algorithm = @innodb_compression_algo; diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index dfd6b4bfbea..9ce172f1d07 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -427,12 +427,9 @@ public: updated then its state must be set to BUF_PAGE_NOT_USED. For compressed tables the page descriptor memory will be at offset: block->frame + UNIV_PAGE_SIZE; - @param offset - physical offset within the file - @param block - block read from file, note it is not from the buffer pool + @param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ - virtual dberr_t operator()( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW = 0; + virtual dberr_t operator()(buf_block_t* block) UNIV_NOTHROW = 0; /** @return the space id of the tablespace */ @@ -689,12 +686,9 @@ struct FetchIndexRootPages : public AbstractCallback { /** Called for each block as it is read from the file. - @param offset - physical offset in the file - @param block - block to convert, it is not from the buffer pool. + @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; /** Update the import configuration that will be used to import the tablespace. */ @@ -712,13 +706,9 @@ Called for each block as it is read from the file. Check index pages to determine the exact row format. We can't get that from the tablespace header flags alone. -@param offset - physical offset in the file -@param block - block to convert, it is not from the buffer pool. +@param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ -dberr_t -FetchIndexRootPages::operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW +dberr_t FetchIndexRootPages::operator()(buf_block_t* block) UNIV_NOTHROW { if (is_interrupted()) return DB_INTERRUPTED; @@ -726,15 +716,7 @@ FetchIndexRootPages::operator() ( ulint page_type = fil_page_get_type(page); - if (block->page.offset * m_page_size != offset) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Page offset doesn't match file offset: " - "page offset: %u, file offset: " ULINTPF, - block->page.offset, - (ulint) (offset / m_page_size)); - - return DB_CORRUPTION; - } else if (page_type == FIL_PAGE_TYPE_XDES) { + if (page_type == FIL_PAGE_TYPE_XDES) { return set_current_xdes(block->page.offset, page); } else if (page_type == FIL_PAGE_INDEX && !is_free(block->page.offset) @@ -877,12 +859,9 @@ public: /** Called for each block as it is read from the file. - @param offset - physical offset in the file - @param block - block to convert, it is not from the buffer pool. + @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; private: /** Update the page, set the space id, max trx id and index id. @@ -2039,10 +2018,9 @@ PageConverter::update_page( /** Called for every page in the tablespace. If the page was not updated then its state must be set to BUF_PAGE_NOT_USED. -@param block - block read from file, note it is not from the buffer pool +@param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ -dberr_t -PageConverter::operator() (os_offset_t, buf_block_t* block) UNIV_NOTHROW +dberr_t PageConverter::operator()(buf_block_t* block) UNIV_NOTHROW { /* If we already had an old page with matching number in the buffer pool, evict it now, because @@ -3430,26 +3408,15 @@ fil_iterate( } bool updated = false; - os_offset_t page_off = offset; ulint n_pages_read = (ulint) n_bytes / iter.page_size; const ulint size = iter.page_size; - block->page.offset = page_off / size; + block->page.offset = offset / size; for (ulint i = 0; i < n_pages_read; - ++i, page_off += size, block->frame += size, - block->page.offset++) { - bool decrypted = false; - dberr_t err = DB_SUCCESS; + ++i, block->frame += size, block->page.offset++) { byte* src = readptr + (i * size); - byte* dst = io_buffer + (i * size); - bool frame_changed = false; - ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); - const bool page_compressed - = page_type - == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_COMPRESSED; const ulint page_no = page_get_page_no(src); - if (!page_no && page_off) { + if (!page_no && block->page.offset) { const ulint* b = reinterpret_cast (src); const ulint* const e = b + size / sizeof *b; @@ -3464,11 +3431,46 @@ fil_iterate( continue; } - if (page_no != page_off / size) { - goto page_corrupted; + if (page_no != block->page.offset) { +page_corrupted: + ib_logf(IB_LOG_LEVEL_WARN, + "%s: Page %lu at offset " + UINT64PF " looks corrupted.", + callback.filename(), + ulong(offset / size), offset); + return DB_CORRUPTION; } - if (encrypted) { + bool decrypted = false; + dberr_t err = DB_SUCCESS; + byte* dst = io_buffer + (i * size); + bool frame_changed = false; + ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); + const bool page_compressed + = page_type + == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED + || page_type == FIL_PAGE_PAGE_COMPRESSED; + + if (!encrypted) { + } else if (!mach_read_from_4( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + src)) { +not_encrypted: + if (!page_compressed + && !block->page.zip.data) { + block->frame = src; + frame_changed = true; + } else { + ut_ad(dst != src); + memcpy(dst, src, size); + } + } else { + if (!fil_space_verify_crypt_checksum( + src, callback.get_zip_size(), + NULL, block->page.offset)) { + goto page_corrupted; + } + decrypted = fil_space_decrypt( iter.crypt_data, dst, iter.page_size, src, &err); @@ -3477,18 +3479,11 @@ fil_iterate( return err; } - if (decrypted) { - updated = true; - } else { - if (!page_compressed - && !block->page.zip.data) { - block->frame = src; - frame_changed = true; - } else { - ut_ad(dst != src); - memcpy(dst, src, size); - } + if (!decrypted) { + goto not_encrypted; } + + updated = true; } /* If the original page is page_compressed, we need @@ -3502,16 +3497,10 @@ fil_iterate( encrypted && !frame_changed ? dst : src, callback.get_zip_size(), NULL)) { -page_corrupted: - ib_logf(IB_LOG_LEVEL_WARN, - "%s: Page %lu at offset " - UINT64PF " looks corrupted.", - callback.filename(), - ulong(offset / size), offset); - return DB_CORRUPTION; + goto page_corrupted; } - if ((err = callback(page_off, block)) != DB_SUCCESS) { + if ((err = callback(block)) != DB_SUCCESS) { return err; } else if (!updated) { updated = buf_block_get_state(block) @@ -3584,19 +3573,15 @@ page_corrupted: buffer pool is not being used at all! */ if (decrypted && encrypted) { byte *dest = writeptr + (i * size); - ulint space = mach_read_from_4( - src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); - ib_uint64_t lsn = mach_read_from_8(src + FIL_PAGE_LSN); byte* tmp = fil_encrypt_buf( - iter.crypt_data, - space, - offset, - lsn, - src, - iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size, - dest); + iter.crypt_data, + callback.get_space_id(), + block->page.offset, + mach_read_from_8(src + FIL_PAGE_LSN), + src, + callback.get_zip_size(), + dest); if (tmp == src) { /* TODO: remove unnecessary memcpy's */ diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index a3478dc2fc4..2b08c819b6d 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -427,12 +427,9 @@ public: updated then its state must be set to BUF_PAGE_NOT_USED. For compressed tables the page descriptor memory will be at offset: block->frame + UNIV_PAGE_SIZE; - @param offset - physical offset within the file - @param block - block read from file, note it is not from the buffer pool + @param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ - virtual dberr_t operator()( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW = 0; + virtual dberr_t operator()(buf_block_t* block) UNIV_NOTHROW = 0; /** @return the space id of the tablespace */ @@ -689,12 +686,9 @@ struct FetchIndexRootPages : public AbstractCallback { /** Called for each block as it is read from the file. - @param offset - physical offset in the file - @param block - block to convert, it is not from the buffer pool. + @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; /** Update the import configuration that will be used to import the tablespace. */ @@ -712,13 +706,9 @@ Called for each block as it is read from the file. Check index pages to determine the exact row format. We can't get that from the tablespace header flags alone. -@param offset - physical offset in the file -@param block - block to convert, it is not from the buffer pool. +@param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ -dberr_t -FetchIndexRootPages::operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW +dberr_t FetchIndexRootPages::operator()(buf_block_t* block) UNIV_NOTHROW { if (is_interrupted()) return DB_INTERRUPTED; @@ -726,15 +716,7 @@ FetchIndexRootPages::operator() ( ulint page_type = fil_page_get_type(page); - if (block->page.offset * m_page_size != offset) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Page offset doesn't match file offset: " - "page offset: %u, file offset: " ULINTPF, - block->page.offset, - (ulint) (offset / m_page_size)); - - return DB_CORRUPTION; - } else if (page_type == FIL_PAGE_TYPE_XDES) { + if (page_type == FIL_PAGE_TYPE_XDES) { return set_current_xdes(block->page.offset, page); } else if (page_type == FIL_PAGE_INDEX && !is_free(block->page.offset) @@ -877,12 +859,9 @@ public: /** Called for each block as it is read from the file. - @param offset - physical offset in the file - @param block - block to convert, it is not from the buffer pool. + @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; private: /** Update the page, set the space id, max trx id and index id. @@ -2038,10 +2017,9 @@ PageConverter::update_page( /** Called for every page in the tablespace. If the page was not updated then its state must be set to BUF_PAGE_NOT_USED. -@param block - block read from file, note it is not from the buffer pool +@param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ -dberr_t -PageConverter::operator() (os_offset_t, buf_block_t* block) UNIV_NOTHROW +dberr_t PageConverter::operator()(buf_block_t* block) UNIV_NOTHROW { /* If we already had an old page with matching number in the buffer pool, evict it now, because @@ -3429,26 +3407,15 @@ fil_iterate( } bool updated = false; - os_offset_t page_off = offset; ulint n_pages_read = (ulint) n_bytes / iter.page_size; const ulint size = iter.page_size; - block->page.offset = page_off / size; + block->page.offset = offset / size; for (ulint i = 0; i < n_pages_read; - ++i, page_off += size, block->frame += size, - block->page.offset++) { - bool decrypted = false; - dberr_t err = DB_SUCCESS; + ++i, block->frame += size, block->page.offset++) { byte* src = readptr + (i * size); - byte* dst = io_buffer + (i * size); - bool frame_changed = false; - ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); - const bool page_compressed - = page_type - == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_COMPRESSED; const ulint page_no = page_get_page_no(src); - if (!page_no && page_off) { + if (!page_no && block->page.offset) { const ulint* b = reinterpret_cast (src); const ulint* const e = b + size / sizeof *b; @@ -3463,11 +3430,46 @@ fil_iterate( continue; } - if (page_no != page_off / size) { - goto page_corrupted; + if (page_no != block->page.offset) { +page_corrupted: + ib_logf(IB_LOG_LEVEL_WARN, + "%s: Page %lu at offset " + UINT64PF " looks corrupted.", + callback.filename(), + ulong(offset / size), offset); + return DB_CORRUPTION; } - if (encrypted) { + bool decrypted = false; + dberr_t err = DB_SUCCESS; + byte* dst = io_buffer + (i * size); + bool frame_changed = false; + ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); + const bool page_compressed + = page_type + == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED + || page_type == FIL_PAGE_PAGE_COMPRESSED; + + if (!encrypted) { + } else if (!mach_read_from_4( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + src)) { +not_encrypted: + if (!page_compressed + && !block->page.zip.data) { + block->frame = src; + frame_changed = true; + } else { + ut_ad(dst != src); + memcpy(dst, src, size); + } + } else { + if (!fil_space_verify_crypt_checksum( + src, callback.get_zip_size(), + NULL, block->page.offset)) { + goto page_corrupted; + } + decrypted = fil_space_decrypt( iter.crypt_data, dst, iter.page_size, src, &err); @@ -3476,18 +3478,11 @@ fil_iterate( return err; } - if (decrypted) { - updated = true; - } else { - if (!page_compressed - && !block->page.zip.data) { - block->frame = src; - frame_changed = true; - } else { - ut_ad(dst != src); - memcpy(dst, src, size); - } + if (!decrypted) { + goto not_encrypted; } + + updated = true; } /* If the original page is page_compressed, we need @@ -3501,16 +3496,10 @@ fil_iterate( encrypted && !frame_changed ? dst : src, callback.get_zip_size(), NULL)) { -page_corrupted: - ib_logf(IB_LOG_LEVEL_WARN, - "%s: Page %lu at offset " - UINT64PF " looks corrupted.", - callback.filename(), - ulong(offset / size), offset); - return DB_CORRUPTION; + goto page_corrupted; } - if ((err = callback(page_off, block)) != DB_SUCCESS) { + if ((err = callback(block)) != DB_SUCCESS) { return err; } else if (!updated) { updated = buf_block_get_state(block) @@ -3583,19 +3572,15 @@ page_corrupted: buffer pool is not being used at all! */ if (decrypted && encrypted) { byte *dest = writeptr + (i * size); - ulint space = mach_read_from_4( - src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); - ib_uint64_t lsn = mach_read_from_8(src + FIL_PAGE_LSN); byte* tmp = fil_encrypt_buf( - iter.crypt_data, - space, - offset, - lsn, - src, - iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size, - dest); + iter.crypt_data, + callback.get_space_id(), + block->page.offset, + mach_read_from_8(src + FIL_PAGE_LSN), + src, + callback.get_zip_size(), + dest); if (tmp == src) { /* TODO: remove unnecessary memcpy's */ From 619c277a6cc11b62e9d84449c0ee8ab39abc54f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 7 Jun 2018 14:11:55 +0300 Subject: [PATCH 099/276] Mariabackup: Make some globals static --- extra/mariabackup/backup_copy.cc | 1 + extra/mariabackup/xtrabackup.cc | 4 ++-- extra/mariabackup/xtrabackup.h | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 2ea34406a6b..1405df9bcc8 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -1205,6 +1205,7 @@ cleanup: +static bool backup_files(const char *from, bool prep_mode) { diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 91c7b5faf0e..e14e65d1679 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -202,9 +202,9 @@ my_bool opt_ssl_verify_server_cert; /* === metadata of backup === */ #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints" char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/ -lsn_t metadata_from_lsn; +static lsn_t metadata_from_lsn; lsn_t metadata_to_lsn; -lsn_t metadata_last_lsn; +static lsn_t metadata_last_lsn; static ds_file_t* dst_log_file; diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index c77485cb4de..2db5bc29b51 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -56,9 +56,7 @@ extern xb_page_bitmap *changed_page_bitmap; extern char *xtrabackup_incremental; extern my_bool xtrabackup_incremental_force_scan; -extern lsn_t metadata_from_lsn; extern lsn_t metadata_to_lsn; -extern lsn_t metadata_last_lsn; extern xb_stream_fmt_t xtrabackup_stream_fmt; extern ibool xtrabackup_stream; From dc9c555415b1d37b713839b9f8143a053f402c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 7 Jun 2018 14:29:35 +0300 Subject: [PATCH 100/276] MDEV-16367 mariabackup: error: failed to copy enough redo log log_copying_thread(): Keep copying redo log until the end has been reached. (Previously, we would stop copying as soon as the first batch of xtrabackup_copy_logfile() returned.) log_copying: Remove. Use log_copying_running instead. copy_logfile: Remove. Log copying will now only be invoked from 2 places: from xtrabackup_backup_func() for the initial batch, and from log_copying_thread() until all of the log has been read. Use the global variable metadata_to_lsn for determining if the final part of the log is being copied. xtrabackup_copy_log(): Add diagnostic messages for terminating the copying. These messages should be dead code, because log_group_read_log_seg() should be checking for the same. xtrabackup_copy_logfile(): Correct the retrying logic. If anything was successfully read, process the portion that was read. On failure, let the caller close dst_log_file. io_watching_thread(): Stop throttling during the last phase of copying the log (metadata_to_lsn!=0). The final copying of the log will now be performed in log_copying_thread(). stop_backup_threads(): Clean up the message about stopping the log copying thread. xtrabackup_backup_low(): Read metadata_to_lsn from the latest checkpoint header page, even if it is the first page. Let the log_copying_thread take care of copying all of the redo log. --- extra/mariabackup/xtrabackup.cc | 113 ++++++++++++++++---------------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index e14e65d1679..65fddedc61a 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -172,7 +172,6 @@ typedef struct xb_filter_entry_struct xb_filter_entry_t; lsn_t checkpoint_lsn_start; lsn_t checkpoint_no_start; static lsn_t log_copy_scanned_lsn; -static bool log_copying; static bool log_copying_running; static bool io_watching_thread_running; @@ -2452,25 +2451,12 @@ skip: return(FALSE); } -/** How to copy a redo log segment in backup */ -enum copy_logfile { - /** Initial copying: copy at least one block */ - COPY_FIRST, - /** Tracking while copying data files */ - COPY_ONLINE, - /** Final copying: copy until the end of the log */ - COPY_LAST -}; - /** Copy redo log blocks to the data sink. -@param[in] copy how to copy the log -@param[in] start_lsn buffer start LSN -@param[in] end_lsn buffer end LSN -@return last scanned LSN (equals to last copied LSN if copy=COPY_LAST) +@param start_lsn buffer start LSN +@param end_lsn buffer end LSN +@return last scanned LSN @retval 0 on failure */ -static -lsn_t -xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) +static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn) { lsn_t scanned_lsn = start_lsn; const byte* log_block = log_sys->buf; @@ -2485,6 +2471,9 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) && scanned_checkpoint - checkpoint >= 0x80000000UL) { /* Garbage from a log buffer flush which was made before the most recent database recovery */ + msg("mariabackup: checkpoint wrap: " + LSN_PF ",%zx,%zx\n", + scanned_lsn, scanned_checkpoint, checkpoint); break; } @@ -2505,6 +2494,8 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) >= OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_TRL_SIZE || data_len <= LOG_BLOCK_HDR_SIZE) { /* We got a garbage block (abrupt end of the log). */ + msg("mariabackup: garbage block: " LSN_PF ",%zu\n", + scanned_lsn, data_len); break; } else { /* We got a partial block (abrupt end of the log). */ @@ -2524,7 +2515,7 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) log_sys->log.scanned_lsn = scanned_lsn; - end_lsn = copy == COPY_LAST + end_lsn = metadata_to_lsn ? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE) : scanned_lsn & ~lsn_t(OS_FILE_LOG_BLOCK_SIZE - 1); @@ -2544,10 +2535,8 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn) } /** Copy redo log until the current end of the log is reached -@param copy how to copy the log @return whether the operation failed */ -static bool -xtrabackup_copy_logfile(copy_logfile copy) +static bool xtrabackup_copy_logfile() { ut_a(dst_log_file != NULL); ut_ad(recv_sys != NULL); @@ -2560,33 +2549,29 @@ xtrabackup_copy_logfile(copy_logfile copy) start_lsn = ut_uint64_align_down(log_copy_scanned_lsn, OS_FILE_LOG_BLOCK_SIZE); - /* When copying the first or last part of the log, retry a few - times to ensure that all log up to the last checkpoint will be - read. */ do { end_lsn = start_lsn + RECV_SCAN_SIZE; xtrabackup_io_throttling(); log_mutex_enter(); - lsn_t lsn= start_lsn; - for(int retries= 0; retries < 100; retries++) { + for (int retries= 0; retries < 100; retries++) { if (log_group_read_log_seg(log_sys->buf, &log_sys->log, - &lsn, end_lsn)){ + &lsn, end_lsn) + || lsn != start_lsn) { break; } - msg("Retrying read of a redo log block"); + msg("Retrying read of log at LSN=" LSN_PF "\n", lsn); my_sleep(1000); } - start_lsn = xtrabackup_copy_log(copy, start_lsn, lsn); + start_lsn = (lsn == start_lsn) + ? 0 : xtrabackup_copy_log(start_lsn, lsn); log_mutex_exit(); if (!start_lsn) { - ds_close(dst_log_file); - dst_log_file = NULL; msg("mariabackup: Error: xtrabackup_copy_logfile()" " failed.\n"); return(true); @@ -2612,12 +2597,23 @@ static os_thread_ret_t log_copying_thread(void*) */ my_thread_init(); - do { + for (;;) { os_event_reset(log_copying_stop); os_event_wait_time_low(log_copying_stop, xtrabackup_log_copy_interval * 1000ULL, 0); - } while (log_copying && xtrabackup_copy_logfile(COPY_ONLINE)); + if (xtrabackup_copy_logfile()) { + break; + } + + log_mutex_enter(); + bool completed = metadata_to_lsn + && metadata_to_lsn < log_copy_scanned_lsn; + log_mutex_exit(); + if (completed) { + break; + } + } log_copying_running = false; my_thread_end(); @@ -2632,7 +2628,7 @@ static os_thread_ret_t io_watching_thread(void*) /* currently, for --backup only */ ut_a(xtrabackup_backup); - while (log_copying) { + while (log_copying_running && !metadata_to_lsn) { os_thread_sleep(1000000); /*1 sec*/ io_ticket = xtrabackup_throttle; os_event_set(wait_throttle); @@ -3662,16 +3658,16 @@ end: static void stop_backup_threads() { - log_copying = false; - - if (log_copying_stop) { + if (log_copying_stop && log_copying_running) { os_event_set(log_copying_stop); - msg("mariabackup: Stopping log copying thread.\n"); + fputs("mariabackup: Stopping log copying thread", stderr); + fflush(stderr); while (log_copying_running) { - msg("."); + putc('.', stderr); + fflush(stderr); os_thread_sleep(200000); /*0.2 sec*/ } - msg("\n"); + putc('\n', stderr); os_event_destroy(log_copying_stop); } @@ -3686,10 +3682,10 @@ static void stop_backup_threads() /** Implement the core of --backup @return whether the operation succeeded */ -static -bool -xtrabackup_backup_low() +static bool xtrabackup_backup_low() { + ut_ad(!metadata_to_lsn); + /* read the latest checkpoint lsn */ { ulint max_cp_field; @@ -3698,13 +3694,16 @@ xtrabackup_backup_low() if (recv_find_max_checkpoint(&max_cp_field) == DB_SUCCESS && log_sys->log.format != 0) { + if (max_cp_field == LOG_CHECKPOINT_1) { + log_group_header_read(&log_sys->log, + max_cp_field); + } metadata_to_lsn = mach_read_from_8( log_sys->checkpoint_buf + LOG_CHECKPOINT_LSN); msg("mariabackup: The latest check point" " (for incremental): '" LSN_PF "'\n", metadata_to_lsn); } else { - metadata_to_lsn = 0; msg("mariabackup: Error: recv_find_max_checkpoint() failed.\n"); } log_mutex_exit(); @@ -3712,11 +3711,7 @@ xtrabackup_backup_low() stop_backup_threads(); - if (!dst_log_file || xtrabackup_copy_logfile(COPY_LAST)) { - return false; - } - - if (ds_close(dst_log_file)) { + if (ds_close(dst_log_file) || !metadata_to_lsn) { dst_log_file = NULL; return false; } @@ -3795,6 +3790,7 @@ xtrabackup_backup_func() srv_read_only_mode = TRUE; srv_operation = SRV_OPERATION_BACKUP; + metadata_to_lsn = 0; if (xb_close_files) msg("mariabackup: warning: close-files specified. Use it " @@ -3805,7 +3801,12 @@ xtrabackup_backup_func() /* initialize components */ if(innodb_init_param()) { fail: + metadata_to_lsn = log_copying_running; stop_backup_threads(); + if (dst_log_file) { + ds_close(dst_log_file); + dst_log_file = NULL; + } if (fil_system) { innodb_shutdown(); } @@ -3814,7 +3815,6 @@ fail: xb_normalize_init_values(); - if (srv_file_flush_method_str == NULL) { /* These are the default options */ srv_file_flush_method = SRV_FSYNC; @@ -4058,9 +4058,7 @@ reread_log_header: goto log_write_fail; } - /* start flag */ - log_copying = TRUE; - + log_copying_running = true; /* start io throttle */ if(xtrabackup_throttle) { os_thread_id_t io_watching_thread_id; @@ -4078,6 +4076,8 @@ reread_log_header: if (err != DB_SUCCESS) { msg("mariabackup: error: xb_load_tablespaces() failed with" " error %s.\n", ut_strerr(err)); +fail_before_log_copying_thread_start: + log_copying_running = false; goto fail; } @@ -4085,11 +4085,10 @@ reread_log_header: log_copy_scanned_lsn = checkpoint_lsn_start; recv_sys->recovered_lsn = log_copy_scanned_lsn; - if (xtrabackup_copy_logfile(COPY_FIRST)) - goto fail; + if (xtrabackup_copy_logfile()) + goto fail_before_log_copying_thread_start; log_copying_stop = os_event_create(0); - log_copying_running = true; os_thread_create(log_copying_thread, NULL, &log_copying_thread_id); /* FLUSH CHANGED_PAGE_BITMAPS call */ From d9b159a2027c56c5c87385cfe1ae43b8c73a97b6 Mon Sep 17 00:00:00 2001 From: Chris Calender Date: Tue, 17 Apr 2018 15:00:15 -0400 Subject: [PATCH 101/276] MDEV-15789 - mysqlslap use incorrect table def The bug arises when one uses --auto-generate-sql-guid-primary (and --auto-generate-sql-secondary-indexes) with mysqlslap and also have sql_mode=STRICT_TRANS_TABLE. When using this option, mysqlslap should create a column with varchar(36), but it appears to create it as a varchar(32) only. Then if one has sql_mode=STRICT_TRANS_TABLES, it throws an error, like: mysqlslap: Cannot run query INSERT INTO t1 VALUES (...) ERROR : Data too long for column 'id' at row 1 Upstream bug report: BUG#80329. --- client/mysqlslap.c | 4 ++-- mysql-test/r/mysqlslap.result | 3 +++ mysql-test/t/mysqlslap.test | 8 ++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index a9746916411..ba7b8cea6a9 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -840,7 +840,7 @@ build_table_string(void) if (auto_generate_sql_guid_primary) { - dynstr_append(&table_string, "id varchar(32) primary key"); + dynstr_append(&table_string, "id varchar(36) primary key"); if (num_int_cols || num_char_cols || auto_generate_sql_guid_primary) dynstr_append(&table_string, ","); @@ -855,7 +855,7 @@ build_table_string(void) if (count) /* Except for the first pass we add a comma */ dynstr_append(&table_string, ","); - if (snprintf(buf, HUGE_STRING_LENGTH, "id%d varchar(32) unique key", count) + if (snprintf(buf, HUGE_STRING_LENGTH, "id%d varchar(36) unique key", count) > HUGE_STRING_LENGTH) { fprintf(stderr, "Memory Allocation error in create table\n"); diff --git a/mysql-test/r/mysqlslap.result b/mysql-test/r/mysqlslap.result index af78677647f..50bd53f4263 100644 --- a/mysql-test/r/mysqlslap.result +++ b/mysql-test/r/mysqlslap.result @@ -255,3 +255,6 @@ Benchmark # MDEV-4684 - Enhancement request: --init-command support for mysqlslap # DROP TABLE t1; +# +# Bug MDEV-15789 (Upstream: #80329): MYSQLSLAP OPTIONS --AUTO-GENERATE-SQL-GUID-PRIMARY and --AUTO-GENERATE-SQL-SECONDARY-INDEXES DONT WORK +# diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index c49c4ab3d7d..81115d59d09 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -80,3 +80,11 @@ DROP DATABASE bug58090; --exec $MYSQL_SLAP --create-schema=test --init-command="CREATE TABLE t1(a INT)" --silent --concurrency=1 --iterations=1 DROP TABLE t1; + +--echo # +--echo # Bug MDEV-15789 (Upstream: #80329): MYSQLSLAP OPTIONS --AUTO-GENERATE-SQL-GUID-PRIMARY and --AUTO-GENERATE-SQL-SECONDARY-INDEXES DONT WORK +--echo # + +--exec $MYSQL_SLAP --concurrency=1 --silent --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --create-schema=slap + +--exec $MYSQL_SLAP --concurrency=1 --silent --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-secondary-indexes=1 --create-schema=slap From ea70586282d902fe067cee11a5aeb7086ed375f8 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 7 Jun 2018 15:12:26 +0100 Subject: [PATCH 102/276] MDEV-16300 : remove rocksdb checkpoint created by mariabackup. Add variable rocksdb_remove_mariabackup_checkpoint. If set, it will remove $rocksdb_datadir/mariabackup-checkpoint directory. The variable is to be used by exclusively by mariabackup, to remove temporary checkpoints. --- storage/rocksdb/ha_rocksdb.cc | 69 ++++++++++++++++++- .../mysql-test/rocksdb/r/rocksdb.result | 1 + ...remove_mariabackup_checkpoint_basic.result | 4 ++ ...b_remove_mariabackup_checkpoint_basic.test | 5 ++ 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_remove_mariabackup_checkpoint_basic.result create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_remove_mariabackup_checkpoint_basic.test diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 96bc7dad3e6..7cddfea33a7 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -479,6 +479,7 @@ static uint32_t rocksdb_access_hint_on_compaction_start; static char *rocksdb_compact_cf_name; static char *rocksdb_checkpoint_name; static my_bool rocksdb_signal_drop_index_thread; +static my_bool rocksdb_signal_remove_mariabackup_checkpoint; static my_bool rocksdb_strict_collation_check = 1; static my_bool rocksdb_ignore_unknown_options = 1; static my_bool rocksdb_enable_2pc = 0; @@ -515,6 +516,67 @@ std::atomic rocksdb_row_lock_wait_timeouts(0); std::atomic rocksdb_snapshot_conflict_errors(0); std::atomic rocksdb_wal_group_syncs(0); + + +/* + Remove directory with files in it. + Used to remove checkpoint created by mariabackup. +*/ +#ifdef _WIN32 +#include /* unlink*/ +#ifndef F_OK +#define F_OK 0 +#endif +#endif + +static int rmdir_force(const char *dir) { + if (access(dir, F_OK)) + return true; + + char path[FN_REFLEN]; + char sep[] = {FN_LIBCHAR, 0}; + int err = 0; + + MY_DIR *dir_info = my_dir(dir, MYF(MY_DONT_SORT | MY_WANT_STAT)); + if (!dir_info) + return 1; + + for (uint i = 0; i < dir_info->number_of_files; i++) { + FILEINFO *file = dir_info->dir_entry + i; + + strxnmov(path, sizeof(path), dir, sep, file->name, NULL); + + err = my_delete(path, 0); + + if (err) { + break; + } + } + + my_dirend(dir_info); + + if (!err) + err = rmdir(dir); + + return (err == 0) ? HA_EXIT_SUCCESS : HA_EXIT_FAILURE; +} + + +static void rocksdb_remove_mariabackup_checkpoint( + my_core::THD *const, + struct st_mysql_sys_var *const , + void *const var_ptr, const void *const) { + std::string mariabackup_checkpoint_dir(rocksdb_datadir); + + mariabackup_checkpoint_dir.append("/mariabackup-checkpoint"); + + if (unlink(mariabackup_checkpoint_dir.c_str()) == 0) + return; + + rmdir_force(mariabackup_checkpoint_dir.c_str()); +} + + static std::unique_ptr rdb_init_rocksdb_db_options(void) { auto o = std::unique_ptr(new rocksdb::DBOptions()); @@ -1312,6 +1374,11 @@ static MYSQL_SYSVAR_STR(create_checkpoint, rocksdb_checkpoint_name, rocksdb_create_checkpoint, rocksdb_create_checkpoint_stub, ""); +static MYSQL_SYSVAR_BOOL(remove_mariabackup_checkpoint, + rocksdb_signal_remove_mariabackup_checkpoint, + PLUGIN_VAR_RQCMDARG, "Remove mariabackup checkpoint", + nullptr, rocksdb_remove_mariabackup_checkpoint, FALSE); + static MYSQL_SYSVAR_BOOL(signal_drop_index_thread, rocksdb_signal_drop_index_thread, PLUGIN_VAR_RQCMDARG, "Wake up drop index thread", nullptr, @@ -1675,7 +1742,7 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = { MYSQL_SYSVAR(datadir), MYSQL_SYSVAR(supported_compression_types), MYSQL_SYSVAR(create_checkpoint), - + MYSQL_SYSVAR(remove_mariabackup_checkpoint), MYSQL_SYSVAR(checksums_pct), MYSQL_SYSVAR(store_row_debug_checksums), MYSQL_SYSVAR(verify_row_debug_checksums), diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result index 6138dac92e5..dc13bb375b8 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result @@ -959,6 +959,7 @@ rocksdb_print_snapshot_conflict_queries OFF rocksdb_rate_limiter_bytes_per_sec 0 rocksdb_read_free_rpl_tables rocksdb_records_in_range 50 +rocksdb_remove_mariabackup_checkpoint OFF rocksdb_reset_stats OFF rocksdb_seconds_between_stat_computes 3600 rocksdb_signal_drop_index_thread OFF diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_remove_mariabackup_checkpoint_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_remove_mariabackup_checkpoint_basic.result new file mode 100644 index 00000000000..01145cd2ab8 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_remove_mariabackup_checkpoint_basic.result @@ -0,0 +1,4 @@ +SET GLOBAL rocksdb_create_checkpoint=CONCAT(@@rocksdb_datadir,'/mariabackup-checkpoint'); +SET GLOBAL rocksdb_remove_mariabackup_checkpoint=ON; +SET GLOBAL rocksdb_create_checkpoint=CONCAT(@@rocksdb_datadir,'/mariabackup-checkpoint'); +SET GLOBAL rocksdb_remove_mariabackup_checkpoint=ON; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_remove_mariabackup_checkpoint_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_remove_mariabackup_checkpoint_basic.test new file mode 100644 index 00000000000..30f38283ba4 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_remove_mariabackup_checkpoint_basic.test @@ -0,0 +1,5 @@ +# Simulate creating and removing mariabackup checkpoint twice +SET GLOBAL rocksdb_create_checkpoint=CONCAT(@@rocksdb_datadir,'/mariabackup-checkpoint'); +SET GLOBAL rocksdb_remove_mariabackup_checkpoint=ON; +SET GLOBAL rocksdb_create_checkpoint=CONCAT(@@rocksdb_datadir,'/mariabackup-checkpoint'); +SET GLOBAL rocksdb_remove_mariabackup_checkpoint=ON; From aba2d7301f30daab9f54e92d21a4a130dda11dc4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 7 Jun 2018 15:13:54 +0100 Subject: [PATCH 103/276] MDEV-13122 Backup myrocksdb with mariabackup. --- extra/mariabackup/backup_copy.cc | 272 +++++++++++++++++- extra/mariabackup/xtrabackup.cc | 17 +- extra/mariabackup/xtrabackup.h | 3 + .../mariabackup/include/have_rocksdb.inc | 4 + mysql-test/suite/mariabackup/xb_rocksdb.opt | 1 + .../suite/mariabackup/xb_rocksdb.result | 22 ++ mysql-test/suite/mariabackup/xb_rocksdb.test | 52 ++++ .../suite/mariabackup/xb_rocksdb_datadir.opt | 1 + .../mariabackup/xb_rocksdb_datadir.result | 9 + .../suite/mariabackup/xb_rocksdb_datadir.test | 34 +++ .../mariabackup/xb_rocksdb_datadir_debug.opt | 1 + .../xb_rocksdb_datadir_debug.result | 9 + .../mariabackup/xb_rocksdb_datadir_debug.test | 13 + 13 files changed, 435 insertions(+), 3 deletions(-) create mode 100644 mysql-test/suite/mariabackup/include/have_rocksdb.inc create mode 100644 mysql-test/suite/mariabackup/xb_rocksdb.opt create mode 100644 mysql-test/suite/mariabackup/xb_rocksdb.result create mode 100644 mysql-test/suite/mariabackup/xb_rocksdb.test create mode 100644 mysql-test/suite/mariabackup/xb_rocksdb_datadir.opt create mode 100644 mysql-test/suite/mariabackup/xb_rocksdb_datadir.result create mode 100644 mysql-test/suite/mariabackup/xb_rocksdb_datadir.test create mode 100644 mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.opt create mode 100644 mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.result create mode 100644 mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.test diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 1405df9bcc8..16a4042d66e 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -59,6 +59,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include "xb0xb.h" +#define ROCKSDB_BACKUP_DIR "#rocksdb" /* list of files to sync for --rsync mode */ static std::set rsync_list; @@ -68,6 +69,21 @@ static std::map tablespace_locations; /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */ bool binlog_locked; +static void rocksdb_create_checkpoint(); +static bool has_rocksdb_plugin(); +static void copy_or_move_dir(const char *from, const char *to, bool copy, bool allow_hardlinks); +static void rocksdb_backup_checkpoint(); +static void rocksdb_copy_back(); + +static bool is_abs_path(const char *path) +{ +#ifdef _WIN32 + return path[0] && path[1] == ':' && (path[2] == '/' || path[2] == '\\'); +#else + return path[0] == '/'; +#endif +} + /************************************************************************ Struct represents file or directory. */ struct datadir_node_t { @@ -1140,7 +1156,8 @@ bool copy_or_move_file(const char *src_file_path, const char *dst_file_path, const char *dst_dir, - uint thread_n) + uint thread_n, + bool copy = xtrabackup_copy_back) { ds_ctxt_t *datasink = ds_data; /* copy to datadir by default */ char filedir[FN_REFLEN]; @@ -1188,7 +1205,7 @@ copy_or_move_file(const char *src_file_path, free(link_filepath); } - ret = (xtrabackup_copy_back ? + ret = (copy ? copy_file(datasink, src_file_path, dst_file_path, thread_n) : move_file(datasink, src_file_path, dst_file_path, dst_dir, thread_n)); @@ -1373,6 +1390,10 @@ bool backup_start() return false; } + if (has_rocksdb_plugin()) { + rocksdb_create_checkpoint(); + } + // There is no need to stop slave thread before coping non-Innodb data when // --no-lock option is used because --no-lock option requires that no DDL or // DML to non-transaction tables can occur. @@ -1458,6 +1479,10 @@ bool backup_finish() } } + if (has_rocksdb_plugin()) { + rocksdb_backup_checkpoint(); + } + msg_ts("Backup created in directory '%s'\n", xtrabackup_target_dir); if (mysql_binlog_position != NULL) { msg("MySQL binlog position: %s\n", mysql_binlog_position); @@ -1773,6 +1798,16 @@ copy_back() int i_tmp; bool is_ibdata_file; + if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/") +#ifdef _WIN32 + || strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\") +#endif + ) + { + // copied at later step + continue; + } + /* create empty directories */ if (node.is_empty_dir) { char path[FN_REFLEN]; @@ -1857,6 +1892,8 @@ copy_back() } } + rocksdb_copy_back(); + cleanup: if (it != NULL) { datadir_iter_free(it); @@ -2033,3 +2070,234 @@ static bool backup_files_from_datadir(const char *dir_path) os_file_closedir(dir); return ret; } + + +static int rocksdb_remove_checkpoint_directory() +{ + xb_mysql_query(mysql_connection, "set global rocksdb_remove_mariabackup_checkpoint=ON", false); + return 0; +} + +static bool has_rocksdb_plugin() +{ + static bool first_time = true; + static bool has_plugin= false; + if (!first_time || !xb_backup_rocksdb) + return has_plugin; + + const char *query = "SELECT COUNT(*) FROM information_schema.plugins WHERE plugin_name='rocksdb'"; + MYSQL_RES* result = xb_mysql_query(mysql_connection, query, true); + MYSQL_ROW row = mysql_fetch_row(result); + if (row) + has_plugin = !strcmp(row[0], "1"); + mysql_free_result(result); + first_time = false; + return has_plugin; +} + +static char *trim_trailing_dir_sep(char *path) +{ + size_t path_len = strlen(path); + while (path_len) + { + char c = path[path_len - 1]; + if (c == '/' IF_WIN(|| c == '\\', )) + path_len--; + else + break; + } + path[path_len] = 0; + return path; +} + +/* +Create a file hardlink. +@return true on success, false on error. +*/ +static bool make_hardlink(const char *from_path, const char *to_path) +{ + DBUG_EXECUTE_IF("no_hardlinks", return false;); + char to_path_full[FN_REFLEN]; + if (!is_abs_path(to_path)) + { + fn_format(to_path_full, to_path, ds_data->root, "", MYF(MY_RELATIVE_PATH)); + } + else + { + strncpy(to_path_full, to_path, sizeof(to_path_full)); + } +#ifdef _WIN32 + return CreateHardLink(to_path_full, from_path, NULL); +#else + return !link(from_path, to_path_full); +#endif +} + +/* + Copies or moves a directory (non-recursively so far). + Helper function used to backup rocksdb checkpoint, or copy-back the + rocksdb files. + + Has optimization that allows to use hardlinks when possible + (source and destination are directories on the same device) +*/ +static void copy_or_move_dir(const char *from, const char *to, bool do_copy, bool allow_hardlinks) +{ + datadir_node_t node; + datadir_node_init(&node); + datadir_iter_t *it = datadir_iter_new(from, false); + + while (datadir_iter_next(it, &node)) + { + char to_path[FN_REFLEN]; + const char *from_path = node.filepath; + snprintf(to_path, sizeof(to_path), "%s/%s", to, base_name(from_path)); + bool rc = false; + if (do_copy && allow_hardlinks) + { + rc = make_hardlink(from_path, to_path); + if (rc) + { + msg_ts("[%02u] Creating hardlink from %s to %s\n", + 1, from_path, to_path); + } + else + { + allow_hardlinks = false; + } + } + + if (!rc) + { + rc = (do_copy ? + copy_file(ds_data, from_path, to_path, 1) : + move_file(ds_data, from_path, node.filepath_rel, + to, 1)); + } + if (!rc) + exit(EXIT_FAILURE); + } + datadir_iter_free(it); + datadir_node_free(&node); + +} + +/* + Obtain user level lock , to protect the checkpoint directory of the server + from being user/overwritten by different backup processes, if backups are + running in parallel. + + This lock will be acquired before rocksdb checkpoint is created, held + while all files from it are being copied to their final backup destination, + and finally released after the checkpoint is removed. +*/ +static void rocksdb_lock_checkpoint() +{ + msg_ts("Obtaining rocksdb checkpoint lock.\n"); + MYSQL_RES *res = + xb_mysql_query(mysql_connection, "SELECT GET_LOCK('mariabackup_rocksdb_checkpoint',3600)", true, true); + + MYSQL_ROW r = mysql_fetch_row(res); + if (r && r[0] && strcmp(r[0], "1")) + { + msg_ts("Could not obtain rocksdb checkpont lock\n"); + exit(EXIT_FAILURE); + } +} + +static void rocksdb_unlock_checkpoint() +{ + xb_mysql_query(mysql_connection, + "SELECT RELEASE_LOCK('mariabackup_rocksdb_checkpoint')", false, true); +} + + +/* + Create temporary checkpoint in $rocksdb_datadir/mariabackup-checkpoint + directory. + A (user-level) lock named 'mariabackup_rocksdb_checkpoint' will also be + acquired be this function. +*/ +#define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint" +static char rocksdb_checkpoint_dir[FN_REFLEN]; + +static void rocksdb_create_checkpoint() +{ + MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true); + MYSQL_ROW row = mysql_fetch_row(result); + + DBUG_ASSERT(row && row[0] && row[1]); + + char *rocksdbdir = row[0]; + char *datadir = row[1]; + + if (is_abs_path(rocksdbdir)) + { + snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir), + "%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(rocksdbdir)); + } + else + { + snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir), + "%s/%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(datadir), + trim_dotslash(rocksdbdir)); + } + mysql_free_result(result); + +#ifdef _WIN32 + for (char *p = rocksdb_checkpoint_dir; *p; p++) + if (*p == '\\') *p = '/'; +#endif + + rocksdb_lock_checkpoint(); + + if (!access(rocksdb_checkpoint_dir, 0)) + { + msg_ts("Removing rocksdb checkpoint from previous backup attempt.\n"); + rocksdb_remove_checkpoint_directory(); + } + + char query[FN_REFLEN + 32]; + snprintf(query, sizeof(query), "SET GLOBAL rocksdb_create_checkpoint='%s'", rocksdb_checkpoint_dir); + xb_mysql_query(mysql_connection, query, false, true); +} + +/* + Copy files from rocksdb temporary checkpoint to final destination. + remove temp.checkpoint directory (in server's datadir) + and release user level lock acquired inside rocksdb_create_checkpoint(). +*/ +static void rocksdb_backup_checkpoint() +{ + msg_ts("Backing up rocksdb files.\n"); + char rocksdb_backup_dir[FN_REFLEN]; + snprintf(rocksdb_backup_dir, sizeof(rocksdb_backup_dir), "%s/" ROCKSDB_BACKUP_DIR , xtrabackup_target_dir); + bool backup_to_directory = xtrabackup_backup && xtrabackup_stream_fmt == XB_STREAM_FMT_NONE; + if (backup_to_directory) + { + if (my_mkdir(rocksdb_backup_dir, 0777, MYF(0))){ + msg_ts("Can't create rocksdb backup directory %s\n", rocksdb_backup_dir); + exit(EXIT_FAILURE); + } + } + copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory); + rocksdb_remove_checkpoint_directory(); + rocksdb_unlock_checkpoint(); +} + +/* + Copies #rocksdb directory to the $rockdb_data_dir, on copy-back +*/ +static void rocksdb_copy_back() { + if (access(ROCKSDB_BACKUP_DIR, 0)) + return; + char rocksdb_home_dir[FN_REFLEN]; + if (xb_rocksdb_datadir && is_abs_path(xb_rocksdb_datadir)) { + strncpy(rocksdb_home_dir, xb_rocksdb_datadir, sizeof(rocksdb_home_dir)); + } else { + snprintf(rocksdb_home_dir, sizeof(rocksdb_home_dir), "%s/%s", mysql_data_home, + xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR); + } + mkdirp(rocksdb_home_dir, 0777, MYF(0)); + copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back); +} diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 65fddedc61a..4d88778f020 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -146,6 +146,8 @@ char *xtrabackup_tmpdir; char *xtrabackup_tables; char *xtrabackup_tables_file; char *xtrabackup_tables_exclude; +char *xb_rocksdb_datadir; +my_bool xb_backup_rocksdb = 1; typedef std::list regex_list_t; static regex_list_t regex_include_list; @@ -687,7 +689,9 @@ enum options_xtrabackup OPT_XTRA_TABLES_EXCLUDE, OPT_XTRA_DATABASES_EXCLUDE, OPT_PROTOCOL, - OPT_LOCK_DDL_PER_TABLE + OPT_LOCK_DDL_PER_TABLE, + OPT_ROCKSDB_DATADIR, + OPT_BACKUP_ROCKSDB }; struct my_option xb_client_options[] = @@ -1234,6 +1238,17 @@ struct my_option xb_server_options[] = (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory." + "This option is only used with --copy-back or --move-back option", + &xb_rocksdb_datadir, &xb_rocksdb_datadir, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, + + { "backup-rocksdb", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed." + "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data", + &xb_backup_rocksdb, &xb_backup_rocksdb, + 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 }, + + { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index 2db5bc29b51..857c253f803 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -44,6 +44,9 @@ extern char *xtrabackup_incremental_basedir; extern char *innobase_data_home_dir; extern char *innobase_buffer_pool_filename; extern char *xb_plugin_dir; +extern char *xb_rocksdb_datadir; +extern my_bool xb_backup_rocksdb; + extern uint opt_protocol; extern ds_ctxt_t *ds_meta; extern ds_ctxt_t *ds_data; diff --git a/mysql-test/suite/mariabackup/include/have_rocksdb.inc b/mysql-test/suite/mariabackup/include/have_rocksdb.inc new file mode 100644 index 00000000000..d59f76f6cf3 --- /dev/null +++ b/mysql-test/suite/mariabackup/include/have_rocksdb.inc @@ -0,0 +1,4 @@ +if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'rocksdb'`) +{ + --skip Requires rocksdb +} \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/xb_rocksdb.opt b/mysql-test/suite/mariabackup/xb_rocksdb.opt new file mode 100644 index 00000000000..e582413e5b5 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_rocksdb.opt @@ -0,0 +1 @@ +--plugin-load=$HA_ROCKSDB_SO \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/xb_rocksdb.result b/mysql-test/suite/mariabackup/xb_rocksdb.result new file mode 100644 index 00000000000..84476eeaba0 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_rocksdb.result @@ -0,0 +1,22 @@ +CREATE TABLE t(i INT) ENGINE ROCKSDB; +INSERT INTO t VALUES(1); +# xtrabackup backup +INSERT INTO t VALUES(2); +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +# xbstream extract +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/xb_rocksdb.test b/mysql-test/suite/mariabackup/xb_rocksdb.test new file mode 100644 index 00000000000..e41f3b2bf7e --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_rocksdb.test @@ -0,0 +1,52 @@ +--source include/have_rocksdb.inc + +CREATE TABLE t(i INT) ENGINE ROCKSDB; +INSERT INTO t VALUES(1); +echo # xtrabackup backup; +# we'll backup to both directory and to stream to restore that later + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +let $stream=$MYSQLTEST_VARDIR/tmp/backup.xb; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir $backup_extra_param; +--enable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --stream=xbstream > $stream 2>$MYSQLTEST_VARDIR/tmp/backup_stream.log; + +INSERT INTO t VALUES(2); + + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * FROM t; + +rmdir $targetdir; +mkdir $targetdir; + + +echo # xbstream extract; + +exec $XBSTREAM -x -C $targetdir < $stream; + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; + +let $_datadir= `SELECT @@datadir`; +echo # shutdown server; +--source include/shutdown_mysqld.inc +echo # remove datadir; +rmdir $_datadir; +echo # xtrabackup move back; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --move-back --datadir=$_datadir --target-dir=$targetdir $copy_back_extra_param; +echo # restart server; +--source include/start_mysqld.inc + +--enable_result_log +SELECT * FROM t; + +DROP TABLE t; +rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/xb_rocksdb_datadir.opt b/mysql-test/suite/mariabackup/xb_rocksdb_datadir.opt new file mode 100644 index 00000000000..0f069018e15 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_rocksdb_datadir.opt @@ -0,0 +1 @@ +--plugin-load=$HA_ROCKSDB_SO --loose-rocksdb-datadir=$MYSQLTEST_VARDIR/tmp/rocksdb_datadir \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/xb_rocksdb_datadir.result b/mysql-test/suite/mariabackup/xb_rocksdb_datadir.result new file mode 100644 index 00000000000..9227198cbec --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_rocksdb_datadir.result @@ -0,0 +1,9 @@ +CREATE TABLE t(i INT) ENGINE ROCKSDB; +INSERT INTO t VALUES(1); +# xtrabackup backup +INSERT INTO t VALUES(2); +# xtrabackup prepare +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/xb_rocksdb_datadir.test b/mysql-test/suite/mariabackup/xb_rocksdb_datadir.test new file mode 100644 index 00000000000..c2e90d9075b --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_rocksdb_datadir.test @@ -0,0 +1,34 @@ +if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'rocksdb'`) +{ + --skip Requires rocksdb +} + + +CREATE TABLE t(i INT) ENGINE ROCKSDB; +INSERT INTO t VALUES(1); +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log + +INSERT INTO t VALUES(2); + + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +let $_datadir= `SELECT @@datadir`; +let $_rocksdb_datadir=`SELECT @@rocksdb_datadir`; +--source include/shutdown_mysqld.inc +rmdir $_datadir; +rmdir $_rocksdb_datadir; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --move-back --target-dir=$targetdir --datadir=$_datadir --rocksdb_datadir=$_rocksdb_datadir; +--enable_result_log +--source include/start_mysqld.inc + + +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; + diff --git a/mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.opt b/mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.opt new file mode 100644 index 00000000000..0f069018e15 --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.opt @@ -0,0 +1 @@ +--plugin-load=$HA_ROCKSDB_SO --loose-rocksdb-datadir=$MYSQLTEST_VARDIR/tmp/rocksdb_datadir \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.result b/mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.result new file mode 100644 index 00000000000..9227198cbec --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.result @@ -0,0 +1,9 @@ +CREATE TABLE t(i INT) ENGINE ROCKSDB; +INSERT INTO t VALUES(1); +# xtrabackup backup +INSERT INTO t VALUES(2); +# xtrabackup prepare +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.test b/mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.test new file mode 100644 index 00000000000..a71c63b98cc --- /dev/null +++ b/mysql-test/suite/mariabackup/xb_rocksdb_datadir_debug.test @@ -0,0 +1,13 @@ +--source include/have_debug.inc +--source include/have_rocksdb.inc + +# Check how rocksdb backup works without hardlinks +let $backup_extra_param='--dbug=+d,no_hardlinks'; +let $copy_back_extra_param='--dbug=+d,no_hardlinks'; + +# Pretend that previous backup crashes, and left checkpoint directory +let $rocksdb_datadir= `SELECT @@rocksdb_datadir`; +mkdir $rocksdb_datadir/mariadb-checkpoint; + +--source xb_rocksdb_datadir.test + From 4461b0f9b301233b245853b9b71b3770c24c5aac Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 8 Jun 2018 14:30:04 +0100 Subject: [PATCH 104/276] MDEV-16424 replace cmake/bison.cmake with cmake's builtin FindBison module --- cmake/bison.cmake | 106 --------------------------------------- cmake/make_dist.cmake.in | 17 ++++--- libmysqld/CMakeLists.txt | 4 +- sql/CMakeLists.txt | 93 +++++++++++++++++----------------- sql/gen_lex_token.cc | 2 +- sql/sql_digest.cc | 2 +- sql/sql_lex.h | 6 +-- 7 files changed, 63 insertions(+), 167 deletions(-) delete mode 100644 cmake/bison.cmake diff --git a/cmake/bison.cmake b/cmake/bison.cmake deleted file mode 100644 index d825e407b22..00000000000 --- a/cmake/bison.cmake +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) 2009 Sun Microsystems, Inc. -# Use is subject to license terms. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -IF(CMAKE_SYSTEM_NAME MATCHES "SunOS") - # On Solaris, /opt/csw often contains a newer bison - IF(NOT BISON_EXECUTABLE AND EXISTS /opt/csw/bin/bison) - SET(BISON_EXECUTABLE /opt/csw/bin/bison) - ENDIF() -ENDIF() -IF(WIN32) - SET(BISON_PATH_HINTS - HINTS - C:/gnuwin32/bin - C:/cygwin64/bin - C:/cygwin/bin) -ENDIF() -FIND_PROGRAM(BISON_EXECUTABLE bison - ${BISON_PATH_HINTS} - DOC "path to the bison executable") -MARK_AS_ADVANCED(BISON_EXECUTABLE "") -IF(NOT BISON_EXECUTABLE) - MESSAGE("Warning: Bison executable not found in PATH") -ELSEIF(BISON_EXECUTABLE AND NOT BISON_USABLE) - # Check version as well - EXEC_PROGRAM(${BISON_EXECUTABLE} ARGS --version OUTPUT_VARIABLE BISON_VERSION_STR) - # Get first line in case it's multiline - STRING(REGEX REPLACE "([^\n]+).*" "\\1" FIRST_LINE "${BISON_VERSION_STR}") - # get version information - STRING(REGEX REPLACE ".* ([0-9]+)\\.([0-9]+)" "\\1" BISON_VERSION_MAJOR "${FIRST_LINE}") - STRING(REGEX REPLACE ".* ([0-9]+)\\.([0-9]+)" "\\2" BISON_VERSION_MINOR "${FIRST_LINE}") - IF (BISON_VERSION_MAJOR LESS 2) - MESSAGE("Warning: bison version is old. please update to version 2") - ELSE() - SET(BISON_USABLE 1 CACHE INTERNAL "Bison version 2 or higher") - ENDIF() -ENDIF() - -# Use bison to generate C++ and header file -MACRO (RUN_BISON input_yy output_cc output_h name_prefix) - IF(BISON_TOO_OLD) - IF(EXISTS ${output_cc} AND EXISTS ${output_h}) - SET(BISON_USABLE FALSE) - ENDIF() - ENDIF() - IF(BISON_USABLE) - # Workaround for VS regenerating output even - # when outputs are up-to-date. At least, fix output timestamp - # after build so that files that depend on generated header are - # not rebuilt. - IF(CMAKE_GENERATOR MATCHES "Visual Studio") - FIND_PROGRAM(TOUCH_EXECUTABLE touch DOC "Path to touch executable" - PATHS "C:/Program Files/Git/usr/bin" - "C:/Program Files (x86)/Git/usr/bin") - IF(TOUCH_EXECUTABLE) - SET(VS_FIX_OUTPUT_TIMESTAMPS - COMMAND ${TOUCH_EXECUTABLE} -r ${input_yy} ${output_cc} - COMMAND ${TOUCH_EXECUTABLE} -r ${input_yy} ${output_h}) - ENDIF() - ENDIF() - - ADD_CUSTOM_COMMAND( - OUTPUT ${output_cc} - ${output_h} - COMMAND ${BISON_EXECUTABLE} -y -p ${name_prefix} - --output=${output_cc} - --defines=${output_h} - ${input_yy} - ${VS_FIX_OUTPUT_TIMESTAMPS} - DEPENDS ${input_yy} - ) - ELSE() - # Bison is missing or not usable, e.g too old - IF(EXISTS ${output_cc} AND EXISTS ${output_h}) - IF(${input_yy} IS_NEWER_THAN ${output_cc} OR ${input_yy} IS_NEWER_THAN ${output_h}) - # Possibly timestamps are messed up in source distribution. - MESSAGE("Warning: no usable bison found, ${input_yy} will not be rebuilt.") - ENDIF() - ELSE() - # Output files are missing, bail out. - SET(ERRMSG - "Bison (GNU parser generator) is required to build MySQL." - "Please install bison." - ) - IF(WIN32) - SET(ERRMSG ${ERRMSG} - "You can download bison from http://gnuwin32.sourceforge.net/packages/bison.htm " - "Choose 'Complete package, except sources' installation. We recommend to " - "install bison into a directory without spaces, e.g C:\\GnuWin32.") - ENDIF() - MESSAGE(FATAL_ERROR ${ERRMSG}) - ENDIF() - ENDIF() -ENDMACRO() diff --git a/cmake/make_dist.cmake.in b/cmake/make_dist.cmake.in index 885cfa2a706..43498f1f888 100644 --- a/cmake/make_dist.cmake.in +++ b/cmake/make_dist.cmake.in @@ -67,8 +67,8 @@ IF(NOT GIT_EXECUTABLE) # Save bison output first. CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.cc ${CMAKE_BINARY_DIR}/sql_yacc.cc COPYONLY) - CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.h - ${CMAKE_BINARY_DIR}/sql_yacc.h COPYONLY) + CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.hh + ${CMAKE_BINARY_DIR}/sql_yacc.hh COPYONLY) IF(CMAKE_GENERATOR MATCHES "Makefiles") # make clean @@ -81,10 +81,10 @@ IF(NOT GIT_EXECUTABLE) # Restore bison output CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql_yacc.cc ${CMAKE_BINARY_DIR}/sql/sql_yacc.cc COPYONLY) - CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql_yacc.h + CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql_yacc.hh ${CMAKE_BINARY_DIR}/sql/sql_yacc.h COPYONLY) FILE(REMOVE ${CMAKE_BINARY_DIR}/sql_yacc.cc) - FILE(REMOVE ${CMAKE_BINARY_DIR}/sql_yacc.h) + FILE(REMOVE ${CMAKE_BINARY_DIR}/sql_yacc.hh) ENDIF() EXECUTE_PROCESS( @@ -102,10 +102,15 @@ IF(NOT GIT_EXECUTABLE) ENDIF() # Copy bison output -CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.h - ${PACKAGE_DIR}/sql/sql_yacc.h COPYONLY) +CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.hh + ${PACKAGE_DIR}/sql/sql_yacc.hh COPYONLY) CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc.cc ${PACKAGE_DIR}/sql/sql_yacc.cc COPYONLY) +# Copy bison output +CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.hh + ${PACKAGE_DIR}/sql/sql_yacc_ora.hh COPYONLY) +CONFIGURE_FILE(${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.cc + ${PACKAGE_DIR}/sql/sql_yacc_orac.cc COPYONLY) # Add documentation, if user has specified where to find them IF(MYSQL_DOCS_LOCATION) diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index f1fcfac58f5..d7450eb3e87 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -28,9 +28,9 @@ ${SSL_INTERNAL_INCLUDE_DIRS} ) SET(GEN_SOURCES -${CMAKE_BINARY_DIR}/sql/sql_yacc.h +${CMAKE_BINARY_DIR}/sql/sql_yacc.hh ${CMAKE_BINARY_DIR}/sql/sql_yacc.cc -${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.h +${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.hh ${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.cc ${CMAKE_BINARY_DIR}/sql/lex_hash.h ) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 7253f7ba447..0037ff23153 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -45,23 +45,10 @@ ${CMAKE_BINARY_DIR}/sql ${WSREP_INCLUDES} ) -SET(GEN_SOURCES -${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h -${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc -${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.h -${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc -${CMAKE_CURRENT_BINARY_DIR}/lex_hash.h -${CMAKE_CURRENT_BINARY_DIR}/lex_token.h -) - -SET_SOURCE_FILES_PROPERTIES(${GEN_SOURCES} - PROPERTIES GENERATED 1) -IF(NOT CMAKE_CROSSCOMPILING) - ADD_EXECUTABLE(gen_lex_token gen_lex_token.cc - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h) -ENDIF() + + ADD_CUSTOM_COMMAND( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lex_token.h @@ -151,7 +138,10 @@ SET (SQL_SOURCE table_cache.cc encryption.cc temporary_tables.cc proxy_protocol.cc ${CMAKE_CURRENT_BINARY_DIR}/sql_builtin.cc - ${GEN_SOURCES} + ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc + ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc + ${CMAKE_CURRENT_BINARY_DIR}/lex_hash.h + ${CMAKE_CURRENT_BINARY_DIR}/lex_token.h ${MYSYS_LIBWRAP_SOURCE} ) @@ -173,7 +163,6 @@ MYSQL_ADD_PLUGIN(sql_sequence ha_sequence.cc STORAGE_ENGINE MANDATORY STATIC_ONL RECOMPILE_FOR_EMBEDDED) ADD_LIBRARY(sql STATIC ${SQL_SOURCE}) -ADD_DEPENDENCIES(sql GenServerSource) DTRACE_INSTRUMENT(sql) TARGET_LINK_LIBRARIES(sql ${MYSQLD_STATIC_PLUGIN_LIBS} mysys mysys_ssl dbug strings vio pcre @@ -339,40 +328,48 @@ IF(WITH_MYSQLD_LDFLAGS) "${MYSQLD_LINK_FLAGS} ${WITH_MYSQLD_LDFLAGS}") ENDIF() -INCLUDE(${CMAKE_SOURCE_DIR}/cmake/bison.cmake) + +FIND_PACKAGE(BISON 2.0) + # Handle out-of-source build from source package with possibly broken # bison. Copy bison output to from source to build directory, if not already # there -IF (NOT BISON_USABLE) -IF (NOT ${CMAKE_CURRENT_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR}) - IF(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.cc) - IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc) - CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.cc - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc COPYONLY) - CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.h - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h COPYONLY) +IF (NOT BISON_FOUND) + IF (NOT ${CMAKE_CURRENT_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR}) + FOREACH(file sql_yacc.cc sql_yacc.hh sql_yacc_ora.cc sql_yacc_ora.hh) + IF(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${file} AND (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${file})) + CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/${file} + ${CMAKE_CURRENT_BINARY_DIR}/${file} COPYONLY) ENDIF() + ENDFOREACH() ENDIF() -ENDIF() + + IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc) + # Output files are missing, bail out. + SET(ERRMSG + "Bison (GNU parser generator) is required to build MySQL." + "Please install bison." + ) + IF(WIN32) + SET(ERRMSG ${ERRMSG} + "You can download bison from http://gnuwin32.sourceforge.net/packages/bison.htm " + "Choose 'Complete package, except sources' installation. We recommend to " + "install bison into a directory without spaces, e.g C:\\GnuWin32.") + ENDIF() + MESSAGE(FATAL_ERROR ${ERRMSG}) + ENDIF() +ELSE() + BISON_TARGET(gen_sql_yacc ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc + COMPILE_FLAGS "-p MYSQL") + + BISON_TARGET(gen_sql_yacc_ora ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc_ora.yy ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc + COMPILE_FLAGS "-p ORA") ENDIF() -RUN_BISON( - ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.cc - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.h - MYSQL -) - -RUN_BISON( - ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc_ora.yy - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc - ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.h - ORA -) - -# Gen_lex_hash IF(NOT CMAKE_CROSSCOMPILING) + ADD_EXECUTABLE(gen_lex_token gen_lex_token.cc + ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc.hh) ADD_EXECUTABLE(gen_lex_hash gen_lex_hash.cc) ENDIF() @@ -388,12 +385,12 @@ TARGET_LINK_LIBRARIES(mysql_tzinfo_to_sql mysys mysys_ssl) ADD_CUSTOM_TARGET( GenServerSource - DEPENDS ${GEN_SOURCES} + DEPENDS + ${CMAKE_CURRENT_BINARY_DIR}/lex_hash.h + ${CMAKE_CURRENT_BINARY_DIR}/lex_token.h + ${CMAKE_CURRENT_BINARY_DIR}/sql_yacc_ora.cc ) -#Need this only for embedded -SET_TARGET_PROPERTIES(GenServerSource PROPERTIES EXCLUDE_FROM_ALL TRUE) - IF(WIN32 OR HAVE_DLOPEN AND NOT DISABLE_SHARED) ADD_LIBRARY(udf_example MODULE udf_example.c udf_example.def) SET_TARGET_PROPERTIES(udf_example PROPERTIES PREFIX "") @@ -406,8 +403,8 @@ CONFIGURE_FILE( ADD_CUSTOM_TARGET(dist COMMAND ${CMAKE_COMMAND} -P ${CMAKE_BINARY_DIR}/make_dist.cmake - DEPENDS ${CMAKE_BINARY_DIR}/sql/sql_yacc.cc ${CMAKE_BINARY_DIR}/sql/sql_yacc.h - DEPENDS ${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.cc ${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.h + DEPENDS ${CMAKE_BINARY_DIR}/sql/sql_yacc.cc ${CMAKE_BINARY_DIR}/sql/sql_yacc.hh + DEPENDS ${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.cc ${CMAKE_BINARY_DIR}/sql/sql_yacc_ora.hh WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) diff --git a/sql/gen_lex_token.cc b/sql/gen_lex_token.cc index b36df5e94be..c03bfe66f47 100644 --- a/sql/gen_lex_token.cc +++ b/sql/gen_lex_token.cc @@ -19,7 +19,7 @@ /* We only need the tokens here */ #define YYSTYPE_IS_DECLARED -#include +#include #include #include /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ diff --git a/sql/sql_digest.cc b/sql/sql_digest.cc index 54ed1c3f225..a8afd33c46a 100644 --- a/sql/sql_digest.cc +++ b/sql/sql_digest.cc @@ -32,7 +32,7 @@ #include "sql_get_diagnostics.h" /* Generated code */ -#include "sql_yacc.h" +#include "sql_yacc.hh" #define LEX_TOKEN_WITH_DEFINITION #include "lex_token.h" diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 70caa3b92b9..77e477b7cb4 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -269,10 +269,10 @@ struct LEX_TYPE #else #include "lex_symbol.h" #ifdef MYSQL_LEX -#include "item_func.h" /* Cast_target used in sql_yacc.h */ -#include "sql_get_diagnostics.h" /* Types used in sql_yacc.h */ +#include "item_func.h" /* Cast_target used in sql_yacc.hh */ +#include "sql_get_diagnostics.h" /* Types used in sql_yacc.hh */ #include "sp_pcontext.h" -#include "sql_yacc.h" +#include "sql_yacc.hh" #define LEX_YYSTYPE YYSTYPE * #else #define LEX_YYSTYPE void * From 141bc58ac9926aff0a4eb5bac204f44f60d58840 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 8 Jun 2018 19:52:30 +0100 Subject: [PATCH 105/276] MDEV-16430: mysql_upgrade_service does not write log file. Fixed CreateFile() for the log file - TRUNCATE_EXISTING does not create a new file, use CREATE_ALWAYS - Make log file handle inheritable --- sql/mysql_upgrade_service.cc | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/sql/mysql_upgrade_service.cc b/sql/mysql_upgrade_service.cc index db916101eb1..0723d1c0356 100644 --- a/sql/mysql_upgrade_service.cc +++ b/sql/mysql_upgrade_service.cc @@ -146,6 +146,11 @@ static void die(const char *fmt, ...) exit(1); } +#define WRITE_LOG(fmt,...) {\ + char log_buf[1024]; \ + snprintf(log_buf,sizeof(log_buf), fmt, __VA_ARGS__);\ + WriteFile(logfile_handle,log_buf, strlen(log_buf), 0 , 0);\ +} /* spawn-like function to run subprocesses. @@ -187,17 +192,22 @@ static intptr_t run_tool(int wait_flag, const char *program,...) { char tmpdir[FN_REFLEN]; GetTempPath(FN_REFLEN, tmpdir); - sprintf_s(logfile_path, "%s\\mysql_upgrade_service.%s.log", tmpdir, + sprintf_s(logfile_path, "%smysql_upgrade_service.%s.log", tmpdir, opt_service); - logfile_handle= CreateFile(logfile_path, GENERIC_WRITE, FILE_SHARE_READ, - NULL, TRUNCATE_EXISTING, 0, NULL); - if (!logfile_handle) + SECURITY_ATTRIBUTES attr= {0}; + attr.nLength= sizeof(SECURITY_ATTRIBUTES); + attr.bInheritHandle= TRUE; + logfile_handle= CreateFile(logfile_path, FILE_APPEND_DATA, + FILE_SHARE_READ|FILE_SHARE_WRITE, &attr, CREATE_ALWAYS, 0, NULL); + if (logfile_handle == INVALID_HANDLE_VALUE) { die("Cannot open log file %s, windows error %u", logfile_path, GetLastError()); } } + WRITE_LOG("Executing %s\r\n", cmdline); + /* Start child process */ STARTUPINFO si= {0}; si.cb= sizeof(si); From 5bfd562a00828035ec873e5a6ee0af2c3f52a33a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 8 Jun 2018 20:42:25 +0100 Subject: [PATCH 106/276] MDEV-16445 mysql_upgrade_service should add skip-slave-start to server start parameters --- sql/mysql_upgrade_service.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/mysql_upgrade_service.cc b/sql/mysql_upgrade_service.cc index 0723d1c0356..809161193a5 100644 --- a/sql/mysql_upgrade_service.cc +++ b/sql/mysql_upgrade_service.cc @@ -468,7 +468,7 @@ int main(int argc, char **argv) log("Phase 3/8: Starting mysqld for upgrade"); mysqld_process= (HANDLE)run_tool(P_NOWAIT, mysqld_path, defaults_file_param, "--skip-networking", "--skip-grant-tables", - "--enable-named-pipe", socket_param, NULL); + "--enable-named-pipe", socket_param,"--skip-slave-start", NULL); if (mysqld_process == INVALID_HANDLE_VALUE) { From 15155ecd3483c47d8f2c05727184f6bf47bbf0b0 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 8 Jun 2018 20:42:39 +0100 Subject: [PATCH 107/276] fix typo --- sql/mysql_install_db.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index de799874a8f..f9343aab011 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -195,7 +195,7 @@ int main(int argc, char **argv) die("database creation failed"); } - printf("Creation of the database was successfull"); + printf("Creation of the database was successful"); return 0; } From cd33280b682692f0517a26178d7b5337db648751 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 9 Jun 2018 11:26:52 +0530 Subject: [PATCH 108/276] MDEV-16374: Filtered shows 0 for materilization scan for a semi join, which makes optimizer always picks materialization scan over materialization lookup For non-mergeable semi-joins we don't store the estimates of the IN subquery in table->file->stats.records. In the function TABLE_LIST::fetch_number_of_rows, we store the number of rows in the tables (estimates in case of derived table/views). Currently we don't store the estimates for non-mergeable semi-joins, which leads to a problem of selecting materialization scan over materialization lookup. Fixed this by storing these estimated appropriately --- mysql-test/r/selectivity.result | 70 ++++++++++++++++++++-- mysql-test/r/selectivity_innodb.result | 70 ++++++++++++++++++++-- mysql-test/r/subselect_sj_nonmerged.result | 6 +- mysql-test/t/selectivity.test | 18 ++++++ sql/table.cc | 8 +++ 5 files changed, 161 insertions(+), 11 deletions(-) diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index 61a77d135e7..3e8fb8e2e41 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -356,13 +356,13 @@ and o_orderkey = l_orderkey group by c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice order by o_totalprice desc, o_orderdate; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL distinct_key NULL NULL NULL 6005 0.00 Using temporary; Using filesort -1 PRIMARY orders eq_ref PRIMARY,i_o_custkey PRIMARY 4 .l_orderkey 1 100.00 Using where +1 PRIMARY orders ALL PRIMARY,i_o_custkey NULL NULL NULL 1500 100.00 Using where; Using temporary; Using filesort +1 PRIMARY eq_ref distinct_key distinct_key 4 dbt3_s001.orders.o_orderkey 1 100.00 1 PRIMARY customer eq_ref PRIMARY PRIMARY 4 dbt3_s001.orders.o_custkey 1 100.00 -1 PRIMARY lineitem ref PRIMARY,i_l_orderkey,i_l_orderkey_quantity PRIMARY 4 .l_orderkey 4 100.00 +1 PRIMARY lineitem ref PRIMARY,i_l_orderkey,i_l_orderkey_quantity i_l_orderkey_quantity 4 dbt3_s001.orders.o_orderkey 4 100.00 Using index 2 MATERIALIZED lineitem index NULL i_l_orderkey_quantity 13 NULL 6005 100.00 Using index Warnings: -Note 1003 select `dbt3_s001`.`customer`.`c_name` AS `c_name`,`dbt3_s001`.`customer`.`c_custkey` AS `c_custkey`,`dbt3_s001`.`orders`.`o_orderkey` AS `o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE` AS `o_orderdate`,`dbt3_s001`.`orders`.`o_totalprice` AS `o_totalprice`,sum(`dbt3_s001`.`lineitem`.`l_quantity`) AS `sum(l_quantity)` from (select `dbt3_s001`.`lineitem`.`l_orderkey` from `dbt3_s001`.`lineitem` group by `dbt3_s001`.`lineitem`.`l_orderkey` having (sum(`dbt3_s001`.`lineitem`.`l_quantity`) > 250)) join `dbt3_s001`.`customer` join `dbt3_s001`.`orders` join `dbt3_s001`.`lineitem` where ((`dbt3_s001`.`customer`.`c_custkey` = `dbt3_s001`.`orders`.`o_custkey`) and (`dbt3_s001`.`orders`.`o_orderkey` = ``.`l_orderkey`) and (`dbt3_s001`.`lineitem`.`l_orderkey` = ``.`l_orderkey`)) group by `dbt3_s001`.`customer`.`c_name`,`dbt3_s001`.`customer`.`c_custkey`,`dbt3_s001`.`orders`.`o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE`,`dbt3_s001`.`orders`.`o_totalprice` order by `dbt3_s001`.`orders`.`o_totalprice` desc,`dbt3_s001`.`orders`.`o_orderDATE` +Note 1003 select `dbt3_s001`.`customer`.`c_name` AS `c_name`,`dbt3_s001`.`customer`.`c_custkey` AS `c_custkey`,`dbt3_s001`.`orders`.`o_orderkey` AS `o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE` AS `o_orderdate`,`dbt3_s001`.`orders`.`o_totalprice` AS `o_totalprice`,sum(`dbt3_s001`.`lineitem`.`l_quantity`) AS `sum(l_quantity)` from (select `dbt3_s001`.`lineitem`.`l_orderkey` from `dbt3_s001`.`lineitem` group by `dbt3_s001`.`lineitem`.`l_orderkey` having (sum(`dbt3_s001`.`lineitem`.`l_quantity`) > 250)) join `dbt3_s001`.`customer` join `dbt3_s001`.`orders` join `dbt3_s001`.`lineitem` where ((`dbt3_s001`.`customer`.`c_custkey` = `dbt3_s001`.`orders`.`o_custkey`) and (``.`l_orderkey` = `dbt3_s001`.`orders`.`o_orderkey`) and (`dbt3_s001`.`lineitem`.`l_orderkey` = `dbt3_s001`.`orders`.`o_orderkey`)) group by `dbt3_s001`.`customer`.`c_name`,`dbt3_s001`.`customer`.`c_custkey`,`dbt3_s001`.`orders`.`o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE`,`dbt3_s001`.`orders`.`o_totalprice` order by `dbt3_s001`.`orders`.`o_totalprice` desc,`dbt3_s001`.`orders`.`o_orderDATE` select c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice, sum(l_quantity) from customer, orders, lineitem @@ -1535,6 +1535,68 @@ t 10:00:00 11:00:00 DROP TABLE t1; +# +# MDEV-16374: filtered shows 0 for materilization scan for a semi join, which makes optimizer +# always pick materialization scan over materialization lookup +# +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 (a int, b int); +insert into t1 values (0,0),(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10), +(11,11),(12,12),(13,13),(14,14),(15,15); +set @@optimizer_use_condition_selectivity=2; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 16 100.00 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t1.a 1 100.00 +2 MATERIALIZED t1 ALL NULL NULL NULL NULL 16 100.00 Using temporary +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t1`.`a`) from `test`.`t1` group by `test`.`t1`.`b`) join `test`.`t1` where (``.`max(a)` = `test`.`t1`.`a`) +select * from t1 where a in (select max(a) from t1 group by b); +a b +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +set @@optimizer_use_condition_selectivity=1; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 16 100.00 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t1.a 1 100.00 +2 MATERIALIZED t1 ALL NULL NULL NULL NULL 16 100.00 Using temporary +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t1`.`a`) from `test`.`t1` group by `test`.`t1`.`b`) join `test`.`t1` where (``.`max(a)` = `test`.`t1`.`a`) +select * from t1 where a in (select max(a) from t1 group by b); +a b +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +drop table t1,t0; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index a026c2e6d92..748ef0cb6ca 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -359,13 +359,13 @@ and o_orderkey = l_orderkey group by c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice order by o_totalprice desc, o_orderdate; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL distinct_key NULL NULL NULL 6005 0.00 Using temporary; Using filesort -1 PRIMARY orders eq_ref PRIMARY,i_o_custkey PRIMARY 4 .l_orderkey 1 100.00 Using where +1 PRIMARY orders ALL PRIMARY,i_o_custkey NULL NULL NULL 1500 100.00 Using where; Using temporary; Using filesort +1 PRIMARY eq_ref distinct_key distinct_key 4 dbt3_s001.orders.o_orderkey 1 100.00 1 PRIMARY customer eq_ref PRIMARY PRIMARY 4 dbt3_s001.orders.o_custkey 1 100.00 -1 PRIMARY lineitem ref PRIMARY,i_l_orderkey,i_l_orderkey_quantity PRIMARY 4 .l_orderkey 4 100.00 +1 PRIMARY lineitem ref PRIMARY,i_l_orderkey,i_l_orderkey_quantity i_l_orderkey_quantity 4 dbt3_s001.orders.o_orderkey 4 100.00 Using index 2 MATERIALIZED lineitem index NULL PRIMARY 8 NULL 6005 100.00 Warnings: -Note 1003 select `dbt3_s001`.`customer`.`c_name` AS `c_name`,`dbt3_s001`.`customer`.`c_custkey` AS `c_custkey`,`dbt3_s001`.`orders`.`o_orderkey` AS `o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE` AS `o_orderdate`,`dbt3_s001`.`orders`.`o_totalprice` AS `o_totalprice`,sum(`dbt3_s001`.`lineitem`.`l_quantity`) AS `sum(l_quantity)` from (select `dbt3_s001`.`lineitem`.`l_orderkey` from `dbt3_s001`.`lineitem` group by `dbt3_s001`.`lineitem`.`l_orderkey` having (sum(`dbt3_s001`.`lineitem`.`l_quantity`) > 250)) join `dbt3_s001`.`customer` join `dbt3_s001`.`orders` join `dbt3_s001`.`lineitem` where ((`dbt3_s001`.`customer`.`c_custkey` = `dbt3_s001`.`orders`.`o_custkey`) and (`dbt3_s001`.`orders`.`o_orderkey` = ``.`l_orderkey`) and (`dbt3_s001`.`lineitem`.`l_orderkey` = ``.`l_orderkey`)) group by `dbt3_s001`.`customer`.`c_name`,`dbt3_s001`.`customer`.`c_custkey`,`dbt3_s001`.`orders`.`o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE`,`dbt3_s001`.`orders`.`o_totalprice` order by `dbt3_s001`.`orders`.`o_totalprice` desc,`dbt3_s001`.`orders`.`o_orderDATE` +Note 1003 select `dbt3_s001`.`customer`.`c_name` AS `c_name`,`dbt3_s001`.`customer`.`c_custkey` AS `c_custkey`,`dbt3_s001`.`orders`.`o_orderkey` AS `o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE` AS `o_orderdate`,`dbt3_s001`.`orders`.`o_totalprice` AS `o_totalprice`,sum(`dbt3_s001`.`lineitem`.`l_quantity`) AS `sum(l_quantity)` from (select `dbt3_s001`.`lineitem`.`l_orderkey` from `dbt3_s001`.`lineitem` group by `dbt3_s001`.`lineitem`.`l_orderkey` having (sum(`dbt3_s001`.`lineitem`.`l_quantity`) > 250)) join `dbt3_s001`.`customer` join `dbt3_s001`.`orders` join `dbt3_s001`.`lineitem` where ((`dbt3_s001`.`customer`.`c_custkey` = `dbt3_s001`.`orders`.`o_custkey`) and (``.`l_orderkey` = `dbt3_s001`.`orders`.`o_orderkey`) and (`dbt3_s001`.`lineitem`.`l_orderkey` = `dbt3_s001`.`orders`.`o_orderkey`)) group by `dbt3_s001`.`customer`.`c_name`,`dbt3_s001`.`customer`.`c_custkey`,`dbt3_s001`.`orders`.`o_orderkey`,`dbt3_s001`.`orders`.`o_orderDATE`,`dbt3_s001`.`orders`.`o_totalprice` order by `dbt3_s001`.`orders`.`o_totalprice` desc,`dbt3_s001`.`orders`.`o_orderDATE` select c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice, sum(l_quantity) from customer, orders, lineitem @@ -1539,6 +1539,68 @@ t 10:00:00 11:00:00 DROP TABLE t1; +# +# MDEV-16374: filtered shows 0 for materilization scan for a semi join, which makes optimizer +# always pick materialization scan over materialization lookup +# +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 (a int, b int); +insert into t1 values (0,0),(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10), +(11,11),(12,12),(13,13),(14,14),(15,15); +set @@optimizer_use_condition_selectivity=2; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 16 100.00 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t1.a 1 100.00 +2 MATERIALIZED t1 ALL NULL NULL NULL NULL 16 100.00 Using temporary +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t1`.`a`) from `test`.`t1` group by `test`.`t1`.`b`) join `test`.`t1` where (``.`max(a)` = `test`.`t1`.`a`) +select * from t1 where a in (select max(a) from t1 group by b); +a b +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +set @@optimizer_use_condition_selectivity=1; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 16 100.00 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t1.a 1 100.00 +2 MATERIALIZED t1 ALL NULL NULL NULL NULL 16 100.00 Using temporary +Warnings: +Note 1003 select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from (select max(`test`.`t1`.`a`) from `test`.`t1` group by `test`.`t1`.`b`) join `test`.`t1` where (``.`max(a)` = `test`.`t1`.`a`) +select * from t1 where a in (select max(a) from t1 group by b); +a b +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +drop table t1,t0; set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/r/subselect_sj_nonmerged.result b/mysql-test/r/subselect_sj_nonmerged.result index c7e04225ffe..47970668ae5 100644 --- a/mysql-test/r/subselect_sj_nonmerged.result +++ b/mysql-test/r/subselect_sj_nonmerged.result @@ -77,9 +77,9 @@ explain select * from t4 where t4.a in (select max(t2.a) from t1, t2 group by t2.b) and t4.b in (select max(t2.a) from t1, t2 group by t2.b); id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY ALL distinct_key NULL NULL NULL 5 -1 PRIMARY ALL distinct_key NULL NULL NULL 5 Using join buffer (flat, BNL join) -1 PRIMARY t4 ref a a 10 .max(t2.a),.max(t2.a) 12 +1 PRIMARY ALL distinct_key NULL NULL NULL 5 +1 PRIMARY t4 ref a a 5 .max(t2.a) 12 Using index condition +1 PRIMARY eq_ref distinct_key distinct_key 4 test.t4.b 1 3 MATERIALIZED t2 ALL NULL NULL NULL NULL 5 Using temporary 3 MATERIALIZED t1 ALL NULL NULL NULL NULL 10 Using join buffer (flat, BNL join) 2 MATERIALIZED t2 ALL NULL NULL NULL NULL 5 Using temporary diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index 548ef295fb2..afaa937c360 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -1043,6 +1043,24 @@ SELECT * FROM (SELECT t FROM t1 WHERE d IS NULL) sq; DROP TABLE t1; +--echo # +--echo # MDEV-16374: filtered shows 0 for materilization scan for a semi join, which makes optimizer +--echo # always pick materialization scan over materialization lookup +--echo # + +create table t0(a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1 (a int, b int); +insert into t1 values (0,0),(1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10), +(11,11),(12,12),(13,13),(14,14),(15,15); +set @@optimizer_use_condition_selectivity=2; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +select * from t1 where a in (select max(a) from t1 group by b); +set @@optimizer_use_condition_selectivity=1; +explain extended select * from t1 where a in (select max(a) from t1 group by b); +select * from t1 where a in (select max(a) from t1 group by b); +drop table t1,t0; + set histogram_size=@save_histogram_size; set optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set use_stat_tables=@save_use_stat_tables; diff --git a/sql/table.cc b/sql/table.cc index bc6e1e754ee..b5082df7076 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -7099,7 +7099,15 @@ int TABLE_LIST::fetch_number_of_rows() { int error= 0; if (jtbm_subselect) + { + if (jtbm_subselect->is_jtbm_merged) + { + table->file->stats.records= jtbm_subselect->jtbm_record_count; + set_if_bigger(table->file->stats.records, 2); + table->used_stat_records= table->file->stats.records; + } return 0; + } if (is_materialized_derived() && !fill_me) { From c17468d4abe59339af11ef4a17a59156741c6a21 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Sat, 9 Jun 2018 19:04:34 +0530 Subject: [PATCH 109/276] MDEV-16191: Analyze format=json gives incorrect value for r_limit inside a dependent subquery when ORDER BY is present Currently for setting r_limit we divide with the number of iterations we invoke the dependent subquery. This is not needed for the case of limit. For varying limits we produce the output that the limit varies with execution. Also there is a type for filtered , we forgot to multiply by 100 as it is represented as a percent. --- mysql-test/r/analyze_stmt_orderby.result | 2 +- sql/sql_analyze_stmt.cc | 2 +- sql/sql_explain.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/r/analyze_stmt_orderby.result b/mysql-test/r/analyze_stmt_orderby.result index be1f01a2a52..238baff50e1 100644 --- a/mysql-test/r/analyze_stmt_orderby.result +++ b/mysql-test/r/analyze_stmt_orderby.result @@ -303,7 +303,7 @@ ANALYZE "r_rows": 10, "r_total_time_ms": "REPLACED", "filtered": 100, - "r_filtered": 1, + "r_filtered": 100, "attached_condition": "(t0.a is not null)" } } diff --git a/sql/sql_analyze_stmt.cc b/sql/sql_analyze_stmt.cc index 098e99e88fc..68299d024fd 100644 --- a/sql/sql_analyze_stmt.cc +++ b/sql/sql_analyze_stmt.cc @@ -39,7 +39,7 @@ void Filesort_tracker::print_json_members(Json_writer *writer) if (r_limit == 0) writer->add_str(varied_str); else - writer->add_ll((longlong) rint(r_limit/get_r_loops())); + writer->add_ll((longlong) rint(r_limit)); } writer->add_member("r_used_priority_queue"); diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc index ac6bee05001..82107f9b922 100644 --- a/sql/sql_explain.cc +++ b/sql/sql_explain.cc @@ -1642,7 +1642,7 @@ void Explain_table_access::print_explain_json(Explain_query *query, { /* Get r_filtered value from filesort */ if (fs_tracker->get_r_loops()) - writer->add_double(fs_tracker->get_r_filtered()); + writer->add_double(fs_tracker->get_r_filtered()*100); else writer->add_null(); } From b8e267c0c5007dbfebc2dee8a4f408f28b637ba5 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Wed, 9 May 2018 15:14:57 +0300 Subject: [PATCH 110/276] MDEV-15778: Manually backport TokuDB macOS fixes from 10.0 Fix build on macOS 10.13: 39dceaae607 MDEV-10983: TokuDB does not compile on OS X 10.12 Make use of a different function to get the current tid. Additionally, librt doesn't exist on OS X. Use System library instead. storage/tokudb/PerconaFT/cmake_modules/TokuFeatureDetection.cmake | 4 +++- storage/tokudb/PerconaFT/portability/portability.cc | 9 ++++++++- storage/tokudb/PerconaFT/portability/tests/test-xid.cc | 9 ++++++++- storage/tokudb/PerconaFT/portability/toku_config.h.in | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) --- .../ft-index/cmake_modules/TokuFeatureDetection.cmake | 4 +++- storage/tokudb/ft-index/portability/portability.cc | 9 ++++++++- storage/tokudb/ft-index/portability/tests/test-xid.cc | 9 ++++++++- storage/tokudb/ft-index/portability/toku_config.h.in | 1 + 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake b/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake index e7fd27525d5..ef7934a5a09 100644 --- a/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake +++ b/storage/tokudb/ft-index/cmake_modules/TokuFeatureDetection.cmake @@ -96,7 +96,7 @@ if (NOT HAVE_BACKTRACE_WITHOUT_EXECINFO) endif () endif () -if(HAVE_CLOCK_REALTIME) +if(HAVE_CLOCK_REALTIME AND (NOT APPLE)) list(APPEND EXTRA_SYSTEM_LIBS rt) else() list(APPEND EXTRA_SYSTEM_LIBS System) @@ -108,6 +108,8 @@ check_function_exists(pthread_rwlockattr_setkind_np HAVE_PTHREAD_RWLOCKATTR_SETK ## check for the right way to yield using pthreads check_function_exists(pthread_yield HAVE_PTHREAD_YIELD) check_function_exists(pthread_yield_np HAVE_PTHREAD_YIELD_NP) +## check if we have pthread_threadid_np() (i.e. osx) +check_function_exists(pthread_threadid_np HAVE_PTHREAD_THREADID_NP) ## check if we have pthread_getthreadid_np() (i.e. freebsd) check_function_exists(pthread_getthreadid_np HAVE_PTHREAD_GETTHREADID_NP) diff --git a/storage/tokudb/ft-index/portability/portability.cc b/storage/tokudb/ft-index/portability/portability.cc index 09c1ccd50be..9863c45c60d 100644 --- a/storage/tokudb/ft-index/portability/portability.cc +++ b/storage/tokudb/ft-index/portability/portability.cc @@ -115,6 +115,9 @@ PATENT RIGHTS GRANT: #if defined(HAVE_SYS_SYSCTL_H) # include #endif +#if defined(HAVE_PTHREAD_H) +# include +#endif #if defined(HAVE_PTHREAD_NP_H) # include #endif @@ -154,7 +157,11 @@ toku_os_getpid(void) { int toku_os_gettid(void) { -#if defined(__NR_gettid) +#if defined(HAVE_PTHREAD_THREADID_NP) + uint64_t result; + pthread_threadid_np(NULL, &result); + return (int) result; // Used for instrumentation so overflow is ok here. +#elif defined(__NR_gettid) return syscall(__NR_gettid); #elif defined(SYS_gettid) return syscall(SYS_gettid); diff --git a/storage/tokudb/ft-index/portability/tests/test-xid.cc b/storage/tokudb/ft-index/portability/tests/test-xid.cc index 9277f984b43..fb1abd33793 100644 --- a/storage/tokudb/ft-index/portability/tests/test-xid.cc +++ b/storage/tokudb/ft-index/portability/tests/test-xid.cc @@ -103,11 +103,18 @@ PATENT RIGHTS GRANT: #if defined(HAVE_PTHREAD_NP_H) # include #endif +#if defined(HAVE_PTHREAD_H) +# include +#endif // since we implement the same thing here as in toku_os_gettid, this test // is pretty pointless static int gettid(void) { -#if defined(__NR_gettid) +#if defined(HAVE_PTHREAD_THREADID_NP) + uint64_t result; + pthread_threadid_np(NULL, &result); + return (int) result; +#elif defined(__NR_gettid) return syscall(__NR_gettid); #elif defined(SYS_gettid) return syscall(SYS_gettid); diff --git a/storage/tokudb/ft-index/portability/toku_config.h.in b/storage/tokudb/ft-index/portability/toku_config.h.in index c56674ece9e..b8a89e81da6 100644 --- a/storage/tokudb/ft-index/portability/toku_config.h.in +++ b/storage/tokudb/ft-index/portability/toku_config.h.in @@ -57,6 +57,7 @@ #cmakedefine HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP 1 #cmakedefine HAVE_PTHREAD_YIELD 1 #cmakedefine HAVE_PTHREAD_YIELD_NP 1 +#cmakedefine HAVE_PTHREAD_THREADID_NP 1 #cmakedefine HAVE_PTHREAD_GETTHREADID_NP 1 #cmakedefine PTHREAD_YIELD_RETURNS_INT 1 From 1735fa340a9d7ca8683f18fe2ecc148423e78ba7 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Wed, 9 May 2018 16:54:16 +0300 Subject: [PATCH 111/276] MDEV-15778: Remove packed attr from omt_ and subtree_ classes This is happening because they are declared as packed and clang has -Waddress-of-packed-member when passing the address of a packed member, a legit concern on different architectures. The easiest way to get rid of the errors is to remove the packed attribute from said structs. --- storage/tokudb/ft-index/util/omt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ft-index/util/omt.h b/storage/tokudb/ft-index/util/omt.h index 02f3f0d759a..6cf3f80d2bc 100644 --- a/storage/tokudb/ft-index/util/omt.h +++ b/storage/tokudb/ft-index/util/omt.h @@ -181,7 +181,7 @@ public: paranoid_invariant(index != NODE_NULL); m_index = index; } -} __attribute__((__packed__,aligned(4))); +} ; template<> class subtree_templated { @@ -238,7 +238,7 @@ public: inline void disable_bit(void) { m_bitfield &= MASK_INDEX; } -} __attribute__((__packed__)) ; +} ; template class omt_node_templated { @@ -251,7 +251,7 @@ public: // this needs to be in both implementations because we don't have // a "static if" the caller can use inline void clear_stolen_bits(void) {} -} __attribute__((__packed__,aligned(4))); +} ; template class omt_node_templated { @@ -288,7 +288,7 @@ public: this->unset_marked_bit(); this->unset_marks_below_bit(); } -} __attribute__((__packed__,aligned(4))); +} ; } From 8f82c4844342f42c27e2336b64f7bdd1a1344cbd Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Wed, 9 May 2018 16:29:18 +0300 Subject: [PATCH 112/276] MDEV-15778: Restore file permissions lost in merge Permissions were probably due to a file copy in: 15f7f5c6bb4 Merge branch 'merge-tokudb-5.6' into 10.0 --- storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/autogen.sh | 0 .../tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/compile | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/config.rpath | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/config.sub | 0 .../tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/depcomp | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/install-sh | 0 .../PerconaFT/third_party/xz-4.999.9beta/build-aux/ltmain.sh | 0 .../tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/missing | 0 .../third_party/xz-4.999.9beta/extra/7z2lzma/7z2lzma.bash | 0 .../PerconaFT/third_party/xz-4.999.9beta/tests/test_compress.sh | 0 .../PerconaFT/third_party/xz-4.999.9beta/tests/test_files.sh | 0 12 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/autogen.sh mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/compile mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.rpath mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.sub mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/depcomp mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/install-sh mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/ltmain.sh mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/missing mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/extra/7z2lzma/7z2lzma.bash mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_compress.sh mode change 100644 => 100755 storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_files.sh diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/autogen.sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/autogen.sh old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/compile b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/compile old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.guess old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.rpath b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.rpath old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.sub b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/config.sub old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/depcomp b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/depcomp old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/install-sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/install-sh old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/ltmain.sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/ltmain.sh old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/missing b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/build-aux/missing old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/extra/7z2lzma/7z2lzma.bash b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/extra/7z2lzma/7z2lzma.bash old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_compress.sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_compress.sh old mode 100644 new mode 100755 diff --git a/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_files.sh b/storage/tokudb/PerconaFT/third_party/xz-4.999.9beta/tests/test_files.sh old mode 100644 new mode 100755 From 7053e26e1869962ef473043686996f40ac0fb88c Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Thu, 10 May 2018 10:00:51 +0300 Subject: [PATCH 113/276] MDEV-15778: Fix TokuDB build issues on macOS 10.13.4 Several issues were encountered and fixed as explained bellow: * missing link to dbug lib; * user proper fprintf format specifier; * ZERO_COND_INITIALIZER was using wrong toku_cond_t struct initializer for first member of type pthread_cond_t and not considering the TOKU_PTHREAD_DEBUG case which has one extra struct member of type pfs_key_t; * Remove likely(!opt_debug_sync_timeout), argument is declared extern and not available to Toku; * pthread_mutex_timedlock() is not available in pthreads for Mac, as it's not part of the POSIX pthreads spec. The encompassing event_t::wait(ms) methods are unused, thus have been removed; --- storage/tokudb/PerconaFT/ft/ft-ops.cc | 4 +- .../PerconaFT/ft/serialize/ft-serialize.cc | 6 +- .../ft/serialize/ft_node-serialize.cc | 58 +++++++++---------- .../PerconaFT/portability/toku_debug_sync.h | 3 - .../PerconaFT/portability/toku_pthread.h | 12 +++- storage/tokudb/PerconaFT/src/CMakeLists.txt | 2 +- storage/tokudb/tokudb_thread.h | 49 ---------------- 7 files changed, 46 insertions(+), 88 deletions(-) diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.cc b/storage/tokudb/PerconaFT/ft/ft-ops.cc index d036366dd63..d7ea5efdff5 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.cc +++ b/storage/tokudb/PerconaFT/ft/ft-ops.cc @@ -821,7 +821,7 @@ int toku_ftnode_fetch_callback(CACHEFILE UU(cachefile), fprintf( stderr, "%s:%d:toku_ftnode_fetch_callback - " - "file[%s], blocknum[%ld], toku_deserialize_ftnode_from " + "file[%s], blocknum[%lld], toku_deserialize_ftnode_from " "failed with a checksum error.\n", __FILE__, __LINE__, @@ -831,7 +831,7 @@ int toku_ftnode_fetch_callback(CACHEFILE UU(cachefile), fprintf( stderr, "%s:%d:toku_ftnode_fetch_callback - " - "file[%s], blocknum[%ld], toku_deserialize_ftnode_from " + "file[%s], blocknum[%lld], toku_deserialize_ftnode_from " "failed with %d.\n", __FILE__, __LINE__, diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc index b24d72a5dff..fe68e1b20e0 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc @@ -656,9 +656,9 @@ exit: fprintf(stderr, \ "%s:%d toku_deserialize_ft_from: " \ "filename[%s] " \ - "r[%d] max_acceptable_lsn[%lu]" \ - "r0[%d] checkpoint_lsn_0[%lu] checkpoint_count_0[%lu] " \ - "r1[%d] checkpoint_lsn_1[%lu] checkpoint_count_1[%lu]\n", \ + "r[%d] max_acceptable_lsn[%llu]" \ + "r0[%d] checkpoint_lsn_0[%llu] checkpoint_count_0[%llu] " \ + "r1[%d] checkpoint_lsn_1[%llu] checkpoint_count_1[%llu]\n", \ __FILE__, \ __LINE__, \ fn, \ diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc index 55899905baf..02d0cc691cd 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc @@ -1170,7 +1170,7 @@ int verify_ftnode_sub_block(struct sub_block *sb, fprintf( stderr, "%s:%d:verify_ftnode_sub_block - " - "file[%s], blocknum[%ld], stored_xsum[%u] != actual_xsum[%u]\n", + "file[%s], blocknum[%lld], stored_xsum[%u] != actual_xsum[%u]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1197,7 +1197,7 @@ static int deserialize_ftnode_info(struct sub_block *sb, FTNODE node) { fprintf( stderr, "%s:%d:deserialize_ftnode_info - " - "file[%s], blocknum[%ld], verify_ftnode_sub_block failed with %d\n", + "file[%s], blocknum[%lld], verify_ftnode_sub_block failed with %d\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1253,7 +1253,7 @@ static int deserialize_ftnode_info(struct sub_block *sb, FTNODE node) { fprintf( stderr, "%s:%d:deserialize_ftnode_info - " - "file[%s], blocknum[%ld], data_size[%d] != rb.ndone[%d]\n", + "file[%s], blocknum[%lld], data_size[%d] != rb.ndone[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1388,7 +1388,7 @@ static int deserialize_ftnode_partition( if (r != 0) { fprintf(stderr, "%s:%d:deserialize_ftnode_partition - " - "file[%s], blocknum[%ld], " + "file[%s], blocknum[%lld], " "verify_ftnode_sub_block failed with %d\n", __FILE__, __LINE__, @@ -1410,7 +1410,7 @@ static int deserialize_ftnode_partition( if (ch != FTNODE_PARTITION_MSG_BUFFER) { fprintf(stderr, "%s:%d:deserialize_ftnode_partition - " - "file[%s], blocknum[%ld], ch[%d] != " + "file[%s], blocknum[%lld], ch[%d] != " "FTNODE_PARTITION_MSG_BUFFER[%d]\n", __FILE__, __LINE__, @@ -1433,7 +1433,7 @@ static int deserialize_ftnode_partition( if (ch != FTNODE_PARTITION_DMT_LEAVES) { fprintf(stderr, "%s:%d:deserialize_ftnode_partition - " - "file[%s], blocknum[%ld], ch[%d] != " + "file[%s], blocknum[%lld], ch[%d] != " "FTNODE_PARTITION_DMT_LEAVES[%d]\n", __FILE__, __LINE__, @@ -1457,7 +1457,7 @@ static int deserialize_ftnode_partition( if (rb.ndone != rb.size) { fprintf(stderr, "%s:%d:deserialize_ftnode_partition - " - "file[%s], blocknum[%ld], rb.ndone[%d] != rb.size[%d]\n", + "file[%s], blocknum[%lld], rb.ndone[%d] != rb.size[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1485,7 +1485,7 @@ static int decompress_and_deserialize_worker(struct rbuf curr_rbuf, const char *fname = toku_ftnode_get_cachefile_fname_in_env(node); fprintf(stderr, "%s:%d:decompress_and_deserialize_worker - " - "file[%s], blocknum[%ld], read_and_decompress_sub_block failed " + "file[%s], blocknum[%lld], read_and_decompress_sub_block failed " "with %d\n", __FILE__, __LINE__, @@ -1502,7 +1502,7 @@ static int decompress_and_deserialize_worker(struct rbuf curr_rbuf, const char *fname = toku_ftnode_get_cachefile_fname_in_env(node); fprintf(stderr, "%s:%d:decompress_and_deserialize_worker - " - "file[%s], blocknum[%ld], deserialize_ftnode_partition failed " + "file[%s], blocknum[%lld], deserialize_ftnode_partition failed " "with %d\n", __FILE__, __LINE__, @@ -1582,7 +1582,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], rb->size[%u] < 24\n", + "file[%s], blocknum[%lld], rb->size[%u] < 24\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1602,7 +1602,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], unrecognized magic number " + "file[%s], blocknum[%lld], unrecognized magic number " "%2.2x %2.2x %2.2x %2.2x %2.2x %2.2x %2.2x %2.2x\n", __FILE__, __LINE__, @@ -1627,7 +1627,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], node->layout_version_read_from_disk[%d] " + "file[%s], blocknum[%lld], node->layout_version_read_from_disk[%d] " "< FT_FIRST_LAYOUT_VERSION_WITH_BASEMENT_NODES[%d]\n", __FILE__, __LINE__, @@ -1667,7 +1667,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], needed_size[%d] > rb->size[%d]\n", + "file[%s], blocknum[%lld], needed_size[%d] > rb->size[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1695,7 +1695,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], stored_checksum[%d] != checksum[%d]\n", + "file[%s], blocknum[%lld], stored_checksum[%d] != checksum[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1717,7 +1717,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], rb->size[%d] - rb->ndone[%d] < " + "file[%s], blocknum[%lld], rb->size[%d] - rb->ndone[%d] < " "sb_node_info.compressed_size[%d] + 8\n", __FILE__, __LINE__, @@ -1744,7 +1744,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], sb_node_info.xsum[%d] != actual_xsum[%d]\n", + "file[%s], blocknum[%lld], sb_node_info.xsum[%d] != actual_xsum[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -1774,7 +1774,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], deserialize_ftnode_info failed with " + "file[%s], blocknum[%lld], deserialize_ftnode_info failed with " "%d\n", __FILE__, __LINE__, @@ -1812,7 +1812,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( fprintf( stderr, "%s:%d:deserialize_ftnode_header_from_rbuf_if_small_enough - " - "file[%s], blocknum[%ld], toku_ftnode_pf_callback failed with " + "file[%s], blocknum[%lld], toku_ftnode_pf_callback failed with " "%d\n", __FILE__, __LINE__, @@ -2164,7 +2164,7 @@ static int deserialize_and_upgrade_ftnode(FTNODE node, const char* fname = toku_cachefile_fname_in_env(bfe->ft->cf); fprintf(stderr, "%s:%d:deserialize_and_upgrade_ftnode - " - "file[%s], blocknum[%ld], " + "file[%s], blocknum[%lld], " "read_and_decompress_block_from_fd_into_rbuf failed with %d\n", __FILE__, __LINE__, @@ -2190,7 +2190,7 @@ static int deserialize_and_upgrade_ftnode(FTNODE node, const char* fname = toku_cachefile_fname_in_env(bfe->ft->cf); fprintf(stderr, "%s:%d:deserialize_and_upgrade_ftnode - " - "file[%s], blocknum[%ld], version[%d] > " + "file[%s], blocknum[%lld], version[%d] > " "FT_LAYOUT_VERSION_14[%d]\n", __FILE__, __LINE__, @@ -2278,7 +2278,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, memcmp(magic, "tokunode", 8) != 0) { fprintf(stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], unrecognized magic number " + "file[%s], blocknum[%lld], unrecognized magic number " "%2.2x %2.2x %2.2x %2.2x %2.2x %2.2x %2.2x %2.2x\n", __FILE__, __LINE__, @@ -2309,7 +2309,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, if (r != 0) { fprintf(stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], deserialize_and_upgrade_ftnode " + "file[%s], blocknum[%lld], deserialize_and_upgrade_ftnode " "failed with %d\n", __FILE__, __LINE__, @@ -2355,7 +2355,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], stored_checksum[%d] != checksum[%d]\n", + "file[%s], blocknum[%lld], stored_checksum[%d] != checksum[%d]\n", __FILE__, __LINE__, fname ? fname : "unknown", @@ -2377,7 +2377,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], read_and_decompress_sub_block failed " + "file[%s], blocknum[%lld], read_and_decompress_sub_block failed " "with %d\n", __FILE__, __LINE__, @@ -2398,7 +2398,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], deserialize_ftnode_info failed with " + "file[%s], blocknum[%lld], deserialize_ftnode_info failed with " "%d\n", __FILE__, __LINE__, @@ -2470,7 +2470,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], childnum[%d], " + "file[%s], blocknum[%lld], childnum[%d], " "decompress_and_deserialize_worker failed with %d\n", __FILE__, __LINE__, @@ -2490,7 +2490,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, fprintf( stderr, "%s:%d:deserialize_ftnode_from_rbuf - " - "file[%s], blocknum[%ld], childnum[%d], " + "file[%s], blocknum[%lld], childnum[%d], " "check_and_copy_compressed_sub_block_worker failed with " "%d\n", __FILE__, @@ -2641,7 +2641,7 @@ int toku_deserialize_bp_from_compressed(FTNODE node, const char* fname = toku_cachefile_fname_in_env(bfe->ft->cf); fprintf(stderr, "%s:%d:toku_deserialize_bp_from_compressed - " - "file[%s], blocknum[%ld], " + "file[%s], blocknum[%lld], " "deserialize_ftnode_partition failed with %d\n", __FILE__, __LINE__, @@ -2689,7 +2689,7 @@ static int deserialize_ftnode_from_fd(int fd, fprintf( stderr, "%s:%d:deserialize_ftnode_from_fd - " - "file[%s], blocknum[%ld], deserialize_ftnode_from_rbuf failed with " + "file[%s], blocknum[%lld], deserialize_ftnode_from_rbuf failed with " "%d\n", __FILE__, __LINE__, diff --git a/storage/tokudb/PerconaFT/portability/toku_debug_sync.h b/storage/tokudb/PerconaFT/portability/toku_debug_sync.h index b5394e58d68..493075c36c3 100644 --- a/storage/tokudb/PerconaFT/portability/toku_debug_sync.h +++ b/storage/tokudb/PerconaFT/portability/toku_debug_sync.h @@ -62,9 +62,6 @@ inline void toku_debug_sync(struct tokutxn *txn, const char *sync_point_name) { void *client_extra; THD *thd; - if (likely(!opt_debug_sync_timeout)) - return; - toku_txn_get_client_id(txn, &client_id, &client_extra); thd = reinterpret_cast(client_extra); DEBUG_SYNC(thd, sync_point_name); diff --git a/storage/tokudb/PerconaFT/portability/toku_pthread.h b/storage/tokudb/PerconaFT/portability/toku_pthread.h index e3bd3bce598..a0dfcc246a7 100644 --- a/storage/tokudb/PerconaFT/portability/toku_pthread.h +++ b/storage/tokudb/PerconaFT/portability/toku_pthread.h @@ -162,10 +162,20 @@ typedef struct toku_mutex_aligned { #define ZERO_COND_INITIALIZER \ { 0 } #elif defined(__APPLE__) +#if TOKU_PTHREAD_DEBUG #define ZERO_COND_INITIALIZER \ { \ - { 0 } \ + { 0 , { 0 } }, \ + nullptr, \ + 0 \ } +#else +#define ZERO_COND_INITIALIZER \ + { \ + { 0 , { 0 } }, \ + nullptr \ + } +#endif #else // __linux__, at least #define ZERO_COND_INITIALIZER \ {} diff --git a/storage/tokudb/PerconaFT/src/CMakeLists.txt b/storage/tokudb/PerconaFT/src/CMakeLists.txt index 65bf4814cf8..bae37389004 100644 --- a/storage/tokudb/PerconaFT/src/CMakeLists.txt +++ b/storage/tokudb/PerconaFT/src/CMakeLists.txt @@ -18,7 +18,7 @@ set(tokudb_srcs ## make the shared library add_library(${LIBTOKUDB} SHARED ${tokudb_srcs}) add_dependencies(${LIBTOKUDB} install_tdb_h generate_log_code) -target_link_libraries(${LIBTOKUDB} LINK_PRIVATE locktree_static ft_static util_static lzma snappy ${LIBTOKUPORTABILITY}) +target_link_libraries(${LIBTOKUDB} LINK_PRIVATE locktree_static ft_static util_static lzma snappy dbug ${LIBTOKUPORTABILITY}) target_link_libraries(${LIBTOKUDB} LINK_PUBLIC ${ZLIB_LIBRARY} ) ## make the static library diff --git a/storage/tokudb/tokudb_thread.h b/storage/tokudb/tokudb_thread.h index dec58f3fd35..5df0159901f 100644 --- a/storage/tokudb/tokudb_thread.h +++ b/storage/tokudb/tokudb_thread.h @@ -111,7 +111,6 @@ public: // wait for the event to become signalled void wait(void); - int wait(ulonglong microseconds); // signal the event void signal(void); @@ -152,7 +151,6 @@ public: // wait for the semaphore to become signalled E_WAIT wait(void); - E_WAIT wait(ulonglong microseconds); // signal the semaphore to increase the count // return true if signalled, false if ignored due to count @@ -372,28 +370,6 @@ inline void event_t::wait(void) { assert_debug(r == 0); return; } -inline int event_t::wait(ulonglong microseconds) { - timespec waittime = time::offset_timespec(microseconds); - int r = pthread_mutex_timedlock(&_mutex, &waittime); - if (r == ETIMEDOUT) return ETIMEDOUT; - assert_debug(r == 0); - while (_signalled == false && _pulsed == false) { - r = pthread_cond_timedwait(&_cond, &_mutex, &waittime); - if (r == ETIMEDOUT) { - r = pthread_mutex_unlock(&_mutex); - assert_debug(r == 0); - return ETIMEDOUT; - } - assert_debug(r == 0); - } - if (_manual_reset == false) - _signalled = false; - if (_pulsed) - _pulsed = false; - r = pthread_mutex_unlock(&_mutex); - assert_debug(r == 0); - return 0; -} inline void event_t::signal(void) { int r MY_ATTRIBUTE((unused)) = pthread_mutex_lock(&_mutex); assert_debug(r == 0); @@ -479,31 +455,6 @@ inline semaphore_t::E_WAIT semaphore_t::wait(void) { assert_debug(r == 0); return ret; } -inline semaphore_t::E_WAIT semaphore_t::wait(ulonglong microseconds) { - E_WAIT ret; - timespec waittime = time::offset_timespec(microseconds); - int r = pthread_mutex_timedlock(&_mutex, &waittime); - if (r == ETIMEDOUT) return E_TIMEDOUT; - assert_debug(r == 0); - while (_signalled == 0 && _interrupted == false) { - r = pthread_cond_timedwait(&_cond, &_mutex, &waittime); - if (r == ETIMEDOUT) { - r = pthread_mutex_unlock(&_mutex); - assert_debug(r == 0); - return E_TIMEDOUT; - } - assert_debug(r == 0); - } - if (_interrupted) { - ret = E_INTERRUPTED; - } else { - _signalled--; - ret = E_SIGNALLED; - } - r = pthread_mutex_unlock(&_mutex); - assert_debug(r == 0); - return ret; -} inline bool semaphore_t::signal(void) { bool ret = false; int r MY_ATTRIBUTE((unused)) = pthread_mutex_lock(&_mutex); From 814a284f22b617e2d1ced74ba10d1a870ee6267b Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Thu, 12 Apr 2018 13:33:39 +0300 Subject: [PATCH 114/276] Ignore .cbp QtCreator && CodeBlocks project files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4e3d2a1e66e..0e00a27cbf0 100644 --- a/.gitignore +++ b/.gitignore @@ -236,3 +236,6 @@ storage/mroonga/vendor/groonga/src/grnslap storage/mroonga/vendor/groonga/src/groonga storage/mroonga/vendor/groonga/src/groonga-benchmark storage/mroonga/vendor/groonga/src/suggest/groonga-suggest-create-dataset + +# QtCreator && CodeBlocks +*.cbp From 0e6d6354bf32ac26d36a5a1bb854f2b22de3a494 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Tue, 15 May 2018 10:25:47 +0300 Subject: [PATCH 115/276] Also ignore macOS .DS_Store Finder junk. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0e00a27cbf0..5f2bf5a0fcd 100644 --- a/.gitignore +++ b/.gitignore @@ -237,5 +237,8 @@ storage/mroonga/vendor/groonga/src/groonga storage/mroonga/vendor/groonga/src/groonga-benchmark storage/mroonga/vendor/groonga/src/suggest/groonga-suggest-create-dataset +# macOS garbage +.DS_Store + # QtCreator && CodeBlocks *.cbp From d39629f01ebdd5b89186e6c8a4a8d3dd528bd26a Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Mon, 7 May 2018 22:40:27 +0300 Subject: [PATCH 116/276] MDEV-16075: Workaround to run MTR test suite for make test Assign all tests added via MY_ADD_TEST to a bogus default_ignore target, so that they are not ran by default when doing bare make test. Add default test named MTR that calls mysql-test-run suite, which is now the single test run by make test. In consequence, modified unit/suite.pm to exclude the MTR test and run the real ctests flagged for default_ignore target, thus no circular loop. --- CMakeLists.txt | 4 ++++ cmake/ctest.cmake | 2 +- mysql-test/suite/unit/suite.pm | 2 +- unittest/sql/CMakeLists.txt | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ab17ebf64c..21ed1fcc24b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -366,6 +366,10 @@ INCLUDE(maintainer) IF(WITH_UNIT_TESTS) ENABLE_TESTING() + # This is the only instance where ADD_TEST should be used, + # to make sure that make test will run MTR, + # use MY_ADD_TEST macro to add other tests + ADD_TEST(NAME MTR COMMAND ./mysql-test-run WORKING_DIRECTORY "mysql-test") ADD_SUBDIRECTORY(unittest/mytap) ADD_SUBDIRECTORY(unittest/strings) ADD_SUBDIRECTORY(unittest/examples) diff --git a/cmake/ctest.cmake b/cmake/ctest.cmake index 08852a366f6..fde7e1632f6 100644 --- a/cmake/ctest.cmake +++ b/cmake/ctest.cmake @@ -2,7 +2,7 @@ INCLUDE(${MYSQL_CMAKE_SCRIPT_DIR}/cmake_parse_arguments.cmake) MACRO(MY_ADD_TEST name) - ADD_TEST(${name} ${name}-t) + ADD_TEST(NAME ${name} COMMAND ${name}-t CONFIGURATIONS default_ignore) ENDMACRO() MACRO (MY_ADD_TESTS) diff --git a/mysql-test/suite/unit/suite.pm b/mysql-test/suite/unit/suite.pm index 78d82ccb31d..43eaa970350 100644 --- a/mysql-test/suite/unit/suite.pm +++ b/mysql-test/suite/unit/suite.pm @@ -31,7 +31,7 @@ sub start_test { return "Not run for embedded server" if $::opt_embedded_server; return "Not configured to run ctest" unless -f "../CTestTestfile.cmake"; my ($ctest_vs)= $opt_vs_config ? "--build-config $opt_vs_config" : ""; - my (@ctest_list)= `cd .. && ctest $opt_vs_config --show-only --verbose`; + my (@ctest_list)= `cd .. && ctest $opt_vs_config -E MTR -C default_ignore --show-only --verbose`; return "No ctest" if $?; my ($command, %tests); diff --git a/unittest/sql/CMakeLists.txt b/unittest/sql/CMakeLists.txt index f80f2a5ae70..1efb0ac9cd1 100644 --- a/unittest/sql/CMakeLists.txt +++ b/unittest/sql/CMakeLists.txt @@ -26,4 +26,4 @@ ELSE() ENDIF() TARGET_LINK_LIBRARIES(explain_filename-t sql mytap) -ADD_TEST(explain_filename explain_filename-t) +MY_ADD_TEST(explain_filename explain_filename-t) From 7fca4b50ffbe750532cfcdb95bcd425ec1b8e22b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 15:20:43 +0300 Subject: [PATCH 117/276] Revert "MDEV-15778: Remove packed attr from omt_ and subtree_ classes" This reverts commit 1735fa340a9d7ca8683f18fe2ecc148423e78ba7. --- storage/tokudb/ft-index/util/omt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ft-index/util/omt.h b/storage/tokudb/ft-index/util/omt.h index 6cf3f80d2bc..02f3f0d759a 100644 --- a/storage/tokudb/ft-index/util/omt.h +++ b/storage/tokudb/ft-index/util/omt.h @@ -181,7 +181,7 @@ public: paranoid_invariant(index != NODE_NULL); m_index = index; } -} ; +} __attribute__((__packed__,aligned(4))); template<> class subtree_templated { @@ -238,7 +238,7 @@ public: inline void disable_bit(void) { m_bitfield &= MASK_INDEX; } -} ; +} __attribute__((__packed__)) ; template class omt_node_templated { @@ -251,7 +251,7 @@ public: // this needs to be in both implementations because we don't have // a "static if" the caller can use inline void clear_stolen_bits(void) {} -} ; +} __attribute__((__packed__,aligned(4))); template class omt_node_templated { @@ -288,7 +288,7 @@ public: this->unset_marked_bit(); this->unset_marks_below_bit(); } -} ; +} __attribute__((__packed__,aligned(4))); } From 21246066b296bd9c6e44272a8f93cb7dc73ea43c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 15:54:57 +0300 Subject: [PATCH 118/276] Make TokuDB compile with GCC-8 GCC-8 introduced multiple warnings and increased the level of strictness. * -Wshadow will warn if a local variable shadows a typedef. * GCC will also warn when memsetting a non-trivial type. In this case a non-trivial type can not have a custom constructor. For all intents and purposes, the class is trivially-copyable. * GCC will also warn if you use too many paranthesses which are not necessary --- storage/tokudb/CMakeLists.txt | 6 ++++++ storage/tokudb/ft-index/portability/memory.h | 2 +- storage/tokudb/ft-index/tools/ba_replay.cc | 2 +- storage/tokudb/ft-index/util/dmt.cc | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 903471b097c..9bff7d4729f 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -17,6 +17,12 @@ IF(NOT LIBJEMALLOC) MESSAGE(WARNING "TokuDB is enabled, but jemalloc is not. This configuration is not supported") ENDIF() +CHECK_C_COMPILER_FLAG("-Wshadow" HAVE_WSHADOW) +IF (HAVE_WSHADOW) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-shadow") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-shadow") +ENDIF() + IF (HAVE_WVLA) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-vla") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-vla") diff --git a/storage/tokudb/ft-index/portability/memory.h b/storage/tokudb/ft-index/portability/memory.h index 837b0a70265..52e0c69575f 100644 --- a/storage/tokudb/ft-index/portability/memory.h +++ b/storage/tokudb/ft-index/portability/memory.h @@ -160,7 +160,7 @@ size_t toku_malloc_usable_size(void *p) __attribute__((__visibility__("default") #define XMALLOC(v) CAST_FROM_VOIDP(v, toku_xmalloc(sizeof(*v))) #define XMALLOC_N(n,v) CAST_FROM_VOIDP(v, toku_xmalloc((n)*sizeof(*v))) #define XCALLOC_N(n,v) CAST_FROM_VOIDP(v, toku_xcalloc((n), (sizeof(*v)))) -#define XCALLOC(v) XCALLOC_N(1,(v)) +#define XCALLOC(v) XCALLOC_N(1,v) #define XREALLOC(v,s) CAST_FROM_VOIDP(v, toku_xrealloc(v, s)) #define XREALLOC_N(n,v) CAST_FROM_VOIDP(v, toku_xrealloc(v, (n)*sizeof(*v))) diff --git a/storage/tokudb/ft-index/tools/ba_replay.cc b/storage/tokudb/ft-index/tools/ba_replay.cc index e274ac0a1e8..8f2bc3a02ec 100644 --- a/storage/tokudb/ft-index/tools/ba_replay.cc +++ b/storage/tokudb/ft-index/tools/ba_replay.cc @@ -638,7 +638,7 @@ int main(int argc, char *argv[]) { it == candidate_strategies.begin() ? &stats : &dummy_stats); struct fragmentation_report aggregate_report; - memset(&aggregate_report, 0, sizeof(aggregate_report)); + memset(static_cast(&aggregate_report), 0, sizeof(aggregate_report)); for (map::iterator rp = reports.begin(); rp != reports.end(); rp++) { const struct fragmentation_report &report = rp->second; diff --git a/storage/tokudb/ft-index/util/dmt.cc b/storage/tokudb/ft-index/util/dmt.cc index 3e0b512d7a7..1a7e2db9f40 100644 --- a/storage/tokudb/ft-index/util/dmt.cc +++ b/storage/tokudb/ft-index/util/dmt.cc @@ -132,8 +132,8 @@ void dmt::create_from_sorted_memory_of_fix paranoid_invariant(numvalues > 0); void *ptr = toku_mempool_malloc(&this->mp, aligned_memsize); paranoid_invariant_notnull(ptr); - uint8_t * const CAST_FROM_VOIDP(dest, ptr); - const uint8_t * const CAST_FROM_VOIDP(src, mem); + uint8_t * CAST_FROM_VOIDP(dest, ptr); + const uint8_t * CAST_FROM_VOIDP(src, mem); if (pad_bytes == 0) { paranoid_invariant(aligned_memsize == mem_length); memcpy(dest, src, aligned_memsize); From 953d70f960421ef4a6a2d711b66ef3d04592efc7 Mon Sep 17 00:00:00 2001 From: Teodor Mircea Ionita Date: Sun, 10 Jun 2018 16:37:49 +0300 Subject: [PATCH 119/276] MDEV-15778: Remove packed attr from omt_ and subtree_ classes Undo the revert that happened by mystake in commit 7fca4b50ffbe750532cfcdb95bcd425ec1b8e22b. --- storage/tokudb/ft-index/util/omt.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/tokudb/ft-index/util/omt.h b/storage/tokudb/ft-index/util/omt.h index 02f3f0d759a..6cf3f80d2bc 100644 --- a/storage/tokudb/ft-index/util/omt.h +++ b/storage/tokudb/ft-index/util/omt.h @@ -181,7 +181,7 @@ public: paranoid_invariant(index != NODE_NULL); m_index = index; } -} __attribute__((__packed__,aligned(4))); +} ; template<> class subtree_templated { @@ -238,7 +238,7 @@ public: inline void disable_bit(void) { m_bitfield &= MASK_INDEX; } -} __attribute__((__packed__)) ; +} ; template class omt_node_templated { @@ -251,7 +251,7 @@ public: // this needs to be in both implementations because we don't have // a "static if" the caller can use inline void clear_stolen_bits(void) {} -} __attribute__((__packed__,aligned(4))); +} ; template class omt_node_templated { @@ -288,7 +288,7 @@ public: this->unset_marked_bit(); this->unset_marks_below_bit(); } -} __attribute__((__packed__,aligned(4))); +} ; } From 1d43f71c7b4c39a6fd196c104b5ebafb65376199 Mon Sep 17 00:00:00 2001 From: Rutuja Surve Date: Sun, 10 Jun 2018 11:19:39 +0300 Subject: [PATCH 120/276] MDEV-15021: mysqldump --tables --routines generates non importable dump file The order of outputting stored procedures is important. Stored procedures must be available on view creation, for views which make use of them. Make sure to print them before outputting tables. --- client/mysqldump.c | 12 ++++++------ mysql-test/r/mysqldump.result | 15 +++++++++++++++ mysql-test/t/mysqldump.test | 22 ++++++++++++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/client/mysqldump.c b/client/mysqldump.c index 4e611ed5cb2..ad71a4dd0ce 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -4701,6 +4701,12 @@ static int dump_selected_tables(char *db, char **table_names, int tables) if (opt_xml) print_xml_tag(md_result_file, "", "\n", "database", "name=", db, NullS); + /* obtain dump of routines (procs/functions) */ + if (opt_routines && mysql_get_server_version(mysql) >= 50009) + { + DBUG_PRINT("info", ("Dumping routines for database %s", db)); + dump_routines_for_db(db); + } /* Dump each selected table */ for (pos= dump_tables; pos < end; pos++) { @@ -4729,12 +4735,6 @@ static int dump_selected_tables(char *db, char **table_names, int tables) DBUG_PRINT("info", ("Dumping events for database %s", db)); dump_events_for_db(db); } - /* obtain dump of routines (procs/functions) */ - if (opt_routines && mysql_get_server_version(mysql) >= 50009) - { - DBUG_PRINT("info", ("Dumping routines for database %s", db)); - dump_routines_for_db(db); - } free_root(&root, MYF(0)); my_free(order_by); order_by= 0; diff --git a/mysql-test/r/mysqldump.result b/mysql-test/r/mysqldump.result index 68de653742c..99b14189282 100644 --- a/mysql-test/r/mysqldump.result +++ b/mysql-test/r/mysqldump.result @@ -5401,3 +5401,18 @@ DELIMITER ; /*!50003 SET collation_connection = @saved_col_connection */ ; ALTER DATABASE `a\"'``b` CHARACTER SET utf8 COLLATE utf8_general_ci ; DROP DATABASE `a\"'``b`; +# +# MDEV-15021: Fix the order in which routines are called +# +use test; +CREATE FUNCTION f() RETURNS INT RETURN 1; +CREATE VIEW v1 AS SELECT f(); +# Running mysqldump -uroot test --routines --tables v1 > **vardir**/test.dmp +DROP VIEW v1; +DROP FUNCTION f; +# Running mysql -uroot test < **vardir**/test.dmp +# +# Cleanup after succesful import. +# +DROP VIEW v1; +DROP FUNCTION f; diff --git a/mysql-test/t/mysqldump.test b/mysql-test/t/mysqldump.test index 39ca5b3c58e..cb12919a823 100644 --- a/mysql-test/t/mysqldump.test +++ b/mysql-test/t/mysqldump.test @@ -2567,3 +2567,25 @@ let SEARCH_FILE=$MYSQLTEST_VARDIR/tmp/bug11505.sql; let SEARCH_PATTERN=Database: mysql; exec $MYSQL_DUMP mysql func > $SEARCH_FILE; source include/search_pattern_in_file.inc; + +--echo # +--echo # MDEV-15021: Fix the order in which routines are called +--echo # +use test; +CREATE FUNCTION f() RETURNS INT RETURN 1; +CREATE VIEW v1 AS SELECT f(); + +--echo # Running mysqldump -uroot test --routines --tables v1 > **vardir**/test.dmp +--exec $MYSQL_DUMP -uroot test --routines --tables v1 > $MYSQLTEST_VARDIR/test.dmp + +DROP VIEW v1; +DROP FUNCTION f; + +--echo # Running mysql -uroot test < **vardir**/test.dmp +--exec $MYSQL -uroot test < $MYSQLTEST_VARDIR/test.dmp + +--echo # +--echo # Cleanup after succesful import. +--echo # +DROP VIEW v1; +DROP FUNCTION f; From 719ed09e5e96b53467ede331631de641f98a348f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 18:25:11 +0300 Subject: [PATCH 121/276] Update test results post-merge --- mysql-test/r/limit.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/r/limit.result b/mysql-test/r/limit.result index deb4ca2ab95..6c295bb86af 100644 --- a/mysql-test/r/limit.result +++ b/mysql-test/r/limit.result @@ -154,7 +154,7 @@ SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Zero limit SELECT * FROM mysql.slow_log WHERE sql_text != 'foo' LIMIT 0; -start_time user_host query_time lock_time rows_sent rows_examined db last_insert_id insert_id server_id sql_text +start_time user_host query_time lock_time rows_sent rows_examined db last_insert_id insert_id server_id sql_text thread_id EXPLAIN SELECT * FROM mysql.help_topic WHERE help_category_id != example LIMIT 0; id select_type table type possible_keys key key_len ref rows Extra From e5a3d24b876a266c67b9a6f4390ae31435ec746d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 21:45:05 +0300 Subject: [PATCH 122/276] Followup for make TokuDB compile with GCC-8 Missed printfs from: 21246066b296bd9c6e44272a8f93cb7dc73ea43c --- storage/tokudb/PerconaFT/ft/ft-ops.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/tokudb/PerconaFT/ft/ft-ops.cc b/storage/tokudb/PerconaFT/ft/ft-ops.cc index d7ea5efdff5..eb8696423ca 100644 --- a/storage/tokudb/PerconaFT/ft/ft-ops.cc +++ b/storage/tokudb/PerconaFT/ft/ft-ops.cc @@ -826,7 +826,7 @@ int toku_ftnode_fetch_callback(CACHEFILE UU(cachefile), __FILE__, __LINE__, toku_cachefile_fname_in_env(cachefile), - blocknum.b); + (longlong)blocknum.b); } else { fprintf( stderr, @@ -836,7 +836,7 @@ int toku_ftnode_fetch_callback(CACHEFILE UU(cachefile), __FILE__, __LINE__, toku_cachefile_fname_in_env(cachefile), - blocknum.b, + (longlong)blocknum.b, r); } // make absolutely sure we crash before doing anything else. From 24d7cbe1e0a16f75e2325c84f23531742e2a035d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Sun, 10 Jun 2018 16:26:57 +0300 Subject: [PATCH 123/276] Ensure TokuDB compiles both on Linux and OS X On OS X, (u)int64_t is defined as (unsigned) long long int while on 74 bit Linux it is defined as (unsigned) long int. Ensure the type matches when doing printf on all systems. --- .../PerconaFT/ft/serialize/ft-serialize.cc | 10 ++-- .../ft/serialize/ft_node-serialize.cc | 58 +++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc index fe68e1b20e0..0d6573972d7 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft-serialize.cc @@ -663,13 +663,13 @@ exit: __LINE__, \ fn, \ r, \ - max_acceptable_lsn.lsn, \ + (ulonglong)max_acceptable_lsn.lsn, \ r0, \ - checkpoint_lsn_0.lsn, \ - checkpoint_count_0, \ + (ulonglong)checkpoint_lsn_0.lsn, \ + (ulonglong)checkpoint_count_0, \ r1, \ - checkpoint_lsn_1.lsn, \ - checkpoint_count_1); + (ulonglong)checkpoint_lsn_1.lsn, \ + (ulonglong)checkpoint_count_1); int toku_deserialize_ft_from(int fd, const char *fn, diff --git a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc index 02d0cc691cd..46bb8f81412 100644 --- a/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc +++ b/storage/tokudb/PerconaFT/ft/serialize/ft_node-serialize.cc @@ -1174,7 +1174,7 @@ int verify_ftnode_sub_block(struct sub_block *sb, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, stored_xsum, actual_xsum); dump_bad_block((Bytef *) sb->uncompressed_ptr, sb->uncompressed_size); @@ -1201,7 +1201,7 @@ static int deserialize_ftnode_info(struct sub_block *sb, FTNODE node) { __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); dump_bad_block(static_cast(sb->uncompressed_ptr), sb->uncompressed_size); @@ -1257,7 +1257,7 @@ static int deserialize_ftnode_info(struct sub_block *sb, FTNODE node) { __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, data_size, rb.ndone); dump_bad_block(rb.buf, rb.size); @@ -1393,7 +1393,7 @@ static int deserialize_ftnode_partition( __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); goto exit; } @@ -1415,7 +1415,7 @@ static int deserialize_ftnode_partition( __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, ch, FTNODE_PARTITION_MSG_BUFFER); dump_bad_block(rb.buf, rb.size); @@ -1438,7 +1438,7 @@ static int deserialize_ftnode_partition( __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, ch, FTNODE_PARTITION_DMT_LEAVES); dump_bad_block(rb.buf, rb.size); @@ -1461,7 +1461,7 @@ static int deserialize_ftnode_partition( __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, rb.ndone, rb.size); dump_bad_block(rb.buf, rb.size); @@ -1490,7 +1490,7 @@ static int decompress_and_deserialize_worker(struct rbuf curr_rbuf, __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); dump_bad_block(curr_rbuf.buf, curr_rbuf.size); goto exit; @@ -1507,7 +1507,7 @@ static int decompress_and_deserialize_worker(struct rbuf curr_rbuf, __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); dump_bad_block(curr_rbuf.buf, curr_rbuf.size); goto exit; @@ -1586,7 +1586,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, rb->size); dump_bad_block(rb->buf, rb->size); // TODO: What error do we return here? @@ -1607,7 +1607,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, static_cast(magic)[0], static_cast(magic)[1], static_cast(magic)[2], @@ -1632,7 +1632,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, node->layout_version_read_from_disk, FT_FIRST_LAYOUT_VERSION_WITH_BASEMENT_NODES); dump_bad_block(rb->buf, rb->size); @@ -1671,7 +1671,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, needed_size, rb->size); dump_bad_block(rb->buf, rb->size); @@ -1699,7 +1699,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, stored_checksum, checksum); dump_bad_block(rb->buf, rb->size); @@ -1722,7 +1722,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, rb->size, rb->ndone, sb_node_info.compressed_size); @@ -1748,7 +1748,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, sb_node_info.xsum, actual_xsum); dump_bad_block(rb->buf, rb->size); @@ -1779,7 +1779,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block( static_cast(sb_node_info.uncompressed_ptr), @@ -1817,7 +1817,7 @@ static int deserialize_ftnode_header_from_rbuf_if_small_enough( __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block(rb->buf, rb->size); goto cleanup; @@ -2169,7 +2169,7 @@ static int deserialize_and_upgrade_ftnode(FTNODE node, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); goto exit; } @@ -2195,7 +2195,7 @@ static int deserialize_and_upgrade_ftnode(FTNODE node, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, version, FT_LAYOUT_VERSION_14); dump_bad_block(rb.buf, rb.size); @@ -2283,7 +2283,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, static_cast(magic)[0], static_cast(magic)[1], static_cast(magic)[2], @@ -2314,7 +2314,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block(rb->buf, rb->size); goto cleanup; @@ -2359,7 +2359,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, stored_checksum, checksum); dump_bad_block(rb->buf, rb->size); @@ -2382,7 +2382,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block( static_cast(sb_node_info.uncompressed_ptr), @@ -2403,7 +2403,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block(rb->buf, rb->size); goto cleanup; @@ -2475,7 +2475,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, i, r); dump_bad_block(rb->buf, rb->size); @@ -2496,7 +2496,7 @@ static int deserialize_ftnode_from_rbuf(FTNODE *ftnode, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, i, r); dump_bad_block(rb->buf, rb->size); @@ -2646,7 +2646,7 @@ int toku_deserialize_bp_from_compressed(FTNODE node, __FILE__, __LINE__, fname ? fname : "unknown", - node->blocknum.b, + (longlong)node->blocknum.b, r); dump_bad_block(static_cast(curr_sb->compressed_ptr), curr_sb->compressed_size); @@ -2694,7 +2694,7 @@ static int deserialize_ftnode_from_fd(int fd, __FILE__, __LINE__, fname ? fname : "unknown", - blocknum.b, + (longlong)blocknum.b, r); dump_bad_block(rb.buf, rb.size); } From e7ca377cb73a64e40832e9306194412fb07b8fa5 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 5 Jun 2018 15:21:45 +0200 Subject: [PATCH 124/276] MDEV-16342 SHOW ENGINES: MyISAM description is useless rewrite tautological engine descriptions --- mysql-test/r/information_schema.result | 2 +- mysql-test/suite/federated/have_federatedx.inc | 2 +- mysql-test/suite/funcs_1/r/is_engines_archive.result | 2 +- mysql-test/suite/funcs_1/r/is_engines_csv.result | 2 +- mysql-test/suite/funcs_1/r/is_engines_federated.result | 2 +- mysql-test/suite/funcs_1/r/is_engines_myisam.result | 2 +- storage/archive/ha_archive.cc | 2 +- storage/csv/ha_tina.cc | 2 +- storage/federated/ha_federated.cc | 2 +- storage/federatedx/ha_federatedx.cc | 2 +- storage/myisam/ha_myisam.cc | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mysql-test/r/information_schema.result b/mysql-test/r/information_schema.result index 1f765a70137..dfa7b12562d 100644 --- a/mysql-test/r/information_schema.result +++ b/mysql-test/r/information_schema.result @@ -1387,7 +1387,7 @@ USE test; End of 5.0 tests. select * from information_schema.engines WHERE ENGINE="MyISAM"; ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -MyISAM DEFAULT MyISAM storage engine NO NO NO +MyISAM DEFAULT Non-transactional engine with good performance and small data footprint NO NO NO grant select on *.* to user3148@localhost; select user,db from information_schema.processlist; user db diff --git a/mysql-test/suite/federated/have_federatedx.inc b/mysql-test/suite/federated/have_federatedx.inc index 56ce31f5b2f..2250dd205bd 100644 --- a/mysql-test/suite/federated/have_federatedx.inc +++ b/mysql-test/suite/federated/have_federatedx.inc @@ -1,5 +1,5 @@ if (!`SELECT count(*) FROM information_schema.plugins WHERE plugin_name = 'federated' AND plugin_status = 'active' AND - plugin_description LIKE '%FederatedX%'`){ + plugin_description LIKE '%transactions%'`){ skip Need FederatedX engine; } diff --git a/mysql-test/suite/funcs_1/r/is_engines_archive.result b/mysql-test/suite/funcs_1/r/is_engines_archive.result index 2772992495c..52802b17acd 100644 --- a/mysql-test/suite/funcs_1/r/is_engines_archive.result +++ b/mysql-test/suite/funcs_1/r/is_engines_archive.result @@ -2,7 +2,7 @@ SELECT * FROM information_schema.engines WHERE ENGINE = 'ARCHIVE'; ENGINE ARCHIVE SUPPORT YES -COMMENT Archive storage engine +COMMENT gzip-compresses tables for a low storage footprint TRANSACTIONS NO XA NO SAVEPOINTS NO diff --git a/mysql-test/suite/funcs_1/r/is_engines_csv.result b/mysql-test/suite/funcs_1/r/is_engines_csv.result index 2a7e61ee4d3..7e413b9af6f 100644 --- a/mysql-test/suite/funcs_1/r/is_engines_csv.result +++ b/mysql-test/suite/funcs_1/r/is_engines_csv.result @@ -2,7 +2,7 @@ SELECT * FROM information_schema.engines WHERE ENGINE = 'CSV'; ENGINE CSV SUPPORT YES -COMMENT CSV storage engine +COMMENT Stores tables as CSV files TRANSACTIONS NO XA NO SAVEPOINTS NO diff --git a/mysql-test/suite/funcs_1/r/is_engines_federated.result b/mysql-test/suite/funcs_1/r/is_engines_federated.result index 8057a0266c5..20926458ed0 100644 --- a/mysql-test/suite/funcs_1/r/is_engines_federated.result +++ b/mysql-test/suite/funcs_1/r/is_engines_federated.result @@ -2,7 +2,7 @@ SELECT * FROM information_schema.engines WHERE ENGINE = 'FEDERATED'; ENGINE FEDERATED SUPPORT YES -COMMENT FederatedX pluggable storage engine +COMMENT Allows to access tables on other MariaDB servers, supports transactions and more TRANSACTIONS YES XA NO SAVEPOINTS YES diff --git a/mysql-test/suite/funcs_1/r/is_engines_myisam.result b/mysql-test/suite/funcs_1/r/is_engines_myisam.result index 7e42c864187..d307ce4be6a 100644 --- a/mysql-test/suite/funcs_1/r/is_engines_myisam.result +++ b/mysql-test/suite/funcs_1/r/is_engines_myisam.result @@ -2,7 +2,7 @@ SELECT * FROM information_schema.engines WHERE ENGINE = 'MyISAM'; ENGINE MyISAM SUPPORT DEFAULT -COMMENT MyISAM storage engine +COMMENT Non-transactional engine with good performance and small data footprint TRANSACTIONS NO XA NO SAVEPOINTS NO diff --git a/storage/archive/ha_archive.cc b/storage/archive/ha_archive.cc index d39d1187ce6..3b8e58134b0 100644 --- a/storage/archive/ha_archive.cc +++ b/storage/archive/ha_archive.cc @@ -1840,7 +1840,7 @@ maria_declare_plugin(archive) &archive_storage_engine, "ARCHIVE", "Brian Aker, MySQL AB", - "Archive storage engine", + "gzip-compresses tables for a low storage footprint", PLUGIN_LICENSE_GPL, archive_db_init, /* Plugin Init */ archive_db_done, /* Plugin Deinit */ diff --git a/storage/csv/ha_tina.cc b/storage/csv/ha_tina.cc index 54a85ec3a44..fe39d0e2fa6 100644 --- a/storage/csv/ha_tina.cc +++ b/storage/csv/ha_tina.cc @@ -1800,7 +1800,7 @@ maria_declare_plugin(csv) &csv_storage_engine, "CSV", "Brian Aker, MySQL AB", - "CSV storage engine", + "Stores tables as CSV files", PLUGIN_LICENSE_GPL, tina_init_func, /* Plugin Init */ tina_done_func, /* Plugin Deinit */ diff --git a/storage/federated/ha_federated.cc b/storage/federated/ha_federated.cc index 8680c3aac25..c621653f9f0 100644 --- a/storage/federated/ha_federated.cc +++ b/storage/federated/ha_federated.cc @@ -3503,7 +3503,7 @@ maria_declare_plugin(federated) &federated_storage_engine, "FEDERATED", "Patrick Galbraith and Brian Aker, MySQL AB", - "Federated MySQL storage engine", + "Allows to access tables on other MariaDB servers", PLUGIN_LICENSE_GPL, federated_db_init, /* Plugin Init */ federated_done, /* Plugin Deinit */ diff --git a/storage/federatedx/ha_federatedx.cc b/storage/federatedx/ha_federatedx.cc index bafae614fab..647cc57e101 100644 --- a/storage/federatedx/ha_federatedx.cc +++ b/storage/federatedx/ha_federatedx.cc @@ -3608,7 +3608,7 @@ maria_declare_plugin(federatedx) &federatedx_storage_engine, "FEDERATED", "Patrick Galbraith", - "FederatedX pluggable storage engine", + "Allows to access tables on other MariaDB servers, supports transactions and more", PLUGIN_LICENSE_GPL, federatedx_db_init, /* Plugin Init */ federatedx_done, /* Plugin Deinit */ diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index f63b9c85372..4305fab1778 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -2346,7 +2346,7 @@ maria_declare_plugin(myisam) &myisam_storage_engine, "MyISAM", "MySQL AB", - "MyISAM storage engine", + "Non-transactional engine with good performance and small data footprint", PLUGIN_LICENSE_GPL, myisam_init, /* Plugin Init */ NULL, /* Plugin Deinit */ From 6da8192174f033c2958ddafb2a15c14360bb1ecc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 10 Jun 2018 17:23:53 +0200 Subject: [PATCH 125/276] mysqltest: Allow HANDLER READ in --ps-protocol tests adjust tests --- client/mysqltest.cc | 1 + mysql-test/r/lock.result | 2 +- mysql-test/suite/handler/handler.inc | 2 ++ mysql-test/suite/handler/interface.result | 2 +- mysql-test/suite/handler/interface.test | 2 +- mysql-test/t/lock.test | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 98e35136b2a..43dbff981d3 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -8672,6 +8672,7 @@ void init_re(void) "[[:space:]]*SELECT[[:space:]]|" "[[:space:]]*CREATE[[:space:]]+TABLE[[:space:]]|" "[[:space:]]*DO[[:space:]]|" + "[[:space:]]*HANDLER[[:space:]]+.*[[:space:]]+READ[[:space:]]|" "[[:space:]]*SET[[:space:]]+OPTION[[:space:]]|" "[[:space:]]*DELETE[[:space:]]+MULTI[[:space:]]|" "[[:space:]]*UPDATE[[:space:]]+MULTI[[:space:]]|" diff --git a/mysql-test/r/lock.result b/mysql-test/r/lock.result index 501c379b257..0dcc0de828f 100644 --- a/mysql-test/r/lock.result +++ b/mysql-test/r/lock.result @@ -407,7 +407,7 @@ LOCK TABLE t1 WRITE; HANDLER t1 OPEN; ERROR HY000: Can't execute the given command because you have active locked tables or an active transaction HANDLER t1 READ FIRST; -ERROR HY000: Can't execute the given command because you have active locked tables or an active transaction +Got one of the listed errors HANDLER t1 CLOSE; ERROR HY000: Can't execute the given command because you have active locked tables or an active transaction UNLOCK TABLES; diff --git a/mysql-test/suite/handler/handler.inc b/mysql-test/suite/handler/handler.inc index 5b2dd5ef7e9..bfc21a3b2c3 100644 --- a/mysql-test/suite/handler/handler.inc +++ b/mysql-test/suite/handler/handler.inc @@ -377,7 +377,9 @@ send optimize table t1; # client 1 --echo proceed with the normal connection connection default; +--disable_ps_protocol handler t1 read next; +--enable_ps_protocol handler t1 close; # client 2 --echo read the result from the other connection diff --git a/mysql-test/suite/handler/interface.result b/mysql-test/suite/handler/interface.result index fb633bb94c4..c9cffba33f7 100644 --- a/mysql-test/suite/handler/interface.result +++ b/mysql-test/suite/handler/interface.result @@ -269,7 +269,7 @@ handler t1 open; lock table t1 write; alter table t1 engine=csv; handler t1 read a next; -ERROR HY000: Table storage engine for 't1' doesn't have this option +Got one of the listed errors handler t1 close; unlock tables; drop table t1; diff --git a/mysql-test/suite/handler/interface.test b/mysql-test/suite/handler/interface.test index 2ef617c3ce7..06797ed6980 100644 --- a/mysql-test/suite/handler/interface.test +++ b/mysql-test/suite/handler/interface.test @@ -326,7 +326,7 @@ let $wait_condition= info = "alter table t1 engine=csv"; --source include/wait_condition.inc connection default; ---error ER_ILLEGAL_HA +--error ER_ILLEGAL_HA,ER_KEY_DOES_NOT_EXITS handler t1 read a next; handler t1 close; connection con1; diff --git a/mysql-test/t/lock.test b/mysql-test/t/lock.test index 78f0e2ecf8d..6cfaf9fc320 100644 --- a/mysql-test/t/lock.test +++ b/mysql-test/t/lock.test @@ -481,7 +481,7 @@ LOCK TABLE t1 WRITE; --echo # HANDLER commands are not allowed in LOCK TABLES mode --error ER_LOCK_OR_ACTIVE_TRANSACTION HANDLER t1 OPEN; ---error ER_LOCK_OR_ACTIVE_TRANSACTION +--error ER_LOCK_OR_ACTIVE_TRANSACTION,ER_UNKNOWN_TABLE HANDLER t1 READ FIRST; --error ER_LOCK_OR_ACTIVE_TRANSACTION HANDLER t1 CLOSE; From ca733d03c82b02cd842ff2a226fee7b12eb86f8d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 10 Jun 2018 21:19:11 +0200 Subject: [PATCH 126/276] MDEV-15729 Server crashes in Field::make_field upon HANDLER READ executed with PS protocol update table->pos_in_table_list during prepare, just like it's done in normal execution. otherwise it'll be a dangling pointer --- mysql-test/suite/handler/ps.result | 9 +++++++++ mysql-test/suite/handler/ps.test | 11 +++++++++++ sql/sql_handler.cc | 1 + 3 files changed, 21 insertions(+) create mode 100644 mysql-test/suite/handler/ps.result create mode 100644 mysql-test/suite/handler/ps.test diff --git a/mysql-test/suite/handler/ps.result b/mysql-test/suite/handler/ps.result new file mode 100644 index 00000000000..54685f9156b --- /dev/null +++ b/mysql-test/suite/handler/ps.result @@ -0,0 +1,9 @@ +create table t1 (i int); +handler test.t1 open handler_a; +flush status; +handler handler_a read first; +i +show status like 'Com_stmt_prepare%'; +Variable_name Value +Com_stmt_prepare OK +drop table t1; diff --git a/mysql-test/suite/handler/ps.test b/mysql-test/suite/handler/ps.test new file mode 100644 index 00000000000..68091190c85 --- /dev/null +++ b/mysql-test/suite/handler/ps.test @@ -0,0 +1,11 @@ +# +# MDEV-15729 Server crashes in Field::make_field upon HANDLER READ executed with PS protocol +# +create table t1 (i int); +handler test.t1 open handler_a; +flush status; +handler handler_a read first; +# handler...read must be prepared in --ps-protocol mode +--replace_result $PS_PROTOCOL OK +show status like 'Com_stmt_prepare%'; +drop table t1; diff --git a/sql/sql_handler.cc b/sql/sql_handler.cc index f5c79e59bf2..778507ebc38 100644 --- a/sql/sql_handler.cc +++ b/sql/sql_handler.cc @@ -980,6 +980,7 @@ SQL_HANDLER *mysql_ha_read_prepare(THD *thd, TABLE_LIST *tables, if (!(handler= mysql_ha_find_handler(thd, tables->alias))) DBUG_RETURN(0); tables->table= handler->table; // This is used by fix_fields + handler->table->pos_in_table_list= tables; if (mysql_ha_fix_cond_and_key(handler, mode, keyname, key_expr, cond, 1)) DBUG_RETURN(0); DBUG_RETURN(handler); From 5227198908d4f12ab3e5c95e4c8f15e6b9805f10 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 11 Jun 2018 16:29:22 +0400 Subject: [PATCH 127/276] MDEV-16190 Server crashes in Item_null_result::field_type on SELECT with time field, ROLLUP and HAVING virtual Item_null_result::get_date() was not overridden. It used the inherited Item::get_date(), which tests field_type(), which in case of Item_null_result calls result_field->field_type(), and result_field is not really always set (e.g. it's not set in the test case from the bug report). Overriding Item_null::get_date() like it's done for other val_xxx() methods. This make the code more symmetric across data types. In the new reduction, get_date() immediately returns NULL without entering into any data type specific code. --- mysql-test/r/olap.result | 22 ++++++++++++++++++++++ mysql-test/t/olap.test | 23 +++++++++++++++++++++++ sql/item.cc | 28 +++++++++++++++++++++------- sql/item.h | 7 +++++++ 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/mysql-test/r/olap.result b/mysql-test/r/olap.result index b10f175b63e..24140583d13 100644 --- a/mysql-test/r/olap.result +++ b/mysql-test/r/olap.result @@ -766,3 +766,25 @@ b NULL DROP TABLE t1, t2; End of 5.0 tests +# +# Start of 10.1 tests +# +# +# MDEV-16190 Server crashes in Item_null_result::field_type on SELECT with time field, ROLLUP and HAVING +# +CREATE TABLE t1 (t TIME) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('12:12:12'); +SELECT t, COUNT(*) FROM t1 GROUP BY t WITH ROLLUP HAVING t > '00:00:00'; +t COUNT(*) +12:12:12 1 +DROP TABLE t1; +CREATE TABLE t1 (t TIME) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('12:12:12'),('12:12:13'); +SELECT t, COUNT(*) FROM t1 GROUP BY t WITH ROLLUP HAVING t > '00:00:00'; +t COUNT(*) +12:12:12 1 +12:12:13 1 +DROP TABLE t1; +# +# End of 10.1 tests +# diff --git a/mysql-test/t/olap.test b/mysql-test/t/olap.test index fec5df1a1c7..4a61cebdc0d 100644 --- a/mysql-test/t/olap.test +++ b/mysql-test/t/olap.test @@ -404,3 +404,26 @@ SELECT DISTINCT b FROM t1, t2 GROUP BY a, b WITH ROLLUP; DROP TABLE t1, t2; --echo End of 5.0 tests + + +--echo # +--echo # Start of 10.1 tests +--echo # + +--echo # +--echo # MDEV-16190 Server crashes in Item_null_result::field_type on SELECT with time field, ROLLUP and HAVING +--echo # +CREATE TABLE t1 (t TIME) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('12:12:12'); +SELECT t, COUNT(*) FROM t1 GROUP BY t WITH ROLLUP HAVING t > '00:00:00'; +DROP TABLE t1; + +CREATE TABLE t1 (t TIME) ENGINE=MyISAM; +INSERT INTO t1 VALUES ('12:12:12'),('12:12:13'); +SELECT t, COUNT(*) FROM t1 GROUP BY t WITH ROLLUP HAVING t > '00:00:00'; +DROP TABLE t1; + + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/sql/item.cc b/sql/item.cc index be04f64862c..e23829c750c 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -1287,7 +1287,7 @@ bool Item::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) ltime, fuzzydate, field_name_or_null())) goto err; - break; + return null_value= false; } case REAL_RESULT: { @@ -1295,7 +1295,7 @@ bool Item::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) if (null_value || double_to_datetime_with_warn(value, ltime, fuzzydate, field_name_or_null())) goto err; - break; + return null_value= false; } case DECIMAL_RESULT: { @@ -1304,7 +1304,7 @@ bool Item::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) decimal_to_datetime_with_warn(res, ltime, fuzzydate, field_name_or_null())) goto err; - break; + return null_value= false; } case STRING_RESULT: { @@ -1314,15 +1314,20 @@ bool Item::get_date(MYSQL_TIME *ltime,ulonglong fuzzydate) str_to_datetime_with_warn(res->charset(), res->ptr(), res->length(), ltime, fuzzydate)) goto err; - break; + return null_value= false; } default: + null_value= true; DBUG_ASSERT(0); } - return null_value= 0; - err: + return null_value|= make_zero_date(ltime, fuzzydate); +} + + +bool Item::make_zero_date(MYSQL_TIME *ltime, ulonglong fuzzydate) +{ /* if the item was not null and convertion failed, we return a zero date if allowed, otherwise - null. @@ -1344,7 +1349,7 @@ err: */ ltime->time_type= MYSQL_TIMESTAMP_TIME; } - return null_value|= !(fuzzydate & TIME_FUZZY_DATES); + return !(fuzzydate & TIME_FUZZY_DATES); } bool Item::get_seconds(ulonglong *sec, ulong *sec_part) @@ -3132,6 +3137,15 @@ my_decimal *Item_null::val_decimal(my_decimal *decimal_value) } +bool Item_null::get_date(MYSQL_TIME *ltime, ulonglong fuzzydate) +{ + // following assert is redundant, because fixed=1 assigned in constructor + DBUG_ASSERT(fixed == 1); + make_zero_date(ltime, fuzzydate); + return (null_value= true); +} + + Item *Item_null::safe_charset_converter(THD *thd, CHARSET_INFO *tocs) { return this; diff --git a/sql/item.h b/sql/item.h index 60073f054c9..139ce40b25b 100644 --- a/sql/item.h +++ b/sql/item.h @@ -663,6 +663,12 @@ protected: SEL_TREE *get_mm_tree_for_const(RANGE_OPT_PARAM *param); Field *create_tmp_field(bool group, TABLE *table, uint convert_int_length); + /* + This method is used if the item was not null but convertion to + TIME/DATE/DATETIME failed. We return a zero date if allowed, + otherwise - null. + */ + bool make_zero_date(MYSQL_TIME *ltime, ulonglong fuzzydate); public: /* @@ -2565,6 +2571,7 @@ public: longlong val_int(); String *val_str(String *str); my_decimal *val_decimal(my_decimal *); + bool get_date(MYSQL_TIME *ltime, ulonglong fuzzydate); int save_in_field(Field *field, bool no_conversions); int save_safe_in_field(Field *field); bool send(Protocol *protocol, String *str); From 147744d455d57c753ca75fccca106d3326d251ab Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Mon, 11 Jun 2018 08:52:26 -0700 Subject: [PATCH 128/276] MDEV-16235 Server crashes in my_utf8_uni or in my_strtod_int upon SELECT .. LIMIT 0 (new variant) This is another attempt to fix the problem of mdev-14515. --- mysql-test/r/having.result | 11 +++++++++++ mysql-test/r/subselect4.result | 4 ++-- mysql-test/t/having.test | 9 +++++++++ sql/opt_subselect.cc | 3 ++- sql/sql_select.cc | 1 - 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/having.result b/mysql-test/r/having.result index 18915da1a09..9e0466efb9f 100644 --- a/mysql-test/r/having.result +++ b/mysql-test/r/having.result @@ -712,3 +712,14 @@ a ct 4 2 set sql_mode=@save_sql_mode; drop table t1; +# +# mdev-16235: impossible HAVING in query without aggregation +# +explain extended +select * from mysql.help_topic where example = 'foo' having description is null; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL Impossible HAVING +Warnings: +Note 1003 select `mysql`.`help_topic`.`help_topic_id` AS `help_topic_id`,`mysql`.`help_topic`.`name` AS `name`,`mysql`.`help_topic`.`help_category_id` AS `help_category_id`,`mysql`.`help_topic`.`description` AS `description`,`mysql`.`help_topic`.`example` AS `example`,`mysql`.`help_topic`.`url` AS `url` from `mysql`.`help_topic` where (`mysql`.`help_topic`.`example` = 'foo') having 0 +select * from mysql.help_topic where example = 'foo' having description is null; +help_topic_id name help_category_id description example url diff --git a/mysql-test/r/subselect4.result b/mysql-test/r/subselect4.result index 8973330c7da..f726bcfa15e 100644 --- a/mysql-test/r/subselect4.result +++ b/mysql-test/r/subselect4.result @@ -1056,7 +1056,7 @@ EXPLAIN SELECT * FROM t1 WHERE (2, 0) NOT IN (SELECT min(f3)+f3, min(f4)+f3+max(f4) FROM t2 WHERE f3 > 10); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE -2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL No matching min/max row SELECT * FROM t1 WHERE (2, 0) NOT IN (SELECT min(f3)+f3, min(f4)+f3+max(f4) FROM t2 WHERE f3 > 10); f1 f2 SET @@optimizer_switch = 'materialization=off,in_to_exists=on,semijoin=off'; @@ -1147,7 +1147,7 @@ EXPLAIN SELECT * FROM t1 WHERE (2, 0) NOT IN (SELECT min(f3)+f3, min(f4)+f3+max(f4) FROM t2 WHERE f3 > 10); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE -2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +2 SUBQUERY NULL NULL NULL NULL NULL NULL NULL No matching min/max row SELECT * FROM t1 WHERE (2, 0) NOT IN (SELECT min(f3)+f3, min(f4)+f3+max(f4) FROM t2 WHERE f3 > 10); f1 f2 set @@optimizer_switch=@save_optimizer_switch; diff --git a/mysql-test/t/having.test b/mysql-test/t/having.test index a470f462d6a..c8bd47940cc 100644 --- a/mysql-test/t/having.test +++ b/mysql-test/t/having.test @@ -744,3 +744,12 @@ SELECT a, COUNT(a) as ct FROM t1 GROUP BY a HAVING ct>0; set sql_mode=@save_sql_mode; drop table t1; + +--echo # +--echo # mdev-16235: impossible HAVING in query without aggregation +--echo # + +explain extended +select * from mysql.help_topic where example = 'foo' having description is null; + +select * from mysql.help_topic where example = 'foo' having description is null; diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index a7edd64e68b..ec7b10f20c8 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -5880,6 +5880,7 @@ bool JOIN::choose_tableless_subquery_plan() functions produce empty subquery result. There is no need to further rewrite the subquery because it will not be executed at all. */ + exec_const_cond= 0; return FALSE; } @@ -5911,6 +5912,6 @@ bool JOIN::choose_tableless_subquery_plan() tmp_having= having; } } - exec_const_cond= conds; + exec_const_cond= zero_result_cause ? 0 : conds; return FALSE; } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 016dd594e08..b79431ae2f6 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1147,7 +1147,6 @@ JOIN::optimize() { DBUG_PRINT("info", ("Zero limit")); zero_result_cause= "Zero limit"; - conds= 0; } table_count= top_join_tab_count= 0; error= 0; From ecd4c2b4a953596e15b90c1108aaa74a87d9b945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Mon, 11 Jun 2018 20:24:41 +0300 Subject: [PATCH 129/276] Add missed change from 7ffa82b03c8da12062223d5e332e972d6f828d44 --- client/mysqltest.cc | 2 +- mysql-test/include/wait_until_connected_again.inc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index b962813f558..803550ba7be 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -1507,7 +1507,7 @@ void free_used_memory() dynstr_free(&(*q)->content); my_free((*q)); } - for (i= 0; i < 10; i++) + for (i= 0; i < 12; i++) { if (var_reg[i].alloced_len) my_free(var_reg[i].str_val); diff --git a/mysql-test/include/wait_until_connected_again.inc b/mysql-test/include/wait_until_connected_again.inc index 2e80ea2ed7d..26168d10558 100644 --- a/mysql-test/include/wait_until_connected_again.inc +++ b/mysql-test/include/wait_until_connected_again.inc @@ -11,7 +11,7 @@ let $counter= 5000; let $mysql_errno= 9999; while ($mysql_errno) { - --error 0,ER_SERVER_SHUTDOWN,ER_CONNECTION_KILLED,2002,2006,2013 + --error 0,ER_SERVER_SHUTDOWN,ER_CONNECTION_KILLED,ER_LOCK_WAIT_TIMEOUT,2002,2006,2013 show status; dec $counter; From d54d06760195ed0117882a7b8dacfb6cfc23e823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Mon, 11 Jun 2018 21:16:37 +0300 Subject: [PATCH 130/276] Undo wrong my_free overflow --- client/mysqltest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 803550ba7be..b962813f558 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -1507,7 +1507,7 @@ void free_used_memory() dynstr_free(&(*q)->content); my_free((*q)); } - for (i= 0; i < 12; i++) + for (i= 0; i < 10; i++) { if (var_reg[i].alloced_len) my_free(var_reg[i].str_val); From e425216045c7a998139bd953b0aed83c3a085c8c Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 31 Jan 2018 09:35:38 +0100 Subject: [PATCH 131/276] MDEV-15113: Hang in Aria loghandler Added unregistering writers in case of log error. Added more debugging control about adding/removing writers to the buffers. --- storage/maria/ma_loghandler.c | 219 +++++++++++++++++++++++++++++----- 1 file changed, 187 insertions(+), 32 deletions(-) diff --git a/storage/maria/ma_loghandler.c b/storage/maria/ma_loghandler.c index 2118b3b6ce6..7cca14d0785 100644 --- a/storage/maria/ma_loghandler.c +++ b/storage/maria/ma_loghandler.c @@ -78,6 +78,32 @@ typedef union uchar buffer[TRANSLOG_PAGE_SIZE]; } TRANSLOG_PAGE_SIZE_BUFF; +#define MAX_TRUNSLOG_USED_BUFFERS 3 + +typedef struct +{ + struct st_translog_buffer *buff[MAX_TRUNSLOG_USED_BUFFERS]; + uint8 wrt_ptr; + uint8 unlck_ptr; +} TRUNSLOG_USED_BUFFERS; + +static void +used_buffs_init(TRUNSLOG_USED_BUFFERS *buffs) +{ + buffs->unlck_ptr= buffs->wrt_ptr= 0; +} + +static void +used_buffs_add(TRUNSLOG_USED_BUFFERS *buffs, + struct st_translog_buffer *buff); + +static void +used_buffs_register_unlock(TRUNSLOG_USED_BUFFERS *buffs, + struct st_translog_buffer *buff); + +static void +used_buffs_urgent_unlock(TRUNSLOG_USED_BUFFERS *buffs); + /* min chunk length */ #define TRANSLOG_MIN_CHUNK 3 /* @@ -156,7 +182,28 @@ struct st_translog_buffer TRANSLOG_FILE *file; /* Threads which are waiting for buffer filling/freeing */ mysql_cond_t waiting_filling_buffer; - /* Number of records which are in copy progress */ + /* + Number of records which are in copy progress. + + Controlled via translog_buffer_increase_writers() and + translog_buffer_decrease_writers(). + + 1 Simple case: translog_force_current_buffer_to_finish both called in + the same procedure. + + 2 Simple case: translog_write_variable_record_1group: + translog_advance_pointer() increase writer of the buffer and + translog_buffer_decrease_writers() decrease it. + + Usual case: + 1) translog_advance_pointer (i.e. reserve place for future writing) + increase writers for all buffers where place reserved. + Simpliest case: just all space reserved in one buffer + complex case: end of the first buffer, all second buffer, beginning + of the third buffer. + 2) When we finish with writing translog_chaser_page_next() will be + called and unlock the buffer by decreasing number of writers. + */ uint copy_to_buffer_in_progress; /* list of waiting buffer ready threads */ struct st_my_thread_var *waiting_flush; @@ -214,6 +261,7 @@ struct st_translog_buffer struct st_buffer_cursor { + TRUNSLOG_USED_BUFFERS buffs; /* pointer into the buffer */ uchar *ptr; /* current buffer */ @@ -1663,15 +1711,12 @@ static my_bool translog_create_new_file() DBUG_PRINT("info", ("file_no: %lu", (ulong)file_no)); if (translog_write_file_header()) - DBUG_RETURN(1); + goto error; if (ma_control_file_write_and_force(last_checkpoint_lsn, file_no, max_trid_in_control_file, recovery_failures)) - { - translog_stop_writing(); - DBUG_RETURN(1); - } + goto error; DBUG_RETURN(0); @@ -1711,10 +1756,6 @@ static void translog_buffer_lock(struct st_translog_buffer *buffer) SYNOPSIS translog_buffer_unlock() buffer This buffer which should be unlocked - - RETURN - 0 OK - 1 Error */ static void translog_buffer_unlock(struct st_translog_buffer *buffer) @@ -1908,7 +1949,10 @@ static void translog_finish_page(TRANSLOG_ADDRESS *horizon, (ulong) cursor->buffer->size, (ulong) (cursor->ptr -cursor->buffer->buffer), (uint) cursor->current_page_fill, (uint) left)); - DBUG_ASSERT(LSN_FILE_NO(*horizon) == LSN_FILE_NO(cursor->buffer->offset)); + DBUG_ASSERT(LSN_FILE_NO(*horizon) == LSN_FILE_NO(cursor->buffer->offset) + || translog_status == TRANSLOG_UNINITED); + if ((LSN_FILE_NO(*horizon) != LSN_FILE_NO(cursor->buffer->offset))) + DBUG_VOID_RETURN; // everything wrong do not write to awoid more problems translog_check_cursor(cursor); if (cursor->protected) { @@ -4607,6 +4651,7 @@ static my_bool translog_chaser_page_next(TRANSLOG_ADDRESS *horizon, { translog_buffer_lock(buffer_to_flush); translog_buffer_decrease_writers(buffer_to_flush); + used_buffs_register_unlock(&cursor->buffs, buffer_to_flush); if (!rc) rc= translog_buffer_flush(buffer_to_flush); translog_buffer_unlock(buffer_to_flush); @@ -4711,7 +4756,8 @@ translog_write_variable_record_chunk3_page(struct st_translog_parts *parts, 1 Error */ -static my_bool translog_advance_pointer(int pages, uint16 last_page_data) +static my_bool translog_advance_pointer(int pages, uint16 last_page_data, + TRUNSLOG_USED_BUFFERS *buffs) { translog_size_t last_page_offset= (log_descriptor.page_overhead + last_page_data); @@ -4728,6 +4774,8 @@ static my_bool translog_advance_pointer(int pages, uint16 last_page_data) (uint) last_page_data)); translog_lock_assert_owner(); + used_buffs_init(buffs); + if (pages == -1) { /* @@ -4805,8 +4853,10 @@ static my_bool translog_advance_pointer(int pages, uint16 last_page_data) translog_wait_for_buffer_free(new_buffer); #ifndef DBUG_OFF /* We keep the handler locked so nobody can start this new buffer */ - DBUG_ASSERT(offset == new_buffer->offset && new_buffer->file == NULL && - (file == NULL ? ver : (uint8)(ver + 1)) == new_buffer->ver); + DBUG_ASSERT((offset == new_buffer->offset && new_buffer->file == NULL && + (file == NULL ? ver : (uint8)(ver + 1)) == + new_buffer->ver) || + translog_status == TRANSLOG_READONLY); } #endif @@ -4827,6 +4877,8 @@ static my_bool translog_advance_pointer(int pages, uint16 last_page_data) DBUG_ASSERT(log_descriptor.bc.buffer->buffer_no == log_descriptor.bc.buffer_no); translog_buffer_increase_writers(log_descriptor.bc.buffer); + // register for case of error + used_buffs_add(buffs, log_descriptor.bc.buffer); if (file_end_offset <= buffer_end_offset) { @@ -4837,6 +4889,10 @@ static my_bool translog_advance_pointer(int pages, uint16 last_page_data) (ulong) LSN_FILE_NO(log_descriptor.horizon))); if (translog_create_new_file()) { + struct st_translog_buffer *ob= log_descriptor.bc.buffer; + translog_buffer_unlock(ob); + used_buffs_urgent_unlock(buffs); + translog_buffer_lock(ob); DBUG_RETURN(1); } } @@ -4858,6 +4914,7 @@ end: log_descriptor.bc.ptr+= offset; log_descriptor.bc.buffer->size+= offset; translog_buffer_increase_writers(log_descriptor.bc.buffer); + used_buffs_add(buffs, log_descriptor.bc.buffer); log_descriptor.horizon+= offset; /* offset increasing */ log_descriptor.bc.current_page_fill= last_page_offset; DBUG_PRINT("info", ("NewP buffer #%u: 0x%lx chaser: %d Size: %lu (%lu) " @@ -4878,6 +4935,56 @@ end: DBUG_RETURN(0); } +static void +used_buffs_add(TRUNSLOG_USED_BUFFERS *buffs, + struct st_translog_buffer *buff) +{ + DBUG_ENTER("used_buffs_add"); + DBUG_PRINT("enter", ("ADD buffs: %p unlk %u (%p) wrt_ptr: %u (%p)" + " buff %p (%u)", + buffs, + buffs->wrt_ptr, buffs->buff[buffs->wrt_ptr], + buffs->unlck_ptr, buffs->buff[buffs->unlck_ptr], + buff, buff->buffer_no)); + DBUG_ASSERT(buffs->wrt_ptr < MAX_TRUNSLOG_USED_BUFFERS); + buffs->buff[buffs->wrt_ptr++]= buff; + DBUG_VOID_RETURN; +} + +static void +used_buffs_register_unlock(TRUNSLOG_USED_BUFFERS *buffs, + struct st_translog_buffer *buff + __attribute__((unused)) ) +{ + DBUG_ENTER("used_buffs_register_unlock"); + DBUG_PRINT("enter", ("SUB buffs: %p unlk %u (%p) wrt_ptr: %u (%p)" + " buff %p (%u)", + buffs, + buffs->wrt_ptr, buffs->buff[buffs->wrt_ptr], + buffs->unlck_ptr, buffs->buff[buffs->unlck_ptr], + buff, buff->buffer_no)); + DBUG_ASSERT(buffs->buff[buffs->unlck_ptr] == buff); + buffs->unlck_ptr++; + DBUG_VOID_RETURN; +} +static void used_buffs_urgent_unlock(TRUNSLOG_USED_BUFFERS *buffs) +{ + uint i; + DBUG_ENTER("used_buffs_urgent_unlock"); + translog_lock(); + translog_stop_writing(); + translog_unlock(); + for (i= buffs->unlck_ptr; i < buffs->wrt_ptr; i++) + { + struct st_translog_buffer *buf= buffs->buff[i]; + translog_buffer_lock(buf); + translog_buffer_decrease_writers(buf); + translog_buffer_unlock(buf); + buffs->buff[i]= NULL; + } + used_buffs_init(buffs); + DBUG_VOID_RETURN; +} /* Get page rest @@ -5016,6 +5123,11 @@ translog_write_variable_record_1group(LSN *lsn, lsn, hook_arg))) { translog_unlock(); + if (buffer_to_flush != NULL) + { + translog_buffer_flush(buffer_to_flush); + translog_buffer_unlock(buffer_to_flush); + } DBUG_RETURN(1); } cursor= log_descriptor.bc; @@ -5046,8 +5158,9 @@ translog_write_variable_record_1group(LSN *lsn, (log_descriptor.page_capacity_chunk_2 - 1), record_rest, parts->record_length)); /* record_rest + 3 is chunk type 3 overhead + record_rest */ - rc|= translog_advance_pointer((int)(full_pages + additional_chunk3_page), - (record_rest ? record_rest + 3 : 0)); + rc= translog_advance_pointer((int)(full_pages + additional_chunk3_page), + (record_rest ? record_rest + 3 : 0), + &cursor.buffs); log_descriptor.bc.buffer->last_lsn= *lsn; DBUG_PRINT("info", ("last_lsn set to (%lu,0x%lx) buffer: 0x%lx", LSN_IN_PARTS(log_descriptor.bc.buffer->last_lsn), @@ -5066,7 +5179,11 @@ translog_write_variable_record_1group(LSN *lsn, translog_buffer_unlock(buffer_to_flush); } if (rc) + { + //translog_advance_pointer decreased writers so it is OK + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); DBUG_RETURN(1); + } translog_write_variable_record_1group_header(parts, type, short_trid, header_length, chunk0_header); @@ -5081,7 +5198,7 @@ translog_write_variable_record_1group(LSN *lsn, for (i= 0; i < full_pages; i++) { if (translog_write_variable_record_chunk2_page(parts, &horizon, &cursor)) - DBUG_RETURN(1); + goto error; DBUG_PRINT("info", ("absolute horizon: (%lu,0x%lx) local: (%lu,0x%lx)", LSN_IN_PARTS(log_descriptor.horizon), @@ -5094,7 +5211,7 @@ translog_write_variable_record_1group(LSN *lsn, log_descriptor. page_capacity_chunk_2 - 2, &horizon, &cursor)) - DBUG_RETURN(1); + goto error; DBUG_PRINT("info", ("absolute horizon: (%lu,0x%lx) local: (%lu,0x%lx)", LSN_IN_PARTS(log_descriptor.horizon), LSN_IN_PARTS(horizon))); @@ -5104,17 +5221,22 @@ translog_write_variable_record_1group(LSN *lsn, if (translog_write_variable_record_chunk3_page(parts, record_rest, &horizon, &cursor)) - DBUG_RETURN(1); - DBUG_PRINT("info", ("absolute horizon: (%lu,0x%lx) local: (%lu,0x%lx)", - (ulong) LSN_FILE_NO(log_descriptor.horizon), - (ulong) LSN_OFFSET(log_descriptor.horizon), - (ulong) LSN_FILE_NO(horizon), - (ulong) LSN_OFFSET(horizon))); + goto error; + DBUG_PRINT("info", ("absolute horizon: (%lu,0x%lx) local: (%lu,0x%lx)", + (ulong) LSN_FILE_NO(log_descriptor.horizon), + (ulong) LSN_OFFSET(log_descriptor.horizon), + (ulong) LSN_FILE_NO(horizon), + (ulong) LSN_OFFSET(horizon))); translog_buffer_lock(cursor.buffer); translog_buffer_decrease_writers(cursor.buffer); + used_buffs_register_unlock(&cursor.buffs, cursor.buffer); translog_buffer_unlock(cursor.buffer); - DBUG_RETURN(rc); + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); + DBUG_RETURN(0); +error: + used_buffs_urgent_unlock(&cursor.buffs); + DBUG_RETURN(1); } @@ -5168,7 +5290,8 @@ translog_write_variable_record_1chunk(LSN *lsn, lsn, hook_arg))) { translog_unlock(); - DBUG_RETURN(1); + rc= 1; + goto err; } rc= translog_write_parts_on_page(&log_descriptor.horizon, @@ -5184,6 +5307,7 @@ translog_write_variable_record_1chunk(LSN *lsn, check if we switched buffer and need process it (current buffer is unlocked already => we will not delay other threads */ +err: if (buffer_to_flush != NULL) { if (!rc) @@ -5523,9 +5647,11 @@ translog_write_variable_record_mgroup(LSN *lsn, uint file_of_the_first_group; int pages_to_skip; struct st_translog_buffer *buffer_of_last_lsn; + my_bool external_buffer_to_flush= TRUE; DBUG_ENTER("translog_write_variable_record_mgroup"); translog_lock_assert_owner(); + used_buffs_init(&cursor.buffs); chunk2_header[0]= TRANSLOG_CHUNK_NOHDR; if (my_init_dynamic_array(&groups, @@ -5533,6 +5659,11 @@ translog_write_variable_record_mgroup(LSN *lsn, 10, 10)) { translog_unlock(); + if (buffer_to_flush != NULL) + { + translog_buffer_flush(buffer_to_flush); + translog_buffer_unlock(buffer_to_flush); + } DBUG_PRINT("error", ("init array failed")); DBUG_RETURN(1); } @@ -5559,6 +5690,7 @@ translog_write_variable_record_mgroup(LSN *lsn, translog_mark_file_unfinished(file_of_the_first_group); do { + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); group.addr= horizon= log_descriptor.horizon; cursor= log_descriptor.bc; cursor.chaser= 1; @@ -5591,21 +5723,26 @@ translog_write_variable_record_mgroup(LSN *lsn, (ulong)(parts->record_length - (first_page - 1 + buffer_rest) - done))); - rc|= translog_advance_pointer((int)full_pages, 0); + rc= translog_advance_pointer((int)full_pages, 0, &cursor.buffs); translog_unlock(); if (buffer_to_flush != NULL) { - translog_buffer_decrease_writers(buffer_to_flush); + if (!external_buffer_to_flush) + translog_buffer_decrease_writers(buffer_to_flush); if (!rc) rc= translog_buffer_flush(buffer_to_flush); translog_buffer_unlock(buffer_to_flush); buffer_to_flush= NULL; } + external_buffer_to_flush= FALSE; + if (rc) { DBUG_PRINT("error", ("flush of unlock buffer failed")); + //translog_advance_pointer decreased writers so it is OK + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); goto err; } @@ -5642,6 +5779,7 @@ translog_write_variable_record_mgroup(LSN *lsn, } translog_buffer_lock(cursor.buffer); translog_buffer_decrease_writers(cursor.buffer); + used_buffs_register_unlock(&cursor.buffs, cursor.buffer); translog_buffer_unlock(cursor.buffer); translog_lock(); @@ -5656,6 +5794,11 @@ translog_write_variable_record_mgroup(LSN *lsn, first_page= translog_get_current_page_rest(); } buffer_rest= translog_get_current_group_size(); + + if (buffer_to_flush) + used_buffs_register_unlock(&cursor.buffs, + buffer_to_flush); // will be unlocked + } while ((translog_size_t)(first_page + buffer_rest) < (translog_size_t)(parts->record_length - done)); @@ -5751,17 +5894,21 @@ translog_write_variable_record_mgroup(LSN *lsn, (ulong) full_pages * log_descriptor.page_capacity_chunk_2, chunk3_pages, (uint) chunk3_size, (uint) record_rest)); + + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); rc= translog_advance_pointer(pages_to_skip + (int)(chunk0_pages - 1), record_rest + header_fixed_part + (groups.elements - ((page_capacity - header_fixed_part) / (7 + 1)) * - (chunk0_pages - 1)) * (7 + 1)); + (chunk0_pages - 1)) * (7 + 1), + &cursor.buffs); buffer_of_last_lsn= log_descriptor.bc.buffer; translog_unlock(); if (buffer_to_flush != NULL) { + DBUG_ASSERT(!external_buffer_to_flush); translog_buffer_decrease_writers(buffer_to_flush); if (!rc) rc= translog_buffer_flush(buffer_to_flush); @@ -5925,8 +6072,10 @@ translog_write_variable_record_mgroup(LSN *lsn, } while (chunk0_pages != 0); translog_buffer_lock(cursor.buffer); translog_buffer_decrease_writers(cursor.buffer); + used_buffs_register_unlock(&cursor.buffs, cursor.buffer); translog_buffer_unlock(cursor.buffer); rc= 0; + DBUG_ASSERT(cursor.buffs.unlck_ptr == cursor.buffs.wrt_ptr); if (translog_set_lsn_for_files(file_of_the_first_group, LSN_FILE_NO(*lsn), *lsn, FALSE)) @@ -5935,17 +6084,22 @@ translog_write_variable_record_mgroup(LSN *lsn, translog_mark_file_finished(file_of_the_first_group); delete_dynamic(&groups); - DBUG_RETURN(rc); + DBUG_RETURN(0); err_unlock: translog_unlock(); err: + + if (cursor.buffs.unlck_ptr != cursor.buffs.wrt_ptr) + used_buffs_urgent_unlock(&cursor.buffs); + if (buffer_to_flush != NULL) { /* This is to prevent locking buffer forever in case of error */ - translog_buffer_decrease_writers(buffer_to_flush); + if (!external_buffer_to_flush) + translog_buffer_decrease_writers(buffer_to_flush); if (!rc) rc= translog_buffer_flush(buffer_to_flush); translog_buffer_unlock(buffer_to_flush); @@ -7500,7 +7654,8 @@ static void translog_force_current_buffer_to_finish() DBUG_ASSERT(log_descriptor.bc.ptr !=NULL); DBUG_ASSERT(LSN_FILE_NO(log_descriptor.horizon) == - LSN_FILE_NO(old_buffer->offset)); + LSN_FILE_NO(old_buffer->offset) || + translog_status == TRANSLOG_READONLY ); translog_check_cursor(&log_descriptor.bc); DBUG_ASSERT(left < TRANSLOG_PAGE_SIZE); if (left) From 26be5072429082e634b8fc102609370975443439 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 12 Jun 2018 10:41:25 +0400 Subject: [PATCH 132/276] MDEV-12060 Crash in EXECUTE IMMEDIATE with an expression returning a GRANT command Backporting (partially) the fix for MDEV-14603. --- mysql-test/r/ps.result | 42 ++++++++++++++++++++++++++++++++ mysql-test/t/ps.test | 55 ++++++++++++++++++++++++++++++++++++++++++ sql/sql_prepare.cc | 38 +++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+) diff --git a/mysql-test/r/ps.result b/mysql-test/r/ps.result index e63b820d4dd..1b40a734263 100644 --- a/mysql-test/r/ps.result +++ b/mysql-test/r/ps.result @@ -4351,3 +4351,45 @@ LINE2 2 LINE3 3 drop table t1; # End of 5.5 tests +# +# Start of 10.1 tests +# +# +# MDEV-12060 Crash in EXECUTE IMMEDIATE with an expression returning a GRANT command +# (the 10.1 part) +# +CREATE PROCEDURE p2 () +BEGIN +SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +END; +/ +CALL p2(); +1 +1 +DROP PROCEDURE p2; +BEGIN NOT ATOMIC +SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +END; +/ +1 +1 +BEGIN NOT ATOMIC +SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; +DEALLOCATE PREPARE stmt; +END; +/ +BEGIN NOT ATOMIC +PREPARE stmt FROM 'SELECT 1'; +SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR EXECUTE stmt; +DEALLOCATE PREPARE stmt; +END; +/ +1 +1 +# +# End of 10.1 tests +# diff --git a/mysql-test/t/ps.test b/mysql-test/t/ps.test index f7008f570b8..e1e8dfeeb17 100644 --- a/mysql-test/t/ps.test +++ b/mysql-test/t/ps.test @@ -3876,3 +3876,58 @@ FROM ) X; drop table t1; --echo # End of 5.5 tests + +--echo # +--echo # Start of 10.1 tests +--echo # + +--echo # +--echo # MDEV-12060 Crash in EXECUTE IMMEDIATE with an expression returning a GRANT command +--echo # (the 10.1 part) +--echo # + +DELIMITER /; +CREATE PROCEDURE p2 () +BEGIN + SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; +END; +/ +DELIMITER ;/ +CALL p2(); +DROP PROCEDURE p2; + + +DELIMITER /; +BEGIN NOT ATOMIC + SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; +END; +/ +DELIMITER ;/ + + +DELIMITER /; +BEGIN NOT ATOMIC + SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR PREPARE stmt FROM 'SELECT 1'; + DEALLOCATE PREPARE stmt; +END; +/ +DELIMITER ;/ + + +DELIMITER /; +BEGIN NOT ATOMIC + PREPARE stmt FROM 'SELECT 1'; + SET STATEMENT join_cache_level=CAST(CONCAT(_utf8'6',_latin1'') AS INT) FOR EXECUTE stmt; + DEALLOCATE PREPARE stmt; +END; +/ +DELIMITER ;/ + + +--echo # +--echo # End of 10.1 tests +--echo # diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index d39ed6aa637..92d2b1c68b0 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -2736,6 +2736,15 @@ void mysql_sql_stmt_prepare(THD *thd) DBUG_VOID_RETURN; } +#if MYSQL_VERSION_ID < 100200 + /* + Backpoiting MDEV-14603 from 10.2 to 10.1 + Remove the code between #if..#endif when merging. + */ + Item_change_list change_list_save_point; + thd->change_list.move_elements_to(&change_list_save_point); +#endif + if (stmt->prepare(query, query_len)) { /* Statement map deletes the statement on erase */ @@ -2744,6 +2753,15 @@ void mysql_sql_stmt_prepare(THD *thd) else my_ok(thd, 0L, 0L, "Statement prepared"); +#if MYSQL_VERSION_ID < 100200 + /* + Backpoiting MDEV-14603 from 10.2 to 10.1 + Remove the code between #if..#endif when merging. + */ + thd->rollback_item_tree_changes(); + change_list_save_point.move_elements_to(&thd->change_list); +#endif + DBUG_VOID_RETURN; } @@ -3039,7 +3057,27 @@ void mysql_sql_stmt_execute(THD *thd) */ Item *free_list_backup= thd->free_list; thd->free_list= NULL; // Hide the external (e.g. "SET STATEMENT") Items + +#if MYSQL_VERSION_ID < 100200 + /* + Backpoiting MDEV-14603 from 10.2 to 10.1 + Remove the code between #if..#endif when merging. + */ + Item_change_list change_list_save_point; + thd->change_list.move_elements_to(&change_list_save_point); +#endif + (void) stmt->execute_loop(&expanded_query, FALSE, NULL, NULL); + +#if MYSQL_VERSION_ID < 100200 + /* + Backpoiting MDEV-14603 from 10.2 to 10.1 + Remove the code between #if..#endif when merging. + */ + thd->rollback_item_tree_changes(); + change_list_save_point.move_elements_to(&thd->change_list); +#endif + thd->free_items(); // Free items created by execute_loop() /* Now restore the "external" (e.g. "SET STATEMENT") Item list. From 0ad9c3a0160d1dd46139b3e6b6b05d0fba01540b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 11 Jun 2018 13:02:47 +0300 Subject: [PATCH 133/276] MDEV-16456 InnoDB error "returned OS error 71" complains about wrong path When attempting to rename a table to a non-existing database, InnoDB would misleadingly report "OS error 71" when in fact the error code is InnoDB's own (OS_FILE_NOT_FOUND), and not report both pathnames. Errors on rename could occur due to reasons connected to either pathname. os_file_handle_rename_error(): New function, to report errors in renaming files. --- mysql-test/suite/innodb/r/rename_table.result | 5 ++++ .../suite/innodb/t/innodb-mdev7046.test | 7 +++--- mysql-test/suite/innodb/t/rename_table.test | 11 +++++++++ storage/innobase/os/os0file.cc | 24 +++++++++++++++---- storage/xtradb/os/os0file.cc | 22 +++++++++++++++-- 5 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 mysql-test/suite/innodb/r/rename_table.result create mode 100644 mysql-test/suite/innodb/t/rename_table.test diff --git a/mysql-test/suite/innodb/r/rename_table.result b/mysql-test/suite/innodb/r/rename_table.result new file mode 100644 index 00000000000..b2e15c87348 --- /dev/null +++ b/mysql-test/suite/innodb/r/rename_table.result @@ -0,0 +1,5 @@ +call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +RENAME TABLE t1 TO non_existing_db.t1; +ERROR HY000: Error on rename of './test/t1' to './non_existing_db/t1' (errno: -1 "Internal error < 0 (Not system error)") +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-mdev7046.test b/mysql-test/suite/innodb/t/innodb-mdev7046.test index b4085228e02..85a257a1739 100644 --- a/mysql-test/suite/innodb/t/innodb-mdev7046.test +++ b/mysql-test/suite/innodb/t/innodb-mdev7046.test @@ -9,9 +9,10 @@ # Ignore OS errors -call mtr.add_suppression("InnoDB: File ./test/t1*"); -call mtr.add_suppression("InnoDB: Error number*"); -call mtr.add_suppression("InnoDB: File ./test/t1#p#p1#sp#p1sp0.ibd: 'rename' returned OS error*"); +call mtr.add_suppression("InnoDB: File ./test/t1"); +call mtr.add_suppression("InnoDB: Error number"); +call mtr.add_suppression("InnoDB: Cannot rename file '.*/test/t1#[Pp]#p1#[Ss][Pp]#p1sp0\\.ibd' to"); +call mtr.add_suppression("InnoDB: Operating system error number .* in a file operation."); # MDEV-7046: MySQL#74480 - Failing assertion: os_file_status(newpath, &exists, &type) # after Operating system error number 36 in a file operation diff --git a/mysql-test/suite/innodb/t/rename_table.test b/mysql-test/suite/innodb/t/rename_table.test new file mode 100644 index 00000000000..60df8b5d2ec --- /dev/null +++ b/mysql-test/suite/innodb/t/rename_table.test @@ -0,0 +1,11 @@ +--source include/have_innodb.inc + +call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); + +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +--replace_result "\\" "/" +--error ER_ERROR_ON_RENAME +RENAME TABLE t1 TO non_existing_db.t1; + +# Cleanup +DROP TABLE t1; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 8f3f3716fc2..038d59c2170 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -1939,6 +1939,24 @@ loop: #endif } +/** Handle RENAME error. +@param name old name of the file +@param new_name new name of the file */ +static void os_file_handle_rename_error(const char* name, const char* new_name) +{ + if (os_file_get_last_error(true) != OS_FILE_DISK_FULL) { + ib_logf(IB_LOG_LEVEL_ERROR, "Cannot rename file '%s' to '%s'", + name, new_name); + } else if (!os_has_said_disk_full) { + os_has_said_disk_full = true; + /* Disk full error is reported irrespective of the + on_error_silent setting. */ + ib_logf(IB_LOG_LEVEL_ERROR, + "Full disk prevents renaming file '%s' to '%s'", + name, new_name); + } +} + /***********************************************************************//** NOTE! Use the corresponding macro os_file_rename(), not directly this function! Renames a file (can also move it to another directory). It is safest that the @@ -1974,8 +1992,7 @@ os_file_rename_func( return(TRUE); } - os_file_handle_error_no_exit(oldpath, "rename", FALSE); - + os_file_handle_rename_error(oldpath, newpath); return(FALSE); #else int ret; @@ -1983,8 +2000,7 @@ os_file_rename_func( ret = rename(oldpath, newpath); if (ret != 0) { - os_file_handle_error_no_exit(oldpath, "rename", FALSE); - + os_file_handle_rename_error(oldpath, newpath); return(FALSE); } diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index c1ddc83d852..9df42470e5a 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2146,6 +2146,24 @@ loop: #endif } +/** Handle RENAME error. +@param name old name of the file +@param new_name new name of the file */ +static void os_file_handle_rename_error(const char* name, const char* new_name) +{ + if (os_file_get_last_error(true) != OS_FILE_DISK_FULL) { + ib_logf(IB_LOG_LEVEL_ERROR, "Cannot rename file '%s' to '%s'", + name, new_name); + } else if (!os_has_said_disk_full) { + os_has_said_disk_full = true; + /* Disk full error is reported irrespective of the + on_error_silent setting. */ + ib_logf(IB_LOG_LEVEL_ERROR, + "Full disk prevents renaming file '%s' to '%s'", + name, new_name); + } +} + /***********************************************************************//** NOTE! Use the corresponding macro os_file_rename(), not directly this function! Renames a file (can also move it to another directory). It is safest that the @@ -2181,7 +2199,7 @@ os_file_rename_func( return(TRUE); } - os_file_handle_error_no_exit(oldpath, "rename", FALSE); + os_file_handle_rename_error(oldpath, newpath); return(FALSE); #else @@ -2190,7 +2208,7 @@ os_file_rename_func( ret = rename(oldpath, newpath); if (ret != 0) { - os_file_handle_error_no_exit(oldpath, "rename", FALSE); + os_file_handle_rename_error(oldpath, newpath); return(FALSE); } From 8f5f0575ab41bd03369ab1dc31425ef7ab1c6b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 11 Jun 2018 13:02:47 +0300 Subject: [PATCH 134/276] MDEV-16456 InnoDB error "returned OS error 71" complains about wrong path When attempting to rename a table to a non-existing database, InnoDB would misleadingly report "OS error 71" when in fact the error code is InnoDB's own (OS_FILE_NOT_FOUND), and not report both pathnames. Errors on rename could occur due to reasons connected to either pathname. os_file_handle_rename_error(): New function, to report errors in renaming files. --- mysql-test/suite/innodb/r/rename_table.result | 6 +++++ .../suite/innodb/t/innodb-mdev7046.test | 6 ++--- mysql-test/suite/innodb/t/rename_table.test | 14 +++++++++++ storage/innobase/os/os0file.cc | 24 +++++++++++++++---- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/innodb/r/rename_table.result b/mysql-test/suite/innodb/r/rename_table.result index 3d7c3ff1b0e..9c45117cf10 100644 --- a/mysql-test/suite/innodb/r/rename_table.result +++ b/mysql-test/suite/innodb/r/rename_table.result @@ -18,3 +18,9 @@ path ./abc_def2/test1.ibd DROP DATABASE abc_def; DROP DATABASE abc_def2; +call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +RENAME TABLE t1 TO non_existing_db.t1; +ERROR HY000: Error on rename of './test/t1' to './non_existing_db/t1' (errno: 168 "Unknown (generic) error from engine") +FOUND 1 /\[ERROR\] InnoDB: Cannot rename file '.*t1\.ibd' to '.*non_existing_db/ in mysqld.1.err +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-mdev7046.test b/mysql-test/suite/innodb/t/innodb-mdev7046.test index 208dcd52f35..85a257a1739 100644 --- a/mysql-test/suite/innodb/t/innodb-mdev7046.test +++ b/mysql-test/suite/innodb/t/innodb-mdev7046.test @@ -9,9 +9,9 @@ # Ignore OS errors -call mtr.add_suppression("InnoDB: File ./test/t1*"); -call mtr.add_suppression("InnoDB: Error number*"); -call mtr.add_suppression("InnoDB: File ./test/t1#p#p1#sp#p1sp0.ibd: 'rename' returned OS error*"); +call mtr.add_suppression("InnoDB: File ./test/t1"); +call mtr.add_suppression("InnoDB: Error number"); +call mtr.add_suppression("InnoDB: Cannot rename file '.*/test/t1#[Pp]#p1#[Ss][Pp]#p1sp0\\.ibd' to"); call mtr.add_suppression("InnoDB: Operating system error number .* in a file operation."); # MDEV-7046: MySQL#74480 - Failing assertion: os_file_status(newpath, &exists, &type) diff --git a/mysql-test/suite/innodb/t/rename_table.test b/mysql-test/suite/innodb/t/rename_table.test index ea9f70bacb0..0191a94def2 100644 --- a/mysql-test/suite/innodb/t/rename_table.test +++ b/mysql-test/suite/innodb/t/rename_table.test @@ -29,3 +29,17 @@ DROP DATABASE abc_def; --source include/restart_mysqld.inc DROP DATABASE abc_def2; + +call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); + +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +--replace_result "\\" "/" +--error ER_ERROR_ON_RENAME +RENAME TABLE t1 TO non_existing_db.t1; + +--let SEARCH_PATTERN= \[ERROR\] InnoDB: Cannot rename file '.*t1\.ibd' to '.*non_existing_db +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +--source include/search_pattern_in_file.inc + +# Cleanup +DROP TABLE t1; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 9fb1c398e6e..613a3cad0e9 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -753,6 +753,23 @@ os_file_handle_error_no_exit( name, operation, false, on_error_silent)); } +/** Handle RENAME error. +@param name old name of the file +@param new_name new name of the file */ +static void os_file_handle_rename_error(const char* name, const char* new_name) +{ + if (os_file_get_last_error(true) != OS_FILE_DISK_FULL) { + ib::error() << "Cannot rename file '" << name << "' to '" + << new_name << "'"; + } else if (!os_has_said_disk_full) { + os_has_said_disk_full = true; + /* Disk full error is reported irrespective of the + on_error_silent setting. */ + ib::error() << "Full disk prevents renaming file '" + << name << "' to '" << new_name << "'"; + } +} + /** Does simulated AIO. This function should be called by an i/o-handler thread. @@ -777,9 +794,7 @@ os_aio_simulated_handler( #ifdef _WIN32 static HANDLE win_get_syncio_event(); -#endif -#ifdef _WIN32 /** Wrapper around Windows DeviceIoControl() function. @@ -3224,7 +3239,7 @@ os_file_rename_func( ret = rename(oldpath, newpath); if (ret != 0) { - os_file_handle_error_no_exit(oldpath, "rename", FALSE); + os_file_handle_rename_error(oldpath, newpath); return(false); } @@ -4555,8 +4570,7 @@ os_file_rename_func( return(true); } - os_file_handle_error_no_exit(oldpath, "rename", false); - + os_file_handle_rename_error(oldpath, newpath); return(false); } From 6b8d34fe0d5e73a469383bcb0818d3ff91ed9a84 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Tue, 12 Jun 2018 12:36:51 +0400 Subject: [PATCH 135/276] MDEV-14668 ADD PRIMARY KEY IF NOT EXISTS on composite key. Check the name of the primary key to be 'PRIMARY'. Than differs it from any implicit primary keys created by an engine. --- mysql-test/r/alter_table.result | 23 +++++++++++++++++++++++ mysql-test/t/alter_table.test | 16 ++++++++++++++++ sql/sql_table.cc | 4 +++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index bb78df907ad..c6e3c7e31d9 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -2225,3 +2225,26 @@ t1 CREATE TABLE `t1` ( `b` int(11) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 DROP TABLE t1; +# +# MDEV-14668 ADD PRIMARY KEY IF NOT EXISTS on composite key +# +CREATE TABLE t1 ( +`ID` BIGINT(20) NOT NULL, +`RANK` MEDIUMINT(4) NOT NULL, +`CHECK_POINT` BIGINT(20) NOT NULL, +UNIQUE INDEX `HORIZON_UIDX01` (`ID`, `RANK`) +) ENGINE=InnoDB; +ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS (`ID`, `CHECK_POINT`); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `ID` bigint(20) NOT NULL, + `RANK` mediumint(4) NOT NULL, + `CHECK_POINT` bigint(20) NOT NULL, + PRIMARY KEY (`ID`,`CHECK_POINT`), + UNIQUE KEY `HORIZON_UIDX01` (`ID`,`RANK`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS (`ID`, `CHECK_POINT`); +Warnings: +Note 1061 Multiple primary key defined +DROP TABLE t1; diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index 585a272de9b..28d8c5bf5e9 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1841,3 +1841,19 @@ CREATE TABLE t1 (a INT, b INT) ENGINE=InnoDB; ALTER TABLE t1 DROP INDEX IF EXISTS fk, DROP COLUMN IF EXISTS c; SHOW CREATE TABLE t1; DROP TABLE t1; + +--echo # +--echo # MDEV-14668 ADD PRIMARY KEY IF NOT EXISTS on composite key +--echo # +CREATE TABLE t1 ( + `ID` BIGINT(20) NOT NULL, + `RANK` MEDIUMINT(4) NOT NULL, + `CHECK_POINT` BIGINT(20) NOT NULL, + UNIQUE INDEX `HORIZON_UIDX01` (`ID`, `RANK`) + ) ENGINE=InnoDB; + +ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS (`ID`, `CHECK_POINT`); +SHOW CREATE TABLE t1; +ALTER TABLE t1 ADD PRIMARY KEY IF NOT EXISTS (`ID`, `CHECK_POINT`); +DROP TABLE t1; + diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 376c1362cc7..6c71067d51b 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -5930,7 +5930,9 @@ drop_create_field: /* Check if the table already has a PRIMARY KEY */ if (key->type == Key::PRIMARY && - table->s->primary_key != MAX_KEY) + table->s->primary_key != MAX_KEY && + (keyname= table->s->key_info[table->s->primary_key].name) && + my_strcasecmp(system_charset_info, keyname, primary_key_name) == 0) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, ER_DUP_KEYNAME, ER(ER_MULTIPLE_PRI_KEY)); From b52bb6eb82db8f3bff88efbf0876d364a3826cc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 12 Jun 2018 12:49:42 +0300 Subject: [PATCH 136/276] MDEV-16469 SET GLOBAL innodb_change_buffering has no effect When type of the settable global variable innodb_change_buffering was changed from string to ENUM in MDEV-12218, the "shadow" variable ibuf_use stopped being updated as a result of SET GLOBAL innodb_change_buffering. Only on InnoDB startup, the parameter innodb_change_buffering would take effect. ibuf_use: Remove, and use the global variable innodb_change_buffering. --- storage/innobase/handler/ha_innodb.cc | 2 -- storage/innobase/ibuf/ibuf0ibuf.cc | 6 +++--- storage/innobase/include/ibuf0ibuf.h | 2 +- storage/innobase/include/ibuf0ibuf.ic | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5c6b992bf9b..66a28ea56b7 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -209,7 +209,6 @@ static char* innobase_disable_monitor_counter; static char* innobase_reset_monitor_counter; static char* innobase_reset_all_monitor_counter; -static ulong innodb_change_buffering; static ulong innodb_flush_method; /* This variable can be set in the server configure file, specifying @@ -3859,7 +3858,6 @@ static int innodb_init_params() } DBUG_ASSERT(innodb_change_buffering <= IBUF_USE_ALL); - ibuf_use = ibuf_use_t(innodb_change_buffering); /* Check that interdependent parameters have sane values. */ if (srv_max_buf_pool_modified_pct < srv_max_dirty_pages_pct_lwm) { diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 04cee74e6ea..7b2bbdcd7c7 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -185,7 +185,7 @@ it uses synchronous aio, it can access any pages, as long as it obeys the access order rules. */ /** Operations that can currently be buffered. */ -ibuf_use_t ibuf_use = IBUF_USE_ALL; +ulong innodb_change_buffering; #if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG /** Flag to control insert buffer debugging. */ @@ -3695,9 +3695,9 @@ ibuf_insert( dberr_t err; ulint entry_size; ibool no_counter; - /* Read the settable global variable ibuf_use only once in + /* Read the settable global variable only once in this function, so that we will have a consistent view of it. */ - ibuf_use_t use = ibuf_use; + ibuf_use_t use = ibuf_use_t(innodb_change_buffering); DBUG_ENTER("ibuf_insert"); DBUG_PRINT("ibuf", ("op: %d, space: " UINT32PF ", page_no: " UINT32PF, diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index b5e3d1eddf7..8233a536abc 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -61,7 +61,7 @@ enum ibuf_use_t { }; /** Operations that can currently be buffered. */ -extern ibuf_use_t ibuf_use; +extern ulong innodb_change_buffering; /** The insert buffer control structure */ extern ibuf_t* ibuf; diff --git a/storage/innobase/include/ibuf0ibuf.ic b/storage/innobase/include/ibuf0ibuf.ic index f91ae5aee4a..355fad62f24 100644 --- a/storage/innobase/include/ibuf0ibuf.ic +++ b/storage/innobase/include/ibuf0ibuf.ic @@ -124,7 +124,7 @@ ibuf_should_try( a secondary index when we decide */ { - return(ibuf_use != IBUF_USE_NONE + return(innodb_change_buffering && ibuf->max_size != 0 && !dict_index_is_clust(index) && !dict_index_is_spatial(index) From ae0aefb1c522455b785c2e43636c482cd161e3de Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 12 Jun 2018 14:12:36 +0400 Subject: [PATCH 137/276] MDEV-12060 Crash in EXECUTE IMMEDIATE with an expression returning a GRANT command This problem was earlier fixed by MDEV-14603. Only adding 10.2 specific tests. --- mysql-test/r/ps.result | 39 ++++++++++++++++++++++++++--- mysql-test/t/ps.test | 56 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 89 insertions(+), 6 deletions(-) diff --git a/mysql-test/r/ps.result b/mysql-test/r/ps.result index 6f86b330251..ac96dd05a2a 100644 --- a/mysql-test/r/ps.result +++ b/mysql-test/r/ps.result @@ -4918,9 +4918,6 @@ DROP TABLE t1; # End of MDEV-10866 Extend PREPARE and EXECUTE IMMEDIATE to understand expressions # # -# End of 10.2 tests -# -# # MDEV-11360 Dynamic SQL: DEFAULT as a bind parameter # CREATE TABLE t1 (a INT DEFAULT 10, b INT DEFAULT NULL); @@ -5174,3 +5171,39 @@ execute stmt; execute stmt; execute stmt; drop table t1; +# +# MDEV-12060 Crash in EXECUTE IMMEDIATE with an expression returning a GRANT command +# +CREATE ROLE testrole; +CREATE OR REPLACE PROCEDURE p1() +BEGIN +END; +/ +CREATE PROCEDURE p2 (wgrp VARCHAR(10)) +BEGIN +EXECUTE IMMEDIATE concat('GRANT EXECUTE ON PROCEDURE p1 TO ',wgrp); +END; +/ +CALL p2('testrole'); +DROP PROCEDURE p2; +CREATE PROCEDURE p2 () +BEGIN +EXECUTE IMMEDIATE concat(_utf8'GRANT EXECUTE ON PROCEDURE p1 TO ',_latin1'testrole'); +END; +/ +CALL p2(); +DROP PROCEDURE p2; +CREATE PROCEDURE p2 () +BEGIN +PREPARE stmt FROM concat(_utf8'GRANT EXECUTE ON PROCEDURE p1 TO ',_latin1' testrole'); +EXECUTE stmt; +DEALLOCATE PREPARE stmt; +END; +/ +CALL p2(); +DROP PROCEDURE p2; +DROP PROCEDURE p1; +DROP ROLE testrole; +# +# End of 10.2 tests +# diff --git a/mysql-test/t/ps.test b/mysql-test/t/ps.test index e051cdce179..1c8595936cd 100644 --- a/mysql-test/t/ps.test +++ b/mysql-test/t/ps.test @@ -4386,9 +4386,6 @@ DROP TABLE t1; --echo # End of MDEV-10866 Extend PREPARE and EXECUTE IMMEDIATE to understand expressions --echo # ---echo # ---echo # End of 10.2 tests ---echo # --echo # @@ -4650,3 +4647,56 @@ execute stmt; execute stmt; execute stmt; drop table t1; + + +--echo # +--echo # MDEV-12060 Crash in EXECUTE IMMEDIATE with an expression returning a GRANT command +--echo # + +CREATE ROLE testrole; +DELIMITER /; +CREATE OR REPLACE PROCEDURE p1() +BEGIN +END; +/ +DELIMITER ;/ + +DELIMITER /; +CREATE PROCEDURE p2 (wgrp VARCHAR(10)) +BEGIN + EXECUTE IMMEDIATE concat('GRANT EXECUTE ON PROCEDURE p1 TO ',wgrp); +END; +/ +DELIMITER ;/ +CALL p2('testrole'); +DROP PROCEDURE p2; + +DELIMITER /; +CREATE PROCEDURE p2 () +BEGIN + EXECUTE IMMEDIATE concat(_utf8'GRANT EXECUTE ON PROCEDURE p1 TO ',_latin1'testrole'); +END; +/ +DELIMITER ;/ +CALL p2(); +DROP PROCEDURE p2; + +DELIMITER /; +CREATE PROCEDURE p2 () +BEGIN + PREPARE stmt FROM concat(_utf8'GRANT EXECUTE ON PROCEDURE p1 TO ',_latin1' testrole'); + EXECUTE stmt; + DEALLOCATE PREPARE stmt; +END; +/ +DELIMITER ;/ +CALL p2(); +DROP PROCEDURE p2; + +DROP PROCEDURE p1; +DROP ROLE testrole; + + +--echo # +--echo # End of 10.2 tests +--echo # From 7bbe324fc17d9734833f717921629c36d1d0c996 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Fri, 8 Jun 2018 22:01:05 +0300 Subject: [PATCH 138/276] MDEV-13577 slave_parallel_mode=optimistic should not report the mode's specific temporary errors The optimistic parallel slave's worker thread could face a run-time error due to the algorithm's specifics which allows for conflicts like the reported "Can't find record in 'table'". A typical stack is like {noformat} #0 handler::print_error (this=0x61c00008f8a0, error=149, errflag=0) at handler.cc:3650 #1 0x0000555555e95361 in write_record (thd=thd@entry=0x62a0000a2208, table=table@entry=0x61f00008ce88, info=info@entry=0x7fffdee356d0) at sql_insert.cc:1944 #2 0x0000555555ea7767 in mysql_insert (thd=thd@entry=0x62a0000a2208, table_list=0x61b00012ada0, fields=..., values_list=..., update_fields=..., update_values=..., duplic=, ignore=) at sql_insert.cc:1039 #3 0x0000555555efda90 in mysql_execute_command (thd=thd@entry=0x62a0000a2208) at sql_parse.cc:3927 #4 0x0000555555f0cc50 in mysql_parse (thd=0x62a0000a2208, rawbuf=, length=, parser_state=) at sql_parse.cc:7449 #5 0x00005555566d4444 in Query_log_event::do_apply_event (this=0x61200005b9c8, rgi=, query_arg=, q_len_arg=) at log_event.cc:4508 #6 0x00005555566d639e in Query_log_event::do_apply_event (this=, rgi=) at log_event.cc:4185 #7 0x0000555555d738cf in Log_event::apply_event (rgi=0x61d0001ea080, this=0x61200005b9c8) at log_event.h:1343 #8 apply_event_and_update_pos_apply (ev=ev@entry=0x61200005b9c8, thd=thd@entry=0x62a0000a2208, rgi=rgi@entry=0x61d0001ea080, reason=) at slave.cc:3479 #9 0x0000555555d8596b in apply_event_and_update_pos_for_parallel (ev=ev@entry=0x61200005b9c8, thd=thd@entry=0x62a0000a2208, rgi=rgi@entry=0x61d0001ea080) at slave.cc:3623 #10 0x00005555562aca83 in rpt_handle_event (qev=qev@entry=0x6190000fa088, rpt=rpt@entry=0x62200002bd68) at rpl_parallel.cc:50 #11 0x00005555562bd04e in handle_rpl_parallel_thread (arg=arg@entry=0x62200002bd68) at rpl_parallel.cc:1258 {noformat} Here {{handler::print_error}} computes whether to error log the current error when --log-warnings > 1. The decision flag is consulted bu {{my_message_sql()}} which can be eventually called. In the bug case the decision is to log. However in the optimistic mode slave applier case any conflict is attempted to resolve with rollback and retry to success. Hence the logging is at least extraneous. The case is fixed with adding a new flag {{ME_LOG_AS_WARN}} which {{handler::print_error}} may propagate further on through {{my_error}} when the error comes from an optimistically running slave worker thread. The new flag effectively requests the warning level for the errlog record, while the thread's DA records the actual error (which is regarded as temporary one by the parallel slave error handler). --- include/my_sys.h | 1 + .../rpl/r/rpl_parallel_optimistic.result | 23 +++++++ .../suite/rpl/t/rpl_parallel_optimistic.test | 64 ++++++++++++++++++- sql/handler.cc | 5 +- sql/mysqld.cc | 10 +++ sql/sql_class.cc | 7 ++ sql/sql_class.h | 6 ++ 7 files changed, 114 insertions(+), 2 deletions(-) diff --git a/include/my_sys.h b/include/my_sys.h index 110a2ee9af3..1c5649812d1 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -112,6 +112,7 @@ typedef struct my_aio_result { #define ME_JUST_INFO 1024 /**< not error but just info */ #define ME_JUST_WARNING 2048 /**< not error but just warning */ #define ME_FATALERROR 4096 /* Fatal statement error */ +#define ME_LOG_AS_WARN 8192 /* is error but error-logged as warning */ /* Bits in last argument to fn_format */ #define MY_REPLACE_DIR 1 /* replace dir in name with 'dir' */ diff --git a/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result b/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result index 0177e65b10f..a6da3399fab 100644 --- a/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result +++ b/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result @@ -1,4 +1,6 @@ include/rpl_init.inc [topology=1->2] +call mtr.add_suppression("Warning.*Deadlock found when trying to get lock; try restarting transaction"); +call mtr.add_suppression("Warning.*mysqld: Can't find record in 't2'"); ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; @@ -543,6 +545,27 @@ a b 57 7 58 8 59 9 +DELETE FROM t1; +DELETE FROM t2; +include/save_master_gtid.inc +include/sync_with_master_gtid.inc +BEGIN; +INSERT INTO t1 SET a=1; +SET @save.binlog_format=@@session.binlog_format; +SET @@SESSION.binlog_format=row; +BEGIN; +INSERT INTO t1 SET a=1; +INSERT INTO t2 SET a=1; +COMMIT; +BEGIN; +DELETE FROM t2; +COMMIT; +ROLLBACK; +SET @@SESSION.binlog_format= @save.binlog_format; +DELETE FROM t1; +DELETE FROM t2; +include/save_master_gtid.inc +include/sync_with_master_gtid.inc include/stop_slave.inc SET GLOBAL slave_parallel_mode=@old_parallel_mode; SET GLOBAL slave_parallel_threads=@old_parallel_threads; diff --git a/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test b/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test index 41fb6ebb72e..28bf4a77fd4 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test @@ -4,6 +4,11 @@ --let $rpl_topology=1->2 --source include/rpl_init.inc +--connection server_2 +call mtr.add_suppression("Warning.*Deadlock found when trying to get lock; try restarting transaction"); +# The following instruction is a part of the proof of MDEV-13577 fixes, below. +call mtr.add_suppression("Warning.*mysqld: Can't find record in 't2'"); + --connection server_1 ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; @@ -480,8 +485,65 @@ SELECT * FROM t2 WHERE a >= 40 ORDER BY a; SELECT * FROM t1 WHERE a >= 40 ORDER BY a; SELECT * FROM t2 WHERE a >= 40 ORDER BY a; -# Clean up. +# partial cleanup to reuse the tables by following tests +--connection server_1 +DELETE FROM t1; +DELETE FROM t2; +--source include/save_master_gtid.inc +--connection server_2 +--source include/sync_with_master_gtid.inc + +# +# MDEV-13577 optimistic parallel slave errors out to error log unnecessary +# + +# The 1st of the following two trx:s a blocker on slave +--connection server_2 +BEGIN; +INSERT INTO t1 SET a=1; + +--connection server_1 +SET @save.binlog_format=@@session.binlog_format; +SET @@SESSION.binlog_format=row; + +BEGIN; + INSERT INTO t1 SET a=1; + INSERT INTO t2 SET a=1; +COMMIT; + +# This transaction is going to win optimistical race with above INSERT +# on slave while being depend on it. That means it will face a kind of temporary error +# and then will retry to succeed. +BEGIN; + DELETE FROM t2; +COMMIT; + +# First make sure DELETE raced indeed to get stuck at retrying stage +# where it runs "realistically" now. There is nomore optimistic error +# in the errorlog, which is downgraded to the warning level (when +# --log-warnings > 1), see above suppression. +--connection server_2 +--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +# Next release the 1st trx to commit. +--connection server_2 +ROLLBACK; + +# MDEV-13577 local cleanup: +--connection server_1 +SET @@SESSION.binlog_format= @save.binlog_format; +DELETE FROM t1; +DELETE FROM t2; +--source include/save_master_gtid.inc + +--connection server_2 +--source include/sync_with_master_gtid.inc + +# +# Clean up. +# --connection server_2 --source include/stop_slave.inc SET GLOBAL slave_parallel_mode=@old_parallel_mode; diff --git a/sql/handler.cc b/sql/handler.cc index 397891ceec5..87f0926ad9a 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3639,12 +3639,15 @@ void handler::print_error(int error, myf errflag) if ((debug_assert_if_crashed_table || global_system_variables.log_warnings > 1)) { + THD *thd= ha_thd(); /* Log error to log before we crash or if extended warnings are requested */ errflag|= ME_NOREFRESH; + if (thd && thd->is_optimistic_slave_worker()) + errflag|= ME_LOG_AS_WARN; } - } + } /* if we got an OS error from a file-based engine, specify a path of error */ if (error < HA_ERR_FIRST && bas_ext()[0]) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5f954f7576d..5bf5e3f73fc 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3517,6 +3517,16 @@ void my_message_sql(uint error, const char *str, myf MyFlags) level= Sql_condition::WARN_LEVEL_WARN; func= sql_print_warning; } + else if (MyFlags & ME_LOG_AS_WARN) + { + /* + Typical use case is optimistic parallel slave where DA needs to hold + an error condition caused by the current error, but the error-log + level is relaxed to the warning one. + */ + level= Sql_condition::WARN_LEVEL_ERROR; + func= sql_print_warning; + } else { level= Sql_condition::WARN_LEVEL_ERROR; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 24140246b96..37f29115171 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -7100,6 +7100,13 @@ bool THD::rgi_have_temporary_tables() return rgi_slave->rli->save_temporary_tables != 0; } +bool THD::is_optimistic_slave_worker() +{ + DBUG_ASSERT(system_thread != SYSTEM_THREAD_SLAVE_SQL || rgi_slave); + + return system_thread == SYSTEM_THREAD_SLAVE_SQL && rgi_slave && + rgi_slave->speculation == rpl_group_info::SPECULATE_OPTIMISTIC; +} void wait_for_commit::reinit() diff --git a/sql/sql_class.h b/sql/sql_class.h index ca6155ec93f..0ced84791a0 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -4167,6 +4167,12 @@ public: (THD_TRANS::DID_WAIT | THD_TRANS::CREATED_TEMP_TABLE | THD_TRANS::DROPPED_TEMP_TABLE | THD_TRANS::DID_DDL)); } + + /* + Returns true when the thread handle belongs to a slave worker thread + running in the optimistic execution mode. + */ + bool is_optimistic_slave_worker(); }; From 4787913db0b49b4a48e337e6b76b8b6c8233a941 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 13 Jun 2018 08:25:16 +0400 Subject: [PATCH 139/276] MDEV-16464 Oracle Comp.: Sql-Error on "SELECT name, comment FROM mysql.proc" --- .../suite/compat/oracle/r/parser.result | 51 +++++++++++++++++++ mysql-test/suite/compat/oracle/t/parser.test | 48 +++++++++++++++++ sql/sql_yacc.yy | 2 +- sql/sql_yacc_ora.yy | 3 +- 4 files changed, 102 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/compat/oracle/r/parser.result b/mysql-test/suite/compat/oracle/r/parser.result index 28ed8eb8185..c8600c29bd4 100644 --- a/mysql-test/suite/compat/oracle/r/parser.result +++ b/mysql-test/suite/compat/oracle/r/parser.result @@ -448,3 +448,54 @@ BEGIN SELECT precedes INTO precedes FROM DUAL; END; / +# +# MDEV-16464 Oracle Comp.: Sql-Error on "SELECT name, comment FROM mysql.proc" +# +SET sql_mode=ORACLE; +SELECT name, comment FROM mysql.proc WHERE db='test'; +name comment +CREATE TABLE comment (comment INT); +SELECT comment FROM comment; +comment +SELECT comment comment FROM comment comment; +comment +SELECT comment AS comment FROM comment AS comment; +comment +DROP TABLE comment; +DECLARE +comment INT; +BEGIN +SELECT comment INTO comment FROM DUAL; +END; +/ +CREATE PROCEDURE comment COMMENT 'test' AS +BEGIN +SELECT 1; +END; +/ +BEGIN +comment; +END; +/ +1 +1 +CALL comment(); +1 +1 +CALL comment; +1 +1 +DROP PROCEDURE comment; +CREATE FUNCTION comment RETURN INT COMMENT 'test' AS +BEGIN +RETURN 1; +END; +/ +Warnings: +Note 1585 This function 'comment' has the same name as a native function +SELECT test.comment() FROM DUAL; +test.comment() +1 +Warnings: +Note 1585 This function 'comment' has the same name as a native function +DROP FUNCTION comment; diff --git a/mysql-test/suite/compat/oracle/t/parser.test b/mysql-test/suite/compat/oracle/t/parser.test index a8f5eda7016..5b219e3bce3 100644 --- a/mysql-test/suite/compat/oracle/t/parser.test +++ b/mysql-test/suite/compat/oracle/t/parser.test @@ -208,3 +208,51 @@ END; / DELIMITER ;/ + + +--echo # +--echo # MDEV-16464 Oracle Comp.: Sql-Error on "SELECT name, comment FROM mysql.proc" +--echo # + +SET sql_mode=ORACLE; +SELECT name, comment FROM mysql.proc WHERE db='test'; + +CREATE TABLE comment (comment INT); +SELECT comment FROM comment; +SELECT comment comment FROM comment comment; +SELECT comment AS comment FROM comment AS comment; +DROP TABLE comment; + +DELIMITER /; +DECLARE + comment INT; +BEGIN + SELECT comment INTO comment FROM DUAL; +END; +/ +DELIMITER ;/ + +DELIMITER /; +CREATE PROCEDURE comment COMMENT 'test' AS +BEGIN + SELECT 1; +END; +/ +BEGIN + comment; +END; +/ +DELIMITER ;/ +CALL comment(); +CALL comment; +DROP PROCEDURE comment; + +DELIMITER /; +CREATE FUNCTION comment RETURN INT COMMENT 'test' AS +BEGIN + RETURN 1; +END; +/ +DELIMITER ;/ +SELECT test.comment() FROM DUAL; +DROP FUNCTION comment; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index b53a89ecc94..0d440bf0c2c 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1213,7 +1213,6 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); Keywords that have different reserved status in std/oracle modes. */ %token BODY_SYM /* Oracle-R */ -%token COMMENT_SYM %token ELSIF_SYM /* Oracle, reserved in PL/SQL*/ %token GOTO_SYM /* Oracle, reserved in PL/SQL*/ %token OTHERS_SYM /* SQL-2011-N */ @@ -1276,6 +1275,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %token COLUMN_GET_SYM %token COLUMN_SYM /* SQL-2003-R */ %token COLUMN_NAME_SYM /* SQL-2003-N */ +%token COMMENT_SYM /* Oracle-R */ %token COMMITTED_SYM /* SQL-2003-N */ %token COMMIT_SYM /* SQL-2003-R */ %token COMPACT_SYM diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index f0db33f67ae..8c4d1e897dc 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -607,7 +607,6 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); Keywords that have different reserved status in std/oracle modes. */ %token BODY_SYM /* Oracle-R */ -%token COMMENT_SYM %token ELSIF_SYM /* Oracle, reserved in PL/SQL*/ %token GOTO_SYM /* Oracle, reserved in PL/SQL*/ %token OTHERS_SYM /* SQL-2011-N */ @@ -670,6 +669,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %token COLUMN_GET_SYM %token COLUMN_SYM /* SQL-2003-R */ %token COLUMN_NAME_SYM /* SQL-2003-N */ +%token COMMENT_SYM /* Oracle-R */ %token COMMITTED_SYM /* SQL-2003-N */ %token COMMIT_SYM /* SQL-2003-R */ %token COMPACT_SYM @@ -15593,6 +15593,7 @@ keyword_sp_var_not_label: | COLUMN_CREATE_SYM | COLUMN_DELETE_SYM | COLUMN_GET_SYM + | COMMENT_SYM | DEALLOCATE_SYM | EXAMINED_SYM | EXCLUDE_SYM From d2e1ed8b936a78ceeec0e64231997d7ed18a4daf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 13 Jun 2018 08:33:25 +0300 Subject: [PATCH 140/276] Fix innodb.rename_table for embedded An embedded run will output the full path name instead of a relative one. Update results to cover both cases. --- mysql-test/suite/innodb/r/rename_table.result | 2 +- mysql-test/suite/innodb/t/rename_table.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/rename_table.result b/mysql-test/suite/innodb/r/rename_table.result index b2e15c87348..49ce3254091 100644 --- a/mysql-test/suite/innodb/r/rename_table.result +++ b/mysql-test/suite/innodb/r/rename_table.result @@ -1,5 +1,5 @@ call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); CREATE TABLE t1 (a INT) ENGINE=InnoDB; RENAME TABLE t1 TO non_existing_db.t1; -ERROR HY000: Error on rename of './test/t1' to './non_existing_db/t1' (errno: -1 "Internal error < 0 (Not system error)") +ERROR HY000: Error on rename of '**path-to-t1**' to '**path-to-non-existing-db-t1**' (errno: -1 "Internal error < 0 (Not system error)") DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/rename_table.test b/mysql-test/suite/innodb/t/rename_table.test index 60df8b5d2ec..695c0915d14 100644 --- a/mysql-test/suite/innodb/t/rename_table.test +++ b/mysql-test/suite/innodb/t/rename_table.test @@ -3,7 +3,7 @@ call mtr.add_suppression("InnoDB: (Operating system error|The error means|Cannot rename file)"); CREATE TABLE t1 (a INT) ENGINE=InnoDB; ---replace_result "\\" "/" +--replace_regex /\'.*t1\' to/'**path-to-t1**' to/ /to \'.*non.*t1\'/to '**path-to-non-existing-db-t1**'/ --error ER_ERROR_ON_RENAME RENAME TABLE t1 TO non_existing_db.t1; From edc1b8e1172abc35644f2b4824a78d49ac17a7d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 13 Jun 2018 08:57:15 +0300 Subject: [PATCH 141/276] Fix wsrep.variables test case Correctly port 09b25f85966f44aae933e86b84b4ebe59ded47c3 from 10.0. Expose check_is_super via include file to allow wsrep_on_check to use it. --- sql/set_var.h | 1 + sql/sys_vars.cc | 2 +- sql/wsrep_var.cc | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/set_var.h b/sql/set_var.h index fe66eaff775..203969d6169 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -426,6 +426,7 @@ CHARSET_INFO *get_old_charset_by_name(const char *old_name); int sys_var_init(); int sys_var_add_options(DYNAMIC_ARRAY *long_options, int parse_flags); void sys_var_end(void); +bool check_has_super(sys_var *self, THD *thd, set_var *var); #endif diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 9f87d66fef1..5470acc7892 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -428,7 +428,7 @@ error_if_in_trans_or_substatement(THD *thd, int in_substatement_error, return false; } -static bool check_has_super(sys_var *self, THD *thd, set_var *var) +bool check_has_super(sys_var *self, THD *thd, set_var *var) { DBUG_ASSERT(self->scope() != sys_var::GLOBAL);// don't abuse check_has_super() #ifndef NO_EMBEDDED_ACCESS_CHECKS diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 2d52396331d..216bab0cdcd 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -69,6 +69,9 @@ bool wsrep_on_check(sys_var *self, THD* thd, set_var* var) { bool new_wsrep_on= (bool)var->save_result.ulonglong_value; + if (check_has_super(self, thd, var)) + return true; + if (new_wsrep_on && innodb_lock_schedule_algorithm != 0) { my_message(ER_WRONG_ARGUMENTS, " WSREP (galera) can't be enabled " "if innodb_lock_schedule_algorithm=VATS. Please configure" From 2412c151916dc65660644a0cd2fe5f34816ea901 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 13 Jun 2018 11:56:56 +0400 Subject: [PATCH 142/276] MDEV-15870 Using aggregate and window function in unexpected places can crash the server --- mysql-test/r/sp.result | 11 +++++++++++ mysql-test/t/sp.test | 14 ++++++++++++++ sql/item_sum.cc | 4 ++-- sql/item_windowfunc.cc | 6 +++--- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index ad8dc15318e..59387b37585 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -8354,3 +8354,14 @@ drop procedure p3; CREATE PROCEDURE foo ( IN i INT UNSIGNED ) BEGIN END; CALL foo( LAST_INSERT_ID() ); DROP PROCEDURE foo; +# +# MDEV-15870 Using aggregate and window function in unexpected places can crash the server +# +CREATE PROCEDURE p1 (a TEXT) BEGIN END; +CALL p1(RANK() OVER (ORDER BY 1)); +ERROR HY000: Window function is allowed only in SELECT list and ORDER BY clause +CALL p1(ROW_NUMBER() OVER ()); +ERROR HY000: Window function is allowed only in SELECT list and ORDER BY clause +CALL p1(SUM(1)); +ERROR HY000: Invalid use of group function +DROP PROCEDURE p1; diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index 549d97ad72b..e8b63c4d791 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9865,3 +9865,17 @@ drop procedure p3; CREATE PROCEDURE foo ( IN i INT UNSIGNED ) BEGIN END; CALL foo( LAST_INSERT_ID() ); DROP PROCEDURE foo; + + +--echo # +--echo # MDEV-15870 Using aggregate and window function in unexpected places can crash the server +--echo # + +CREATE PROCEDURE p1 (a TEXT) BEGIN END; +--error ER_WRONG_PLACEMENT_OF_WINDOW_FUNCTION +CALL p1(RANK() OVER (ORDER BY 1)); +--error ER_WRONG_PLACEMENT_OF_WINDOW_FUNCTION +CALL p1(ROW_NUMBER() OVER ()); +--error ER_INVALID_GROUP_FUNC_USE +CALL p1(SUM(1)); +DROP PROCEDURE p1; diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 4cf11e81d3d..cb150db3031 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -68,14 +68,14 @@ size_t Item_sum::ram_limitation(THD *thd) bool Item_sum::init_sum_func_check(THD *thd) { SELECT_LEX *curr_sel= thd->lex->current_select; - if (!curr_sel->name_visibility_map) + if (curr_sel && !curr_sel->name_visibility_map) { for (SELECT_LEX *sl= curr_sel; sl; sl= sl->context.outer_select()) { curr_sel->name_visibility_map|= (1 << sl-> nest_level); } } - if (!(thd->lex->allow_sum_func & curr_sel->name_visibility_map)) + if (!curr_sel || !(thd->lex->allow_sum_func & curr_sel->name_visibility_map)) { my_message(ER_INVALID_GROUP_FUNC_USE, ER_THD(thd, ER_INVALID_GROUP_FUNC_USE), MYF(0)); diff --git a/sql/item_windowfunc.cc b/sql/item_windowfunc.cc index 5fbfb2651af..52738bfab87 100644 --- a/sql/item_windowfunc.cc +++ b/sql/item_windowfunc.cc @@ -71,9 +71,9 @@ Item_window_func::fix_fields(THD *thd, Item **ref) { DBUG_ASSERT(fixed == 0); - enum_parsing_place place= thd->lex->current_select->context_analysis_place; - - if (!(place == SELECT_LIST || place == IN_ORDER_BY)) + if (!thd->lex->current_select || + (thd->lex->current_select->context_analysis_place != SELECT_LIST && + thd->lex->current_select->context_analysis_place != IN_ORDER_BY)) { my_error(ER_WRONG_PLACEMENT_OF_WINDOW_FUNCTION, MYF(0)); return true; From 931daaf79b9bea447a28abff01f04cdf2f3a07f8 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 13 Jun 2018 14:50:25 +0300 Subject: [PATCH 143/276] MDEV-15319: [SQL Layer] Server crashes in Field::set_null / myrocks::ha_rocksdb ... Added a testcase --- .../mysql-test/rocksdb/r/mariadb_port_fixes.result | 9 +++++++++ .../mysql-test/rocksdb/t/mariadb_port_fixes.test | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index d9e2bf5eea5..aab6e82ae67 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@ -107,3 +107,12 @@ a b 101 foo 102 bar DROP TABLE t1; +# +# MDEV-15319: [SQL Layer] Server crashes in Field::set_null / myrocks::ha_rocksdb::convert_record_from_storage_format +# (just a testcase) +# +CREATE TABLE t1 (i INT); +INSERT INTO t1 VALUES (1); +CREATE TABLE t2 ENGINE=RocksDB AS SELECT VALUES(i) AS a FROM t1; +DELETE FROM t2; +DROP TABLE t1,t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test b/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test index 7282ec237c2..569f33c94b2 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test @@ -101,3 +101,14 @@ UPDATE t1 SET a=a+100; SELECT * FROM t1; DROP TABLE t1; + +--echo # +--echo # MDEV-15319: [SQL Layer] Server crashes in Field::set_null / myrocks::ha_rocksdb::convert_record_from_storage_format +--echo # (just a testcase) +--echo # + +CREATE TABLE t1 (i INT); +INSERT INTO t1 VALUES (1); +CREATE TABLE t2 ENGINE=RocksDB AS SELECT VALUES(i) AS a FROM t1; +DELETE FROM t2; +DROP TABLE t1,t2; From 8662015c90718501d504f4c7aeb94b8626902a9c Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Wed, 13 Jun 2018 15:26:50 +0300 Subject: [PATCH 144/276] MDEV-15304: Server crash in print_keydup_error / key_unpack or unexpected ER_DUP_KEY Adjust the patch to match the variant accepted into the upstream: undo the changes in ha_rocksdb::load_hidden_pk_value(). --- storage/rocksdb/ha_rocksdb.cc | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 7cddfea33a7..c1c3191514f 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -5302,17 +5302,8 @@ int ha_rocksdb::load_hidden_pk_value() { active_index = m_tbl_def->m_key_count - 1; const uint8 save_table_status = table->status; - /* - We should read the latest committed value in the database. - That is, if we have an open transaction with a snapshot, we should not use - it as we may get old data. Start a new transaction to read the latest - value. - */ - Rdb_transaction *const temp_tx = new Rdb_transaction_impl(table->in_use); - temp_tx->start_tx(); - Rdb_transaction *&tx = get_tx_from_thd(table->in_use); - Rdb_transaction *save_tx= tx; - tx= temp_tx; + Rdb_transaction *const tx = get_or_create_tx(table->in_use); + const bool is_new_snapshot = !tx->has_snapshot(); longlong hidden_pk_id = 1; // Do a lookup. @@ -5322,8 +5313,9 @@ int ha_rocksdb::load_hidden_pk_value() { */ auto err = read_hidden_pk_id_from_rowkey(&hidden_pk_id); if (err) { - delete tx; - tx= save_tx; + if (is_new_snapshot) { + tx->release_snapshot(); + } return err; } @@ -5335,8 +5327,9 @@ int ha_rocksdb::load_hidden_pk_value() { !m_tbl_def->m_hidden_pk_val.compare_exchange_weak(old, hidden_pk_id)) { } - delete tx; - tx= save_tx; + if (is_new_snapshot) { + tx->release_snapshot(); + } table->status = save_table_status; active_index = save_active_index; From 92bd177fe9f8b297a464891a25bbbd6a992d655f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 11 Jun 2018 10:33:09 +0300 Subject: [PATCH 145/276] Correct a typo in a comment --- storage/innobase/include/fsp0fsp.h | 2 +- storage/xtradb/include/fsp0fsp.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 275a5785ce2..c11c48d58cf 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -922,7 +922,7 @@ fsp_flags_convert_from_101(ulint flags) /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they should contain one of the values 3,4,6,7, that is, be of the form - 0b0011 or 0b01xx (except 0b0110). + 0b0011 or 0b01xx (except 0b0101). In correct versions, these bits should be 0bc0se where c is the MariaDB COMPRESSED flag and e is the MySQL 5.7 ENCRYPTION flag diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 7395a248bed..bc81a9c126b 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -921,7 +921,7 @@ fsp_flags_convert_from_101(ulint flags) /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they should contain one of the values 3,4,6,7, that is, be of the form - 0b0011 or 0b01xx (except 0b0110). + 0b0011 or 0b01xx (except 0b0101). In correct versions, these bits should be 0bc0se where c is the MariaDB COMPRESSED flag and e is the MySQL 5.7 ENCRYPTION flag From 3fcc11fbb41aca43298e39c96525491699e686e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 11 Jun 2018 10:46:15 +0300 Subject: [PATCH 146/276] Remove traces of the non-working MDEV-6354 MariaDB never supported the MySQL 5.7 compression format. FIL_PAGE_TYPE_COMPRESSED: Remove. This was originally added as FIL_PAGE_COMPRESSED. --- storage/innobase/fil/fil0pagecompress.cc | 5 ++--- storage/innobase/include/fil0fil.h | 3 +-- storage/innobase/include/fil0fil.ic | 7 ++----- storage/xtradb/fil/fil0pagecompress.cc | 5 ++--- storage/xtradb/include/fil0fil.h | 3 +-- storage/xtradb/include/fil0fil.ic | 7 ++----- 6 files changed, 10 insertions(+), 20 deletions(-) diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index edc932f36f5..0ac764d5a32 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -501,8 +501,7 @@ fil_decompress_page( /* Do not try to uncompressed pages that are not compressed */ if (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && - ptype != FIL_PAGE_TYPE_COMPRESSED) { + ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { return; } diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9b6178bb8f1..ae2629b5b04 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -173,8 +173,7 @@ extern fil_addr_t fil_addr_null; #define FIL_PAGE_TYPE_BLOB 10 /*!< Uncompressed BLOB page */ #define FIL_PAGE_TYPE_ZBLOB 11 /*!< First compressed BLOB page */ #define FIL_PAGE_TYPE_ZBLOB2 12 /*!< Subsequent compressed BLOB page */ -#define FIL_PAGE_TYPE_COMPRESSED 13 /*!< Compressed page */ -#define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_COMPRESSED +#define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_ZBLOB2 /*!< Last page type */ /* @} */ diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index 6c2504c9f8c..49fdff4f3e5 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -65,8 +65,6 @@ fil_get_page_type_name( return "ZBLOB"; case FIL_PAGE_TYPE_ZBLOB2: return "ZBLOB2"; - case FIL_PAGE_TYPE_COMPRESSED: - return "ORACLE PAGE COMPRESSED"; } return "PAGE TYPE CORRUPTED"; @@ -112,8 +110,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_XDES || page_type == FIL_PAGE_TYPE_BLOB || page_type == FIL_PAGE_TYPE_ZBLOB || - page_type == FIL_PAGE_TYPE_ZBLOB2 || - page_type == FIL_PAGE_TYPE_COMPRESSED))) { + page_type == FIL_PAGE_TYPE_ZBLOB2))) { ulint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index edc932f36f5..0ac764d5a32 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -501,8 +501,7 @@ fil_decompress_page( /* Do not try to uncompressed pages that are not compressed */ if (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && - ptype != FIL_PAGE_TYPE_COMPRESSED) { + ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { return; } diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index a0db48f7037..91d46afe7a8 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -170,8 +170,7 @@ extern fil_addr_t fil_addr_null; #define FIL_PAGE_TYPE_BLOB 10 /*!< Uncompressed BLOB page */ #define FIL_PAGE_TYPE_ZBLOB 11 /*!< First compressed BLOB page */ #define FIL_PAGE_TYPE_ZBLOB2 12 /*!< Subsequent compressed BLOB page */ -#define FIL_PAGE_TYPE_COMPRESSED 13 /*!< Compressed page */ -#define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_COMPRESSED +#define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_ZBLOB2 /*!< Last page type */ /* @} */ diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic index 6c2504c9f8c..49fdff4f3e5 100644 --- a/storage/xtradb/include/fil0fil.ic +++ b/storage/xtradb/include/fil0fil.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -65,8 +65,6 @@ fil_get_page_type_name( return "ZBLOB"; case FIL_PAGE_TYPE_ZBLOB2: return "ZBLOB2"; - case FIL_PAGE_TYPE_COMPRESSED: - return "ORACLE PAGE COMPRESSED"; } return "PAGE TYPE CORRUPTED"; @@ -112,8 +110,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_XDES || page_type == FIL_PAGE_TYPE_BLOB || page_type == FIL_PAGE_TYPE_ZBLOB || - page_type == FIL_PAGE_TYPE_ZBLOB2 || - page_type == FIL_PAGE_TYPE_COMPRESSED))) { + page_type == FIL_PAGE_TYPE_ZBLOB2))) { ulint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); From 72005b7a1c42cb31294ead822b812909e5885f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Jun 2018 08:26:46 +0300 Subject: [PATCH 147/276] MDEV-13103: Improve 'cannot be decrypted' error message buf_page_check_corrupt(): Display the file name. --- .../encryption/r/innodb-bad-key-change.result | 8 ++++---- .../encryption/r/innodb-bad-key-change2.result | 5 ++--- .../encryption/r/innodb-bad-key-change4.result | 2 +- .../encryption/r/innodb-compressed-blob.result | 3 +-- .../r/innodb-encryption-disable.result | 3 +-- .../encryption/r/innodb-force-corrupt.result | 8 ++------ .../encryption/r/innodb-missing-key.result | 4 +--- .../encryption/r/innodb-redo-badkey.result | 17 +++++++---------- .../encryption/r/innodb-redo-nokeys.result | 13 +++++-------- .../encryption/t/innodb-bad-key-change.test | 9 +++++---- .../encryption/t/innodb-bad-key-change2.test | 5 ++--- .../encryption/t/innodb-bad-key-change4.test | 2 +- .../encryption/t/innodb-compressed-blob.test | 3 +-- .../encryption/t/innodb-encryption-disable.test | 3 +-- .../encryption/t/innodb-force-corrupt.test | 8 ++------ .../suite/encryption/t/innodb-missing-key.test | 4 +--- .../suite/encryption/t/innodb-redo-badkey.test | 17 +++++++---------- .../suite/encryption/t/innodb-redo-nokeys.test | 13 +++++-------- storage/innobase/buf/buf0buf.cc | 4 ++-- storage/xtradb/buf/buf0buf.cc | 4 ++-- 20 files changed, 53 insertions(+), 82 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index d581b98513d..51fc7a8cbc4 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -1,7 +1,7 @@ -call mtr.add_suppression("InnoDB: The page .*"); -call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* "); -call mtr.add_suppression("Plugin 'file_key_management' .*"); -call mtr.add_suppression("mysqld: File .*"); +call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); +call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t[12]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("mysqld: File .*keysbad3.txt' not found "); # Start server with keys2.txt SET GLOBAL innodb_file_format = `Barracuda`; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result index 94ed922a0ec..1ed78a9efd2 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result @@ -1,6 +1,5 @@ -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1new cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("Couldn't load plugins from 'file_key_management.*"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t1(new)?\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Table \'\"test\".\"t1\"\' tablespace is set as discarded."); SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result index 227b8cc3deb..a491b084764 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t1\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Cannot open table .*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management.*"); diff --git a/mysql-test/suite/encryption/r/innodb-compressed-blob.result b/mysql-test/suite/encryption/r/innodb-compressed-blob.result index 15c2a410948..ce73b80820f 100644 --- a/mysql-test/suite/encryption/r/innodb-compressed-blob.result +++ b/mysql-test/suite/encryption/r/innodb-compressed-blob.result @@ -1,5 +1,4 @@ -call mtr.add_suppression("InnoDB: However key management plugin or used key_version .*"); -call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file test/t[1-3] cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '..test.t[1-3]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("Unable to decompress space ..test.t[1-3].ibd \\[[1-9][0-9]*:[0-9]+\\]"); # Restart mysqld --file-key-management-filename=keys2.txt SET GLOBAL innodb_file_format = `Barracuda`; diff --git a/mysql-test/suite/encryption/r/innodb-encryption-disable.result b/mysql-test/suite/encryption/r/innodb-encryption-disable.result index dd2b025553e..4d15098d936 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-disable.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-disable.result @@ -1,5 +1,4 @@ -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t5 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("Couldn't load plugins from 'file_key_management*"); create table t5 ( `intcol1` int(32) DEFAULT NULL, diff --git a/mysql-test/suite/encryption/r/innodb-force-corrupt.result b/mysql-test/suite/encryption/r/innodb-force-corrupt.result index 3f3a2afb02d..7d63f47d17e 100644 --- a/mysql-test/suite/encryption/r/innodb-force-corrupt.result +++ b/mysql-test/suite/encryption/r/innodb-force-corrupt.result @@ -1,9 +1,5 @@ -CALL mtr.add_suppression("InnoDB: Database page corruption on disk or a failed .*"); -CALL mtr.add_suppression("InnoDB: Corruption: Block in space_id .*"); -CALL mtr.add_suppression("InnoDB: However key management plugin or used key_version .*"); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t[0-9]+ page \[page id: space=[0-9]+, page number=[0-9]+\]. You may have to recover from a backup."); SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; diff --git a/mysql-test/suite/encryption/r/innodb-missing-key.result b/mysql-test/suite/encryption/r/innodb-missing-key.result index 2f2cc025973..53ca0845c25 100644 --- a/mysql-test/suite/encryption/r/innodb-missing-key.result +++ b/mysql-test/suite/encryption/r/innodb-missing-key.result @@ -1,6 +1,4 @@ -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); # Start server with keys2.txt CREATE TABLE t1(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=19; diff --git a/mysql-test/suite/encryption/r/innodb-redo-badkey.result b/mysql-test/suite/encryption/r/innodb-redo-badkey.result index e12dad6d0d7..ed578d80b3a 100644 --- a/mysql-test/suite/encryption/r/innodb-redo-badkey.result +++ b/mysql-test/suite/encryption/r/innodb-redo-badkey.result @@ -1,16 +1,13 @@ call mtr.add_suppression("InnoDB: Block in space_id .* in file .* encrypted."); -call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("Plugin 'file_key_management' "); call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); -call mtr.add_suppression("InnoDB: Read operation failed for tablespace .*"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); -call mtr.add_suppression("InnoDB: Recovery read page .*"); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE .*"); -call mtr.add_suppression("InnoDB: Plugin initialization aborted .*"); +call mtr.add_suppression("InnoDB: Read operation failed for tablespace "); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed"); +call mtr.add_suppression("InnoDB: Recovery read page "); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[1234]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE "); +call mtr.add_suppression("InnoDB: Plugin initialization aborted "); call mtr.add_suppression("InnoDB: ############### CORRUPT LOG RECORD FOUND ##################"); # Restart mysqld --file-key-management-filename=keys2.txt # Wait max 10 min for key encryption threads to encrypt all spaces diff --git a/mysql-test/suite/encryption/r/innodb-redo-nokeys.result b/mysql-test/suite/encryption/r/innodb-redo-nokeys.result index dcbe1f5a395..1ea706dbde4 100644 --- a/mysql-test/suite/encryption/r/innodb-redo-nokeys.result +++ b/mysql-test/suite/encryption/r/innodb-redo-nokeys.result @@ -1,13 +1,10 @@ -call mtr.add_suppression("InnoDB: Block in space_id .*"); -call mtr.add_suppression("mysqld: File .*"); -call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("InnoDB: Block in space_id "); +call mtr.add_suppression("mysqld: File "); +call mtr.add_suppression("Plugin 'file_key_management' "); call mtr.add_suppression("InnoDB: cannot enable encryption, encryption plugin is not available"); -call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error\\."); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[1234]\\.ibd' cannot be decrypted\\."); # Restart mysqld --file-key-management-filename=keys2.txt SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index 6fa5fc9847f..9d61770f543 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -8,10 +8,11 @@ # table exists and encryption service is not available. # -call mtr.add_suppression("InnoDB: The page .*"); -call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* "); -call mtr.add_suppression("Plugin 'file_key_management' .*"); -call mtr.add_suppression("mysqld: File .*"); +call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); +call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t[12]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("mysqld: File .*keysbad3.txt' not found "); + --echo --echo # Start server with keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test index 52ee442c725..0c312f1f449 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test @@ -8,10 +8,9 @@ # MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys # MDEV-8727: Server/InnoDB hangs on shutdown after trying to read an encrypted table with a wrong key # -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1new cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t1(new)?\\.ibd' cannot be decrypted\\."); # Suppression for builds where file_key_management plugin is linked statically -call mtr.add_suppression("Couldn't load plugins from 'file_key_management.*"); +call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Table \'\"test\".\"t1\"\' tablespace is set as discarded."); --let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test index 14d88614f55..0c52bfe3fe6 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test @@ -7,7 +7,7 @@ # MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys # -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '.*test.t1\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Cannot open table .*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); # Suppression for builds where file_key_management plugin is linked statically diff --git a/mysql-test/suite/encryption/t/innodb-compressed-blob.test b/mysql-test/suite/encryption/t/innodb-compressed-blob.test index cb69f22f745..4a81dae5a0f 100644 --- a/mysql-test/suite/encryption/t/innodb-compressed-blob.test +++ b/mysql-test/suite/encryption/t/innodb-compressed-blob.test @@ -4,8 +4,7 @@ # embedded does not support restart -- source include/not_embedded.inc -call mtr.add_suppression("InnoDB: However key management plugin or used key_version .*"); -call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file test/t[1-3] cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[0-9]+\\] in file '..test.t[1-3]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("Unable to decompress space ..test.t[1-3].ibd \\[[1-9][0-9]*:[0-9]+\\]"); --echo # Restart mysqld --file-key-management-filename=keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-encryption-disable.test b/mysql-test/suite/encryption/t/innodb-encryption-disable.test index 38f36076c73..6ade8d6e3b2 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-disable.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-disable.test @@ -7,8 +7,7 @@ # MDEV-9559: Server without encryption configs crashes if selecting from an implicitly encrypted table # -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t5 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management*"); diff --git a/mysql-test/suite/encryption/t/innodb-force-corrupt.test b/mysql-test/suite/encryption/t/innodb-force-corrupt.test index dd4ee51b1eb..cb6440127cc 100644 --- a/mysql-test/suite/encryption/t/innodb-force-corrupt.test +++ b/mysql-test/suite/encryption/t/innodb-force-corrupt.test @@ -7,12 +7,8 @@ # Don't test under embedded -- source include/not_embedded.inc -CALL mtr.add_suppression("InnoDB: Database page corruption on disk or a failed .*"); -CALL mtr.add_suppression("InnoDB: Corruption: Block in space_id .*"); -CALL mtr.add_suppression("InnoDB: However key management plugin or used key_version .*"); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t[0-9]+ page \[page id: space=[0-9]+, page number=[0-9]+\]. You may have to recover from a backup."); --disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.test b/mysql-test/suite/encryption/t/innodb-missing-key.test index 07a2b16211c..6ea0528825b 100644 --- a/mysql-test/suite/encryption/t/innodb-missing-key.test +++ b/mysql-test/suite/encryption/t/innodb-missing-key.test @@ -7,9 +7,7 @@ # MDEV-11004: Unable to start (Segfault or os error 2) when encryption key missing # -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); --echo --echo # Start server with keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-redo-badkey.test b/mysql-test/suite/encryption/t/innodb-redo-badkey.test index 159646541c7..69de4f0f921 100644 --- a/mysql-test/suite/encryption/t/innodb-redo-badkey.test +++ b/mysql-test/suite/encryption/t/innodb-redo-badkey.test @@ -4,18 +4,15 @@ -- source include/not_embedded.inc call mtr.add_suppression("InnoDB: Block in space_id .* in file .* encrypted."); -call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("Plugin 'file_key_management' "); call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); -call mtr.add_suppression("InnoDB: Read operation failed for tablespace .*"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); -call mtr.add_suppression("InnoDB: Recovery read page .*"); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); -call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE .*"); -call mtr.add_suppression("InnoDB: Plugin initialization aborted .*"); +call mtr.add_suppression("InnoDB: Read operation failed for tablespace "); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed"); +call mtr.add_suppression("InnoDB: Recovery read page "); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[1234]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE "); +call mtr.add_suppression("InnoDB: Plugin initialization aborted "); call mtr.add_suppression("InnoDB: ############### CORRUPT LOG RECORD FOUND ##################"); --echo # Restart mysqld --file-key-management-filename=keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-redo-nokeys.test b/mysql-test/suite/encryption/t/innodb-redo-nokeys.test index e55e2ade153..68d831fcd17 100644 --- a/mysql-test/suite/encryption/t/innodb-redo-nokeys.test +++ b/mysql-test/suite/encryption/t/innodb-redo-nokeys.test @@ -3,16 +3,13 @@ # embedded does not support restart -- source include/not_embedded.inc -call mtr.add_suppression("InnoDB: Block in space_id .*"); -call mtr.add_suppression("mysqld: File .*"); -call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("InnoDB: Block in space_id "); +call mtr.add_suppression("mysqld: File "); +call mtr.add_suppression("Plugin 'file_key_management' "); call mtr.add_suppression("InnoDB: cannot enable encryption, encryption plugin is not available"); -call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error\\."); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); -call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[1234]\\.ibd' cannot be decrypted\\."); --echo # Restart mysqld --file-key-management-filename=keys2.txt -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 50c052fc690..ee8f40e7d39 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4746,9 +4746,9 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) ib_logf(IB_LOG_LEVEL_ERROR, "The page [page id: space=%u" ", page number=%u]" - " in file %s cannot be decrypted.", + " in file '%s' cannot be decrypted.", bpage->space, bpage->offset, - space->name); + space->chain.start->name); ib_logf(IB_LOG_LEVEL_INFO, "However key management plugin or used key_version " ULINTPF diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 3814d08f19a..5728b8a9fa3 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -4693,9 +4693,9 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) ib_logf(IB_LOG_LEVEL_ERROR, "The page [page id: space=%u" ", page number=%u]" - " in file %s cannot be decrypted.", + " in file '%s' cannot be decrypted.", bpage->space, bpage->offset, - space->name); + space->chain.start->name); ib_logf(IB_LOG_LEVEL_INFO, "However key management plugin or used key_version " ULINTPF From 51254da52c144781695250a98f365fbc090ef619 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 12 Jun 2018 12:37:28 +0200 Subject: [PATCH 148/276] MDEV-15359 Thread stay in "cleaning up" status after finishing make thd_get_error_context_description() to show not just thd->proc_info, but exactly the same thread state that SHOW PROCESSLIST shows. --- mysql-test/suite/plugins/r/processlist.result | 8 ++ mysql-test/suite/plugins/t/processlist.test | 18 ++++ sql/sql_class.cc | 97 ------------------- sql/sql_show.cc | 85 ++++++++++++++++ 4 files changed, 111 insertions(+), 97 deletions(-) create mode 100644 mysql-test/suite/plugins/r/processlist.result create mode 100644 mysql-test/suite/plugins/t/processlist.test diff --git a/mysql-test/suite/plugins/r/processlist.result b/mysql-test/suite/plugins/r/processlist.result new file mode 100644 index 00000000000..d08ea9d3523 --- /dev/null +++ b/mysql-test/suite/plugins/r/processlist.result @@ -0,0 +1,8 @@ +create table t1 (a int) engine=innodb; +start transaction; +insert t1 values (1); +state from show engine innodb status + +state from show processlist + +drop table t1; diff --git a/mysql-test/suite/plugins/t/processlist.test b/mysql-test/suite/plugins/t/processlist.test new file mode 100644 index 00000000000..5aacef317b9 --- /dev/null +++ b/mysql-test/suite/plugins/t/processlist.test @@ -0,0 +1,18 @@ +# +# MDEV-15359 Thread stay in "cleaning up" status after finishing +# +source include/have_innodb.inc; + +create table t1 (a int) engine=innodb; +start transaction; +insert t1 values (1); +let id=`select connection_id()`; +connect con2,localhost,root; +let s=query_get_value(show engine innodb status,Status,1); +disable_query_log; +eval select regexp_replace("$s", '(?s)^.*MySQL thread id $id,.*root([^\n]*)\n.*', '\\\\1') as `state from show engine innodb status`; +eval select state as `state from show processlist` from information_schema.processlist where id = $id; +enable_query_log; +disconnect con2; +connection default; +drop table t1; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index a644df9771c..3b293935048 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -717,103 +717,6 @@ extern "C" } -/** - Dumps a text description of a thread, its security context - (user, host) and the current query. - - @param thd thread context - @param buffer pointer to preferred result buffer - @param length length of buffer - @param max_query_len how many chars of query to copy (0 for all) - - @return Pointer to string -*/ - -extern "C" -char *thd_get_error_context_description(THD *thd, char *buffer, - unsigned int length, - unsigned int max_query_len) -{ - String str(buffer, length, &my_charset_latin1); - const Security_context *sctx= &thd->main_security_ctx; - char header[256]; - int len; - - mysql_mutex_lock(&LOCK_thread_count); - - /* - The pointers thd->query and thd->proc_info might change since they are - being modified concurrently. This is acceptable for proc_info since its - values doesn't have to very accurate and the memory it points to is static, - but we need to attempt a snapshot on the pointer values to avoid using NULL - values. The pointer to thd->query however, doesn't point to static memory - and has to be protected by thd->LOCK_thd_data or risk pointing to - uninitialized memory. - */ - const char *proc_info= thd->proc_info; - - len= my_snprintf(header, sizeof(header), - "MySQL thread id %lu, OS thread handle 0x%lx, query id %lu", - thd->thread_id, (ulong) thd->real_id, (ulong) thd->query_id); - str.length(0); - str.append(header, len); - - if (sctx->host) - { - str.append(' '); - str.append(sctx->host); - } - - if (sctx->ip) - { - str.append(' '); - str.append(sctx->ip); - } - - if (sctx->user) - { - str.append(' '); - str.append(sctx->user); - } - - if (proc_info) - { - str.append(' '); - str.append(proc_info); - } - - /* Don't wait if LOCK_thd_data is used as this could cause a deadlock */ - if (!mysql_mutex_trylock(&thd->LOCK_thd_data)) - { - if (thd->query()) - { - if (max_query_len < 1) - len= thd->query_length(); - else - len= MY_MIN(thd->query_length(), max_query_len); - str.append('\n'); - str.append(thd->query(), len); - } - mysql_mutex_unlock(&thd->LOCK_thd_data); - } - mysql_mutex_unlock(&LOCK_thread_count); - - if (str.c_ptr_safe() == buffer) - return buffer; - - /* - We have to copy the new string to the destination buffer because the string - was reallocated to a larger buffer to be able to fit. - */ - DBUG_ASSERT(buffer != NULL); - length= MY_MIN(str.length(), length-1); - memcpy(buffer, str.c_ptr_quick(), length); - /* Make sure that the new string is null terminated */ - buffer[length]= '\0'; - return buffer; -} - - #if MARIA_PLUGIN_INTERFACE_VERSION < 0x0200 /** TODO: This function is for API compatibility, remove it eventually. diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 965bed80148..d7641a6a96d 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -9773,3 +9773,88 @@ static void get_cs_converted_string_value(THD *thd, return; } #endif + +/** + Dumps a text description of a thread, its security context + (user, host) and the current query. + + @param thd thread context + @param buffer pointer to preferred result buffer + @param length length of buffer + @param max_query_len how many chars of query to copy (0 for all) + + @return Pointer to string +*/ + +extern "C" +char *thd_get_error_context_description(THD *thd, char *buffer, + unsigned int length, + unsigned int max_query_len) +{ + String str(buffer, length, &my_charset_latin1); + const Security_context *sctx= &thd->main_security_ctx; + char header[256]; + int len; + + mysql_mutex_lock(&LOCK_thread_count); + + len= my_snprintf(header, sizeof(header), + "MySQL thread id %lu, OS thread handle 0x%lx, query id %lu", + thd->thread_id, (ulong) thd->real_id, (ulong) thd->query_id); + str.length(0); + str.append(header, len); + + if (sctx->host) + { + str.append(' '); + str.append(sctx->host); + } + + if (sctx->ip) + { + str.append(' '); + str.append(sctx->ip); + } + + if (sctx->user) + { + str.append(' '); + str.append(sctx->user); + } + + /* Don't wait if LOCK_thd_data is used as this could cause a deadlock */ + if (!mysql_mutex_trylock(&thd->LOCK_thd_data)) + { + if (const char *info= thread_state_info(thd)) + { + str.append(' '); + str.append(info); + } + + if (thd->query()) + { + if (max_query_len < 1) + len= thd->query_length(); + else + len= MY_MIN(thd->query_length(), max_query_len); + str.append('\n'); + str.append(thd->query(), len); + } + mysql_mutex_unlock(&thd->LOCK_thd_data); + } + mysql_mutex_unlock(&LOCK_thread_count); + + if (str.c_ptr_safe() == buffer) + return buffer; + + /* + We have to copy the new string to the destination buffer because the string + was reallocated to a larger buffer to be able to fit. + */ + DBUG_ASSERT(buffer != NULL); + length= MY_MIN(str.length(), length-1); + memcpy(buffer, str.c_ptr_quick(), length); + /* Make sure that the new string is null terminated */ + buffer[length]= '\0'; + return buffer; +} From 3661d9882224dd485556ce937c1294eaeda02ef8 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 13 Jun 2018 20:31:40 +0200 Subject: [PATCH 149/276] fix SHOW PROCESSLIST for --embedded make it return the same Info values as for the standalone server. This fixes plugins.processlist for --embedded --- sql/sql_show.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/sql_show.cc b/sql/sql_show.cc index d7641a6a96d..af02c74c58b 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2267,16 +2267,16 @@ static const char *thread_state_info(THD *tmp) else return "Reading from net"; } - else +#else + if (tmp->get_command() == COM_SLEEP) + return ""; #endif - { - if (tmp->proc_info) - return tmp->proc_info; - else if (tmp->mysys_var && tmp->mysys_var->current_cond) - return "Waiting on cond"; - else - return NULL; - } + if (tmp->proc_info) + return tmp->proc_info; + else if (tmp->mysys_var && tmp->mysys_var->current_cond) + return "Waiting on cond"; + else + return NULL; } void mysqld_list_processes(THD *thd,const char *user, bool verbose) From 23ced2f846c6a8b9b303c1365780999888fa438f Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 13 Jun 2018 23:37:09 +0400 Subject: [PATCH 150/276] MDEV-16311 Server crash when using a NAME_CONST() with a CURSOR Problem: The problem was most likely introduced by a fix for MDEV-11597 (commit 5f0c31f928338e8a6ffde098b7ffd3d1a8b02903) which removed the assignment "killed= KILL_BAD_DATA" from THD::raise_condition(). Before MDEV-11597, sp_head::execute() tested thd->killed after looping through the SP instructions and exited with an error if thd->killed is set. After MDEV-11597, sp_head::execute() stopped to notice errors and set the OK status on top of the error status, which crashed on assert. Fix: Making sp_cursor::fetch() return -1 if server_side_cursor->fetch(1) left an error in the diagnostics area. This makes the statement "err_status= i->execute(thd, &ip)" in sp_head::execute() set the error code and correctly break the SP instruction loop and return on error without setting the OK status. --- mysql-test/r/sp.result | 17 +++++++++++++++++ mysql-test/t/sp.test | 22 ++++++++++++++++++++++ sql/sp_rcontext.cc | 6 ++++++ 3 files changed, 45 insertions(+) diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index 59387b37585..7e466f23c72 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -8365,3 +8365,20 @@ ERROR HY000: Window function is allowed only in SELECT list and ORDER BY clause CALL p1(SUM(1)); ERROR HY000: Invalid use of group function DROP PROCEDURE p1; +# +# MDEV-16311 Server crash when using a NAME_CONST() with a CURSOR +# +SET sql_mode=STRICT_ALL_TABLES; +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (10); +BEGIN NOT ATOMIC +DECLARE a INT; +DECLARE c CURSOR FOR SELECT NAME_CONST('x','y') FROM t1; +OPEN c; +FETCH c INTO a; +CLOSE c; +END; +$$ +ERROR 22007: Incorrect integer value: 'y' for column 'a' at row 1 +DROP TABLE t1; +SET sql_mode=DEFAULT; diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index e8b63c4d791..a2155d3fe79 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9879,3 +9879,25 @@ CALL p1(ROW_NUMBER() OVER ()); --error ER_INVALID_GROUP_FUNC_USE CALL p1(SUM(1)); DROP PROCEDURE p1; + + +--echo # +--echo # MDEV-16311 Server crash when using a NAME_CONST() with a CURSOR +--echo # + +SET sql_mode=STRICT_ALL_TABLES; +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (10); +DELIMITER $$; +--error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD +BEGIN NOT ATOMIC + DECLARE a INT; + DECLARE c CURSOR FOR SELECT NAME_CONST('x','y') FROM t1; + OPEN c; + FETCH c INTO a; + CLOSE c; +END; +$$ +DELIMITER ;$$ +DROP TABLE t1; +SET sql_mode=DEFAULT; diff --git a/sql/sp_rcontext.cc b/sql/sp_rcontext.cc index 396f5b448fc..d612e15c000 100644 --- a/sql/sp_rcontext.cc +++ b/sql/sp_rcontext.cc @@ -509,9 +509,15 @@ int sp_cursor::fetch(THD *thd, List *vars) result.set_spvar_list(vars); + DBUG_ASSERT(!thd->is_error()); + /* Attempt to fetch one row */ if (server_side_cursor->is_open()) + { server_side_cursor->fetch(1); + if (thd->is_error()) + return -1; // e.g. data type conversion failed + } /* If the cursor was pointing after the last row, the fetch will From 2cdb483bc4ed5bf816ff609cce91745babc9657a Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 14 Jun 2018 13:13:23 +0400 Subject: [PATCH 151/276] MDEV-15352 AUTO_INCREMENT breaks after updating a column value to a negative number --- mysql-test/include/autoinc_mdev15353.inc | 29 +++++ mysql-test/r/auto_increment.result | 120 +++++++++++++++++ .../suite/heap/heap_auto_increment.result | 120 +++++++++++++++++ .../suite/heap/heap_auto_increment.test | 7 + .../suite/innodb/r/innodb-autoinc.result | 121 ++++++++++++++++++ mysql-test/suite/innodb/t/innodb-autoinc.test | 10 ++ mysql-test/suite/maria/maria-autoinc.result | 120 +++++++++++++++++ mysql-test/suite/maria/maria-autoinc.test | 8 ++ mysql-test/t/auto_increment.test | 8 ++ sql/field.cc | 10 ++ sql/field.h | 117 ++++++++++------- 11 files changed, 624 insertions(+), 46 deletions(-) create mode 100644 mysql-test/include/autoinc_mdev15353.inc create mode 100644 mysql-test/suite/maria/maria-autoinc.result create mode 100644 mysql-test/suite/maria/maria-autoinc.test diff --git a/mysql-test/include/autoinc_mdev15353.inc b/mysql-test/include/autoinc_mdev15353.inc new file mode 100644 index 00000000000..9085cb29f20 --- /dev/null +++ b/mysql-test/include/autoinc_mdev15353.inc @@ -0,0 +1,29 @@ +DELIMITER $$; +CREATE PROCEDURE autoinc_mdev15353_one(engine VARCHAR(64), t VARCHAR(64)) +BEGIN + DECLARE query TEXT DEFAULT 'CREATE TABLE t1 (' + ' id TTT NOT NULL AUTO_INCREMENT,' + ' name CHAR(30) NOT NULL,' + ' PRIMARY KEY (id)) ENGINE=EEE'; + EXECUTE IMMEDIATE REPLACE(REPLACE(query,'TTT', t), 'EEE', engine); + SHOW CREATE TABLE t1; + INSERT INTO t1 (name) VALUES ('dog'); + SELECT * FROM t1; + UPDATE t1 SET id=-1 WHERE id=1; + SELECT * FROM t1; + INSERT INTO t1 (name) VALUES ('cat'); + SELECT * FROM t1; + DROP TABLE t1; +END; +$$ +DELIMITER ;$$ + +CALL autoinc_mdev15353_one(@engine, 'tinyint'); +CALL autoinc_mdev15353_one(@engine, 'smallint'); +CALL autoinc_mdev15353_one(@engine, 'mediumint'); +CALL autoinc_mdev15353_one(@engine, 'int'); +CALL autoinc_mdev15353_one(@engine, 'bigint'); +CALL autoinc_mdev15353_one(@engine, 'float'); +CALL autoinc_mdev15353_one(@engine, 'double'); + +DROP PROCEDURE autoinc_mdev15353_one; diff --git a/mysql-test/r/auto_increment.result b/mysql-test/r/auto_increment.result index 12cbf294b69..660a93b1b30 100644 --- a/mysql-test/r/auto_increment.result +++ b/mysql-test/r/auto_increment.result @@ -537,3 +537,123 @@ pk -5 1 drop table t1; +# +# MDEV-15352 AUTO_INCREMENT breaks after updating a column value to a negative number +# +SET @engine='MyISAM'; +CREATE PROCEDURE autoinc_mdev15353_one(engine VARCHAR(64), t VARCHAR(64)) +BEGIN +DECLARE query TEXT DEFAULT 'CREATE TABLE t1 (' + ' id TTT NOT NULL AUTO_INCREMENT,' + ' name CHAR(30) NOT NULL,' + ' PRIMARY KEY (id)) ENGINE=EEE'; +EXECUTE IMMEDIATE REPLACE(REPLACE(query,'TTT', t), 'EEE', engine); +SHOW CREATE TABLE t1; +INSERT INTO t1 (name) VALUES ('dog'); +SELECT * FROM t1; +UPDATE t1 SET id=-1 WHERE id=1; +SELECT * FROM t1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +DROP TABLE t1; +END; +$$ +CALL autoinc_mdev15353_one(@engine, 'tinyint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` tinyint(4) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'smallint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` smallint(6) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'mediumint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` mediumint(9) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'int'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'bigint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'float'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` float NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'double'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` double NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +DROP PROCEDURE autoinc_mdev15353_one; diff --git a/mysql-test/suite/heap/heap_auto_increment.result b/mysql-test/suite/heap/heap_auto_increment.result index 5b04a77e9eb..d6504349d92 100644 --- a/mysql-test/suite/heap/heap_auto_increment.result +++ b/mysql-test/suite/heap/heap_auto_increment.result @@ -39,3 +39,123 @@ _rowid _rowid skey sval 1 1 1 hello 2 2 2 hey drop table t1; +# +# MDEV-15352 AUTO_INCREMENT breaks after updating a column value to a negative number +# +SET @engine='MEMORY'; +CREATE PROCEDURE autoinc_mdev15353_one(engine VARCHAR(64), t VARCHAR(64)) +BEGIN +DECLARE query TEXT DEFAULT 'CREATE TABLE t1 (' + ' id TTT NOT NULL AUTO_INCREMENT,' + ' name CHAR(30) NOT NULL,' + ' PRIMARY KEY (id)) ENGINE=EEE'; +EXECUTE IMMEDIATE REPLACE(REPLACE(query,'TTT', t), 'EEE', engine); +SHOW CREATE TABLE t1; +INSERT INTO t1 (name) VALUES ('dog'); +SELECT * FROM t1; +UPDATE t1 SET id=-1 WHERE id=1; +SELECT * FROM t1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +DROP TABLE t1; +END; +$$ +CALL autoinc_mdev15353_one(@engine, 'tinyint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` tinyint(4) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MEMORY DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'smallint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` smallint(6) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MEMORY DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'mediumint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` mediumint(9) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MEMORY DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'int'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MEMORY DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'bigint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MEMORY DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'float'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` float NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MEMORY DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'double'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` double NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=MEMORY DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +DROP PROCEDURE autoinc_mdev15353_one; diff --git a/mysql-test/suite/heap/heap_auto_increment.test b/mysql-test/suite/heap/heap_auto_increment.test index 016bc946209..6ec150f48da 100644 --- a/mysql-test/suite/heap/heap_auto_increment.test +++ b/mysql-test/suite/heap/heap_auto_increment.test @@ -33,3 +33,10 @@ select _rowid,t1._rowid,skey,sval from t1; drop table t1; # End of 4.1 tests + +--echo # +--echo # MDEV-15352 AUTO_INCREMENT breaks after updating a column value to a negative number +--echo # + +SET @engine='MEMORY'; +--source include/autoinc_mdev15353.inc diff --git a/mysql-test/suite/innodb/r/innodb-autoinc.result b/mysql-test/suite/innodb/r/innodb-autoinc.result index ee1adc07661..eb7ff026b1a 100644 --- a/mysql-test/suite/innodb/r/innodb-autoinc.result +++ b/mysql-test/suite/innodb/r/innodb-autoinc.result @@ -1351,6 +1351,7 @@ t CREATE TABLE `t` ( KEY `i` (`i`) ) ENGINE=InnoDB AUTO_INCREMENT=401 DEFAULT CHARSET=latin1 DROP TABLE t; +SET auto_increment_increment = DEFAULT; # # MDEV-14008 Assertion failing: `!is_set() || (m_status == DA_OK_BULK && is_bulk_op()) # @@ -1369,3 +1370,123 @@ SELECT * FROM t1; a -1 DROP TABLE t1; +# +# MDEV-15352 AUTO_INCREMENT breaks after updating a column value to a negative number +# +SET @engine='INNODB'; +CREATE PROCEDURE autoinc_mdev15353_one(engine VARCHAR(64), t VARCHAR(64)) +BEGIN +DECLARE query TEXT DEFAULT 'CREATE TABLE t1 (' + ' id TTT NOT NULL AUTO_INCREMENT,' + ' name CHAR(30) NOT NULL,' + ' PRIMARY KEY (id)) ENGINE=EEE'; +EXECUTE IMMEDIATE REPLACE(REPLACE(query,'TTT', t), 'EEE', engine); +SHOW CREATE TABLE t1; +INSERT INTO t1 (name) VALUES ('dog'); +SELECT * FROM t1; +UPDATE t1 SET id=-1 WHERE id=1; +SELECT * FROM t1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +DROP TABLE t1; +END; +$$ +CALL autoinc_mdev15353_one(@engine, 'tinyint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` tinyint(4) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'smallint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` smallint(6) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'mediumint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` mediumint(9) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'int'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'bigint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'float'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` float NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'double'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` double NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +DROP PROCEDURE autoinc_mdev15353_one; diff --git a/mysql-test/suite/innodb/t/innodb-autoinc.test b/mysql-test/suite/innodb/t/innodb-autoinc.test index db265ff8f67..b8f2d75c876 100644 --- a/mysql-test/suite/innodb/t/innodb-autoinc.test +++ b/mysql-test/suite/innodb/t/innodb-autoinc.test @@ -683,6 +683,8 @@ INSERT INTO t VALUES (NULL); SELECT * FROM t; SHOW CREATE TABLE t; DROP TABLE t; +SET auto_increment_increment = DEFAULT; + --echo # --echo # MDEV-14008 Assertion failing: `!is_set() || (m_status == DA_OK_BULK && is_bulk_op()) @@ -700,3 +702,11 @@ CREATE TABLE t1 (a DOUBLE PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB; INSERT INTO t1 VALUES (-1); SELECT * FROM t1; DROP TABLE t1; + + +--echo # +--echo # MDEV-15352 AUTO_INCREMENT breaks after updating a column value to a negative number +--echo # + +SET @engine='INNODB'; +--source include/autoinc_mdev15353.inc diff --git a/mysql-test/suite/maria/maria-autoinc.result b/mysql-test/suite/maria/maria-autoinc.result new file mode 100644 index 00000000000..a4ae1f72c1e --- /dev/null +++ b/mysql-test/suite/maria/maria-autoinc.result @@ -0,0 +1,120 @@ +# +# MDEV-15352 AUTO_INCREMENT breaks after updating a column value to a negative number +# +SET @engine='ARIA'; +CREATE PROCEDURE autoinc_mdev15353_one(engine VARCHAR(64), t VARCHAR(64)) +BEGIN +DECLARE query TEXT DEFAULT 'CREATE TABLE t1 (' + ' id TTT NOT NULL AUTO_INCREMENT,' + ' name CHAR(30) NOT NULL,' + ' PRIMARY KEY (id)) ENGINE=EEE'; +EXECUTE IMMEDIATE REPLACE(REPLACE(query,'TTT', t), 'EEE', engine); +SHOW CREATE TABLE t1; +INSERT INTO t1 (name) VALUES ('dog'); +SELECT * FROM t1; +UPDATE t1 SET id=-1 WHERE id=1; +SELECT * FROM t1; +INSERT INTO t1 (name) VALUES ('cat'); +SELECT * FROM t1; +DROP TABLE t1; +END; +$$ +CALL autoinc_mdev15353_one(@engine, 'tinyint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` tinyint(4) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'smallint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` smallint(6) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'mediumint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` mediumint(9) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'int'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'bigint'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'float'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` float NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +CALL autoinc_mdev15353_one(@engine, 'double'); +Table Create Table +t1 CREATE TABLE `t1` ( + `id` double NOT NULL AUTO_INCREMENT, + `name` char(30) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 +id name +1 dog +id name +-1 dog +id name +-1 dog +2 cat +DROP PROCEDURE autoinc_mdev15353_one; diff --git a/mysql-test/suite/maria/maria-autoinc.test b/mysql-test/suite/maria/maria-autoinc.test new file mode 100644 index 00000000000..e7dc10b503b --- /dev/null +++ b/mysql-test/suite/maria/maria-autoinc.test @@ -0,0 +1,8 @@ +-- source include/have_maria.inc + +--echo # +--echo # MDEV-15352 AUTO_INCREMENT breaks after updating a column value to a negative number +--echo # + +SET @engine='ARIA'; +--source include/autoinc_mdev15353.inc diff --git a/mysql-test/t/auto_increment.test b/mysql-test/t/auto_increment.test index 7f0ab5dc169..6f678ed309f 100644 --- a/mysql-test/t/auto_increment.test +++ b/mysql-test/t/auto_increment.test @@ -397,3 +397,11 @@ insert into t1 values(null); select last_insert_id(); select * from t1; drop table t1; + + +--echo # +--echo # MDEV-15352 AUTO_INCREMENT breaks after updating a column value to a negative number +--echo # + +SET @engine='MyISAM'; +--source include/autoinc_mdev15353.inc diff --git a/sql/field.cc b/sql/field.cc index a8d82170d52..56948acd8ba 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -3328,6 +3328,16 @@ longlong Field_new_decimal::val_int(void) } +ulonglong Field_new_decimal::val_uint(void) +{ + ASSERT_COLUMN_MARKED_FOR_READ; + longlong i; + my_decimal decimal_value; + my_decimal2int(E_DEC_FATAL_ERROR, val_decimal(&decimal_value), true, &i); + return i; +} + + my_decimal* Field_new_decimal::val_decimal(my_decimal *decimal_value) { ASSERT_COLUMN_MARKED_FOR_READ; diff --git a/sql/field.h b/sql/field.h index 4bf92faecdd..ca85a7c276f 100644 --- a/sql/field.h +++ b/sql/field.h @@ -851,9 +851,14 @@ public: { return store(ls->str, (uint32) ls->length, cs); } virtual double val_real(void)=0; virtual longlong val_int(void)=0; + /* + Get ulonglong representation. + Negative values are truncated to 0. + */ virtual ulonglong val_uint(void) { - return (ulonglong) val_int(); + longlong nr= val_int(); + return nr < 0 ? 0 : (ulonglong) nr; } virtual bool val_bool(void)= 0; virtual my_decimal *val_decimal(my_decimal *); @@ -1898,6 +1903,7 @@ public: int store_decimal(const my_decimal *); double val_real(void); longlong val_int(void); + ulonglong val_uint(void); my_decimal *val_decimal(my_decimal *); String *val_str(String*, String *); bool get_date(MYSQL_TIME *ltime, ulonglong fuzzydate); @@ -1925,15 +1931,34 @@ public: }; -class Field_tiny :public Field_num { +class Field_integer: public Field_num +{ +public: + Field_integer(uchar *ptr_arg, uint32 len_arg, + uchar *null_ptr_arg, uchar null_bit_arg, + enum utype unireg_check_arg, const char *field_name_arg, + bool zero_arg, bool unsigned_arg) + :Field_num(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, + unireg_check_arg, field_name_arg, 0, + zero_arg, unsigned_arg) + { } + ulonglong val_uint() + { + longlong nr= val_int(); + return nr < 0 && !unsigned_flag ? 0 : (ulonglong) nr; + } +}; + + +class Field_tiny :public Field_integer { public: Field_tiny(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, - uchar null_bit_arg, - enum utype unireg_check_arg, const char *field_name_arg, - bool zero_arg, bool unsigned_arg) - :Field_num(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, - unireg_check_arg, field_name_arg, - 0, zero_arg,unsigned_arg) + uchar null_bit_arg, + enum utype unireg_check_arg, const char *field_name_arg, + bool zero_arg, bool unsigned_arg) + :Field_integer(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, + unireg_check_arg, field_name_arg, + zero_arg, unsigned_arg) {} enum_field_types type() const { return MYSQL_TYPE_TINY;} enum ha_base_keytype key_type() const @@ -1969,20 +1994,20 @@ public: }; -class Field_short :public Field_num { +class Field_short :public Field_integer { public: Field_short(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, - uchar null_bit_arg, - enum utype unireg_check_arg, const char *field_name_arg, - bool zero_arg, bool unsigned_arg) - :Field_num(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, - unireg_check_arg, field_name_arg, - 0, zero_arg,unsigned_arg) + uchar null_bit_arg, + enum utype unireg_check_arg, const char *field_name_arg, + bool zero_arg, bool unsigned_arg) + :Field_integer(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, + unireg_check_arg, field_name_arg, + zero_arg, unsigned_arg) {} Field_short(uint32 len_arg,bool maybe_null_arg, const char *field_name_arg, - bool unsigned_arg) - :Field_num((uchar*) 0, len_arg, maybe_null_arg ? (uchar*) "": 0,0, - NONE, field_name_arg, 0, 0, unsigned_arg) + bool unsigned_arg) + :Field_integer((uchar*) 0, len_arg, maybe_null_arg ? (uchar*) "" : 0, 0, + NONE, field_name_arg, 0, unsigned_arg) {} enum_field_types type() const { return MYSQL_TYPE_SHORT;} enum ha_base_keytype key_type() const @@ -2009,15 +2034,15 @@ public: { return unpack_int16(to, from, from_end); } }; -class Field_medium :public Field_num { +class Field_medium :public Field_integer { public: Field_medium(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, - uchar null_bit_arg, - enum utype unireg_check_arg, const char *field_name_arg, - bool zero_arg, bool unsigned_arg) - :Field_num(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, - unireg_check_arg, field_name_arg, - 0, zero_arg,unsigned_arg) + uchar null_bit_arg, + enum utype unireg_check_arg, const char *field_name_arg, + bool zero_arg, bool unsigned_arg) + :Field_integer(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, + unireg_check_arg, field_name_arg, + zero_arg, unsigned_arg) {} enum_field_types type() const { return MYSQL_TYPE_INT24;} enum ha_base_keytype key_type() const @@ -2043,20 +2068,20 @@ public: }; -class Field_long :public Field_num { +class Field_long :public Field_integer { public: Field_long(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, - uchar null_bit_arg, - enum utype unireg_check_arg, const char *field_name_arg, - bool zero_arg, bool unsigned_arg) - :Field_num(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, - unireg_check_arg, field_name_arg, - 0, zero_arg,unsigned_arg) + uchar null_bit_arg, + enum utype unireg_check_arg, const char *field_name_arg, + bool zero_arg, bool unsigned_arg) + :Field_integer(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, + unireg_check_arg, field_name_arg, + zero_arg, unsigned_arg) {} Field_long(uint32 len_arg,bool maybe_null_arg, const char *field_name_arg, - bool unsigned_arg) - :Field_num((uchar*) 0, len_arg, maybe_null_arg ? (uchar*) "": 0,0, - NONE, field_name_arg,0,0,unsigned_arg) + bool unsigned_arg) + :Field_integer((uchar*) 0, len_arg, maybe_null_arg ? (uchar*) "" : 0, 0, + NONE, field_name_arg, 0, unsigned_arg) {} enum_field_types type() const { return MYSQL_TYPE_LONG;} enum ha_base_keytype key_type() const @@ -2088,21 +2113,21 @@ public: }; -class Field_longlong :public Field_num { +class Field_longlong :public Field_integer { public: Field_longlong(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, - uchar null_bit_arg, - enum utype unireg_check_arg, const char *field_name_arg, - bool zero_arg, bool unsigned_arg) - :Field_num(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, - unireg_check_arg, field_name_arg, - 0, zero_arg,unsigned_arg) + uchar null_bit_arg, + enum utype unireg_check_arg, const char *field_name_arg, + bool zero_arg, bool unsigned_arg) + :Field_integer(ptr_arg, len_arg, null_ptr_arg, null_bit_arg, + unireg_check_arg, field_name_arg, + zero_arg, unsigned_arg) {} Field_longlong(uint32 len_arg,bool maybe_null_arg, - const char *field_name_arg, - bool unsigned_arg) - :Field_num((uchar*) 0, len_arg, maybe_null_arg ? (uchar*) "": 0,0, - NONE, field_name_arg,0,0,unsigned_arg) + const char *field_name_arg, + bool unsigned_arg) + :Field_integer((uchar*) 0, len_arg, maybe_null_arg ? (uchar*) "" : 0, 0, + NONE, field_name_arg,0, unsigned_arg) {} enum_field_types type() const { return MYSQL_TYPE_LONGLONG;} enum ha_base_keytype key_type() const From f5eb37129f24893ab095e78c6fd2ef87e2c460cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Jun 2018 16:15:21 +0300 Subject: [PATCH 152/276] MDEV-13103 Deal with page_compressed page corruption fil_page_decompress(): Replaces fil_decompress_page(). Allow the caller detect errors. Remove duplicated code. Use the "safe" instead of "fast" variants of decompression routines. fil_page_compress(): Replaces fil_compress_page(). The length of the input buffer always was srv_page_size (innodb_page_size). Remove printouts, and remove the fil_space_t* parameter. buf_tmp_buffer_t::reserved: Make private; the accessors acquire() and release() will use atomic memory access. buf_pool_reserve_tmp_slot(): Make static. Remove the second parameter. Do not acquire any mutex. Remove the allocation of the buffers. buf_tmp_reserve_crypt_buf(), buf_tmp_reserve_compression_buf(): Refactored away from buf_pool_reserve_tmp_slot(). buf_page_decrypt_after_read(): Make static, and simplify the logic. Use the encryption buffer also for decompressing. buf_page_io_complete(), buf_dblwr_process(): Check more failures. fil_space_encrypt(): Simplify the debug checks. fil_space_t::printed_compression_failure: Remove. fil_get_compression_alg_name(): Remove. fil_iterate(): Allocate a buffer for compression and decompression only once, instead of allocating and freeing it for every page that uses compression, during IMPORT TABLESPACE. fil_node_get_space_id(), fil_page_is_index_page(), fil_page_is_lzo_compressed(): Remove (unused code). --- .../r/innodb-page_compression_default.result | 1 - .../r/innodb-page_compression_snappy.result | 1 - .../t/innodb-page_compression_default.test | 2 - .../t/innodb-page_compression_snappy.test | 2 - storage/innobase/buf/buf0buf.cc | 370 ++++------ storage/innobase/buf/buf0dblwr.cc | 37 +- storage/innobase/fil/fil0crypt.cc | 59 +- storage/innobase/fil/fil0fil.cc | 13 - storage/innobase/fil/fil0pagecompress.cc | 683 +++++------------- storage/innobase/ibuf/ibuf0ibuf.cc | 5 + storage/innobase/include/buf0buf.h | 53 +- storage/innobase/include/fil0fil.h | 3 - storage/innobase/include/fil0fil.ic | 1 - storage/innobase/include/fil0pagecompress.h | 93 +-- storage/innobase/include/fsp0pagecompress.ic | 70 +- storage/innobase/row/row0import.cc | 77 +- storage/xtradb/buf/buf0buf.cc | 380 +++++----- storage/xtradb/buf/buf0dblwr.cc | 40 +- storage/xtradb/fil/fil0crypt.cc | 59 +- storage/xtradb/fil/fil0fil.cc | 13 - storage/xtradb/fil/fil0pagecompress.cc | 683 +++++------------- storage/xtradb/ibuf/ibuf0ibuf.cc | 4 + storage/xtradb/include/buf0buf.h | 53 +- storage/xtradb/include/fil0fil.h | 3 - storage/xtradb/include/fil0fil.ic | 1 - storage/xtradb/include/fil0pagecompress.h | 93 +-- storage/xtradb/include/fsp0pagecompress.ic | 70 +- storage/xtradb/row/row0import.cc | 77 +- 28 files changed, 945 insertions(+), 2001 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-page_compression_default.result b/mysql-test/suite/innodb/r/innodb-page_compression_default.result index 18f7fb5e04a..19d0a875ce6 100644 --- a/mysql-test/suite/innodb/r/innodb-page_compression_default.result +++ b/mysql-test/suite/innodb/r/innodb-page_compression_default.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); set global innodb_file_format = `Barracuda`; set global innodb_file_per_table = on; create table innodb_normal (c1 int not null auto_increment primary key, b char(200)) engine=innodb; diff --git a/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result b/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result index 74cfa9bce3a..824b671eea3 100644 --- a/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result +++ b/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); set global innodb_compression_algorithm = snappy; set global innodb_file_format = `Barracuda`; set global innodb_file_per_table = on; diff --git a/mysql-test/suite/innodb/t/innodb-page_compression_default.test b/mysql-test/suite/innodb/t/innodb-page_compression_default.test index 1cc6c917548..34b1829485e 100644 --- a/mysql-test/suite/innodb/t/innodb-page_compression_default.test +++ b/mysql-test/suite/innodb/t/innodb-page_compression_default.test @@ -1,8 +1,6 @@ --source include/have_innodb.inc --source include/not_embedded.inc -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); - # All page compression test use the same --source include/innodb-page-compression.inc diff --git a/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test b/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test index 532ec294d28..0186c24ef2e 100644 --- a/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test +++ b/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test @@ -2,8 +2,6 @@ -- source include/have_innodb_snappy.inc --source include/not_embedded.inc -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); - # snappy set global innodb_compression_algorithm = snappy; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ee8f40e7d39..1386fd63cc3 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -354,14 +354,142 @@ on the io_type */ ? (counter##_READ) \ : (counter##_WRITTEN)) + +/** Reserve a buffer slot for encryption, decryption or page compression. +@param[in,out] buf_pool buffer pool +@return reserved buffer slot */ +static buf_tmp_buffer_t* buf_pool_reserve_tmp_slot(buf_pool_t* buf_pool) +{ + for (ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { + buf_tmp_buffer_t* slot = &buf_pool->tmp_arr->slots[i]; + if (slot->acquire()) { + return slot; + } + } + + /* We assume that free slot is found */ + ut_error; + return NULL; +} + +/** Reserve a buffer for encryption, decryption or decompression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_crypt_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->crypt_buf) { + slot->crypt_buf = static_cast( + aligned_malloc(srv_page_size, srv_page_size)); + } +} + +/** Reserve a buffer for compression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_compression_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->comp_buf) { + /* Both snappy and lzo compression methods require that + output buffer used for compression is bigger than input + buffer. Increase the allocated buffer size accordingly. */ + ulint size = srv_page_size; +#ifdef HAVE_LZO + size += LZO1X_1_15_MEM_COMPRESS; +#elif defined HAVE_SNAPPY + size = snappy_max_compressed_length(size); +#endif + slot->comp_buf = static_cast( + aligned_malloc(size, srv_page_size)); + } +} + /** Decrypt a page. @param[in,out] bpage Page control block @param[in,out] space tablespace @return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) - MY_ATTRIBUTE((nonnull)); +static bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) +{ + ut_ad(space->n_pending_ios > 0); + ut_ad(space->id == bpage->space); + + byte* dst_frame = bpage->zip.data ? bpage->zip.data : + ((buf_block_t*) bpage)->frame; + bool page_compressed = fil_page_is_compressed(dst_frame); + buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); + + if (bpage->offset == 0) { + /* File header pages are not encrypted/compressed */ + return true; + } + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + + buf_tmp_buffer_t* slot; + + if (page_compressed) { + /* the page we read is unencrypted */ + /* Find free slot from temporary memory array */ +decompress: + slot = buf_pool_reserve_tmp_slot(buf_pool); + /* For decompression, use crypt_buf. */ + buf_tmp_reserve_crypt_buf(slot); +decompress_with_slot: + ut_d(fil_page_type_validate(dst_frame)); + + bpage->write_size = fil_page_decompress(slot->crypt_buf, + dst_frame); + slot->release(); + + ut_ad(!bpage->write_size || fil_page_type_validate(dst_frame)); + ut_ad(space->n_pending_ios > 0); + return bpage->write_size != 0; + } + + if (space->crypt_data + && mach_read_from_4(FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + dst_frame)) { + /* Verify encryption checksum before we even try to + decrypt. */ + if (!fil_space_verify_crypt_checksum( + dst_frame, buf_page_get_zip_size(bpage), NULL, + bpage->offset)) { +decrypt_failed: + /* Mark page encrypted in case it should be. */ + if (space->crypt_data->type + != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted = true; + } + + return false; + } + + /* Find free slot from temporary memory array */ + slot = buf_pool_reserve_tmp_slot(buf_pool); + buf_tmp_reserve_crypt_buf(slot); + + ut_d(fil_page_type_validate(dst_frame)); + + /* decrypt using crypt_buf to dst_frame */ + if (!fil_space_decrypt(space, slot->crypt_buf, + dst_frame, &bpage->encrypted)) { + slot->release(); + goto decrypt_failed; + } + + ut_d(fil_page_type_validate(dst_frame)); + + if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress_with_slot; + } + + slot->release(); + } else if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress; + } + + ut_ad(space->n_pending_ios > 0); + return true; +} /********************************************************************//** Mark a table with the specified space pointed by bpage->space corrupted. @@ -4784,7 +4912,6 @@ buf_page_io_complete(buf_page_t* bpage, bool evict) buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); - byte* frame = NULL; dberr_t err = DB_SUCCESS; ut_a(buf_page_in_file(bpage)); @@ -4802,19 +4929,18 @@ buf_page_io_complete(buf_page_t* bpage, bool evict) ulint read_page_no = 0; ulint read_space_id = 0; uint key_version = 0; - - ut_ad(bpage->zip.data || ((buf_block_t*)bpage)->frame); + byte* frame = bpage->zip.data + ? bpage->zip.data + : reinterpret_cast(bpage)->frame; + ut_ad(frame); fil_space_t* space = fil_space_acquire_for_io(bpage->space); if (!space) { return(DB_TABLESPACE_DELETED); } - buf_page_decrypt_after_read(bpage, space); - - if (buf_page_get_zip_size(bpage)) { - frame = bpage->zip.data; - } else { - frame = ((buf_block_t*) bpage)->frame; + if (!buf_page_decrypt_after_read(bpage, space)) { + err = DB_DECRYPTION_FAILED; + goto database_corrupted; } if (buf_page_get_zip_size(bpage)) { @@ -4978,7 +5104,7 @@ database_corrupted: /* io_type == BUF_IO_WRITE */ if (bpage->slot) { /* Mark slot free */ - bpage->slot->reserved = false; + bpage->slot->release(); bpage->slot = NULL; } } @@ -6233,66 +6359,6 @@ buf_page_init_for_backup_restore( } #endif /* !UNIV_HOTBACKUP */ -/********************************************************************//** -Reserve unused slot from temporary memory array and allocate necessary -temporary memory if not yet allocated. -@return reserved slot */ -UNIV_INTERN -buf_tmp_buffer_t* -buf_pool_reserve_tmp_slot( -/*======================*/ - buf_pool_t* buf_pool, /*!< in: buffer pool where to - reserve */ - bool compressed) /*!< in: is file space compressed */ -{ - buf_tmp_buffer_t *free_slot=NULL; - - /* Array is protected by buf_pool mutex */ - buf_pool_mutex_enter(buf_pool); - - for(ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { - buf_tmp_buffer_t *slot = &buf_pool->tmp_arr->slots[i]; - - if(slot->reserved == false) { - free_slot = slot; - break; - } - } - - /* We assume that free slot is found */ - ut_a(free_slot != NULL); - free_slot->reserved = true; - /* Now that we have reserved this slot we can release - buf_pool mutex */ - buf_pool_mutex_exit(buf_pool); - - /* Allocate temporary memory for encryption/decryption */ - if (free_slot->crypt_buf == NULL) { - free_slot->crypt_buf = static_cast(aligned_malloc(UNIV_PAGE_SIZE, UNIV_PAGE_SIZE)); - memset(free_slot->crypt_buf, 0, UNIV_PAGE_SIZE); - } - - /* For page compressed tables allocate temporary memory for - compression/decompression */ - if (compressed && free_slot->comp_buf == NULL) { - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - size = snappy_max_compressed_length(size); -#endif -#if HAVE_LZO - size += LZO1X_1_15_MEM_COMPRESS; -#endif - free_slot->comp_buf = static_cast(aligned_malloc(size, UNIV_PAGE_SIZE)); - memset(free_slot->comp_buf, 0, size); - } - - return (free_slot); -} - /** Encryption and page_compression hook that is called just before a page is written to disk. @param[in,out] space tablespace @@ -6342,16 +6408,18 @@ buf_page_encrypt_before_write( } ulint zip_size = buf_page_get_zip_size(bpage); - ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + ut_ad(!zip_size || !page_compressed); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool); slot->out_buf = NULL; bpage->slot = slot; + buf_tmp_reserve_crypt_buf(slot); byte *dst_frame = slot->crypt_buf; if (!page_compressed) { +not_compressed: /* Encrypt page content */ byte* tmp = fil_space_encrypt(space, bpage->offset, @@ -6359,32 +6427,28 @@ buf_page_encrypt_before_write( src_frame, dst_frame); - bpage->real_size = page_size; + bpage->real_size = UNIV_PAGE_SIZE; slot->out_buf = dst_frame = tmp; ut_d(fil_page_type_validate(tmp)); } else { /* First we compress the page content */ - ulint out_len = 0; - - byte *tmp = fil_compress_page( - space, - (byte *)src_frame, - slot->comp_buf, - page_size, + buf_tmp_reserve_compression_buf(slot); + byte* tmp = slot->comp_buf; + ulint out_len = fil_page_compress( + src_frame, tmp, fsp_flags_get_page_compression_level(space->flags), fil_space_get_block_size(space, bpage->offset), - encrypted, - &out_len); + encrypted); + if (!out_len) { + goto not_compressed; + } bpage->real_size = out_len; -#ifdef UNIV_DEBUG - fil_page_type_validate(tmp); -#endif - - if(encrypted) { + ut_d(fil_page_type_validate(tmp)); + if (encrypted) { /* And then we encrypt the page content */ tmp = fil_space_encrypt(space, bpage->offset, @@ -6396,131 +6460,9 @@ buf_page_encrypt_before_write( slot->out_buf = dst_frame = tmp; } -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif + ut_d(fil_page_type_validate(dst_frame)); // return dst_frame which will be written return dst_frame; } - -/** Decrypt a page. -@param[in,out] bpage Page control block -@param[in,out] space tablespace -@return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) -{ - ut_ad(space->n_pending_ios > 0); - ut_ad(space->id == bpage->space); - - ulint zip_size = buf_page_get_zip_size(bpage); - ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - - byte* dst_frame = (zip_size) ? bpage->zip.data : - ((buf_block_t*) bpage)->frame; - unsigned key_version = - mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed = fil_page_is_compressed(dst_frame); - bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); - buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); - bool success = true; - - if (bpage->offset == 0) { - /* File header pages are not encrypted/compressed */ - return (true); - } - - /* Page is encrypted if encryption information is found from - tablespace and page contains used key_version. This is true - also for pages first compressed and then encrypted. */ - if (!space->crypt_data) { - key_version = 0; - } - - if (page_compressed) { - /* the page we read is unencrypted */ - /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size), - &bpage->write_size); - - /* Mark this slot as free */ - slot->reserved = false; - key_version = 0; - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - } else { - buf_tmp_buffer_t* slot = NULL; - - if (key_version) { - /* Verify encryption checksum before we even try to - decrypt. */ - if (!fil_space_verify_crypt_checksum(dst_frame, - zip_size, NULL, bpage->offset)) { - - /* Mark page encrypted in case it should - be. */ - if (space->crypt_data->type - != CRYPT_SCHEME_UNENCRYPTED) { - bpage->encrypted = true; - } - - return (false); - } - - /* Find free slot from temporary memory array */ - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - - /* decrypt using crypt_buf to dst_frame */ - if (!fil_space_decrypt(space, slot->crypt_buf, - dst_frame, &bpage->encrypted)) { - success = false; - } - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - } - - if (page_compressed_encrypted && success) { - if (!slot) { - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - } - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size), - &bpage->write_size); - ut_d(fil_page_type_validate(dst_frame)); - } - - /* Mark this slot as free */ - if (slot) { - slot->reserved = false; - } - } - - ut_ad(space->n_pending_ios > 0); - return (success); -} #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 70422671190..5f126ebb2a9 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -510,10 +510,11 @@ buf_dblwr_process() "Restoring possible half-written data pages " "from the doublewrite buffer..."); - unaligned_read_buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + unaligned_read_buf = static_cast(ut_malloc(3 * UNIV_PAGE_SIZE)); read_buf = static_cast( ut_align(unaligned_read_buf, UNIV_PAGE_SIZE)); + byte* const buf = read_buf + UNIV_PAGE_SIZE; for (std::list::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) { @@ -562,24 +563,23 @@ buf_dblwr_process() ignore this page (there should be redo log records to initialize it). */ } else { - if (fil_page_is_compressed_encrypted(read_buf) || - fil_page_is_compressed(read_buf)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, read_buf, srv_page_size, - NULL, true); + /* Decompress the page before + validating the checksum. */ + ulint decomp = fil_page_decompress(buf, read_buf); + if (!decomp || (decomp != srv_page_size && zip_size)) { + goto bad; } if (fil_space_verify_crypt_checksum( - read_buf, zip_size, NULL, page_no) - || !buf_page_is_corrupted( - true, read_buf, zip_size, space())) { + read_buf, zip_size, NULL, page_no) + || !buf_page_is_corrupted( + true, read_buf, zip_size, space())) { /* The page is good; there is no need to consult the doublewrite buffer. */ continue; } +bad: /* We intentionally skip this message for is_all_zero pages. */ ib_logf(IB_LOG_LEVEL_INFO, @@ -588,18 +588,15 @@ buf_dblwr_process() space_id, page_no); } - /* Next, validate the doublewrite page. */ - if (fil_page_is_compressed_encrypted(page) || - fil_page_is_compressed(page)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, page, srv_page_size, NULL, true); + ulint decomp = fil_page_decompress(buf, page); + if (!decomp || (decomp != srv_page_size && zip_size)) { + goto bad_doublewrite; } - - if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no) + if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, + page_no) && buf_page_is_corrupted(true, page, zip_size, space)) { if (!is_all_zero) { +bad_doublewrite: ib_logf(IB_LOG_LEVEL_WARN, "A doublewrite copy of page " ULINTPF ":" ULINTPF " is corrupted.", diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 2d149f07433..932d7b9e312 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -708,60 +708,39 @@ fil_space_encrypt( #ifdef UNIV_DEBUG if (tmp) { /* Verify that encrypted buffer is not corrupted */ - byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE); dberr_t err = DB_SUCCESS; byte* src = src_frame; bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - byte* comp_mem = NULL; - byte* uncomp_mem = NULL; + byte uncomp_mem[UNIV_PAGE_SIZE_MAX]; + byte tmp_mem[UNIV_PAGE_SIZE_MAX]; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; if (page_compressed_encrypted) { - comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); - fil_decompress_page(uncomp_mem, comp_mem, - srv_page_size, NULL); - src = uncomp_mem; + memcpy(uncomp_mem, src, srv_page_size); + ulint unzipped1 = fil_page_decompress( + tmp_mem, uncomp_mem); + ut_ad(unzipped1); + if (unzipped1 != srv_page_size) { + src = uncomp_mem; + } } - bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, space); - bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); + ut_ad(!buf_page_is_corrupted(true, src, zip_size, space)); + ut_ad(fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err)); + ut_ad(err == DB_SUCCESS); /* Need to decompress the page if it was also compressed */ if (page_compressed_encrypted) { - memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); - fil_decompress_page(tmp_mem, comp_mem, - srv_page_size, NULL); + byte buf[UNIV_PAGE_SIZE_MAX]; + memcpy(buf, tmp_mem, srv_page_size); + ulint unzipped2 = fil_page_decompress(tmp_mem, buf); + ut_ad(unzipped2); } - bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space); - memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); - bool different = memcmp(src, tmp_mem, size); - - if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { - fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", - ok , corrupted, corrupted1, err, different); - fprintf(stderr, "src_frame\n"); - buf_page_print(src_frame, zip_size); - fprintf(stderr, "encrypted_frame\n"); - buf_page_print(tmp, zip_size); - fprintf(stderr, "decrypted_frame\n"); - buf_page_print(tmp_mem, zip_size); - ut_ad(0); - } - - free(tmp_mem); - - if (comp_mem) { - free(comp_mem); - } - - if (uncomp_mem) { - free(uncomp_mem); - } + memcpy(tmp_mem + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + src + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); + ut_ad(!memcmp(src, tmp_mem, size)); } - #endif /* UNIV_DEBUG */ return tmp; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 112320bd3f5..5891db62758 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -342,19 +342,6 @@ fil_space_get_by_id( return(space); } -/****************************************************************//** -Get space id from fil node */ -ulint -fil_node_get_space_id( -/*==================*/ - fil_node_t* node) /*!< in: Compressed node*/ -{ - ut_ad(node); - ut_ad(node->space); - - return (node->space->id); -} - /*******************************************************************//** Returns the table space by a given name, NULL if not found. */ UNIV_INLINE diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 0ac764d5a32..25cd8e28a91 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -80,73 +80,26 @@ static ulint srv_data_read, srv_data_written; #include "snappy-c.h" #endif -/* Used for debugging */ -//#define UNIV_PAGECOMPRESS_DEBUG 1 - -/****************************************************************//** -For page compressed pages compress the page before actual write -operation. -@return compressed page to be written*/ -UNIV_INTERN -byte* -fil_compress_page( -/*==============*/ - fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: compressed buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint level, /* in: compression level */ - ulint block_size, /*!< in: block size */ - bool encrypted, /*!< in: is page also encrypted */ - ulint* out_len) /*!< out: actual length of compressed - page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) { - int err = Z_OK; - int comp_level = level; + int comp_level = int(level); ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; - ulint write_size = 0; -#if HAVE_LZO - lzo_uint write_size_lzo = write_size; -#endif /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; - bool allocated = false; - - /* page_compression does not apply to tables or tablespaces - that use ROW_FORMAT=COMPRESSED */ - ut_ad(!space || !FSP_FLAGS_GET_ZIP_SSIZE(space->flags)); if (encrypted) { header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; } - if (!out_buf) { - allocated = true; - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - if (comp_method == PAGE_SNAPPY_ALGORITHM) { - size = snappy_max_compressed_length(size); - } -#endif -#if HAVE_LZO - if (comp_method == PAGE_LZO_ALGORITHM) { - size += LZO1X_1_15_MEM_COMPRESS; - } -#endif - - out_buf = static_cast(ut_malloc(size)); - } - - ut_ad(buf); - ut_ad(out_buf); - ut_ad(len); - ut_ad(out_len); - /* Let's not compress file space header or extent descriptor */ switch (fil_page_get_type(buf)) { @@ -154,8 +107,7 @@ fil_compress_page( case FIL_PAGE_TYPE_FSP_HDR: case FIL_PAGE_TYPE_XDES: case FIL_PAGE_PAGE_COMPRESSED: - *out_len = len; - goto err_exit; + return 0; } /* If no compression level was provided to this table, use system @@ -164,204 +116,113 @@ fil_compress_page( comp_level = page_zip_level; } - DBUG_PRINT("compress", - ("Preparing for space " ULINTPF " '%s' len " ULINTPF, - space ? space->id : 0, - space ? space->name : "(import)", - len)); + ulint write_size = srv_page_size - header_len; - write_size = UNIV_PAGE_SIZE - header_len; - - switch(comp_method) { + switch (comp_method) { + default: + ut_ad(!"unknown compression method"); + /* fall through */ + case PAGE_UNCOMPRESSED: + return 0; + case PAGE_ZLIB_ALGORITHM: + { + ulong len = uLong(write_size); + if (Z_OK == compress2( + out_buf + header_len, &len, + buf, uLong(srv_page_size), comp_level)) { + write_size = len; + goto success; + } + } + break; #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: +# ifdef HAVE_LZ4_COMPRESS_DEFAULT + write_size = LZ4_compress_default( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# else + write_size = LZ4_compress_limitedOutput( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# endif -#ifdef HAVE_LZ4_COMPRESS_DEFAULT - err = LZ4_compress_default((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#else - err = LZ4_compress_limitedOutput((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#endif /* HAVE_LZ4_COMPRESS_DEFAULT */ - write_size = err; - - if (err == 0) { - /* If error we leave the actual page as it was */ - -#ifndef UNIV_PAGECOMPRESS_DEBUG - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; -#endif - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); -#ifndef UNIV_PAGECOMPRESS_DEBUG - } -#endif - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (write_size) { + goto success; } break; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO - case PAGE_LZO_ALGORITHM: - err = lzo1x_1_15_compress( - buf, len, out_buf+header_len, &write_size_lzo, out_buf+UNIV_PAGE_SIZE); + case PAGE_LZO_ALGORITHM: { + lzo_uint len = write_size; - write_size = write_size_lzo; - - if (err != LZO_E_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (LZO_E_OK == lzo1x_1_15_compress( + buf, srv_page_size, + out_buf + header_len, &len, + out_buf + srv_page_size) + && len <= write_size) { + write_size = len; + goto success; } - break; + } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - size_t out_pos=0; + size_t out_pos = 0; - err = lzma_easy_buffer_encode( - comp_level, - LZMA_CHECK_NONE, - NULL, /* No custom allocator, use malloc/free */ - reinterpret_cast(buf), - len, - reinterpret_cast(out_buf + header_len), - &out_pos, - (size_t)write_size); - - if (err != LZMA_OK || out_pos > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, out_pos); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (LZMA_OK == lzma_easy_buffer_encode( + comp_level, LZMA_CHECK_NONE, NULL, + buf, srv_page_size, out_buf + header_len, + &out_pos, write_size) + && out_pos <= write_size) { + write_size = out_pos; + goto success; } - - write_size = out_pos; - break; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - - err = BZ2_bzBuffToBuffCompress( - (char *)(out_buf + header_len), - (unsigned int *)&write_size, - (char *)buf, - len, - 1, - 0, - 0); - - if (err != BZ_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + unsigned len = unsigned(write_size); + if (BZ_OK == BZ2_bzBuffToBuffCompress( + reinterpret_cast(out_buf + header_len), + &len, + const_cast( + reinterpret_cast(buf)), + unsigned(srv_page_size), 1, 0, 0) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - write_size = snappy_max_compressed_length(UNIV_PAGE_SIZE); + case PAGE_SNAPPY_ALGORITHM: { + size_t len = snappy_max_compressed_length(srv_page_size); - cstatus = snappy_compress( - (const char *)buf, - (size_t)len, - (char *)(out_buf+header_len), - (size_t*)&write_size); - - if (cstatus != SNAPPY_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - (int)cstatus, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (SNAPPY_OK == snappy_compress( + reinterpret_cast(buf), + srv_page_size, + reinterpret_cast(out_buf) + header_len, + &len) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_SNAPPY */ - - case PAGE_ZLIB_ALGORITHM: - err = compress2(out_buf+header_len, (ulong*)&write_size, buf, - uLong(len), comp_level); - - if (err != Z_OK) { - /* If error we leave the actual page as it was */ - - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " rt %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; - } - break; - - case PAGE_UNCOMPRESSED: - *out_len = len; - return (buf); - break; - default: - ut_error; - break; } + srv_stats.pages_page_compression_error.inc(); + return 0; +success: /* Set up the page header */ memcpy(out_buf, buf, FIL_PAGE_DATA); /* Set up the checksum */ @@ -392,22 +253,11 @@ fil_compress_page( /* Verify that page can be decompressed */ { - byte *comp_page; - byte *uncomp_page; - - comp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - uncomp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); - - fil_decompress_page(uncomp_page, comp_page, ulong(len), NULL); - - if (buf_page_is_corrupted(false, uncomp_page, 0, space)) { - buf_page_print(uncomp_page, 0); - ut_ad(0); - } - - ut_free(comp_page); - ut_free(uncomp_page); + page_t tmp_buf[UNIV_PAGE_SIZE_MAX]; + page_t page[UNIV_PAGE_SIZE_MAX]; + memcpy(page, out_buf, srv_page_size); + ut_ad(fil_page_decompress(tmp_buf, page)); + ut_ad(!buf_page_is_corrupted(false, page, 0, NULL)); } #endif /* UNIV_DEBUG */ @@ -431,323 +281,144 @@ fil_compress_page( #endif } - DBUG_PRINT("compress", - ("Succeeded for space " ULINTPF - " '%s' len " ULINTPF " out_len " ULINTPF, - space ? space->id : 0, - space ? space->name : "(import)", - len, write_size)); - - srv_stats.page_compression_saved.add((len - write_size)); + srv_stats.page_compression_saved.add(srv_page_size - write_size); srv_stats.pages_page_compressed.inc(); /* If we do not persistently trim rest of page, we need to write it all */ if (!srv_use_trim) { - memset(out_buf+write_size,0,len-write_size); - write_size = len; + memset(out_buf + write_size, 0, srv_page_size - write_size); } - *out_len = write_size; - - if (allocated) { - /* TODO: reduce number of memcpy's */ - memcpy(buf, out_buf, len); - } else { - return(out_buf); - } - -err_exit: - if (allocated) { - ut_free(out_buf); - } - - return (buf); - + return write_size; } -/****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -UNIV_INTERN -void -fil_decompress_page( -/*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< out: buffer from which to read; in aio - this must be appropriately aligned */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of - the compressed data. */ - bool return_error) /*!< in: true if only an error should - be produced when decompression fails. - By default this parameter is false. */ +/** Decompress a page that may be subject to page_compressed compression. +@param[in,out] tmp_buf temporary buffer (of innodb_page_size) +@param[in,out] buf possibly compressed page buffer +@return size of the compressed data +@retval 0 if decompression failed +@retval srv_page_size if the page was not compressed */ +UNIV_INTERN ulint fil_page_decompress(byte* tmp_buf, byte* buf) { - int err = 0; - ulint actual_size = 0; - ulint compression_alg = 0; - byte *in_buf; - ulint ptype; - ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; - - ut_ad(buf); - ut_ad(len); - - ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); - - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; + const unsigned ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); + ulint header_len; + ib_uint64_t compression_alg; + switch (ptype) { + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + + FIL_PAGE_COMPRESSION_METHOD_SIZE; + compression_alg = mach_read_from_2( + FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + buf); + break; + case FIL_PAGE_PAGE_COMPRESSED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; + compression_alg = mach_read_from_8( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + buf); + break; + default: + return srv_page_size; } - /* Do not try to uncompressed pages that are not compressed */ - if (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return; + if (mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM) + != BUF_NO_CHECKSUM_MAGIC) { + return 0; } - // If no buffer was given, we need to allocate temporal buffer - if (page_buf == NULL) { - in_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - memset(in_buf, 0, UNIV_PAGE_SIZE); - } else { - in_buf = page_buf; - } + ulint actual_size = mach_read_from_2(buf + FIL_PAGE_DATA); - /* Before actual decompress, make sure that page type is correct */ - - if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || - (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page" - " CRC " ULINTPF " type " ULINTPF " len " ULINTPF ".", - mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM), - mach_read_from_2(buf+FIL_PAGE_TYPE), len); - - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - } - - /* Get compression algorithm */ - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - compression_alg = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE); - } else { - compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } - - /* Get the actual size of compressed page */ - actual_size = mach_read_from_2(buf+FIL_PAGE_DATA); /* Check if payload size is corrupted */ - if (actual_size == 0 || actual_size > UNIV_PAGE_SIZE) { + if (actual_size == 0 || actual_size > srv_page_size - header_len) { + return 0; + } + + switch (compression_alg) { + default: ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page" - " actual size " ULINTPF " compression %s.", - actual_size, fil_get_compression_alg_name(compression_alg)); - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - } - - /* Store actual payload size of the compressed data. This pointer - points to buffer pool. */ - if (write_size) { - *write_size = actual_size; - } - - DBUG_PRINT("compress", - ("Preparing for decompress for len " ULINTPF ".", - actual_size)); - - switch(compression_alg) { + "Unknown compression algorithm " UINT64PF, + compression_alg); + return 0; case PAGE_ZLIB_ALGORITHM: - err= uncompress(in_buf, &len, buf+header_len, (unsigned long)actual_size); - - /* If uncompress fails it means that page is corrupted */ - if (err != Z_OK) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; + { + uLong len = srv_page_size; + if (Z_OK != uncompress(tmp_buf, &len, + buf + header_len, + uLong(actual_size)) + && len != srv_page_size) { + return 0; } - ut_error; } break; - #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: - err = LZ4_decompress_fast((const char *)buf+header_len, (char *)in_buf, len); - - if (err != (int)actual_size) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (LZ4_decompress_safe(reinterpret_cast(buf) + + header_len, + reinterpret_cast(tmp_buf), + actual_size, srv_page_size) + == int(srv_page_size)) { + break; } - break; + return 0; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO case PAGE_LZO_ALGORITHM: { - ulint olen = 0; - lzo_uint olen_lzo = olen; - err = lzo1x_decompress((const unsigned char *)buf+header_len, - actual_size,(unsigned char *)in_buf, &olen_lzo, NULL); - - olen = olen_lzo; - - if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + lzo_uint len_lzo = srv_page_size; + if (LZO_E_OK == lzo1x_decompress_safe( + buf + header_len, + actual_size, tmp_buf, &len_lzo, NULL) + && len_lzo == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - - lzma_ret ret; size_t src_pos = 0; size_t dst_pos = 0; uint64_t memlimit = UINT64_MAX; - ret = lzma_stream_buffer_decode( - &memlimit, - 0, - NULL, - buf+header_len, - &src_pos, - actual_size, - in_buf, - &dst_pos, - len); - - - if (ret != LZMA_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size " ULINTPF "len " ULINTPF ".", - dst_pos, actual_size, len); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (LZMA_OK == lzma_stream_buffer_decode( + &memlimit, 0, NULL, buf + header_len, + &src_pos, actual_size, tmp_buf, &dst_pos, + srv_page_size) + && dst_pos == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - unsigned int dst_pos = UNIV_PAGE_SIZE; - - err = BZ2_bzBuffToBuffDecompress( - (char *)in_buf, - &dst_pos, - (char *)(buf+header_len), - actual_size, - 1, - 0); - - if (err != BZ_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %du bytes" - " size " ULINTPF " len " ULINTPF " err %d.", - dst_pos, actual_size, len, err); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + unsigned int dst_pos = srv_page_size; + if (BZ_OK == BZ2_bzBuffToBuffDecompress( + reinterpret_cast(tmp_buf), + &dst_pos, + reinterpret_cast(buf) + header_len, + actual_size, 1, 0) + && dst_pos == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - ulint olen = UNIV_PAGE_SIZE; + case PAGE_SNAPPY_ALGORITHM: { + size_t olen = srv_page_size; - cstatus = snappy_uncompress( - (const char *)(buf+header_len), - (size_t)actual_size, - (char *)in_buf, - (size_t*)&olen); - - if (cstatus != SNAPPY_OK || olen != UNIV_PAGE_SIZE) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only " ULINTPF " bytes" - " size " ULINTPF " len " ULINTPF " err %d.", - olen, actual_size, len, (int)cstatus); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (SNAPPY_OK == snappy_uncompress( + reinterpret_cast(buf) + header_len, + actual_size, + reinterpret_cast(tmp_buf), &olen) + && olen == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_SNAPPY */ - default: - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but compression algorithm %s" - " is not known." - ,fil_get_compression_alg_name(compression_alg)); - - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - break; } srv_stats.pages_page_decompressed.inc(); - - /* Copy the uncompressed page to the buffer pool, not - really any other options. */ - memcpy(buf, in_buf, len); - -error_return: - if (page_buf != in_buf) { - ut_free(in_buf); - } + memcpy(buf, tmp_buf, srv_page_size); + return actual_size; } diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 2b406e116b0..17fe3ae7511 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -5177,6 +5177,11 @@ ibuf_check_bitmap_on_import( bitmap_page = ibuf_bitmap_get_map_page( space_id, page_no, zip_size, &mtr); + if (!bitmap_page) { + mutex_exit(&ibuf_mutex); + return DB_CORRUPTION; + } + for (i = FSP_IBUF_BITMAP_OFFSET + 1; i < page_size; i++) { const ulint offset = page_no + i; diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 857bdf9d2be..aca4e58810c 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -39,6 +39,7 @@ Created 11/5/1995 Heikki Tuuri #include "ut0rbt.h" #include "os0proc.h" #include "log0log.h" +#include "my_atomic.h" /** @name Modes for buf_page_get_gen */ /* @{ */ @@ -1506,45 +1507,16 @@ buf_page_encrypt_before_write( buf_page_t* bpage, byte* src_frame); -/********************************************************************** -The hook that is called after page is written to disk. -The function releases any resources needed for encryption that was allocated -in buf_page_encrypt_before_write */ -UNIV_INTERN -ibool -buf_page_encrypt_after_write( -/*=========================*/ - buf_page_t* page); /*!< in/out: buffer page that was flushed */ - -/********************************************************************//** -The hook that is called just before a page is read from disk. -The function allocates memory that is used to temporarily store disk content -before getting decrypted */ -UNIV_INTERN -byte* -buf_page_decrypt_before_read( -/*=========================*/ - buf_page_t* page, /*!< in/out: buffer page read from disk */ - ulint zip_size); /*!< in: compressed page size, or 0 */ - -/********************************************************************//** -The hook that is called just after a page is read from disk. -The function decrypt disk content into buf_page_t and releases the -temporary buffer that was allocated in buf_page_decrypt_before_read */ -UNIV_INTERN -bool -buf_page_decrypt_after_read( -/*========================*/ - buf_page_t* page); /*!< in/out: buffer page read from disk */ - /** @brief The temporary memory structure. NOTE! The definition appears here only for other modules of this directory (buf) to see it. Do not use from outside! */ typedef struct { - bool reserved; /*!< true if this slot is reserved +private: + int32 reserved; /*!< true if this slot is reserved */ +public: byte* crypt_buf; /*!< for encryption the data needs to be copied to a separate buffer before it's encrypted&written. this as a page can be @@ -1555,6 +1527,21 @@ typedef struct { byte* out_buf; /*!< resulting buffer after encryption/compression. This is a pointer and not allocated. */ + + /** Release the slot */ + void release() + { + my_atomic_store32_explicit(&reserved, false, + MY_MEMORY_ORDER_RELAXED); + } + + /** Acquire the slot + @return whether the slot was acquired */ + bool acquire() + { + return !my_atomic_fas32_explicit(&reserved, true, + MY_MEMORY_ORDER_RELAXED); + } } buf_tmp_buffer_t; /** The common buffer control block structure diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index ae2629b5b04..a8dc9e14381 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -341,9 +341,6 @@ struct fil_space_t { bool is_in_unflushed_spaces; /*!< true if this space is currently in unflushed_spaces */ - bool printed_compression_failure; - /*!< true if we have already printed - compression failure */ fil_space_crypt_t* crypt_data; /*!< tablespace crypt data or NULL */ ulint file_block_size; diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index 49fdff4f3e5..3f21c529308 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -68,7 +68,6 @@ fil_get_page_type_name( } return "PAGE TYPE CORRUPTED"; - } /****************************************************************//** diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index 03e16699ce3..934372c55b2 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018 MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -30,70 +30,26 @@ atomic writes information to table space. Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ -/*******************************************************************//** -Find out wheather the page is index page or not -@return true if page type index page, false if not */ -UNIV_INLINE -ibool -fil_page_is_index_page( -/*===================*/ - byte *buf); /*!< in: page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) + MY_ATTRIBUTE((nonnull, warn_unused_result)); -/****************************************************************//** -Get the name of the compression algorithm used for page -compression. -@return compression algorithm name or "UNKNOWN" if not known*/ -UNIV_INLINE -const char* -fil_get_compression_alg_name( -/*=========================*/ - ulint comp_alg); /*! @@ -3365,15 +3371,30 @@ fil_iterate( os_offset_t offset; ulint n_bytes = iter.n_io_buffers * iter.page_size; + const ulint buf_size = srv_page_size +#ifdef HAVE_LZO + + LZO1X_1_15_MEM_COMPRESS +#elif defined HAVE_SNAPPY + + snappy_max_compressed_length(srv_page_size) +#endif + ; + byte* page_compress_buf = static_cast( + ut_malloc_low(buf_size, false)); ut_ad(!srv_read_only_mode); + if (!page_compress_buf) { + return DB_OUT_OF_MEMORY; + } + /* TODO: For ROW_FORMAT=COMPRESSED tables we do a lot of useless copying for non-index pages. Unfortunately, it is required by buf_zip_decompress() */ + dberr_t err = DB_SUCCESS; for (offset = iter.start; offset < iter.end; offset += n_bytes) { if (callback.is_interrupted()) { - return DB_INTERRUPTED; + err = DB_INTERRUPTED; + goto func_exit; } byte* io_buffer = iter.io_buffer; @@ -3404,12 +3425,13 @@ fil_iterate( if (!os_file_read_no_error_handling(iter.file, readptr, offset, n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed"); - return DB_IO_ERROR; + err = DB_IO_ERROR; + goto func_exit; } bool updated = false; - ulint n_pages_read = (ulint) n_bytes / iter.page_size; const ulint size = iter.page_size; + ulint n_pages_read = ulint(n_bytes) / size; block->page.offset = offset / size; for (ulint i = 0; i < n_pages_read; @@ -3438,11 +3460,11 @@ page_corrupted: UINT64PF " looks corrupted.", callback.filename(), ulong(offset / size), offset); - return DB_CORRUPTION; + err = DB_CORRUPTION; + goto func_exit; } bool decrypted = false; - dberr_t err = DB_SUCCESS; byte* dst = io_buffer + (i * size); bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -3451,6 +3473,10 @@ page_corrupted: == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED || page_type == FIL_PAGE_PAGE_COMPRESSED; + if (page_compressed && block->page.zip.data) { + goto page_corrupted; + } + if (!encrypted) { } else if (!mach_read_from_4( FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION @@ -3476,7 +3502,7 @@ not_encrypted: iter.page_size, src, &err); if (err != DB_SUCCESS) { - return err; + goto func_exit; } if (!decrypted) { @@ -3489,8 +3515,12 @@ not_encrypted: /* If the original page is page_compressed, we need to decompress it before adjusting further. */ if (page_compressed) { - fil_decompress_page(NULL, dst, ulong(size), - NULL); + ulint compress_length = fil_page_decompress( + page_compress_buf, dst); + ut_ad(compress_length != srv_page_size); + if (compress_length == 0) { + goto page_corrupted; + } updated = true; } else if (buf_page_is_corrupted( false, @@ -3501,7 +3531,7 @@ not_encrypted: } if ((err = callback(block)) != DB_SUCCESS) { - return err; + goto func_exit; } else if (!updated) { updated = buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE; @@ -3551,19 +3581,17 @@ not_encrypted: src = io_buffer + (i * size); if (page_compressed) { - ulint len = 0; - - fil_compress_page( - NULL, - src, - NULL, - size, - 0,/* FIXME: compression level */ - 512,/* FIXME: use proper block size */ - encrypted, - &len); - updated = true; + if (fil_page_compress( + src, + page_compress_buf, + 0,/* FIXME: compression level */ + 512,/* FIXME: proper block size */ + encrypted)) { + /* FIXME: remove memcpy() */ + memcpy(src, page_compress_buf, + srv_page_size); + } } /* If tablespace is encrypted, encrypt page before we @@ -3600,11 +3628,14 @@ not_encrypted: offset, (ulint) n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed"); - return DB_IO_ERROR; + err = DB_IO_ERROR; + goto func_exit; } } - return DB_SUCCESS; +func_exit: + ut_free(page_compress_buf); + return err; } /********************************************************************//** diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 5728b8a9fa3..3ae6dd56586 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -77,15 +77,6 @@ Created 11/5/1995 Heikki Tuuri #include "snappy-c.h" #endif -/** Decrypt a page. -@param[in,out] bpage Page control block -@param[in,out] space tablespace -@return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) - MY_ATTRIBUTE((nonnull)); - /********************************************************************//** Mark a table with the specified space pointed by bpage->space corrupted. Also remove the bpage from LRU list. @@ -390,6 +381,143 @@ on the io_type */ ? (counter##_READ) \ : (counter##_WRITTEN)) + +/** Reserve a buffer slot for encryption, decryption or page compression. +@param[in,out] buf_pool buffer pool +@return reserved buffer slot */ +static buf_tmp_buffer_t* buf_pool_reserve_tmp_slot(buf_pool_t* buf_pool) +{ + for (ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { + buf_tmp_buffer_t* slot = &buf_pool->tmp_arr->slots[i]; + if (slot->acquire()) { + return slot; + } + } + + /* We assume that free slot is found */ + ut_error; + return NULL; +} + +/** Reserve a buffer for encryption, decryption or decompression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_crypt_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->crypt_buf) { + slot->crypt_buf = static_cast( + aligned_malloc(srv_page_size, srv_page_size)); + } +} + +/** Reserve a buffer for compression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_compression_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->comp_buf) { + /* Both snappy and lzo compression methods require that + output buffer used for compression is bigger than input + buffer. Increase the allocated buffer size accordingly. */ + ulint size = srv_page_size; +#ifdef HAVE_LZO + size += LZO1X_1_15_MEM_COMPRESS; +#elif defined HAVE_SNAPPY + size = snappy_max_compressed_length(size); +#endif + slot->comp_buf = static_cast( + aligned_malloc(size, srv_page_size)); + } +} + +/** Decrypt a page. +@param[in,out] bpage Page control block +@param[in,out] space tablespace +@return whether the operation was successful */ +static bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) +{ + ut_ad(space->n_pending_ios > 0); + ut_ad(space->id == bpage->space); + + byte* dst_frame = bpage->zip.data ? bpage->zip.data : + ((buf_block_t*) bpage)->frame; + bool page_compressed = fil_page_is_compressed(dst_frame); + buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); + + if (bpage->offset == 0) { + /* File header pages are not encrypted/compressed */ + return true; + } + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + + buf_tmp_buffer_t* slot; + + if (page_compressed) { + /* the page we read is unencrypted */ + /* Find free slot from temporary memory array */ +decompress: + slot = buf_pool_reserve_tmp_slot(buf_pool); + /* For decompression, use crypt_buf. */ + buf_tmp_reserve_crypt_buf(slot); +decompress_with_slot: + ut_d(fil_page_type_validate(dst_frame)); + + bpage->write_size = fil_page_decompress(slot->crypt_buf, + dst_frame); + slot->release(); + + ut_ad(!bpage->write_size || fil_page_type_validate(dst_frame)); + ut_ad(space->n_pending_ios > 0); + return bpage->write_size != 0; + } + + if (space->crypt_data + && mach_read_from_4(FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + dst_frame)) { + /* Verify encryption checksum before we even try to + decrypt. */ + if (!fil_space_verify_crypt_checksum( + dst_frame, buf_page_get_zip_size(bpage), NULL, + bpage->offset)) { +decrypt_failed: + /* Mark page encrypted in case it should be. */ + if (space->crypt_data->type + != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted = true; + } + + return false; + } + + /* Find free slot from temporary memory array */ + slot = buf_pool_reserve_tmp_slot(buf_pool); + buf_tmp_reserve_crypt_buf(slot); + + ut_d(fil_page_type_validate(dst_frame)); + + /* decrypt using crypt_buf to dst_frame */ + if (!fil_space_decrypt(space, slot->crypt_buf, + dst_frame, &bpage->encrypted)) { + slot->release(); + goto decrypt_failed; + } + + ut_d(fil_page_type_validate(dst_frame)); + + if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress_with_slot; + } + + slot->release(); + } else if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress; + } + + ut_ad(space->n_pending_ios > 0); + return true; +} + /********************************************************************//** Gets the smallest oldest_modification lsn for any page in the pool. Returns zero if all modified pages have been flushed to disk. @@ -4730,7 +4858,6 @@ buf_page_io_complete(buf_page_t* bpage) const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); bool have_LRU_mutex = false; - byte* frame = NULL; dberr_t err = DB_SUCCESS; ut_a(buf_page_in_file(bpage)); @@ -4748,19 +4875,18 @@ buf_page_io_complete(buf_page_t* bpage) ulint read_page_no = 0; ulint read_space_id = 0; uint key_version = 0; - - ut_ad(bpage->zip.data || ((buf_block_t*)bpage)->frame); + byte* frame = bpage->zip.data + ? bpage->zip.data + : reinterpret_cast(bpage)->frame; + ut_ad(frame); fil_space_t* space = fil_space_acquire_for_io(bpage->space); if (!space) { return(DB_TABLESPACE_DELETED); } - buf_page_decrypt_after_read(bpage, space); - - if (buf_page_get_zip_size(bpage)) { - frame = bpage->zip.data; - } else { - frame = ((buf_block_t*) bpage)->frame; + if (!buf_page_decrypt_after_read(bpage, space)) { + err = DB_DECRYPTION_FAILED; + goto database_corrupted; } if (buf_page_get_zip_size(bpage)) { @@ -4941,7 +5067,7 @@ database_corrupted: /* io_type == BUF_IO_WRITE */ if (bpage->slot) { /* Mark slot free */ - bpage->slot->reserved = false; + bpage->slot->release(); bpage->slot = NULL; } } @@ -6240,66 +6366,6 @@ buf_pool_mutex_exit( mutex_exit(&buf_pool->LRU_list_mutex); } -/********************************************************************//** -Reserve unused slot from temporary memory array and allocate necessary -temporary memory if not yet allocated. -@return reserved slot */ -UNIV_INTERN -buf_tmp_buffer_t* -buf_pool_reserve_tmp_slot( -/*======================*/ - buf_pool_t* buf_pool, /*!< in: buffer pool where to - reserve */ - bool compressed) /*!< in: is file space compressed */ -{ - buf_tmp_buffer_t *free_slot=NULL; - - /* Array is protected by buf_pool mutex */ - buf_pool_mutex_enter(buf_pool); - - for(ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { - buf_tmp_buffer_t *slot = &buf_pool->tmp_arr->slots[i]; - - if(slot->reserved == false) { - free_slot = slot; - break; - } - } - - /* We assume that free slot is found */ - ut_a(free_slot != NULL); - free_slot->reserved = true; - /* Now that we have reserved this slot we can release - buf_pool mutex */ - buf_pool_mutex_exit(buf_pool); - - /* Allocate temporary memory for encryption/decryption */ - if (free_slot->crypt_buf == NULL) { - free_slot->crypt_buf = static_cast(aligned_malloc(UNIV_PAGE_SIZE, UNIV_PAGE_SIZE)); - memset(free_slot->crypt_buf, 0, UNIV_PAGE_SIZE); - } - - /* For page compressed tables allocate temporary memory for - compression/decompression */ - if (compressed && free_slot->comp_buf == NULL) { - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - size = snappy_max_compressed_length(size); -#endif -#if HAVE_LZO - size += LZO1X_1_15_MEM_COMPRESS; -#endif - free_slot->comp_buf = static_cast(aligned_malloc(size, UNIV_PAGE_SIZE)); - memset(free_slot->comp_buf, 0, size); - } - - return (free_slot); -} - /** Encryption and page_compression hook that is called just before a page is written to disk. @param[in,out] space tablespace @@ -6349,16 +6415,18 @@ buf_page_encrypt_before_write( } ulint zip_size = buf_page_get_zip_size(bpage); - ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + ut_ad(!zip_size || !page_compressed); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool); slot->out_buf = NULL; bpage->slot = slot; + buf_tmp_reserve_crypt_buf(slot); byte *dst_frame = slot->crypt_buf; if (!page_compressed) { +not_compressed: /* Encrypt page content */ byte* tmp = fil_space_encrypt(space, bpage->offset, @@ -6366,32 +6434,28 @@ buf_page_encrypt_before_write( src_frame, dst_frame); - bpage->real_size = page_size; + bpage->real_size = UNIV_PAGE_SIZE; slot->out_buf = dst_frame = tmp; ut_d(fil_page_type_validate(tmp)); } else { /* First we compress the page content */ - ulint out_len = 0; - - byte *tmp = fil_compress_page( - space, - (byte *)src_frame, - slot->comp_buf, - page_size, + buf_tmp_reserve_compression_buf(slot); + byte* tmp = slot->comp_buf; + ulint out_len = fil_page_compress( + src_frame, tmp, fsp_flags_get_page_compression_level(space->flags), fil_space_get_block_size(space, bpage->offset), - encrypted, - &out_len); + encrypted); + if (!out_len) { + goto not_compressed; + } bpage->real_size = out_len; -#ifdef UNIV_DEBUG - fil_page_type_validate(tmp); -#endif - - if(encrypted) { + ut_d(fil_page_type_validate(tmp)); + if (encrypted) { /* And then we encrypt the page content */ tmp = fil_space_encrypt(space, bpage->offset, @@ -6403,130 +6467,8 @@ buf_page_encrypt_before_write( slot->out_buf = dst_frame = tmp; } -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif + ut_d(fil_page_type_validate(dst_frame)); // return dst_frame which will be written return dst_frame; } - -/** Decrypt a page. -@param[in,out] bpage Page control block -@param[in,out] space tablespace -@return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) -{ - ut_ad(space->n_pending_ios > 0); - ut_ad(space->id == bpage->space); - - ulint zip_size = buf_page_get_zip_size(bpage); - ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - - byte* dst_frame = (zip_size) ? bpage->zip.data : - ((buf_block_t*) bpage)->frame; - unsigned key_version = - mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed = fil_page_is_compressed(dst_frame); - bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); - buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); - bool success = true; - - if (bpage->offset == 0) { - /* File header pages are not encrypted/compressed */ - return (true); - } - - /* Page is encrypted if encryption information is found from - tablespace and page contains used key_version. This is true - also for pages first compressed and then encrypted. */ - if (!space->crypt_data) { - key_version = 0; - } - - if (page_compressed) { - /* the page we read is unencrypted */ - /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size), - &bpage->write_size); - - /* Mark this slot as free */ - slot->reserved = false; - key_version = 0; - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - } else { - buf_tmp_buffer_t* slot = NULL; - - if (key_version) { - /* Verify encryption checksum before we even try to - decrypt. */ - if (!fil_space_verify_crypt_checksum(dst_frame, - zip_size, NULL, bpage->offset)) { - - /* Mark page encrypted in case it should - be. */ - if (space->crypt_data->type - != CRYPT_SCHEME_UNENCRYPTED) { - bpage->encrypted = true; - } - - return (false); - } - - /* Find free slot from temporary memory array */ - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - - /* decrypt using crypt_buf to dst_frame */ - if (!fil_space_decrypt(space, slot->crypt_buf, - dst_frame, &bpage->encrypted)) { - success = false; - } - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - } - - if (page_compressed_encrypted && success) { - if (!slot) { - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - } - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size), - &bpage->write_size); - ut_d(fil_page_type_validate(dst_frame)); - } - - /* Mark this slot as free */ - if (slot) { - slot->reserved = false; - } - } - - ut_ad(space->n_pending_ios > 0); - return (success); -} diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index ee8c85446e9..0ec23cbdbe1 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -510,10 +510,11 @@ buf_dblwr_process() "Restoring possible half-written data pages " "from the doublewrite buffer..."); - unaligned_read_buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + unaligned_read_buf = static_cast(ut_malloc(3 * UNIV_PAGE_SIZE)); read_buf = static_cast( ut_align(unaligned_read_buf, UNIV_PAGE_SIZE)); + byte* const buf = read_buf + UNIV_PAGE_SIZE; for (std::list::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) { @@ -562,24 +563,26 @@ buf_dblwr_process() ignore this page (there should be redo log records to initialize it). */ } else { - if (fil_page_is_compressed_encrypted(read_buf) || - fil_page_is_compressed(read_buf)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, read_buf, srv_page_size, - NULL, true); + /* Decompress the page before + validating the checksum. */ + ulint decomp = fil_page_decompress(buf, read_buf); + if (!decomp) { + goto bad; + } + if (!decomp || (decomp != srv_page_size && zip_size)) { + goto bad; } if (fil_space_verify_crypt_checksum( - read_buf, zip_size, NULL, page_no) - || !buf_page_is_corrupted( - true, read_buf, zip_size, space())) { + read_buf, zip_size, NULL, page_no) + || !buf_page_is_corrupted( + true, read_buf, zip_size, space())) { /* The page is good; there is no need to consult the doublewrite buffer. */ continue; } +bad: /* We intentionally skip this message for is_all_zero pages. */ ib_logf(IB_LOG_LEVEL_INFO, @@ -588,18 +591,15 @@ buf_dblwr_process() space_id, page_no); } - /* Next, validate the doublewrite page. */ - if (fil_page_is_compressed_encrypted(page) || - fil_page_is_compressed(page)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, page, srv_page_size, NULL, true); + ulint decomp = fil_page_decompress(buf, page); + if (!decomp || (decomp != srv_page_size && zip_size)) { + goto bad_doublewrite; } - - if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no) + if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, + page_no) && buf_page_is_corrupted(true, page, zip_size, space)) { if (!is_all_zero) { +bad_doublewrite: ib_logf(IB_LOG_LEVEL_WARN, "A doublewrite copy of page " ULINTPF ":" ULINTPF " is corrupted.", diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 2d149f07433..932d7b9e312 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -708,60 +708,39 @@ fil_space_encrypt( #ifdef UNIV_DEBUG if (tmp) { /* Verify that encrypted buffer is not corrupted */ - byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE); dberr_t err = DB_SUCCESS; byte* src = src_frame; bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - byte* comp_mem = NULL; - byte* uncomp_mem = NULL; + byte uncomp_mem[UNIV_PAGE_SIZE_MAX]; + byte tmp_mem[UNIV_PAGE_SIZE_MAX]; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; if (page_compressed_encrypted) { - comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); - fil_decompress_page(uncomp_mem, comp_mem, - srv_page_size, NULL); - src = uncomp_mem; + memcpy(uncomp_mem, src, srv_page_size); + ulint unzipped1 = fil_page_decompress( + tmp_mem, uncomp_mem); + ut_ad(unzipped1); + if (unzipped1 != srv_page_size) { + src = uncomp_mem; + } } - bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, space); - bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); + ut_ad(!buf_page_is_corrupted(true, src, zip_size, space)); + ut_ad(fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err)); + ut_ad(err == DB_SUCCESS); /* Need to decompress the page if it was also compressed */ if (page_compressed_encrypted) { - memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); - fil_decompress_page(tmp_mem, comp_mem, - srv_page_size, NULL); + byte buf[UNIV_PAGE_SIZE_MAX]; + memcpy(buf, tmp_mem, srv_page_size); + ulint unzipped2 = fil_page_decompress(tmp_mem, buf); + ut_ad(unzipped2); } - bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space); - memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); - bool different = memcmp(src, tmp_mem, size); - - if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { - fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", - ok , corrupted, corrupted1, err, different); - fprintf(stderr, "src_frame\n"); - buf_page_print(src_frame, zip_size); - fprintf(stderr, "encrypted_frame\n"); - buf_page_print(tmp, zip_size); - fprintf(stderr, "decrypted_frame\n"); - buf_page_print(tmp_mem, zip_size); - ut_ad(0); - } - - free(tmp_mem); - - if (comp_mem) { - free(comp_mem); - } - - if (uncomp_mem) { - free(uncomp_mem); - } + memcpy(tmp_mem + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + src + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); + ut_ad(!memcmp(src, tmp_mem, size)); } - #endif /* UNIV_DEBUG */ return tmp; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index bd7dfdf5ac8..a8835083165 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -351,19 +351,6 @@ fil_space_get_by_id( return(space); } -/****************************************************************//** -Get space id from fil node */ -ulint -fil_node_get_space_id( -/*==================*/ - fil_node_t* node) /*!< in: Compressed node*/ -{ - ut_ad(node); - ut_ad(node->space); - - return (node->space->id); -} - /*******************************************************************//** Returns the table space by a given name, NULL if not found. */ fil_space_t* diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 0ac764d5a32..25cd8e28a91 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -80,73 +80,26 @@ static ulint srv_data_read, srv_data_written; #include "snappy-c.h" #endif -/* Used for debugging */ -//#define UNIV_PAGECOMPRESS_DEBUG 1 - -/****************************************************************//** -For page compressed pages compress the page before actual write -operation. -@return compressed page to be written*/ -UNIV_INTERN -byte* -fil_compress_page( -/*==============*/ - fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: compressed buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint level, /* in: compression level */ - ulint block_size, /*!< in: block size */ - bool encrypted, /*!< in: is page also encrypted */ - ulint* out_len) /*!< out: actual length of compressed - page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) { - int err = Z_OK; - int comp_level = level; + int comp_level = int(level); ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; - ulint write_size = 0; -#if HAVE_LZO - lzo_uint write_size_lzo = write_size; -#endif /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; - bool allocated = false; - - /* page_compression does not apply to tables or tablespaces - that use ROW_FORMAT=COMPRESSED */ - ut_ad(!space || !FSP_FLAGS_GET_ZIP_SSIZE(space->flags)); if (encrypted) { header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; } - if (!out_buf) { - allocated = true; - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - if (comp_method == PAGE_SNAPPY_ALGORITHM) { - size = snappy_max_compressed_length(size); - } -#endif -#if HAVE_LZO - if (comp_method == PAGE_LZO_ALGORITHM) { - size += LZO1X_1_15_MEM_COMPRESS; - } -#endif - - out_buf = static_cast(ut_malloc(size)); - } - - ut_ad(buf); - ut_ad(out_buf); - ut_ad(len); - ut_ad(out_len); - /* Let's not compress file space header or extent descriptor */ switch (fil_page_get_type(buf)) { @@ -154,8 +107,7 @@ fil_compress_page( case FIL_PAGE_TYPE_FSP_HDR: case FIL_PAGE_TYPE_XDES: case FIL_PAGE_PAGE_COMPRESSED: - *out_len = len; - goto err_exit; + return 0; } /* If no compression level was provided to this table, use system @@ -164,204 +116,113 @@ fil_compress_page( comp_level = page_zip_level; } - DBUG_PRINT("compress", - ("Preparing for space " ULINTPF " '%s' len " ULINTPF, - space ? space->id : 0, - space ? space->name : "(import)", - len)); + ulint write_size = srv_page_size - header_len; - write_size = UNIV_PAGE_SIZE - header_len; - - switch(comp_method) { + switch (comp_method) { + default: + ut_ad(!"unknown compression method"); + /* fall through */ + case PAGE_UNCOMPRESSED: + return 0; + case PAGE_ZLIB_ALGORITHM: + { + ulong len = uLong(write_size); + if (Z_OK == compress2( + out_buf + header_len, &len, + buf, uLong(srv_page_size), comp_level)) { + write_size = len; + goto success; + } + } + break; #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: +# ifdef HAVE_LZ4_COMPRESS_DEFAULT + write_size = LZ4_compress_default( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# else + write_size = LZ4_compress_limitedOutput( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# endif -#ifdef HAVE_LZ4_COMPRESS_DEFAULT - err = LZ4_compress_default((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#else - err = LZ4_compress_limitedOutput((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#endif /* HAVE_LZ4_COMPRESS_DEFAULT */ - write_size = err; - - if (err == 0) { - /* If error we leave the actual page as it was */ - -#ifndef UNIV_PAGECOMPRESS_DEBUG - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; -#endif - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); -#ifndef UNIV_PAGECOMPRESS_DEBUG - } -#endif - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (write_size) { + goto success; } break; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO - case PAGE_LZO_ALGORITHM: - err = lzo1x_1_15_compress( - buf, len, out_buf+header_len, &write_size_lzo, out_buf+UNIV_PAGE_SIZE); + case PAGE_LZO_ALGORITHM: { + lzo_uint len = write_size; - write_size = write_size_lzo; - - if (err != LZO_E_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (LZO_E_OK == lzo1x_1_15_compress( + buf, srv_page_size, + out_buf + header_len, &len, + out_buf + srv_page_size) + && len <= write_size) { + write_size = len; + goto success; } - break; + } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - size_t out_pos=0; + size_t out_pos = 0; - err = lzma_easy_buffer_encode( - comp_level, - LZMA_CHECK_NONE, - NULL, /* No custom allocator, use malloc/free */ - reinterpret_cast(buf), - len, - reinterpret_cast(out_buf + header_len), - &out_pos, - (size_t)write_size); - - if (err != LZMA_OK || out_pos > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, out_pos); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (LZMA_OK == lzma_easy_buffer_encode( + comp_level, LZMA_CHECK_NONE, NULL, + buf, srv_page_size, out_buf + header_len, + &out_pos, write_size) + && out_pos <= write_size) { + write_size = out_pos; + goto success; } - - write_size = out_pos; - break; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - - err = BZ2_bzBuffToBuffCompress( - (char *)(out_buf + header_len), - (unsigned int *)&write_size, - (char *)buf, - len, - 1, - 0, - 0); - - if (err != BZ_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + unsigned len = unsigned(write_size); + if (BZ_OK == BZ2_bzBuffToBuffCompress( + reinterpret_cast(out_buf + header_len), + &len, + const_cast( + reinterpret_cast(buf)), + unsigned(srv_page_size), 1, 0, 0) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - write_size = snappy_max_compressed_length(UNIV_PAGE_SIZE); + case PAGE_SNAPPY_ALGORITHM: { + size_t len = snappy_max_compressed_length(srv_page_size); - cstatus = snappy_compress( - (const char *)buf, - (size_t)len, - (char *)(out_buf+header_len), - (size_t*)&write_size); - - if (cstatus != SNAPPY_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " err %d write_size " ULINTPF ".", - space->id, space->name, len, - (int)cstatus, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; + if (SNAPPY_OK == snappy_compress( + reinterpret_cast(buf), + srv_page_size, + reinterpret_cast(out_buf) + header_len, + &len) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_SNAPPY */ - - case PAGE_ZLIB_ALGORITHM: - err = compress2(out_buf+header_len, (ulong*)&write_size, buf, - uLong(len), comp_level); - - if (err != Z_OK) { - /* If error we leave the actual page as it was */ - - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " ULINTPF - " name %s len " ULINTPF - " rt %d write_size " ULINTPF ".", - space->id, space->name, len, - err, write_size); - } - - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - goto err_exit; - } - break; - - case PAGE_UNCOMPRESSED: - *out_len = len; - return (buf); - break; - default: - ut_error; - break; } + srv_stats.pages_page_compression_error.inc(); + return 0; +success: /* Set up the page header */ memcpy(out_buf, buf, FIL_PAGE_DATA); /* Set up the checksum */ @@ -392,22 +253,11 @@ fil_compress_page( /* Verify that page can be decompressed */ { - byte *comp_page; - byte *uncomp_page; - - comp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - uncomp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); - - fil_decompress_page(uncomp_page, comp_page, ulong(len), NULL); - - if (buf_page_is_corrupted(false, uncomp_page, 0, space)) { - buf_page_print(uncomp_page, 0); - ut_ad(0); - } - - ut_free(comp_page); - ut_free(uncomp_page); + page_t tmp_buf[UNIV_PAGE_SIZE_MAX]; + page_t page[UNIV_PAGE_SIZE_MAX]; + memcpy(page, out_buf, srv_page_size); + ut_ad(fil_page_decompress(tmp_buf, page)); + ut_ad(!buf_page_is_corrupted(false, page, 0, NULL)); } #endif /* UNIV_DEBUG */ @@ -431,323 +281,144 @@ fil_compress_page( #endif } - DBUG_PRINT("compress", - ("Succeeded for space " ULINTPF - " '%s' len " ULINTPF " out_len " ULINTPF, - space ? space->id : 0, - space ? space->name : "(import)", - len, write_size)); - - srv_stats.page_compression_saved.add((len - write_size)); + srv_stats.page_compression_saved.add(srv_page_size - write_size); srv_stats.pages_page_compressed.inc(); /* If we do not persistently trim rest of page, we need to write it all */ if (!srv_use_trim) { - memset(out_buf+write_size,0,len-write_size); - write_size = len; + memset(out_buf + write_size, 0, srv_page_size - write_size); } - *out_len = write_size; - - if (allocated) { - /* TODO: reduce number of memcpy's */ - memcpy(buf, out_buf, len); - } else { - return(out_buf); - } - -err_exit: - if (allocated) { - ut_free(out_buf); - } - - return (buf); - + return write_size; } -/****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -UNIV_INTERN -void -fil_decompress_page( -/*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< out: buffer from which to read; in aio - this must be appropriately aligned */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of - the compressed data. */ - bool return_error) /*!< in: true if only an error should - be produced when decompression fails. - By default this parameter is false. */ +/** Decompress a page that may be subject to page_compressed compression. +@param[in,out] tmp_buf temporary buffer (of innodb_page_size) +@param[in,out] buf possibly compressed page buffer +@return size of the compressed data +@retval 0 if decompression failed +@retval srv_page_size if the page was not compressed */ +UNIV_INTERN ulint fil_page_decompress(byte* tmp_buf, byte* buf) { - int err = 0; - ulint actual_size = 0; - ulint compression_alg = 0; - byte *in_buf; - ulint ptype; - ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; - - ut_ad(buf); - ut_ad(len); - - ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); - - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; + const unsigned ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); + ulint header_len; + ib_uint64_t compression_alg; + switch (ptype) { + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + + FIL_PAGE_COMPRESSION_METHOD_SIZE; + compression_alg = mach_read_from_2( + FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + buf); + break; + case FIL_PAGE_PAGE_COMPRESSED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; + compression_alg = mach_read_from_8( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + buf); + break; + default: + return srv_page_size; } - /* Do not try to uncompressed pages that are not compressed */ - if (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return; + if (mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM) + != BUF_NO_CHECKSUM_MAGIC) { + return 0; } - // If no buffer was given, we need to allocate temporal buffer - if (page_buf == NULL) { - in_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - memset(in_buf, 0, UNIV_PAGE_SIZE); - } else { - in_buf = page_buf; - } + ulint actual_size = mach_read_from_2(buf + FIL_PAGE_DATA); - /* Before actual decompress, make sure that page type is correct */ - - if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || - (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page" - " CRC " ULINTPF " type " ULINTPF " len " ULINTPF ".", - mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM), - mach_read_from_2(buf+FIL_PAGE_TYPE), len); - - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - } - - /* Get compression algorithm */ - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - compression_alg = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE); - } else { - compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } - - /* Get the actual size of compressed page */ - actual_size = mach_read_from_2(buf+FIL_PAGE_DATA); /* Check if payload size is corrupted */ - if (actual_size == 0 || actual_size > UNIV_PAGE_SIZE) { + if (actual_size == 0 || actual_size > srv_page_size - header_len) { + return 0; + } + + switch (compression_alg) { + default: ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page" - " actual size " ULINTPF " compression %s.", - actual_size, fil_get_compression_alg_name(compression_alg)); - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - } - - /* Store actual payload size of the compressed data. This pointer - points to buffer pool. */ - if (write_size) { - *write_size = actual_size; - } - - DBUG_PRINT("compress", - ("Preparing for decompress for len " ULINTPF ".", - actual_size)); - - switch(compression_alg) { + "Unknown compression algorithm " UINT64PF, + compression_alg); + return 0; case PAGE_ZLIB_ALGORITHM: - err= uncompress(in_buf, &len, buf+header_len, (unsigned long)actual_size); - - /* If uncompress fails it means that page is corrupted */ - if (err != Z_OK) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; + { + uLong len = srv_page_size; + if (Z_OK != uncompress(tmp_buf, &len, + buf + header_len, + uLong(actual_size)) + && len != srv_page_size) { + return 0; } - ut_error; } break; - #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: - err = LZ4_decompress_fast((const char *)buf+header_len, (char *)in_buf, len); - - if (err != (int)actual_size) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (LZ4_decompress_safe(reinterpret_cast(buf) + + header_len, + reinterpret_cast(tmp_buf), + actual_size, srv_page_size) + == int(srv_page_size)) { + break; } - break; + return 0; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO case PAGE_LZO_ALGORITHM: { - ulint olen = 0; - lzo_uint olen_lzo = olen; - err = lzo1x_decompress((const unsigned char *)buf+header_len, - actual_size,(unsigned char *)in_buf, &olen_lzo, NULL); - - olen = olen_lzo; - - if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but uncompress failed with error %d " - " size " ULINTPF " len " ULINTPF ".", - err, actual_size, len); - - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + lzo_uint len_lzo = srv_page_size; + if (LZO_E_OK == lzo1x_decompress_safe( + buf + header_len, + actual_size, tmp_buf, &len_lzo, NULL) + && len_lzo == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - - lzma_ret ret; size_t src_pos = 0; size_t dst_pos = 0; uint64_t memlimit = UINT64_MAX; - ret = lzma_stream_buffer_decode( - &memlimit, - 0, - NULL, - buf+header_len, - &src_pos, - actual_size, - in_buf, - &dst_pos, - len); - - - if (ret != LZMA_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size " ULINTPF "len " ULINTPF ".", - dst_pos, actual_size, len); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (LZMA_OK == lzma_stream_buffer_decode( + &memlimit, 0, NULL, buf + header_len, + &src_pos, actual_size, tmp_buf, &dst_pos, + srv_page_size) + && dst_pos == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - unsigned int dst_pos = UNIV_PAGE_SIZE; - - err = BZ2_bzBuffToBuffDecompress( - (char *)in_buf, - &dst_pos, - (char *)(buf+header_len), - actual_size, - 1, - 0); - - if (err != BZ_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %du bytes" - " size " ULINTPF " len " ULINTPF " err %d.", - dst_pos, actual_size, len, err); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + unsigned int dst_pos = srv_page_size; + if (BZ_OK == BZ2_bzBuffToBuffDecompress( + reinterpret_cast(tmp_buf), + &dst_pos, + reinterpret_cast(buf) + header_len, + actual_size, 1, 0) + && dst_pos == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - ulint olen = UNIV_PAGE_SIZE; + case PAGE_SNAPPY_ALGORITHM: { + size_t olen = srv_page_size; - cstatus = snappy_uncompress( - (const char *)(buf+header_len), - (size_t)actual_size, - (char *)in_buf, - (size_t*)&olen); - - if (cstatus != SNAPPY_OK || olen != UNIV_PAGE_SIZE) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only " ULINTPF " bytes" - " size " ULINTPF " len " ULINTPF " err %d.", - olen, actual_size, len, (int)cstatus); - fflush(stderr); - - if (return_error) { - goto error_return; - } - ut_error; + if (SNAPPY_OK == snappy_uncompress( + reinterpret_cast(buf) + header_len, + actual_size, + reinterpret_cast(tmp_buf), &olen) + && olen == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_SNAPPY */ - default: - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but compression algorithm %s" - " is not known." - ,fil_get_compression_alg_name(compression_alg)); - - fflush(stderr); - if (return_error) { - goto error_return; - } - ut_error; - break; } srv_stats.pages_page_decompressed.inc(); - - /* Copy the uncompressed page to the buffer pool, not - really any other options. */ - memcpy(buf, in_buf, len); - -error_return: - if (page_buf != in_buf) { - ut_free(in_buf); - } + memcpy(buf, tmp_buf, srv_page_size); + return actual_size; } diff --git a/storage/xtradb/ibuf/ibuf0ibuf.cc b/storage/xtradb/ibuf/ibuf0ibuf.cc index b169916c34e..96a86eea4c1 100644 --- a/storage/xtradb/ibuf/ibuf0ibuf.cc +++ b/storage/xtradb/ibuf/ibuf0ibuf.cc @@ -5218,6 +5218,10 @@ ibuf_check_bitmap_on_import( bitmap_page = ibuf_bitmap_get_map_page( space_id, page_no, zip_size, &mtr); + if (!bitmap_page) { + mutex_exit(&ibuf_mutex); + return DB_CORRUPTION; + } for (i = FSP_IBUF_BITMAP_OFFSET + 1; i < page_size; i++) { const ulint offset = page_no + i; diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 7661ba1785d..0944b5d4067 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -38,6 +38,7 @@ Created 11/5/1995 Heikki Tuuri #include "ut0rbt.h" #include "os0proc.h" #include "log0log.h" +#include "my_atomic.h" /** @name Modes for buf_page_get_gen */ /* @{ */ @@ -1528,45 +1529,16 @@ buf_page_encrypt_before_write( buf_page_t* bpage, byte* src_frame); -/********************************************************************** -The hook that is called after page is written to disk. -The function releases any resources needed for encryption that was allocated -in buf_page_encrypt_before_write */ -UNIV_INTERN -ibool -buf_page_encrypt_after_write( -/*=========================*/ - buf_page_t* page); /*!< in/out: buffer page that was flushed */ - -/********************************************************************//** -The hook that is called just before a page is read from disk. -The function allocates memory that is used to temporarily store disk content -before getting decrypted */ -UNIV_INTERN -byte* -buf_page_decrypt_before_read( -/*=========================*/ - buf_page_t* page, /*!< in/out: buffer page read from disk */ - ulint zip_size); /*!< in: compressed page size, or 0 */ - -/********************************************************************//** -The hook that is called just after a page is read from disk. -The function decrypt disk content into buf_page_t and releases the -temporary buffer that was allocated in buf_page_decrypt_before_read */ -UNIV_INTERN -bool -buf_page_decrypt_after_read( -/*========================*/ - buf_page_t* page); /*!< in/out: buffer page read from disk */ - /** @brief The temporary memory structure. NOTE! The definition appears here only for other modules of this directory (buf) to see it. Do not use from outside! */ typedef struct { - bool reserved; /*!< true if this slot is reserved +private: + int32 reserved; /*!< true if this slot is reserved */ +public: byte* crypt_buf; /*!< for encryption the data needs to be copied to a separate buffer before it's encrypted&written. this as a page can be @@ -1577,6 +1549,21 @@ typedef struct { byte* out_buf; /*!< resulting buffer after encryption/compression. This is a pointer and not allocated. */ + + /** Release the slot */ + void release() + { + my_atomic_store32_explicit(&reserved, false, + MY_MEMORY_ORDER_RELAXED); + } + + /** Acquire the slot + @return whether the slot was acquired */ + bool acquire() + { + return !my_atomic_fas32_explicit(&reserved, true, + MY_MEMORY_ORDER_RELAXED); + } } buf_tmp_buffer_t; /** The common buffer control block structure diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 91d46afe7a8..7b8339fec7d 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -340,9 +340,6 @@ struct fil_space_t { corrupted page. */ bool is_corrupt; /*!< true if tablespace corrupted */ - bool printed_compression_failure; - /*!< true if we have already printed - compression failure */ fil_space_crypt_t* crypt_data; /*!< tablespace crypt data or NULL */ ulint file_block_size; diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic index 49fdff4f3e5..3f21c529308 100644 --- a/storage/xtradb/include/fil0fil.ic +++ b/storage/xtradb/include/fil0fil.ic @@ -68,7 +68,6 @@ fil_get_page_type_name( } return "PAGE TYPE CORRUPTED"; - } /****************************************************************//** diff --git a/storage/xtradb/include/fil0pagecompress.h b/storage/xtradb/include/fil0pagecompress.h index 03e16699ce3..934372c55b2 100644 --- a/storage/xtradb/include/fil0pagecompress.h +++ b/storage/xtradb/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018 MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -30,70 +30,26 @@ atomic writes information to table space. Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ -/*******************************************************************//** -Find out wheather the page is index page or not -@return true if page type index page, false if not */ -UNIV_INLINE -ibool -fil_page_is_index_page( -/*===================*/ - byte *buf); /*!< in: page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) + MY_ATTRIBUTE((nonnull, warn_unused_result)); -/****************************************************************//** -Get the name of the compression algorithm used for page -compression. -@return compression algorithm name or "UNKNOWN" if not known*/ -UNIV_INLINE -const char* -fil_get_compression_alg_name( -/*=========================*/ - ulint comp_alg); /*! @@ -3364,15 +3370,30 @@ fil_iterate( os_offset_t offset; ulint n_bytes = iter.n_io_buffers * iter.page_size; + const ulint buf_size = srv_page_size +#ifdef HAVE_LZO + + LZO1X_1_15_MEM_COMPRESS +#elif defined HAVE_SNAPPY + + snappy_max_compressed_length(srv_page_size) +#endif + ; + byte* page_compress_buf = static_cast( + ut_malloc_low(buf_size, false)); ut_ad(!srv_read_only_mode); + if (!page_compress_buf) { + return DB_OUT_OF_MEMORY; + } + /* TODO: For ROW_FORMAT=COMPRESSED tables we do a lot of useless copying for non-index pages. Unfortunately, it is required by buf_zip_decompress() */ + dberr_t err = DB_SUCCESS; for (offset = iter.start; offset < iter.end; offset += n_bytes) { if (callback.is_interrupted()) { - return DB_INTERRUPTED; + err = DB_INTERRUPTED; + goto func_exit; } byte* io_buffer = iter.io_buffer; @@ -3403,12 +3424,13 @@ fil_iterate( if (!os_file_read_no_error_handling(iter.file, readptr, offset, n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed"); - return DB_IO_ERROR; + err = DB_IO_ERROR; + goto func_exit; } bool updated = false; - ulint n_pages_read = (ulint) n_bytes / iter.page_size; const ulint size = iter.page_size; + ulint n_pages_read = ulint(n_bytes) / size; block->page.offset = offset / size; for (ulint i = 0; i < n_pages_read; @@ -3437,11 +3459,11 @@ page_corrupted: UINT64PF " looks corrupted.", callback.filename(), ulong(offset / size), offset); - return DB_CORRUPTION; + err = DB_CORRUPTION; + goto func_exit; } bool decrypted = false; - dberr_t err = DB_SUCCESS; byte* dst = io_buffer + (i * size); bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -3450,6 +3472,10 @@ page_corrupted: == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED || page_type == FIL_PAGE_PAGE_COMPRESSED; + if (page_compressed && block->page.zip.data) { + goto page_corrupted; + } + if (!encrypted) { } else if (!mach_read_from_4( FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION @@ -3475,7 +3501,7 @@ not_encrypted: iter.page_size, src, &err); if (err != DB_SUCCESS) { - return err; + goto func_exit; } if (!decrypted) { @@ -3488,8 +3514,12 @@ not_encrypted: /* If the original page is page_compressed, we need to decompress it before adjusting further. */ if (page_compressed) { - fil_decompress_page(NULL, dst, ulong(size), - NULL); + ulint compress_length = fil_page_decompress( + page_compress_buf, dst); + ut_ad(compress_length != srv_page_size); + if (compress_length == 0) { + goto page_corrupted; + } updated = true; } else if (buf_page_is_corrupted( false, @@ -3500,7 +3530,7 @@ not_encrypted: } if ((err = callback(block)) != DB_SUCCESS) { - return err; + goto func_exit; } else if (!updated) { updated = buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE; @@ -3550,19 +3580,17 @@ not_encrypted: src = io_buffer + (i * size); if (page_compressed) { - ulint len = 0; - - fil_compress_page( - NULL, - src, - NULL, - size, - 0,/* FIXME: compression level */ - 512,/* FIXME: use proper block size */ - encrypted, - &len); - updated = true; + if (fil_page_compress( + src, + page_compress_buf, + 0,/* FIXME: compression level */ + 512,/* FIXME: proper block size */ + encrypted)) { + /* FIXME: remove memcpy() */ + memcpy(src, page_compress_buf, + srv_page_size); + } } /* If tablespace is encrypted, encrypt page before we @@ -3599,11 +3627,14 @@ not_encrypted: offset, (ulint) n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed"); - return DB_IO_ERROR; + err = DB_IO_ERROR; + goto func_exit; } } - return DB_SUCCESS; +func_exit: + ut_free(page_compress_buf); + return err; } /********************************************************************//** From 2ca904f0ca981666b90151ff3fcdad4307b96a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Jun 2018 16:15:21 +0300 Subject: [PATCH 153/276] MDEV-13103 Deal with page_compressed page corruption fil_page_decompress(): Replaces fil_decompress_page(). Allow the caller detect errors. Remove duplicated code. Use the "safe" instead of "fast" variants of decompression routines. fil_page_compress(): Replaces fil_compress_page(). The length of the input buffer always was srv_page_size (innodb_page_size). Remove printouts, and remove the fil_space_t* parameter. buf_tmp_buffer_t::reserved: Make private; the accessors acquire() and release() will use atomic memory access. buf_pool_reserve_tmp_slot(): Make static. Remove the second parameter. Do not acquire any mutex. Remove the allocation of the buffers. buf_tmp_reserve_crypt_buf(), buf_tmp_reserve_compression_buf(): Refactored away from buf_pool_reserve_tmp_slot(). buf_page_decrypt_after_read(): Make static, and simplify the logic. Use the encryption buffer also for decompressing. buf_page_io_complete(), buf_dblwr_process(): Check more failures. fil_space_encrypt(): Simplify the debug checks. fil_space_t::printed_compression_failure: Remove. fil_get_compression_alg_name(): Remove. fil_iterate(): Allocate a buffer for compression and decompression only once, instead of allocating and freeing it for every page that uses compression, during IMPORT TABLESPACE. Also, validate the page checksum before decryption, and reduce the scope of some variables. fil_page_is_index_page(), fil_page_is_lzo_compressed(): Remove (unused). AbstractCallback::operator()(): Remove the parameter 'offset'. The check for it in FetchIndexRootPages::operator() was basically redundant and dead code since the previous refactoring. --- .../r/innodb-page_compression_default.result | 1 - .../r/innodb-page_compression_snappy.result | 1 - .../t/innodb-page_compression_default.test | 2 - .../t/innodb-page_compression_snappy.test | 2 - storage/innobase/buf/buf0buf.cc | 352 +++++------ storage/innobase/buf/buf0dblwr.cc | 34 +- storage/innobase/fil/fil0crypt.cc | 59 +- storage/innobase/fil/fil0pagecompress.cc | 575 +++++------------- storage/innobase/ibuf/ibuf0ibuf.cc | 5 + storage/innobase/include/buf0buf.h | 20 +- storage/innobase/include/fil0fil.h | 3 - storage/innobase/include/fil0pagecompress.h | 56 +- storage/innobase/include/fsp0pagecompress.ic | 70 +-- storage/innobase/row/row0import.cc | 180 +++--- 14 files changed, 503 insertions(+), 857 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-page_compression_default.result b/mysql-test/suite/innodb/r/innodb-page_compression_default.result index 413450e1a6d..ca7129fd3eb 100644 --- a/mysql-test/suite/innodb/r/innodb-page_compression_default.result +++ b/mysql-test/suite/innodb/r/innodb-page_compression_default.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); set global innodb_file_format = `Barracuda`; set global innodb_file_per_table = on; create table innodb_normal (c1 int not null auto_increment primary key, b char(200)) engine=innodb; diff --git a/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result b/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result index 83a17f678e4..6e0ee94e69e 100644 --- a/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result +++ b/mysql-test/suite/innodb/r/innodb-page_compression_snappy.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); set global innodb_compression_algorithm = snappy; set global innodb_file_format = `Barracuda`; set global innodb_file_per_table = on; diff --git a/mysql-test/suite/innodb/t/innodb-page_compression_default.test b/mysql-test/suite/innodb/t/innodb-page_compression_default.test index 1cc6c917548..34b1829485e 100644 --- a/mysql-test/suite/innodb/t/innodb-page_compression_default.test +++ b/mysql-test/suite/innodb/t/innodb-page_compression_default.test @@ -1,8 +1,6 @@ --source include/have_innodb.inc --source include/not_embedded.inc -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); - # All page compression test use the same --source include/innodb-page-compression.inc diff --git a/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test b/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test index 532ec294d28..0186c24ef2e 100644 --- a/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test +++ b/mysql-test/suite/innodb/t/innodb-page_compression_snappy.test @@ -2,8 +2,6 @@ -- source include/have_innodb_snappy.inc --source include/not_embedded.inc -call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+."); - # snappy set global innodb_compression_algorithm = snappy; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 845bb17bfa5..30fe1f8f616 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -423,6 +423,53 @@ on the io_type */ ? (counter##_READ) \ : (counter##_WRITTEN)) + +/** Reserve a buffer slot for encryption, decryption or page compression. +@param[in,out] buf_pool buffer pool +@return reserved buffer slot */ +static buf_tmp_buffer_t* buf_pool_reserve_tmp_slot(buf_pool_t* buf_pool) +{ + for (ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { + buf_tmp_buffer_t* slot = &buf_pool->tmp_arr->slots[i]; + if (slot->acquire()) { + return slot; + } + } + + /* We assume that free slot is found */ + ut_error; + return NULL; +} + +/** Reserve a buffer for encryption, decryption or decompression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_crypt_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->crypt_buf) { + slot->crypt_buf = static_cast( + aligned_malloc(srv_page_size, srv_page_size)); + } +} + +/** Reserve a buffer for compression. +@param[in,out] slot reserved slot */ +static void buf_tmp_reserve_compression_buf(buf_tmp_buffer_t* slot) +{ + if (!slot->comp_buf) { + /* Both snappy and lzo compression methods require that + output buffer used for compression is bigger than input + buffer. Increase the allocated buffer size accordingly. */ + ulint size = srv_page_size; +#ifdef HAVE_LZO + size += LZO1X_1_15_MEM_COMPRESS; +#elif defined HAVE_SNAPPY + size = snappy_max_compressed_length(size); +#endif + slot->comp_buf = static_cast( + aligned_malloc(size, srv_page_size)); + } +} + /** Registers a chunk to buf_pool_chunk_map @param[in] chunk chunk of buffers */ static @@ -438,10 +485,91 @@ buf_pool_register_chunk( @param[in,out] bpage Page control block @param[in,out] space tablespace @return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) - MY_ATTRIBUTE((nonnull)); +static bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) +{ + ut_ad(space->n_pending_ios > 0); + ut_ad(space->id == bpage->id.space()); + + byte* dst_frame = bpage->zip.data ? bpage->zip.data : + ((buf_block_t*) bpage)->frame; + bool page_compressed = fil_page_is_compressed(dst_frame); + buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); + + if (bpage->id.page_no() == 0) { + /* File header pages are not encrypted/compressed */ + return (true); + } + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + + buf_tmp_buffer_t* slot; + + if (page_compressed) { + /* the page we read is unencrypted */ + /* Find free slot from temporary memory array */ +decompress: + slot = buf_pool_reserve_tmp_slot(buf_pool); + /* For decompression, use crypt_buf. */ + buf_tmp_reserve_crypt_buf(slot); +decompress_with_slot: + ut_d(fil_page_type_validate(dst_frame)); + + bpage->write_size = fil_page_decompress(slot->crypt_buf, + dst_frame); + slot->release(); + + ut_ad(!bpage->write_size || fil_page_type_validate(dst_frame)); + ut_ad(space->n_pending_ios > 0); + return bpage->write_size != 0; + } + + if (space->crypt_data + && mach_read_from_4(FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + dst_frame)) { + /* Verify encryption checksum before we even try to + decrypt. */ + if (!fil_space_verify_crypt_checksum( + dst_frame, bpage->size, bpage->id.space(), + bpage->id.page_no())) { +decrypt_failed: + /* Mark page encrypted in case it should be. */ + if (space->crypt_data->type + != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted = true; + } + + return false; + } + + /* Find free slot from temporary memory array */ + slot = buf_pool_reserve_tmp_slot(buf_pool); + buf_tmp_reserve_crypt_buf(slot); + + ut_d(fil_page_type_validate(dst_frame)); + + /* decrypt using crypt_buf to dst_frame */ + if (!fil_space_decrypt(space, slot->crypt_buf, + dst_frame, &bpage->encrypted)) { + slot->release(); + goto decrypt_failed; + } + + ut_d(fil_page_type_validate(dst_frame)); + + if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress_with_slot; + } + + slot->release(); + } else if (fil_page_is_compressed_encrypted(dst_frame)) { + goto decompress; + } + + ut_ad(space->n_pending_ios > 0); + return true; +} /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); @@ -5931,21 +6059,23 @@ buf_page_io_complete(buf_page_t* bpage, bool dblwr, bool evict) ulint read_page_no = 0; ulint read_space_id = 0; uint key_version = 0; - - ut_ad(bpage->zip.data != NULL || ((buf_block_t*)bpage)->frame != NULL); + byte* frame = bpage->zip.data + ? bpage->zip.data + : reinterpret_cast(bpage)->frame; + ut_ad(frame); fil_space_t* space = fil_space_acquire_for_io( bpage->id.space()); if (!space) { return DB_TABLESPACE_DELETED; } - buf_page_decrypt_after_read(bpage, space); - - byte* frame = bpage->zip.data - ? bpage->zip.data - : reinterpret_cast(bpage)->frame; dberr_t err; + if (!buf_page_decrypt_after_read(bpage, space)) { + err = DB_DECRYPTION_FAILED; + goto database_corrupted; + } + if (bpage->zip.data && uncompressed) { my_atomic_addlint(&buf_pool->n_pend_unzip, 1); ibool ok = buf_zip_decompress((buf_block_t*) bpage, @@ -6097,7 +6227,7 @@ database_corrupted: /* io_type == BUF_IO_WRITE */ if (bpage->slot) { /* Mark slot free */ - bpage->slot->reserved = false; + bpage->slot->release(); bpage->slot = NULL; } } @@ -7268,66 +7398,6 @@ operator<<( return(out); } -/********************************************************************//** -Reserve unused slot from temporary memory array and allocate necessary -temporary memory if not yet allocated. -@return reserved slot */ -UNIV_INTERN -buf_tmp_buffer_t* -buf_pool_reserve_tmp_slot( -/*======================*/ - buf_pool_t* buf_pool, /*!< in: buffer pool where to - reserve */ - bool compressed) /*!< in: is file space compressed */ -{ - buf_tmp_buffer_t *free_slot=NULL; - - /* Array is protected by buf_pool mutex */ - buf_pool_mutex_enter(buf_pool); - - for(ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { - buf_tmp_buffer_t *slot = &buf_pool->tmp_arr->slots[i]; - - if(slot->reserved == false) { - free_slot = slot; - break; - } - } - - /* We assume that free slot is found */ - ut_a(free_slot != NULL); - free_slot->reserved = true; - /* Now that we have reserved this slot we can release - buf_pool mutex */ - buf_pool_mutex_exit(buf_pool); - - /* Allocate temporary memory for encryption/decryption */ - if (free_slot->crypt_buf == NULL) { - free_slot->crypt_buf = static_cast(aligned_malloc(UNIV_PAGE_SIZE, UNIV_PAGE_SIZE)); - memset(free_slot->crypt_buf, 0, UNIV_PAGE_SIZE); - } - - /* For page compressed tables allocate temporary memory for - compression/decompression */ - if (compressed && free_slot->comp_buf == NULL) { - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - size = snappy_max_compressed_length(size); -#endif -#if HAVE_LZO - size += LZO1X_1_15_MEM_COMPRESS; -#endif - free_slot->comp_buf = static_cast(aligned_malloc(size, UNIV_PAGE_SIZE)); - memset(free_slot->comp_buf, 0, size); - } - - return (free_slot); -} - /** Encryption and page_compression hook that is called just before a page is written to disk. @param[in,out] space tablespace @@ -7376,15 +7446,18 @@ buf_page_encrypt_before_write( return src_frame; } + ut_ad(!bpage->size.is_compressed() || !page_compressed); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool); slot->out_buf = NULL; bpage->slot = slot; + buf_tmp_reserve_crypt_buf(slot); byte *dst_frame = slot->crypt_buf; if (!page_compressed) { +not_compressed: /* Encrypt page content */ byte* tmp = fil_space_encrypt(space, bpage->id.page_no(), @@ -7392,33 +7465,30 @@ buf_page_encrypt_before_write( src_frame, dst_frame); + bpage->real_size = UNIV_PAGE_SIZE; slot->out_buf = dst_frame = tmp; ut_d(fil_page_type_validate(tmp)); } else { /* First we compress the page content */ - ulint out_len = 0; - - byte *tmp = fil_compress_page( - space, - (byte *)src_frame, - slot->comp_buf, - srv_page_size, + buf_tmp_reserve_compression_buf(slot); + byte* tmp = slot->comp_buf; + ulint out_len = fil_page_compress( + src_frame, tmp, fsp_flags_get_page_compression_level(space->flags), fil_space_get_block_size(space, bpage->id.page_no()), - encrypted, - &out_len); + encrypted); + if (!out_len) { + goto not_compressed; + } bpage->real_size = out_len; /* Workaround for MDEV-15527. */ memset(tmp + out_len, 0 , srv_page_size - out_len); -#ifdef UNIV_DEBUG - fil_page_type_validate(tmp); -#endif - - if(encrypted) { + ut_d(fil_page_type_validate(tmp)); + if (encrypted) { /* And then we encrypt the page content */ tmp = fil_space_encrypt(space, bpage->id.page_no(), @@ -7436,112 +7506,6 @@ buf_page_encrypt_before_write( return dst_frame; } -/** Decrypt a page. -@param[in,out] bpage Page control block -@param[in,out] space tablespace -@return whether the operation was successful */ -static -bool -buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) -{ - ut_ad(space->n_pending_ios > 0); - ut_ad(space->id == bpage->id.space()); - - bool compressed = bpage->size.is_compressed(); - const page_size_t& size = bpage->size; - byte* dst_frame = compressed ? bpage->zip.data : - ((buf_block_t*) bpage)->frame; - unsigned key_version = - mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed = fil_page_is_compressed(dst_frame); - bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); - buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); - bool success = true; - - if (bpage->id.page_no() == 0) { - /* File header pages are not encrypted/compressed */ - return (true); - } - - /* Page is encrypted if encryption information is found from - tablespace and page contains used key_version. This is true - also for pages first compressed and then encrypted. */ - if (!space->crypt_data) { - key_version = 0; - } - - if (page_compressed) { - /* the page we read is unencrypted */ - /* Find free slot from temporary memory array */ - buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - - ut_d(fil_page_type_validate(dst_frame)); - - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size.logical()), - &bpage->write_size); - - /* Mark this slot as free */ - slot->reserved = false; - key_version = 0; - - ut_d(fil_page_type_validate(dst_frame)); - } else { - buf_tmp_buffer_t* slot = NULL; - - if (key_version) { - /* Verify encryption checksum before we even try to - decrypt. */ - if (!fil_space_verify_crypt_checksum( - dst_frame, size, - bpage->id.space(), bpage->id.page_no())) { - if (space->crypt_data->type - != CRYPT_SCHEME_UNENCRYPTED) { - bpage->encrypted = true; - } - return (false); - } - - /* Find free slot from temporary memory array */ - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - - ut_d(fil_page_type_validate(dst_frame)); - - /* decrypt using crypt_buf to dst_frame */ - if (!fil_space_decrypt(space, slot->crypt_buf, - dst_frame, &bpage->encrypted)) { - success = false; - } - - ut_d(fil_page_type_validate(dst_frame)); - } - - if (page_compressed_encrypted && success) { - if (!slot) { - slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - } - - ut_d(fil_page_type_validate(dst_frame)); - /* decompress using comp_buf to dst_frame */ - fil_decompress_page(slot->comp_buf, - dst_frame, - ulong(size.logical()), - &bpage->write_size); - ut_d(fil_page_type_validate(dst_frame)); - } - - /* Mark this slot as free */ - if (slot) { - slot->reserved = false; - } - } - - ut_ad(space->n_pending_ios > 0); - return (success); -} - /** Should we punch hole to deallocate unused portion of the page. @param[in] bpage Page control block @@ -7565,6 +7529,4 @@ buf_page_get_trim_length( { return (bpage->size.physical() - write_length); } - - #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 96000c4eb92..f6a325a15fb 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -533,10 +533,11 @@ buf_dblwr_process() } unaligned_read_buf = static_cast( - ut_malloc_nokey(2 * UNIV_PAGE_SIZE)); + ut_malloc_nokey(3 * UNIV_PAGE_SIZE)); read_buf = static_cast( ut_align(unaligned_read_buf, UNIV_PAGE_SIZE)); + byte* const buf = read_buf + UNIV_PAGE_SIZE; for (recv_dblwr_t::list::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); @@ -604,24 +605,24 @@ buf_dblwr_process() ignore this page (there should be redo log records to initialize it). */ } else { - if (fil_page_is_compressed_encrypted(read_buf) || - fil_page_is_compressed(read_buf)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, read_buf, srv_page_size, - NULL, true); + /* Decompress the page before + validating the checksum. */ + ulint decomp = fil_page_decompress(buf, read_buf); + if (!decomp || (decomp != srv_page_size + && page_size.is_compressed())) { + goto bad; } if (fil_space_verify_crypt_checksum( read_buf, page_size, space_id, page_no) - || !buf_page_is_corrupted( - true, read_buf, page_size, space)) { + || !buf_page_is_corrupted( + true, read_buf, page_size, space)) { /* The page is good; there is no need to consult the doublewrite buffer. */ continue; } +bad: /* We intentionally skip this message for is_all_zero pages. */ ib::info() @@ -629,19 +630,16 @@ buf_dblwr_process() << " from the doublewrite buffer."; } - /* Next, validate the doublewrite page. */ - if (fil_page_is_compressed_encrypted(page) || - fil_page_is_compressed(page)) { - /* Decompress the page before - validating the checksum. */ - fil_decompress_page( - NULL, page, srv_page_size, NULL, true); + ulint decomp = fil_page_decompress(buf, page); + if (!decomp || (decomp != srv_page_size + && page_size.is_compressed())) { + goto bad_doublewrite; } - if (!fil_space_verify_crypt_checksum(page, page_size, space_id, page_no) && buf_page_is_corrupted(true, page, page_size, space)) { if (!is_all_zero) { +bad_doublewrite: ib::warn() << "A doublewrite copy of page " << page_id << " is corrupted."; } diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 4c53441df58..3c9640c8e15 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -665,57 +665,38 @@ fil_space_encrypt( #ifdef UNIV_DEBUG if (tmp) { /* Verify that encrypted buffer is not corrupted */ - byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE); dberr_t err = DB_SUCCESS; byte* src = src_frame; bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - byte* comp_mem = NULL; - byte* uncomp_mem = NULL; + byte uncomp_mem[UNIV_PAGE_SIZE_MAX]; + byte tmp_mem[UNIV_PAGE_SIZE_MAX]; if (page_compressed_encrypted) { - comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); - memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); - fil_decompress_page(uncomp_mem, comp_mem, - srv_page_size, NULL); - src = uncomp_mem; + memcpy(uncomp_mem, src, srv_page_size); + ulint unzipped1 = fil_page_decompress( + tmp_mem, uncomp_mem); + ut_ad(unzipped1); + if (unzipped1 != srv_page_size) { + src = uncomp_mem; + } } - bool corrupted1 = buf_page_is_corrupted(true, src, page_size, space); - bool ok = fil_space_decrypt(crypt_data, tmp_mem, page_size, tmp, &err); + ut_ad(!buf_page_is_corrupted(true, src, page_size, space)); + ut_ad(fil_space_decrypt(crypt_data, tmp_mem, page_size, tmp, + &err)); + ut_ad(err == DB_SUCCESS); /* Need to decompress the page if it was also compressed */ if (page_compressed_encrypted) { - memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); - fil_decompress_page(tmp_mem, comp_mem, - srv_page_size, NULL); + byte buf[UNIV_PAGE_SIZE_MAX]; + memcpy(buf, tmp_mem, srv_page_size); + ulint unzipped2 = fil_page_decompress(tmp_mem, buf); + ut_ad(unzipped2); } - bool corrupted = buf_page_is_corrupted(true, tmp_mem, page_size, space); - memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); - bool different = memcmp(src, tmp_mem, page_size.physical()); - - if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { - fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", - ok , corrupted, corrupted1, err, different); - fprintf(stderr, "src_frame\n"); - buf_page_print(src_frame, page_size); - fprintf(stderr, "encrypted_frame\n"); - buf_page_print(tmp, page_size); - fprintf(stderr, "decrypted_frame\n"); - buf_page_print(tmp_mem, page_size); - ut_ad(0); - } - - free(tmp_mem); - - if (comp_mem) { - free(comp_mem); - } - - if (uncomp_mem) { - free(uncomp_mem); - } + memcpy(tmp_mem + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + src + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); + ut_ad(!memcmp(src, tmp_mem, page_size.physical())); } #endif /* UNIV_DEBUG */ diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index cddeb6a2d28..491de512156 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017, MariaDB Corporation. +Copyright (C) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -74,73 +74,26 @@ Updated 14/02/2015 #include "snappy-c.h" #endif -/* Used for debugging */ -//#define UNIV_PAGECOMPRESS_DEBUG 1 - -/****************************************************************//** -For page compressed pages compress the page before actual write -operation. -@return compressed page to be written*/ -UNIV_INTERN -byte* -fil_compress_page( -/*==============*/ - fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: compressed buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint level, /* in: compression level */ - ulint block_size, /*!< in: block size */ - bool encrypted, /*!< in: is page also encrypted */ - ulint* out_len) /*!< out: actual length of compressed - page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) { - int err = Z_OK; int comp_level = int(level); ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; - ulint write_size = 0; -#if HAVE_LZO - lzo_uint write_size_lzo = write_size; -#endif /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; - bool allocated = false; - - /* page_compression does not apply to tables or tablespaces - that use ROW_FORMAT=COMPRESSED */ - ut_ad(!space || !FSP_FLAGS_GET_ZIP_SSIZE(space->flags)); if (encrypted) { header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; } - if (!out_buf) { - allocated = true; - ulint size = UNIV_PAGE_SIZE; - - /* Both snappy and lzo compression methods require that - output buffer used for compression is bigger than input - buffer. Increase the allocated buffer size accordingly. */ -#if HAVE_SNAPPY - if (comp_method == PAGE_SNAPPY_ALGORITHM) { - size = snappy_max_compressed_length(size); - } -#endif -#if HAVE_LZO - if (comp_method == PAGE_LZO_ALGORITHM) { - size += LZO1X_1_15_MEM_COMPRESS; - } -#endif - - out_buf = static_cast(ut_malloc_nokey(size)); - } - - ut_ad(buf); - ut_ad(out_buf); - ut_ad(len); - ut_ad(out_len); - /* Let's not compress file space header or extent descriptor */ switch (fil_page_get_type(buf)) { @@ -148,8 +101,7 @@ fil_compress_page( case FIL_PAGE_TYPE_FSP_HDR: case FIL_PAGE_TYPE_XDES: case FIL_PAGE_PAGE_COMPRESSED: - *out_len = len; - goto err_exit; + return 0; } /* If no compression level was provided to this table, use system @@ -158,125 +110,113 @@ fil_compress_page( comp_level = page_zip_level; } - DBUG_LOG("compress", "Preparing for space " - << (space ? space->id : 0) << " '" - << (space ? space->name : "(import)") << "' len " << len); + ulint write_size = srv_page_size - header_len; - write_size = UNIV_PAGE_SIZE - header_len; - - switch(comp_method) { + switch (comp_method) { + default: + ut_ad(!"unknown compression method"); + /* fall through */ + case PAGE_UNCOMPRESSED: + return 0; + case PAGE_ZLIB_ALGORITHM: + { + ulong len = uLong(write_size); + if (Z_OK == compress2( + out_buf + header_len, &len, + buf, uLong(srv_page_size), comp_level)) { + write_size = len; + goto success; + } + } + break; #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: +# ifdef HAVE_LZ4_COMPRESS_DEFAULT + write_size = LZ4_compress_default( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# else + write_size = LZ4_compress_limitedOutput( + reinterpret_cast(buf), + reinterpret_cast(out_buf) + header_len, + int(srv_page_size), int(write_size)); +# endif -#ifdef HAVE_LZ4_COMPRESS_DEFAULT - err = LZ4_compress_default((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#else - err = LZ4_compress_limitedOutput((const char *)buf, - (char *)out_buf+header_len, len, write_size); -#endif /* HAVE_LZ4_COMPRESS_DEFAULT */ - write_size = err; - - if (err == 0) { - goto err_exit; + if (write_size) { + goto success; } break; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO - case PAGE_LZO_ALGORITHM: - err = lzo1x_1_15_compress( - buf, len, out_buf+header_len, &write_size_lzo, out_buf+UNIV_PAGE_SIZE); + case PAGE_LZO_ALGORITHM: { + lzo_uint len = write_size; - write_size = write_size_lzo; - - if (err != LZO_E_OK || write_size > UNIV_PAGE_SIZE-header_len) { - goto err_exit; + if (LZO_E_OK == lzo1x_1_15_compress( + buf, srv_page_size, + out_buf + header_len, &len, + out_buf + srv_page_size) + && len <= write_size) { + write_size = len; + goto success; } - break; + } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - size_t out_pos=0; + size_t out_pos = 0; - err = lzma_easy_buffer_encode( - comp_level, - LZMA_CHECK_NONE, - NULL, /* No custom allocator, use malloc/free */ - reinterpret_cast(buf), - len, - reinterpret_cast(out_buf + header_len), - &out_pos, - (size_t)write_size); - - if (err != LZMA_OK || out_pos > UNIV_PAGE_SIZE-header_len) { + if (LZMA_OK == lzma_easy_buffer_encode( + comp_level, LZMA_CHECK_NONE, NULL, + buf, srv_page_size, out_buf + header_len, + &out_pos, write_size) + && out_pos <= write_size) { write_size = out_pos; - goto err_exit; + goto success; } - - write_size = out_pos; - break; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - - err = BZ2_bzBuffToBuffCompress( - (char *)(out_buf + header_len), - (unsigned int *)&write_size, - (char *)buf, - len, - 1, - 0, - 0); - - if (err != BZ_OK || write_size > UNIV_PAGE_SIZE-header_len) { - goto err_exit; + unsigned len = unsigned(write_size); + if (BZ_OK == BZ2_bzBuffToBuffCompress( + reinterpret_cast(out_buf + header_len), + &len, + const_cast( + reinterpret_cast(buf)), + unsigned(srv_page_size), 1, 0, 0) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - write_size = snappy_max_compressed_length(UNIV_PAGE_SIZE); + case PAGE_SNAPPY_ALGORITHM: { + size_t len = snappy_max_compressed_length(srv_page_size); - cstatus = snappy_compress( - (const char *)buf, - (size_t)len, - (char *)(out_buf+header_len), - (size_t*)&write_size); - - if (cstatus != SNAPPY_OK || write_size > UNIV_PAGE_SIZE-header_len) { - err = (int)cstatus; - goto err_exit; + if (SNAPPY_OK == snappy_compress( + reinterpret_cast(buf), + srv_page_size, + reinterpret_cast(out_buf) + header_len, + &len) + && len <= write_size) { + write_size = len; + goto success; } break; } #endif /* HAVE_SNAPPY */ - - case PAGE_ZLIB_ALGORITHM: - err = compress2(out_buf+header_len, (ulong*)&write_size, buf, - uLong(len), comp_level); - - if (err != Z_OK) { - goto err_exit; - } - break; - - case PAGE_UNCOMPRESSED: - *out_len = len; - return (buf); - break; - default: - ut_error; - break; } + srv_stats.pages_page_compression_error.inc(); + return 0; +success: /* Set up the page header */ memcpy(out_buf, buf, FIL_PAGE_DATA); /* Set up the checksum */ @@ -307,23 +247,12 @@ fil_compress_page( /* Verify that page can be decompressed */ { - byte *comp_page; - byte *uncomp_page; - - comp_page = static_cast(ut_malloc_nokey(UNIV_PAGE_SIZE)); - uncomp_page = static_cast(ut_malloc_nokey(UNIV_PAGE_SIZE)); - memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); - - fil_decompress_page(uncomp_page, comp_page, ulong(len), NULL); - - if (buf_page_is_corrupted(false, uncomp_page, univ_page_size, - space)) { - buf_page_print(uncomp_page, univ_page_size); - ut_ad(0); - } - - ut_free(comp_page); - ut_free(uncomp_page); + page_t tmp_buf[UNIV_PAGE_SIZE_MAX]; + page_t page[UNIV_PAGE_SIZE_MAX]; + memcpy(page, out_buf, srv_page_size); + ut_ad(fil_page_decompress(tmp_buf, page)); + ut_ad(!buf_page_is_corrupted(false, page, univ_page_size, + NULL)); } #endif /* UNIV_DEBUG */ @@ -347,317 +276,143 @@ fil_compress_page( #endif } - DBUG_LOG("compress", "Succeeded for space " - << (space ? space->id : 0) << " '" - << (space ? space->name : "(import)") - << "' len " << len << " out_len " << write_size); - - srv_stats.page_compression_saved.add((len - write_size)); + srv_stats.page_compression_saved.add(srv_page_size - write_size); srv_stats.pages_page_compressed.inc(); /* If we do not persistently trim rest of page, we need to write it all */ if (!srv_use_trim) { - memset(out_buf+write_size,0,len-write_size); - write_size = len; + memset(out_buf + write_size, 0, srv_page_size - write_size); } - *out_len = write_size; - - if (allocated) { - /* TODO: reduce number of memcpy's */ - memcpy(buf, out_buf, len); - goto exit_free; - } else { - return(out_buf); - } - -err_exit: - /* If error we leave the actual page as it was */ - -#ifndef UNIV_PAGECOMPRESS_DEBUG - if (space && !space->printed_compression_failure) { - space->printed_compression_failure = true; -#endif - ib::warn() << "Compression failed for space: " - << space->id << " name: " - << space->name << " len: " - << len << " err: " << err << " write_size: " - << write_size - << " compression method: " - << fil_get_compression_alg_name(comp_method) - << "."; -#ifndef UNIV_PAGECOMPRESS_DEBUG - } -#endif - srv_stats.pages_page_compression_error.inc(); - *out_len = len; - -exit_free: - if (allocated) { - ut_free(out_buf); - } - - return (buf); - + return write_size; } -/****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -UNIV_INTERN -void -fil_decompress_page( -/*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< out: buffer from which to read; in aio - this must be appropriately aligned */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of - the compressed data. */ - bool return_error) /*!< in: true if only an error should - be produced when decompression fails. - By default this parameter is false. */ +/** Decompress a page that may be subject to page_compressed compression. +@param[in,out] tmp_buf temporary buffer (of innodb_page_size) +@param[in,out] buf possibly compressed page buffer +@return size of the compressed data +@retval 0 if decompression failed +@retval srv_page_size if the page was not compressed */ +ulint fil_page_decompress(byte* tmp_buf, byte* buf) { - int err = 0; - ulint actual_size = 0; - ib_uint64_t compression_alg = 0; - byte *in_buf; - ulint ptype; + const unsigned ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); ulint header_len; - - ut_ad(buf); - ut_ad(len); - - ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); - + uint64_t compression_alg; switch (ptype) { case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + FIL_PAGE_COMPRESSION_METHOD_SIZE; + compression_alg = mach_read_from_2( + FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + buf); break; case FIL_PAGE_PAGE_COMPRESSED: header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; + compression_alg = mach_read_from_8( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + buf); break; default: - /* The page is not in our format. */ - return; + return srv_page_size; } - // If no buffer was given, we need to allocate temporal buffer - if (page_buf == NULL) { - in_buf = static_cast(ut_malloc_nokey(UNIV_PAGE_SIZE)); - memset(in_buf, 0, UNIV_PAGE_SIZE); - } else { - in_buf = page_buf; + if (mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM) + != BUF_NO_CHECKSUM_MAGIC) { + return 0; } - /* Before actual decompress, make sure that page type is correct */ + ulint actual_size = mach_read_from_2(buf + FIL_PAGE_DATA); - if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) - != BUF_NO_CHECKSUM_MAGIC - || (ptype != FIL_PAGE_PAGE_COMPRESSED - && ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)) { - ib::error() << "Corruption: We try to uncompress corrupted " - "page CRC " - << mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) - << " type " << ptype << " len " << len << "."; - - if (return_error) { - goto error_return; - } - ut_error; - } - - /* Get compression algorithm */ - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - compression_alg = static_cast(mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE)); - } else { - compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } - - /* Get the actual size of compressed page */ - actual_size = mach_read_from_2(buf+FIL_PAGE_DATA); /* Check if payload size is corrupted */ - if (actual_size == 0 || actual_size > UNIV_PAGE_SIZE) { - ib::error() << "Corruption: We try to uncompress corrupted page" - << " actual size: " << actual_size - << " compression method: " - << fil_get_compression_alg_name(compression_alg) - << "."; - if (return_error) { - goto error_return; - } - ut_error; + if (actual_size == 0 || actual_size > srv_page_size - header_len) { + return 0; } - /* Store actual payload size of the compressed data. This pointer - points to buffer pool. */ - if (write_size) { - *write_size = actual_size; - } - - DBUG_LOG("compress", "Preparing for decompress for len " - << actual_size << "."); - - switch(compression_alg) { + switch (compression_alg) { + default: + ib::error() << "Unknown compression algorithm " + << compression_alg; + return 0; case PAGE_ZLIB_ALGORITHM: - err= uncompress(in_buf, &len, buf+header_len, (unsigned long)actual_size); - - /* If uncompress fails it means that page is corrupted */ - if (err != Z_OK) { - goto err_exit; - if (return_error) { - goto error_return; + { + uLong len = srv_page_size; + if (Z_OK != uncompress(tmp_buf, &len, + buf + header_len, + uLong(actual_size)) + && len != srv_page_size) { + return 0; } } break; - #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: - err = LZ4_decompress_fast((const char *)buf+header_len, (char *)in_buf, len); - - if (err != (int)actual_size) { - goto err_exit; - if (return_error) { - goto error_return; - } + if (LZ4_decompress_safe(reinterpret_cast(buf) + + header_len, + reinterpret_cast(tmp_buf), + actual_size, srv_page_size) + == int(srv_page_size)) { + break; } - break; + return 0; #endif /* HAVE_LZ4 */ #ifdef HAVE_LZO case PAGE_LZO_ALGORITHM: { - ulint olen = 0; - lzo_uint olen_lzo = olen; - err = lzo1x_decompress((const unsigned char *)buf+header_len, - actual_size,(unsigned char *)in_buf, &olen_lzo, NULL); - - olen = olen_lzo; - - if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - len = olen; - goto err_exit; - if (return_error) { - goto error_return; - } + lzo_uint len_lzo = srv_page_size; + if (LZO_E_OK == lzo1x_decompress_safe( + buf + header_len, + actual_size, tmp_buf, &len_lzo, NULL) + && len_lzo == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_LZO */ #ifdef HAVE_LZMA case PAGE_LZMA_ALGORITHM: { - - lzma_ret ret; size_t src_pos = 0; size_t dst_pos = 0; uint64_t memlimit = UINT64_MAX; - ret = lzma_stream_buffer_decode( - &memlimit, - 0, - NULL, - buf+header_len, - &src_pos, - actual_size, - in_buf, - &dst_pos, - len); - - - if (ret != LZMA_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - len = dst_pos; - goto err_exit; - if (return_error) { - goto error_return; - } + if (LZMA_OK == lzma_stream_buffer_decode( + &memlimit, 0, NULL, buf + header_len, + &src_pos, actual_size, tmp_buf, &dst_pos, + srv_page_size) + && dst_pos == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_LZMA */ #ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { - unsigned int dst_pos = UNIV_PAGE_SIZE; - - err = BZ2_bzBuffToBuffDecompress( - (char *)in_buf, - &dst_pos, - (char *)(buf+header_len), - actual_size, - 1, - 0); - - if (err != BZ_OK || (dst_pos == 0 || dst_pos > UNIV_PAGE_SIZE)) { - len = dst_pos; - goto err_exit; - if (return_error) { - goto error_return; - } + unsigned int dst_pos = srv_page_size; + if (BZ_OK == BZ2_bzBuffToBuffDecompress( + reinterpret_cast(tmp_buf), + &dst_pos, + reinterpret_cast(buf) + header_len, + actual_size, 1, 0) + && dst_pos == srv_page_size) { + break; } - break; + return 0; } #endif /* HAVE_BZIP2 */ #ifdef HAVE_SNAPPY - case PAGE_SNAPPY_ALGORITHM: - { - snappy_status cstatus; - ulint olen = UNIV_PAGE_SIZE; + case PAGE_SNAPPY_ALGORITHM: { + size_t olen = srv_page_size; - cstatus = snappy_uncompress( - (const char *)(buf+header_len), - (size_t)actual_size, - (char *)in_buf, - (size_t*)&olen); - - if (cstatus != SNAPPY_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - err = (int)cstatus; - len = olen; - goto err_exit; - if (return_error) { - goto error_return; - } + if (SNAPPY_OK == snappy_uncompress( + reinterpret_cast(buf) + header_len, + actual_size, + reinterpret_cast(tmp_buf), &olen) + && olen == srv_page_size) { + break; } - - break; + return 0; } #endif /* HAVE_SNAPPY */ - default: - goto err_exit; - if (return_error) { - goto error_return; - } - break; } srv_stats.pages_page_decompressed.inc(); - - /* Copy the uncompressed page to the buffer pool, not - really any other options. */ - memcpy(buf, in_buf, len); - -error_return: - if (page_buf != in_buf) { - ut_free(in_buf); - } - - return; - -err_exit: - /* Note that as we have found the page is corrupted, so - all this could be incorrect. */ - ulint space_id = mach_read_from_4(buf+FIL_PAGE_SPACE_ID); - fil_space_t* space = fil_space_acquire_for_io(space_id); - - ib::error() << "Corruption: Page is marked as compressed" - << " space: " << space_id << " name: " - << (space ? space->name : "NULL") - << " but uncompress failed with error: " << err - << " size: " << actual_size - << " len: " << len - << " compression method: " - << fil_get_compression_alg_name(compression_alg) << "."; - - buf_page_print(buf, univ_page_size); - fil_space_release_for_io(space); - ut_ad(0); + memcpy(buf, tmp_buf, srv_page_size); + return actual_size; } diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index f20354e7588..fe0b4556fa2 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -5040,6 +5040,11 @@ ibuf_check_bitmap_on_import( continue; } + if (!bitmap_page) { + mutex_exit(&ibuf_mutex); + return DB_CORRUPTION; + } + for (i = FSP_IBUF_BITMAP_OFFSET + 1; i < page_size.physical(); i++) { diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 1001f2ca807..a79b39235f3 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -41,6 +41,7 @@ Created 11/5/1995 Heikki Tuuri #include "os0proc.h" #include "log0log.h" #include "srv0srv.h" +#include "my_atomic.h" #include // Forward declaration @@ -1516,8 +1517,10 @@ NOTE! The definition appears here only for other modules of this directory (buf) to see it. Do not use from outside! */ typedef struct { - bool reserved; /*!< true if this slot is reserved +private: + int32 reserved; /*!< true if this slot is reserved */ +public: byte* crypt_buf; /*!< for encryption the data needs to be copied to a separate buffer before it's encrypted&written. this as a page can be @@ -1528,6 +1531,21 @@ typedef struct { byte* out_buf; /*!< resulting buffer after encryption/compression. This is a pointer and not allocated. */ + + /** Release the slot */ + void release() + { + my_atomic_store32_explicit(&reserved, false, + MY_MEMORY_ORDER_RELAXED); + } + + /** Acquire the slot + @return whether the slot was acquired */ + bool acquire() + { + return !my_atomic_fas32_explicit(&reserved, true, + MY_MEMORY_ORDER_RELAXED); + } } buf_tmp_buffer_t; /** The common buffer control block structure diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9797de6df0b..8aa8a746ce1 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -175,9 +175,6 @@ struct fil_space_t { /** MariaDB encryption data */ fil_space_crypt_t* crypt_data; - /** True if we have already printed compression failure */ - bool printed_compression_failure; - /** True if the device this filespace is on supports atomic writes */ bool atomic_write_supported; diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index be10f99d0f0..1046d720102 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018 MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -30,40 +30,24 @@ atomic writes information to table space. Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ -/****************************************************************//** -For page compressed pages compress the page before actual write -operation. -@return compressed page to be written*/ -UNIV_INTERN -byte* -fil_compress_page( -/*==============*/ - fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: compressed buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint level, /* in: compression level */ - ulint block_size, /*!< in: block size */ - bool encrypted, /*!< in: is page also encrypted */ - ulint* out_len); /*!< out: actual length of compressed - page */ +/** Compress a page_compressed page before writing to a data file. +@param[in] buf page to be compressed +@param[out] out_buf compressed page +@param[in] level compression level +@param[in] block_size file system block size +@param[in] encrypted whether the page will be subsequently encrypted +@return actual length of compressed page +@retval 0 if the page was not compressed */ +ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level, + ulint block_size, bool encrypted) + MY_ATTRIBUTE((nonnull, warn_unused_result)); -/****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -UNIV_INTERN -void -fil_decompress_page( -/*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< out: buffer from which to read; in aio - this must be appropriately aligned */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of - the compressed data. */ - bool return_error=false); - /*!< in: true if only an error should - be produced when decompression fails. - By default this parameter is false. */ +/** Decompress a page that may be subject to page_compressed compression. +@param[in,out] tmp_buf temporary buffer (of innodb_page_size) +@param[in,out] buf compressed page buffer +@return size of the compressed data +@retval 0 if decompression failed +@retval srv_page_size if the page was not compressed */ +ulint fil_page_decompress(byte* tmp_buf, byte* buf) + MY_ATTRIBUTE((nonnull, warn_unused_result)); #endif diff --git a/storage/innobase/include/fsp0pagecompress.ic b/storage/innobase/include/fsp0pagecompress.ic index d1f2ea45fbd..5285a7c238f 100644 --- a/storage/innobase/include/fsp0pagecompress.ic +++ b/storage/innobase/include/fsp0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -38,18 +38,6 @@ fsp_flags_get_page_compression_level( } -/*******************************************************************//** -Find out wheather the page is index page or not -@return true if page type index page, false if not */ -UNIV_INLINE -bool -fil_page_is_index_page( -/*===================*/ - byte* buf) /*!< in: page */ -{ - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_INDEX); -} - /*******************************************************************//** Find out wheather the page is page compressed @return true if page is page compressed, false if not */ @@ -73,59 +61,3 @@ fil_page_is_compressed_encrypted( { return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); } - -/****************************************************************//** -Get the name of the compression algorithm used for page -compression. -@return compression algorithm name or "UNKNOWN" if not known*/ -UNIV_INLINE -const char* -fil_get_compression_alg_name( -/*=========================*/ - ib_uint64_t comp_alg) /*! @@ -406,12 +412,9 @@ public: updated then its state must be set to BUF_PAGE_NOT_USED. For compressed tables the page descriptor memory will be at offset: block->frame + UNIV_PAGE_SIZE; - @param offset - physical offset within the file - @param block - block read from file, note it is not from the buffer pool + @param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ - virtual dberr_t operator()( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW = 0; + virtual dberr_t operator()(buf_block_t* block) UNIV_NOTHROW = 0; /** @return the space id of the tablespace */ @@ -635,12 +638,9 @@ struct FetchIndexRootPages : public AbstractCallback { } /** Called for each block as it is read from the file. - @param offset physical offset in the file @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; /** Update the import configuration that will be used to import the tablespace. */ @@ -657,13 +657,9 @@ struct FetchIndexRootPages : public AbstractCallback { determine the exact row format. We can't get that from the tablespace header flags alone. -@param offset physical offset in the file @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ -dberr_t -FetchIndexRootPages::operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW +dberr_t FetchIndexRootPages::operator()(buf_block_t* block) UNIV_NOTHROW { if (is_interrupted()) return DB_INTERRUPTED; @@ -671,15 +667,7 @@ FetchIndexRootPages::operator() ( ulint page_type = fil_page_get_type(page); - if (block->page.id.page_no() * m_page_size.physical() != offset) { - - ib::error() << "Page offset doesn't match file offset:" - " page offset: " << block->page.id.page_no() - << ", file offset: " - << (offset / m_page_size.physical()); - - return DB_CORRUPTION; - } else if (page_type == FIL_PAGE_TYPE_XDES) { + if (page_type == FIL_PAGE_TYPE_XDES) { return set_current_xdes(block->page.id.page_no(), page); } else if (fil_page_index_page_check(page) && !is_free(block->page.id.page_no()) @@ -825,12 +813,9 @@ public: } /** Called for each block as it is read from the file. - @param offset physical offset in the file @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ - virtual dberr_t operator() ( - os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW; + dberr_t operator()(buf_block_t* block) UNIV_NOTHROW; private: /** Update the page, set the space id, max trx id and index id. @param block block read from file @@ -1954,8 +1939,7 @@ PageConverter::update_page( updated then its state must be set to BUF_PAGE_NOT_USED. @param block block read from file, note it is not from the buffer pool @retval DB_SUCCESS or error code. */ -dberr_t -PageConverter::operator() (os_offset_t, buf_block_t* block) UNIV_NOTHROW +dberr_t PageConverter::operator()(buf_block_t* block) UNIV_NOTHROW { /* If we already had an old page with matching number in the buffer pool, evict it now, because @@ -3299,15 +3283,29 @@ fil_iterate( const ulint size = callback.get_page_size().physical(); ulint n_bytes = iter.n_io_buffers * size; + const ulint buf_size = srv_page_size +#ifdef HAVE_LZO + + LZO1X_1_15_MEM_COMPRESS +#elif defined HAVE_SNAPPY + + snappy_max_compressed_length(srv_page_size) +#endif + ; + byte* page_compress_buf = static_cast(malloc(buf_size)); ut_ad(!srv_read_only_mode); + if (!page_compress_buf) { + return DB_OUT_OF_MEMORY; + } + /* TODO: For ROW_FORMAT=COMPRESSED tables we do a lot of useless copying for non-index pages. Unfortunately, it is required by buf_zip_decompress() */ + dberr_t err = DB_SUCCESS; for (offset = iter.start; offset < iter.end; offset += n_bytes) { if (callback.is_interrupted()) { - return DB_INTERRUPTED; + err = DB_INTERRUPTED; + goto func_exit; } byte* io_buffer = iter.io_buffer; @@ -3337,11 +3335,12 @@ fil_iterate( IORequest read_request(IORequest::READ); read_request.disable_partial_io_warnings(); - dberr_t err = os_file_read_no_error_handling( + err = os_file_read_no_error_handling( read_request, iter.file, readptr, offset, n_bytes, 0); if (err != DB_SUCCESS) { ib::error() << iter.filepath << ": os_file_read() failed"; + goto func_exit; } bool updated = false; @@ -3352,18 +3351,9 @@ fil_iterate( for (ulint i = 0; i < n_pages_read; block->page.id.set_page_no(block->page.id.page_no() + 1), ++i, page_off += size, block->frame += size) { - bool decrypted = false; - err = DB_SUCCESS; byte* src = readptr + i * size; - byte* dst = io_buffer + i * size; - bool frame_changed = false; - ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); - const bool page_compressed - = page_type - == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_COMPRESSED; const ulint page_no = page_get_page_no(src); - if (!page_no && page_off) { + if (!page_no && block->page.id.page_no()) { const ulint* b = reinterpret_cast (src); const ulint* const e = b + size / sizeof *b; @@ -3378,54 +3368,84 @@ fil_iterate( continue; } - if (page_no != page_off / size) { + if (page_no != block->page.id.page_no()) { +page_corrupted: + ib::warn() << callback.filename() + << ": Page " << (offset / size) + << " at offset " << offset + << " looks corrupted."; + err = DB_CORRUPTION; + goto func_exit; + } + + const bool page_compressed + = fil_page_is_compressed_encrypted(src) + || fil_page_is_compressed(src); + + if (page_compressed && block->page.zip.data) { goto page_corrupted; } - if (encrypted) { + bool decrypted = false; + byte* dst = io_buffer + i * size; + bool frame_changed = false; + + if (!encrypted) { + } else if (!mach_read_from_4( + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + + src)) { +not_encrypted: + if (!page_compressed + && !block->page.zip.data) { + block->frame = src; + frame_changed = true; + } else { + ut_ad(dst != src); + memcpy(dst, src, size); + } + } else { + if (!fil_space_verify_crypt_checksum( + src, callback.get_page_size(), + block->page.id.space(), + block->page.id.page_no())) { + goto page_corrupted; + } + decrypted = fil_space_decrypt( iter.crypt_data, dst, callback.get_page_size(), src, &err); if (err != DB_SUCCESS) { - return err; + goto func_exit; } - if (decrypted) { - updated = true; - } else { - if (!page_compressed - && !block->page.zip.data) { - block->frame = src; - frame_changed = true; - } else { - ut_ad(dst != src); - memcpy(dst, src, size); - } + if (!decrypted) { + goto not_encrypted; } + + updated = true; } /* If the original page is page_compressed, we need to decompress it before adjusting further. */ if (page_compressed) { - fil_decompress_page(NULL, dst, ulong(size), - NULL); + ulint compress_length = fil_page_decompress( + page_compress_buf, dst); + ut_ad(compress_length != srv_page_size); + if (compress_length == 0) { + goto page_corrupted; + } updated = true; } else if (buf_page_is_corrupted( false, encrypted && !frame_changed ? dst : src, callback.get_page_size(), NULL)) { -page_corrupted: - ib::warn() << callback.filename() - << ": Page " << (offset / size) - << " at offset " << offset - << " looks corrupted."; - return DB_CORRUPTION; + goto page_corrupted; } - if ((err = callback(page_off, block)) != DB_SUCCESS) { - return err; + if ((err = callback(block)) != DB_SUCCESS) { + goto func_exit; } else if (!updated) { updated = buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE; @@ -3475,20 +3495,18 @@ page_corrupted: src = io_buffer + (i * size); if (page_compressed) { - ulint len = 0; - - fil_compress_page( - NULL, - src, - NULL, - size, - 0,/* FIXME: compression level */ - 512,/* FIXME: use proper block size */ - encrypted, - &len); - ut_ad(len <= size); - memset(src + len, 0, size - len); updated = true; + if (ulint len = fil_page_compress( + src, + page_compress_buf, + 0,/* FIXME: compression level */ + 512,/* FIXME: proper block size */ + encrypted)) { + /* FIXME: remove memcpy() */ + memcpy(src, page_compress_buf, len); + memset(src + len, 0, + srv_page_size - len); + } } /* Encrypt the page if encryption was used. */ @@ -3520,12 +3538,14 @@ page_corrupted: writeptr, offset, n_bytes); if (err != DB_SUCCESS) { - return err; + goto func_exit; } } } - return DB_SUCCESS; +func_exit: + free(page_compress_buf); + return err; } /********************************************************************//** From a79b033b359b882f2fe88051781c6e850bd71780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 14 Jun 2018 14:23:20 +0300 Subject: [PATCH 154/276] MDEV-16457 mariabackup 10.2+ should default to innodb_checksum_algorithm=crc32 Since MariaDB Server 10.2.2 (and MySQL 5.7), the default value of innodb_checksum_algorithm is crc32 (CRC-32C), not the inefficient "innodb" checksum. Change Mariabackup to use the same default, so that checksum validation (when using the default algorithm on the server) will take less time during mariabackup --backup. Also, mariabackup --prepare should be a little faster, and the server should read backups faster, because the page checksums would only be validated against CRC-32C. --- extra/mariabackup/xtrabackup.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 4d88778f020..bd5e28a4f5f 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1193,7 +1193,7 @@ struct my_option xb_server_options[] = "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, " "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm, &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM, - REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0}, + REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_CRC32, 0, 0, 0, 0, 0}, {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY, "Directory where undo tablespace files live, this path can be absolute.", From 776fc876862ccf6bc77938246bbab2b349d621a9 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 14 Jun 2018 09:08:41 +0200 Subject: [PATCH 155/276] fix compilation w/o partitioning followup for d8da920264a --- sql/sql_alter.cc | 1 - sql/sql_parse.cc | 1 - sql/sql_table.cc | 3 --- sql/table.cc | 2 ++ 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/sql/sql_alter.cc b/sql/sql_alter.cc index 6f21fb4b931..f17931ae919 100644 --- a/sql/sql_alter.cc +++ b/sql/sql_alter.cc @@ -303,7 +303,6 @@ bool Sql_cmd_alter_table::execute(THD *thd) create_info.data_file_name= create_info.index_file_name= NULL; thd->enable_slow_log= opt_log_slow_admin_statements; - thd->work_part_info= 0; #ifdef WITH_WSREP TABLE *find_temporary_table(THD *thd, const TABLE_LIST *tl); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index b2b2bfb5a27..89b5adff60c 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -3315,7 +3315,6 @@ mysql_execute_command(THD *thd) create_info.add(DDL_options_st::OPT_OR_REPLACE_SLAVE_GENERATED); } - thd->work_part_info= 0; #ifdef WITH_PARTITION_STORAGE_ENGINE { partition_info *part_info= thd->lex->part_info; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 9637702d836..3c9146354d6 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -6060,7 +6060,6 @@ remove_key: } } - DBUG_ASSERT(thd->work_part_info == 0); #ifdef WITH_PARTITION_STORAGE_ENGINE partition_info *tab_part_info= table->part_info; thd->work_part_info= thd->lex->part_info; @@ -8415,8 +8414,6 @@ bool mysql_alter_table(THD *thd,char *new_db, char *new_name, { DBUG_ENTER("mysql_alter_table"); - thd->work_part_info= 0; // Used by partitioning - /* Check if we attempt to alter mysql.slow_log or mysql.general_log table and return an error if diff --git a/sql/table.cc b/sql/table.cc index f6a27b1b1b3..4407d631db8 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -2233,7 +2233,9 @@ int TABLE_SHARE::init_from_sql_statement_string(THD *thd, bool write, goto ret; thd->lex->create_info.db_type= hton; +#ifdef WITH_PARTITION_STORAGE_ENGINE thd->work_part_info= 0; // For partitioning +#endif if (tabledef_version.str) thd->lex->create_info.tabledef_version= tabledef_version; From c22ab56f0d690feee471e173a3d95acb642cd6dc Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 14 Jun 2018 15:12:13 +0200 Subject: [PATCH 156/276] fix galera sst tests note that ${A#foo} is $A if there's no prefix foo. That's why galera nodes tried to connect to 127.0.0.1:127.0.0.1 if there was no port in the address Followup for 2b35db5ac4e --- scripts/wsrep_sst_common.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 50264fe560a..e160bf5c776 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -41,15 +41,14 @@ case "$1" in addr_no_bracket=${WSREP_SST_OPT_ADDR#\[} readonly WSREP_SST_OPT_HOST_UNESCAPED=${addr_no_bracket%%\]*} readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" - remain=${WSREP_SST_OPT_ADDR#*\]} - remain=${remain#*:} ;; *) readonly WSREP_SST_OPT_HOST=${WSREP_SST_OPT_ADDR%%[:/]*} readonly WSREP_SST_OPT_HOST_UNESCAPED=$WSREP_SST_OPT_HOST - remain=${WSREP_SST_OPT_ADDR#*:} ;; esac + remain=${WSREP_SST_OPT_ADDR#${WSREP_SST_OPT_HOST}} + remain=${remain#:} readonly WSREP_SST_OPT_ADDR_PORT=${remain%%/*} remain=${remain#*/} readonly WSREP_SST_OPT_MODULE=${remain%%/*} From ec4fdd574964d3611e904fa5f1751d88d8e8286f Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Wed, 13 Jun 2018 16:32:25 +0200 Subject: [PATCH 157/276] MDEV-16386: Wrong result when pushdown into the HAVING clause of the materialized derived table/view that uses aliases is done The problem appears when a column alias inside the materialized derived table/view t1 definition coincides with the column name used in the GROUP BY clause of t1. If the condition that can be pushed into t1 uses that ambiguous column name this column is determined as a column that is used in the GROUP BY clause instead of the alias used in the projection list of t1. That causes wrong result. To prevent it resolve_ref_in_select_and_group() was changed. --- mysql-test/r/derived_cond_pushdown.result | 196 ++++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 56 +++++++ sql/item.cc | 11 +- sql/sql_lex.cc | 1 + sql/sql_lex.h | 5 + sql/sql_select.cc | 2 + 6 files changed, 267 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 3fbc81019cc..83e70dd634c 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -9473,3 +9473,199 @@ WHERE (a>0 AND a<2 OR a IN (2,3)) AND a 2 DROP TABLE t1; +# +# MDEV-16386: pushing condition into the HAVING clause when ambiguous +# fields warning appears +# +CREATE TABLE t1 (a INT, b INT); +INSERT INTO t1 VALUES (1,2),(2,3),(3,4); +SELECT * FROM +( +SELECT t1.b AS a +FROM t1 +GROUP BY t1.a +) dt +WHERE (dt.a=2); +a +2 +EXPLAIN FORMAT=JSON SELECT * FROM +( +SELECT t1.b AS a +FROM t1 +GROUP BY t1.a +) dt +WHERE (dt.a=2); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "dt.a = 2", + "materialized": { + "query_block": { + "select_id": 2, + "having_condition": "a = 2", + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + } + } +} +SELECT * FROM +( +SELECT t1.b AS a +FROM t1 +GROUP BY t1.a +HAVING (t1.a<3) +) dt +WHERE (dt.a>1); +a +2 +3 +EXPLAIN FORMAT=JSON SELECT * FROM +( +SELECT t1.b AS a +FROM t1 +GROUP BY t1.a +HAVING (t1.a<3) +) dt +WHERE (dt.a>1); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "dt.a > 1", + "materialized": { + "query_block": { + "select_id": 2, + "having_condition": "t1.a < 3 and a > 1", + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + } + } +} +SELECT * FROM +( +SELECT 'ab' AS a +FROM t1 +GROUP BY t1.a +) dt +WHERE (dt.a='ab'); +a +ab +ab +ab +EXPLAIN FORMAT=JSON SELECT * FROM +( +SELECT 'ab' AS a +FROM t1 +GROUP BY t1.a +) dt +WHERE (dt.a='ab'); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "dt.a = 'ab'", + "materialized": { + "query_block": { + "select_id": 2, + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + } + } +} +SELECT * FROM +( +SELECT 1 AS a +FROM t1 +GROUP BY t1.a +) dt +WHERE (dt.a=1); +a +1 +1 +1 +EXPLAIN FORMAT=JSON SELECT * FROM +( +SELECT 1 AS a +FROM t1 +GROUP BY t1.a +) dt +WHERE (dt.a=1); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "dt.a = 1", + "materialized": { + "query_block": { + "select_id": 2, + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + } + } +} +DROP TABLE t1; diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index d3832ce1ec3..718140d3a77 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -1745,3 +1745,59 @@ WHERE (a>0 AND a<2 OR a IN (2,3)) AND (a=2 OR 0); DROP TABLE t1; + +--echo # +--echo # MDEV-16386: pushing condition into the HAVING clause when ambiguous +--echo # fields warning appears +--echo # + +CREATE TABLE t1 (a INT, b INT); + +INSERT INTO t1 VALUES (1,2),(2,3),(3,4); + +LET $query= +SELECT * FROM +( + SELECT t1.b AS a + FROM t1 + GROUP BY t1.a +) dt +WHERE (dt.a=2); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +LET $query= +SELECT * FROM +( + SELECT t1.b AS a + FROM t1 + GROUP BY t1.a + HAVING (t1.a<3) +) dt +WHERE (dt.a>1); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +LET $query= +SELECT * FROM +( + SELECT 'ab' AS a + FROM t1 + GROUP BY t1.a +) dt +WHERE (dt.a='ab'); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +LET $query= +SELECT * FROM +( + SELECT 1 AS a + FROM t1 + GROUP BY t1.a +) dt +WHERE (dt.a=1); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +DROP TABLE t1; diff --git a/sql/item.cc b/sql/item.cc index f9200ccf56d..4bc6fe7e5bc 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -4984,9 +4984,11 @@ static Item** find_field_in_group_list(Item *find_item, ORDER *group_list) in the SELECT clause of Q. - Search for a column named col_ref_i [in table T_j] in the GROUP BY clause of Q. - - If found different columns with the same name in GROUP BY and SELECT - - issue a warning and return the GROUP BY column, - - otherwise + - If found different columns with the same name in GROUP BY and SELECT: + - if the condition that uses this column name is pushed down into + the HAVING clause return the SELECT column + - else issue a warning and return the GROUP BY column. + - Otherwise - if the MODE_ONLY_FULL_GROUP_BY mode is enabled return error - else return the found SELECT column. @@ -5025,7 +5027,8 @@ resolve_ref_in_select_and_group(THD *thd, Item_ident *ref, SELECT_LEX *select) /* Check if the fields found in SELECT and GROUP BY are the same field. */ if (group_by_ref && (select_ref != not_found_item) && - !((*group_by_ref)->eq(*select_ref, 0))) + !((*group_by_ref)->eq(*select_ref, 0)) && + (!select->having_fix_field_for_pushed_cond)) { ambiguous_fields= TRUE; push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 750fb75bfb9..d3ddd9e208c 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -2107,6 +2107,7 @@ void st_select_lex::init_query() cond_pushed_into_where= cond_pushed_into_having= 0; olap= UNSPECIFIED_OLAP_TYPE; having_fix_field= 0; + having_fix_field_for_pushed_cond= 0; context.select_lex= this; context.init(); /* diff --git a/sql/sql_lex.h b/sql/sql_lex.h index ed5e423fd5a..a1f6b202ae6 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -887,6 +887,11 @@ public: bool braces; /* SELECT ... UNION (SELECT ... ) <- this braces */ /* TRUE when having fix field called in processing of this SELECT */ bool having_fix_field; + /* + TRUE when fix field is called for a new condition pushed into the + HAVING clause of this SELECT + */ + bool having_fix_field_for_pushed_cond; /* List of references to fields referenced from inner selects */ List inner_refs_list; /* Number of Item_sum-derived objects in this SELECT */ diff --git a/sql/sql_select.cc b/sql/sql_select.cc index ee5ba4ade54..4ebd5ef4954 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1350,9 +1350,11 @@ JOIN::optimize_inner() if (having) { select_lex->having_fix_field= 1; + select_lex->having_fix_field_for_pushed_cond= 1; if (having->fix_fields(thd, &having)) DBUG_RETURN(1); select_lex->having_fix_field= 0; + select_lex->having_fix_field_for_pushed_cond= 0; } } From f4387288ab54837aa84ce728b2ce61e52850e4ec Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Fri, 15 Jun 2018 03:49:04 +0300 Subject: [PATCH 158/276] Updated list of unstable tests for 10.1.34 release --- mysql-test/unstable-tests | 137 ++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 65 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 5c795bf79c8..9b00579944c 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,74 +23,80 @@ # ############################################################################## -# Based on 10.1 0db66ab18ffef6d8920e2e6ff66e99516a458a4d +# Based on 10.1 c22ab56f0d690feee471e173a3d95acb642cd6dc +main.alter_table : Modified in 10.1.34 main.alter_table_trans : MDEV-12084 - timeout main.analyze_stmt_slow_query_log : MDEV-12237 - Wrong result +main.assign_key_cache : Added in 10.1.34 +main.assign_key_cache_debug : Added in 10.1.34 main.auth_named_pipe : MDEV-14724 - System error 2 main.connect : Modified in 10.1.33 main.connect_debug : Added in 10.1.33 main.create_delayed : MDEV-10605 - failed with timeout -main.ctype_latin1 : Modified in 10.1.32 +main.create_or_replace : Modified in 10.1.34 main.ctype_ucs : Modified in 10.1.33 main.ctype_utf16le : MDEV-10675: timeout or extra warnings -main.ctype_utf8 : Modified in 10.1.32 main.ctype_utf8mb4 : Modified in 10.1.33 -main.dyncol : Modified in 10.1.32 +main.grant : Modified in 10.1.34 +main.grant_not_windows : Added in 10.1.34 main.events_2 : MDEV-13277 - Server crash main.events_bugs : MDEV-12892 - Crash in fill_schema_processlist main.events_slowlog : MDEV-12821 - Wrong result main.events_restart : MDEV-12236 - Server shutdown problem -main.fast_prefix_index_fetch_innodb : Modified in 10.1.32 -main.func_date_add : Modified in 10.1.32 main.func_misc : Modified in 10.1.33 main.func_str : Modified in 10.1.33 -main.func_time : Modified in 10.1.32 -main.having : Modified in 10.1.32 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout main.index_merge_innodb : MDEV-7142 - Wrong execution plan, timeout with valgrind main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure -main.join_outer : Modified in 10.1.32 +main.insert_select : Modified in 10.1.34 main.kill_processlist-6619 : MDEV-10793 - wrong result in processlist +main.limit : Modified in 10.1.34 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect"; modified in 10.1.33 main.merge : MDEV-10607 - sporadic "can't connect" -main.myisam : Modified in 10.1.33 +main.myisam : Modified in 10.1.34 main.myisam_recover : Modified in 10.1.33 +main.mysql : Modified in 10.1.34 main.mysql_client_test_nonblock : MDEV-15096 - exec failed +main.mysql_cp932 : Modified in 10.1.34 main.mysql_upgrade_noengine : MDEV-14355 - Plugin is busy -main.mysqlslap : MDEV-11801 - timeout +main.mysqldump : Modified in 10.1.34 +main.mysqlslap : MDEV-11801 - timeout; modified in 10.1.34 main.mysqltest : MDEV-9269 - fails on Alpha +main.olap : Modified in 10.1.34 main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan main.parser : Modified in 10.1.33 main.partition_debug_sync : MDEV-15669 - Deadlock found when trying to get lock main.partition_innodb_plugin : MDEV-12901 - Valgrind warnings -main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count -main.ps_qc_innodb : Added in 10.1.32 +main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count; modified in 10.1.34 main.query_cache : MDEV-12895 - Wrong result main.query_cache_debug : MDEV-15281 - Resize or similar command in progress main.range_vs_index_merge_innodb : MDEV-15283 - Server has gone away -main.read_only_innodb : Modified in 10.1.33 +main.read_only_innodb : Modified in 10.1.34 +main.rename : Modified in 10.1.34 +main.selectivity : Modified in 10.1.34 main.set_statement : MDEV-13183 - Wrong result main.show_explain : MDEV-10674 - sporadic failure -main.sp : Modified in 10.1.33 +main.sp : Modified in 10.1.34 main.sp-destruct : Modified in 10.1.33 main.sp-innodb : Modified in 10.1.33 main.sp-security : MDEV-10607 - sporadic "can't connect" +main.stat_tables_par_innodb : MDEV-14155 - Wrong rounding main.statistics : Modified in 10.1.33 +main.statistics_close : Added in 10.1.34 main.status : MDEV-8510 - sporadic wrong result main.subselect4 : Modified in 10.1.33 main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.subselect_mat : Modified in 10.1.32 main.subselect_sj : Modified in 10.1.33 +main.subselect_sj2_mat : Modified in 10.1.34 main.symlink-aria-11902 : MDEV-15098 - error 40 from storage engine main.symlink-myisam-11902 : MDEV-15098 - error 40 from storage engine main.tc_heuristic_recover : MDEV-15200 - wrong error on mysqld_stub_cmd +main.trigger : Modified in 10.1.34 main.type_blob : MDEV-15195 - Wrong result main.type_datetime_hires : MDEV-10687 - timeout -main.type_time : Modified in 10.1.32 -main.type_time_6065 : Modified in 10.1.32 main.variables : Modified in 10.1.33 main.view : Modified in 10.1.33 @@ -138,29 +144,32 @@ connect.zip : MDEV-13884 - Wrong result #---------------------------------------------------------------- -disks.disks : Added in 10.1.32 - -#---------------------------------------------------------------- - encryption.create_or_replace : MDEV-16115 - Trying to access tablespace encryption.debug_key_management : MDEV-13841 - Timeout on wait condition encryption.encrypt_and_grep : MDEV-13765 - Wrong result -encryption.innodb-bad-key-change : Modified in 10.1.32 -encryption.innodb-compressed-blob : MDEV-14728 - Unable to get certificate +encryption.innodb-bad-key-change : Modified in 10.1.34 +encryption.innodb-bad-key-change2 : Modified in 10.1.34 +encryption.innodb-bad-key-change4 : Modified in 10.1.34 +encryption.innodb-compressed-blob : MDEV-14728 - Unable to get certificate; modified in 10.1.34 +encryption.innodb-discard-import : Modified in 10.1.34 +encryption.innodb-encryption-disable : Modified in 10.1.34 encryption.innodb_encryption_discard_import : MDEV-16116 - Wrong result; modified in 10.1.33 encryption.innodb_encryption_filekeys : Modified in 10.1.33 -encryption.innodb_encryption-page-compression : MDEV-12630 - crash or assertion failure; modified in 10.1.32 +encryption.innodb_encryption-page-compression : MDEV-12630 - crash or assertion failure encryption.innodb_encryption_row_compressed : MDEV-16113 - Crash encryption.innodb_first_page : MDEV-10689 - Crash encryption.innodb-first-page-read : MDEV-14356 - Timeout on wait condition +encryption.innodb-force-corrupt : Modified in 10.1.34 encryption.innodb_lotoftables : MDEV-16111 - Wrong result +encryption.innodb-missing-key : Modified in 10.1.34 encryption.innodb-page_encryption : MDEV-10641 - mutex problem -encryption.innodb-redo-badkey : MDEV-12898 - Server hang on startup -encryption.innodb-remove-encryption : Added in 10.1.33 +encryption.innodb-read-only : MDEV-14728 - Unable to get certificate +encryption.innodb-redo-badkey : MDEV-12898 - Server hang on startup; modified in 10.1.34 +encryption.innodb-redo-nokeys : Modified in 10.1.34 +encryption.innodb-remove-encryption : MDEV-16493 - Timeout in wait condition; a dded in 10.1.33 encryption.innodb_scrub : MDEV-8139 - scrubbing tests need fixing encryption.innodb_scrub_background : MDEV-8139 - scrubbing tests need fixing encryption.innodb_scrub_compressed : MDEV-8139 - scrubbing tests need fixing -encryption.tempfiles : Modified in 10.1.32 #---------------------------------------------------------------- @@ -202,56 +211,60 @@ galera.MW-328A : MDEV-13876 - Wrong result #---------------------------------------------------------------- +handler.ps : Added in 10.1.34 + +#---------------------------------------------------------------- + innodb.alter_partitioned_xa : Added in 10.1.33 innodb.binlog_consistent : MDEV-10618 - Server fails to start -innodb.default_row_format_alter : Added in 10.1.32 -innodb.default_row_format_compatibility : Added in 10.1.32 -innodb.default_row_format_create : Added in 10.1.32 innodb.doublewrite : MDEV-12905 - Lost connection to MySQL server -innodb.file_format_defaults : Added in 10.1.32 -innodb_fts.fulltext2 : MDEV-14727 - Long semaphore wait innodb.innodb-64k-crash : MDEV-13872 - Failure and crash on startup +innodb.innodb-alter : Added in 10.1.34 innodb.innodb-alter-debug : MDEV-13182 - InnoDB: adjusting FSP_SPACE_FLAGS innodb.innodb-alter-nullable : Modified in 10.1.33 innodb.innodb-alter-table : MDEV-10619 - Testcase timeout -innodb.innodb-blob : MDEV-12053 - Client crash +innodb.innodb-blob : MDEV-12053 - Client crash; modified in 10.1.34 innodb.innodb_bug14147491 : MDEV-11808 - wrong error codes -innodb.innodb_bug27216817 : Added in 10.1.33 innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan innodb.innodb_bug48024 : MDEV-14352 - Assertion failure +innodb.innodb_bug54044 : Modified in 10.1.34 +innodb.innodb_defragment_small : Modified in 10.1.34 innodb.innodb-fk : MDEV-13832 - Assertion failure on shutdown innodb.innodb_max_recordsize_64k : MDEV-15203 - wrong result +innodb.innodb-mdev7046 : Modified in 10.1.34 +innodb.innodb-page_compression_default : Modified in 10.1.34 innodb.innodb-page_compression_lzma : MDEV-14353 - wrong result on Fedora 25 +innodb.innodb-page_compression_snappy : Modified in 10.1.34 innodb.innodb-page_compression_zip : MDEV-10641 - mutex problem innodb.innodb_stats : MDEV-10682 - wrong result innodb.innodb_sys_semaphore_waits : MDEV-10331 - wrong result innodb.innodb_zip_innochecksum2 : MDEV-13882 - Warning: difficult to find free blocks +innodb.lock_deleted : Added in 10.1.34 innodb.log_file_size : MDEV-15668 - Not found pattern -innodb.mvcc : Added in 10.1.32 -innodb.read_only_recover_committed : Added in 10.1.32 -innodb.recovery_shutdown : MDEV-15671 - Warning: database page corruption; modified in 10.1.32 -innodb.restart : Added in 10.1.32 +innodb.recovery_shutdown : MDEV-15671 - Warning: database page corruption +innodb.rename_table : Added in 10.1.34 innodb.row_format_redundant : MDEV-15192 - Trying to access missing tablespace innodb.table_definition_cache_debug : MDEV-14206 - Extra warning innodb.table_flags : MDEV-14363 - Operating system error number 2 -innodb_fts.fulltext_misc : MDEV-12636 - Valgrind warnings +#---------------------------------------------------------------- -innodb_zip.innodb-create-options : Modified in 10.1.32 -innodb_zip.innodb-zip : Modified in 10.1.32 +innodb_fts.basic : Added in 10.1.34 #---------------------------------------------------------------- -maria.dynamic : Added in 10.1.32 +maria.alter : Modified in 10.1.34 maria.insert_select : MDEV-12757 - Timeout -maria.maria : MDEV-14430 - Wrong result +maria.insert_select-7314 : MDEV-16492 - Timeout +maria.lock : Modified in 10.1.34 +maria.maria : MDEV-14430 - Wrong result; modified in 10.1.34 #---------------------------------------------------------------- mariabackup.absolute_ibdata_paths : Added in 10.1.33 +mariabackup.backup_ssl : Added in 10.1.34 mariabackup.incremental_encrypted : MDEV-15667 - Timeout mariabackup.mdev-14447 : MDEV-15201 - Timeout -mariabackup.undo_space_id : Added in 10.1.32 mariabackup.xb_compressed_encrypted : MDEV-14812 - Segfault #---------------------------------------------------------------- @@ -282,8 +295,8 @@ parts.partition_alter_innodb : Include file modified in 10.1.33 parts.partition_alter_maria : Include file modified in 10.1.33 parts.partition_alter_myisam : Include file modified in 10.1.33 parts.partition_alter2_2_maria : MDEV-14364 - Lost connection to MySQL server during query +parts.partition_auto_increment_archive : MDEV-16491 - Table marked as crashed parts.partition_auto_increment_maria : MDEV-14430 - Wrong result -parts.partition_basic_symlink_innodb : Modified in 10.1.32 parts.partition_debug_innodb : MDEV-15095 - table does not exist parts.partition_innodb_status_file : MDEV-12901 - Valgrind @@ -293,8 +306,8 @@ perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.hostcache_ipv4_max_con : Modified in 10.1.33 perfschema.hostcache_ipv6_max_con : Modified in 10.1.33 +perfschema.partition : Added in 10.1.34 perfschema.privilege_table_io : MDEV-13184 - Extra lines -perfschema.setup_actors : MDEV-10679 - rare crash perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match perfschema.stage_mdl_global : MDEV-11803 - wrong result on slow builders perfschema.stage_mdl_table : MDEV-12638 - Wrong result @@ -303,12 +316,14 @@ perfschema.threads_mysql : MDEV-10677 - sporadic wrong resul #---------------------------------------------------------------- plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url -plugins.server_audit : MDEV-9562 - crashes on sol10-sparc; modified in 10.1.32 +plugins.processlist : Added in 10.1.34 +plugins.server_audit : MDEV-9562 - crashes on sol10-sparc; modified in 10.1.34 plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc #---------------------------------------------------------------- rpl.last_insert_id : MDEV-10625 - warnings in error log +rpl.rename : Added in 10.1.34 rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log @@ -331,15 +346,15 @@ rpl.rpl_insert_id : MDEV-15197 - Wrong result rpl.rpl_insert_ignore : MDEV-14365 - Lost connection to MySQL server during query rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips rpl.rpl_mariadb_slave_capability : MDEV-11018 - sporadic wrong events in binlog -rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips +rpl.rpl_mdev6020 : MDEV-10417 - Fails on Mips +rpl.rpl_mixed_implicit_commit_binlog : Modified in 10.1.34 rpl.rpl_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_non_direct_mixed_mixing_engines : MDEV-14489 - Sync slave with master failed -rpl.rpl_non_direct_row_mixing_engines : MDEV-14491 - Long semaphore wait rpl.rpl_non_direct_stm_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_parallel : MDEV-10653 - Timeouts rpl.rpl_parallel_mdev6589 : MDEV-12979 - Assertion failure rpl.rpl_parallel_multilevel2 : MDEV-14723 - Timeout -rpl.rpl_parallel_optimistic : MDEV-10511 - Timeout +rpl.rpl_parallel_optimistic : MDEV-10511 - Timeout; modified in 10.1.34 rpl.rpl_parallel_retry : MDEV-11119 - Server crash rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips @@ -349,20 +364,20 @@ rpl.rpl_row_drop_create_temp_table : MDEV-14487 - Wrong result rpl.rpl_row_img_blobs : MDEV-13875 - command "diff_files" failed rpl.rpl_row_img_eng_min : MDEV-13875 - command "diff_files" failed rpl.rpl_row_img_eng_noblob : MDEV-13875 - command "diff_files" failed +rpl.rpl_row_implicit_commit_binlog : Modified in 10.1.34 rpl.rpl_row_index_choice : MDEV-15196 - Slave crash -rpl.rpl_row_mixing_engines : MDEV-14491 - Long semaphore wait rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_semi_sync : MDEV-11220 - Wrong result rpl.rpl_semi_sync_after_sync : MDEV-14366 - Wrong result rpl.rpl_semi_sync_after_sync_row : MDEV-14366 - Wrong result rpl.rpl_semi_sync_event_after_sync : MDEV-11806 - warnings -rpl.rpl_semi_sync_skip_repl : Added in 10.1.32 rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status rpl.rpl_semi_sync_wait_point : MDEV-11807 - timeout in wait condition rpl.rpl_show_slave_hosts : MDEV-10681 - server startup problem rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock rpl.rpl_start_stop_slave : MDEV-13567 - Replication failure +rpl.rpl_stm_implicit_commit_binlog : Modified in 10.1.34 rpl.rpl_stm_mixing_engines : MDEV-14489 - Sync slave with master failed rpl.rpl_stm_relay_ign_space : MDEV-14360 - Test assertion rpl.rpl_sync : MDEV-10633 - Database page corruption @@ -373,16 +388,11 @@ rpl.sec_behind_master-5114 : MDEV-13878 - Wrong result spider.* : MDEV-9329 - tests are too memory-consuming -spider.spider_fixes : MDEV-12900 - Valgrind -spider.spider_fixes_part : MDEV-12900 - Valgrind - -spider/bg.basic_sql : MDEV-12900 - Valgrind spider/bg.direct_aggregate : MDEV-7098 - Trying to unlock mutex that wasn't locked spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't locked -spider/bg.function : MDEV-12900 - Valgrind -spider/bg.ha : MDEV-7914 - Crash (only fixed in 10.3), MDEV-9329 - failures on s390x -spider/bg.ha_part : MDEV-7914 - Crash (only fixed in 10.3), MDEV-9329 - Fails on Ubuntu/s390x -spider/bg.spider_fixes : MDEV-7098 -Mutex problem, MDEV-9329 - failures on s390x, MDEV-12900 - Valgrind +spider/bg.ha : MDEV-9329 - failures on s390x +spider/bg.ha_part : MDEV-9329 - Fails on Ubuntu/s390x +spider/bg.spider_fixes : MDEV-7098 -Mutex problem, MDEV-9329 - failures on s390x spider/bg.spider3_fixes : MDEV-12639 - Packets out of order spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x @@ -404,7 +414,6 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x sys_vars.keep_files_on_create_basic : MDEV-10676 - timeout -sys_vars.innodb_default_row_format_basic : Added in 10.1.32 sys_vars.log_slow_admin_statements_func : MDEV-12235 - Server crash sys_vars.max_prepared_stmt_count_basic : Modified in 10.1.33 sys_vars.rpl_init_slave_func : MDEV-10149 - wrong results @@ -413,7 +422,6 @@ sys_vars.sysvars_server_embedded : Opt file added in 10.1.33 sys_vars.sysvars_server_notembedded : Opt file added in 10.1.33 sys_vars.thread_cache_size_func : MDEV-11775 - Wrong result sys_vars.wait_timeout_func : MDEV-12896 - Wrong result -sys_vars.wsrep_sst_receive_address_basic : Modified in 10.1.32 #---------------------------------------------------------------- @@ -440,8 +448,6 @@ tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output tokudb_bugs.db917 : Modified in 10.1.33 tokudb_bugs.xa : MDEV-11804 - Lock wait timeout -tokudb_mariadb.mdev6657 : Modified in 10.1.32 - #---------------------------------------------------------------- unit.lf : MDEV-12897 - Signal 11 thrown @@ -459,5 +465,6 @@ wsrep.binlog_format : MDEV-11532 - WSREP has not yet prepared node wsrep.foreign_key : MDEV-14725 - WSREP has not yet prepared node wsrep.mdev_6832 : MDEV-14195 - Failure upon check-testcase wsrep.pool_of_threads : MDEV-12234 - Library problem on Power +wsrep.variables : Modified in 10.1.34 wsrep_info.plugin : MDEV-12909 - Wrong result From f61909e19e58525583316f8edb55335f094af7dd Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 15 Jun 2018 06:33:34 +0400 Subject: [PATCH 159/276] MDEV-10182 Bad value when inserting COALESCE(CURRENT_TIMESTAMP) into a DECIMAL column This problem was earlier fixed by the patch cb16d753b2db936afff844cca0dd434fa7fe736b for MDEV-11337 Split Item::save_in_field() into virtual methods in Type_handler. Adding tests only. --- mysql-test/main/func_time_hires.result | 19 +++++++++++++++++++ mysql-test/main/func_time_hires.test | 21 +++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/mysql-test/main/func_time_hires.result b/mysql-test/main/func_time_hires.result index 0f822456724..a76571a67b5 100644 --- a/mysql-test/main/func_time_hires.result +++ b/mysql-test/main/func_time_hires.result @@ -206,3 +206,22 @@ time(f1) 21:00:00.000000 21:00:01.000000 drop table t1; +# +# Start of 10.3 tests +# +# +# MDEV-10182 Bad value when inserting COALESCE(CURRENT_TIMESTAMP) into a DECIMAL column +# +SET timestamp=UNIX_TIMESTAMP('2001-01-01 10:20:30.000000'); +CREATE TABLE t1 (a DECIMAL(30,0)); +INSERT INTO t1 VALUES (CURRENT_TIMESTAMP(6)); +INSERT INTO t1 VALUES (COALESCE(CURRENT_TIMESTAMP(6))); +SELECT * FROM t1; +a +20010101102030 +20010101102030 +DROP TABLE t1; +SET timestamp=DEFAULT; +# +# End of 10.3 tests +# diff --git a/mysql-test/main/func_time_hires.test b/mysql-test/main/func_time_hires.test index 4dcd51a85ba..35fd36b8456 100644 --- a/mysql-test/main/func_time_hires.test +++ b/mysql-test/main/func_time_hires.test @@ -106,3 +106,24 @@ alter table t1 modify f1 varchar(100); select time(f1) from t1; select time(f1) from t1 union all select time(f1 + interval 1 second) from t1; drop table t1; + + +--echo # +--echo # Start of 10.3 tests +--echo # + +--echo # +--echo # MDEV-10182 Bad value when inserting COALESCE(CURRENT_TIMESTAMP) into a DECIMAL column +--echo # + +SET timestamp=UNIX_TIMESTAMP('2001-01-01 10:20:30.000000'); +CREATE TABLE t1 (a DECIMAL(30,0)); +INSERT INTO t1 VALUES (CURRENT_TIMESTAMP(6)); +INSERT INTO t1 VALUES (COALESCE(CURRENT_TIMESTAMP(6))); +SELECT * FROM t1; +DROP TABLE t1; +SET timestamp=DEFAULT; + +--echo # +--echo # End of 10.3 tests +--echo # From c69357d8d4592b746a2dd7074c8fb6a437098b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 14 Jun 2018 15:47:39 +0300 Subject: [PATCH 160/276] MDEV-15611 Due to the failure of foreign key detection, Galera slave node killed himself. Merge following change from 10.2 revision-id: d52cff9f10aeea208a1058f7b5527e602125584c (mariadb-10.2.14-25-gd52cff9) parent(s): bc2501453c3ab9a2cf3516bc3557de8665bc2776 author: Sachin Setiya committer: Sachin Setiya timestamp: 2018-04-04 12:26:06 +0530 message: MDEV-15611 Due to the failure of foreign key detection, Galera... slave node killed himself. Problem:- If we try to delete table with foreign key and table whom it is referring with wsrep_slave_threads>1 then galera tries to execute both Delete_rows_log-event in parallel, which should not happen. Solution:- This is happening because we do not have foreign key info in write set. Upto version 10.2.7 it used to work fine. Actually it happening because of issue in commit 2f342c4. wsrep_must_process_fk should be used with negation. --- .../suite/galera/r/galera_mdev_15611.result | 15 ++ .../suite/galera/t/galera_mdev_15611.cnf | 5 + .../suite/galera/t/galera_mdev_15611.test | 30 ++++ storage/innobase/row/row0upd.cc | 141 ++++++++++-------- storage/xtradb/row/row0upd.cc | 141 ++++++++++-------- 5 files changed, 200 insertions(+), 132 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_mdev_15611.result create mode 100644 mysql-test/suite/galera/t/galera_mdev_15611.cnf create mode 100644 mysql-test/suite/galera/t/galera_mdev_15611.test diff --git a/mysql-test/suite/galera/r/galera_mdev_15611.result b/mysql-test/suite/galera/r/galera_mdev_15611.result new file mode 100644 index 00000000000..677ed98202d --- /dev/null +++ b/mysql-test/suite/galera/r/galera_mdev_15611.result @@ -0,0 +1,15 @@ +CREATE TABLE t1 ( +id int primary key +); +CREATE TABLE t2 ( +id int primary key , +f_id int DEFAULT NULL, FOREIGN KEY(f_id) REFERENCES t1 (id) +); +insert into t1 select 1; +#Running 200 insert in t2 table +select count(*) from t2; +count(*) +200 +delete from t2; +delete from t1; +drop table t2,t1; diff --git a/mysql-test/suite/galera/t/galera_mdev_15611.cnf b/mysql-test/suite/galera/t/galera_mdev_15611.cnf new file mode 100644 index 00000000000..b6f601c56b1 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_mdev_15611.cnf @@ -0,0 +1,5 @@ +!include ../galera_2nodes.cnf +[mysqld.1] + +[mysqld.2] +wsrep_slave_threads=6 diff --git a/mysql-test/suite/galera/t/galera_mdev_15611.test b/mysql-test/suite/galera/t/galera_mdev_15611.test new file mode 100644 index 00000000000..d32d7e75262 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_mdev_15611.test @@ -0,0 +1,30 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +--connection node_1 +CREATE TABLE t1 ( + id int primary key +); + +CREATE TABLE t2 ( + id int primary key , + f_id int DEFAULT NULL, FOREIGN KEY(f_id) REFERENCES t1 (id) +); + +insert into t1 select 1; + +--disable_query_log +--let $count=200 +--echo #Running 200 insert in t2 table +while($count) +{ + #Repeatedly execute the following SQL until you generate thousands of data + --eval insert into t2 values ($count, 1); + --dec $count +} +--enable_query_log + +select count(*) from t2; +delete from t2; +delete from t1; +drop table t2,t1; diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index e2de47bf86a..33f46882651 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1803,6 +1803,23 @@ row_upd_store_row( } } +#ifdef WITH_WSREP +/** Determine if a FOREIGN KEY constraint needs to be processed. +@param[in] node query node +@param[in] trx transaction +@return whether the node cannot be ignored */ + +inline bool wsrep_must_process_fk(const upd_node_t* node, const trx_t* trx) +{ + if (!wsrep_on_trx(trx)) { + return false; + } + return que_node_get_type(node->common.parent) != QUE_NODE_UPDATE + || static_cast(node->common.parent)->cascade_node + != node; +} +#endif /* WITH_WSREP */ + /***********************************************************//** Updates a secondary index entry of a row. @return DB_SUCCESS if operation successfully completed, else error @@ -1833,7 +1850,7 @@ row_upd_sec_index_entry( referenced = row_upd_index_is_referenced(index, trx); #ifdef WITH_WSREP - ibool foreign = wsrep_row_upd_index_is_foreign(index, trx); + bool foreign = wsrep_row_upd_index_is_foreign(index, trx); #endif /* WITH_WSREP */ heap = mem_heap_create(1024); @@ -1962,61 +1979,61 @@ row_upd_sec_index_entry( row_ins_sec_index_entry() below */ if (!rec_get_deleted_flag( rec, dict_table_is_comp(index->table))) { -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ err = btr_cur_del_mark_set_sec_rec( 0, btr_cur, TRUE, thr, &mtr); - if (err == DB_SUCCESS && referenced) { - - ulint* offsets; - - offsets = rec_get_offsets( - rec, index, NULL, ULINT_UNDEFINED, - &heap); - - /* NOTE that the following call loses - the position of pcur ! */ - err = row_upd_check_references_constraints( - node, &pcur, index->table, - index, offsets, thr, &mtr); + if (err != DB_SUCCESS) { + break; } + #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && - wsrep_on_trx(trx) && - !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && - !(parent && que_node_get_type(parent) == - QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { - ulint* offsets = - rec_get_offsets( + if (!referenced && foreign + && wsrep_must_process_fk(node, trx) + && !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + ulint* offsets = rec_get_offsets( rec, index, NULL, ULINT_UNDEFINED, &heap); + err = wsrep_row_upd_check_foreign_constraints( node, &pcur, index->table, index, offsets, thr, &mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: sec index FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } } #endif /* WITH_WSREP */ } - break; + + if (referenced) { + + ulint* offsets; + + offsets = rec_get_offsets( + rec, index, NULL, ULINT_UNDEFINED, + &heap); + + /* NOTE that the following call loses + the position of pcur ! */ + err = row_upd_check_references_constraints( + node, &pcur, index->table, + index, offsets, thr, &mtr); + } } btr_pcur_close(&pcur); @@ -2185,9 +2202,6 @@ row_upd_clust_rec_by_insert( rec_t* rec; ulint* offsets = NULL; -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ ut_ad(node); ut_ad(dict_index_is_clust(index)); @@ -2269,35 +2283,31 @@ err_exit: if (err != DB_SUCCESS) { goto err_exit; } - } #ifdef WITH_WSREP - if (!referenced && wsrep_on_trx(trx) && - !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { + } else if ((foreign && wsrep_must_process_fk(node, trx))) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: insert FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } - if (err != DB_SUCCESS) { - goto err_exit; - } - } #endif /* WITH_WSREP */ + } } mtr_commit(mtr); @@ -2500,7 +2510,7 @@ row_upd_del_mark_clust_rec( dberr_t err; #ifdef WITH_WSREP rec_t* rec; - que_node_t *parent = que_node_get_parent(node); + trx_t* trx = thr_get_trx(thr) ; #endif /* WITH_WSREP */ ut_ad(node); @@ -2529,38 +2539,37 @@ row_upd_del_mark_clust_rec( btr_cur_get_block(btr_cur), btr_cur_get_rec(btr_cur), #endif /* WITH_WSREP */ index, offsets, thr, mtr); - if (err == DB_SUCCESS && referenced) { + if (err != DB_SUCCESS) { + } else if (referenced) { /* NOTE that the following call loses the position of pcur ! */ err = row_upd_check_references_constraints( node, pcur, index->table, index, offsets, thr, mtr); - } #ifdef WITH_WSREP - trx_t* trx = thr_get_trx(thr) ; - if (err == DB_SUCCESS && !referenced && wsrep_on_trx(trx) && - !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { + } else if (foreign && wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: clust rec FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: clust rec referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } - } #endif /* WITH_WSREP */ + } mtr_commit(mtr); diff --git a/storage/xtradb/row/row0upd.cc b/storage/xtradb/row/row0upd.cc index 9ac72f8d068..93ccc07c9af 100644 --- a/storage/xtradb/row/row0upd.cc +++ b/storage/xtradb/row/row0upd.cc @@ -1806,6 +1806,23 @@ row_upd_store_row( } } +#ifdef WITH_WSREP +/** Determine if a FOREIGN KEY constraint needs to be processed. +@param[in] node query node +@param[in] trx transaction +@return whether the node cannot be ignored */ + +inline bool wsrep_must_process_fk(const upd_node_t* node, const trx_t* trx) +{ + if (!wsrep_on_trx(trx)) { + return false; + } + return que_node_get_type(node->common.parent) != QUE_NODE_UPDATE + || static_cast(node->common.parent)->cascade_node + != node; +} +#endif /* WITH_WSREP */ + /***********************************************************//** Updates a secondary index entry of a row. @return DB_SUCCESS if operation successfully completed, else error @@ -1836,7 +1853,7 @@ row_upd_sec_index_entry( referenced = row_upd_index_is_referenced(index, trx); #ifdef WITH_WSREP - ibool foreign = wsrep_row_upd_index_is_foreign(index, trx); + bool foreign = wsrep_row_upd_index_is_foreign(index, trx); #endif /* WITH_WSREP */ heap = mem_heap_create(1024); @@ -1968,61 +1985,61 @@ row_upd_sec_index_entry( row_ins_sec_index_entry() below */ if (!rec_get_deleted_flag( rec, dict_table_is_comp(index->table))) { -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ err = btr_cur_del_mark_set_sec_rec( 0, btr_cur, TRUE, thr, &mtr); - if (err == DB_SUCCESS && referenced) { - - ulint* offsets; - - offsets = rec_get_offsets( - rec, index, NULL, ULINT_UNDEFINED, - &heap); - - /* NOTE that the following call loses - the position of pcur ! */ - err = row_upd_check_references_constraints( - node, &pcur, index->table, - index, offsets, thr, &mtr); + if (err != DB_SUCCESS) { + break; } + #ifdef WITH_WSREP - if (err == DB_SUCCESS && !referenced && - wsrep_on_trx(trx) && - !wsrep_thd_is_BF(trx->mysql_thd, FALSE) && - !(parent && que_node_get_type(parent) == - QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { - ulint* offsets = - rec_get_offsets( + if (!referenced && foreign + && wsrep_must_process_fk(node, trx) + && !wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + ulint* offsets = rec_get_offsets( rec, index, NULL, ULINT_UNDEFINED, &heap); + err = wsrep_row_upd_check_foreign_constraints( node, &pcur, index->table, index, offsets, thr, &mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: sec index FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } } #endif /* WITH_WSREP */ } - break; + + if (referenced) { + + ulint* offsets; + + offsets = rec_get_offsets( + rec, index, NULL, ULINT_UNDEFINED, + &heap); + + /* NOTE that the following call loses + the position of pcur ! */ + err = row_upd_check_references_constraints( + node, &pcur, index->table, + index, offsets, thr, &mtr); + } } btr_pcur_close(&pcur); @@ -2191,9 +2208,6 @@ row_upd_clust_rec_by_insert( rec_t* rec; ulint* offsets = NULL; -#ifdef WITH_WSREP - que_node_t *parent = que_node_get_parent(node); -#endif /* WITH_WSREP */ ut_ad(node); ut_ad(dict_index_is_clust(index)); @@ -2278,35 +2292,31 @@ err_exit: if (err != DB_SUCCESS) { goto err_exit; } - } #ifdef WITH_WSREP - if (!referenced && wsrep_on_trx(trx) && - !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { + } else if ((foreign && wsrep_must_process_fk(node, trx))) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: insert FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } - if (err != DB_SUCCESS) { - goto err_exit; - } - } #endif /* WITH_WSREP */ + } } mtr_commit(mtr); @@ -2512,7 +2522,7 @@ row_upd_del_mark_clust_rec( dberr_t err; #ifdef WITH_WSREP rec_t* rec; - que_node_t *parent = que_node_get_parent(node); + trx_t* trx = thr_get_trx(thr) ; #endif /* WITH_WSREP */ ut_ad(node); @@ -2541,38 +2551,37 @@ row_upd_del_mark_clust_rec( btr_cur_get_block(btr_cur), btr_cur_get_rec(btr_cur), #endif /* WITH_WSREP */ index, offsets, thr, mtr); - if (err == DB_SUCCESS && referenced) { + if (err != DB_SUCCESS) { + } else if (referenced) { /* NOTE that the following call loses the position of pcur ! */ err = row_upd_check_references_constraints( node, pcur, index->table, index, offsets, thr, mtr); - } #ifdef WITH_WSREP - trx_t* trx = thr_get_trx(thr) ; - if (err == DB_SUCCESS && !referenced && wsrep_on_trx(trx) && - !(parent && que_node_get_type(parent) == QUE_NODE_UPDATE && - ((upd_node_t*)parent)->cascade_node == node) && - foreign - ) { + } else if (foreign && wsrep_must_process_fk(node, trx)) { err = wsrep_row_upd_check_foreign_constraints( node, pcur, index->table, index, offsets, thr, mtr); + switch (err) { case DB_SUCCESS: case DB_NO_REFERENCED_ROW: err = DB_SUCCESS; break; case DB_DEADLOCK: - if (wsrep_debug) fprintf (stderr, - "WSREP: clust rec FK check fail for deadlock"); + if (wsrep_debug) { + ib_logf(IB_LOG_LEVEL_WARN, + "WSREP: sec index FK check fail for deadlock: " + " index %s table %s", index->name, index->table->name); + } break; default: - fprintf (stderr, - "WSREP: clust rec referenced FK check fail: %d", - (int)err); + ib_logf(IB_LOG_LEVEL_ERROR, + "WSREP: referenced FK check fail: %s index %s table %s", + ut_strerr(err), index->name, index->table->name); break; } - } #endif /* WITH_WSREP */ + } mtr_commit(mtr); From c55de8d40bba29503773a6a56d6f13f19ca7e339 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 15 Jun 2018 10:11:51 +0400 Subject: [PATCH 161/276] MDEV-9334 ALTER from DECIMAL to BIGINT UNSIGNED returns a wrong result When altering from DECIMAL to *INT UNIGNED or to BIT, go through val_decimal(), to avoid truncation to the biggest possible signed integer (0x7FFFFFFFFFFFFFFF / 9223372036854775807). --- mysql-test/r/type_bit.result | 19 +++++++++++++++++++ mysql-test/r/type_int.result | 13 +++++++++++++ mysql-test/t/type_bit.test | 25 +++++++++++++++++++++++++ mysql-test/t/type_int.test | 12 ++++++++++++ sql/field.h | 4 ++++ 5 files changed, 73 insertions(+) diff --git a/mysql-test/r/type_bit.result b/mysql-test/r/type_bit.result index 30cd94c9277..eeedc501dc4 100644 --- a/mysql-test/r/type_bit.result +++ b/mysql-test/r/type_bit.result @@ -830,3 +830,22 @@ def COALESCE(val, 1) 246 2 1 Y 32896 0 63 COALESCE(val, 1) 0 DROP TABLE t1; +# +# End of 10.1 tests +# +# +# Start of 10.2 tests +# +# +# MDEV-9334 ALTER from DECIMAL to BIGINT UNSIGNED returns a wrong result +# +CREATE TABLE t1 (a DECIMAL(30,0)); +INSERT INTO t1 VALUES (CAST(0xFFFFFFFFFFFFFFFF AS UNSIGNED)); +ALTER TABLE t1 MODIFY a BIT(64); +SELECT a+0 FROM t1; +a+0 +18446744073709551615 +DROP TABLE IF EXISTS t1; +# +# End of 10.2 tests +# diff --git a/mysql-test/r/type_int.result b/mysql-test/r/type_int.result index 39e2e91ecc7..77e6ee14c25 100644 --- a/mysql-test/r/type_int.result +++ b/mysql-test/r/type_int.result @@ -91,5 +91,18 @@ a 10 DROP TABLE t1; # +# MDEV-9334 ALTER from DECIMAL to BIGINT UNSIGNED returns a wrong result +# +CREATE TABLE t1 (a DECIMAL(30,0)); +INSERT INTO t1 VALUES (CAST(0xFFFFFFFFFFFFFFFF AS UNSIGNED)); +SELECT * FROM t1; +a +18446744073709551615 +ALTER TABLE t1 MODIFY a BIGINT UNSIGNED; +SELECT * FROM t1; +a +18446744073709551615 +DROP TABLE t1; +# # End of 10.2 tests # diff --git a/mysql-test/t/type_bit.test b/mysql-test/t/type_bit.test index bb282fc15e5..04db1511833 100644 --- a/mysql-test/t/type_bit.test +++ b/mysql-test/t/type_bit.test @@ -458,3 +458,28 @@ DROP TABLE t2; SELECT COALESCE(val, 1) FROM t1; --disable_metadata DROP TABLE t1; + +--echo # +--echo # End of 10.1 tests +--echo # + + +--echo # +--echo # Start of 10.2 tests +--echo # + +--echo # +--echo # MDEV-9334 ALTER from DECIMAL to BIGINT UNSIGNED returns a wrong result +--echo # + +CREATE TABLE t1 (a DECIMAL(30,0)); +INSERT INTO t1 VALUES (CAST(0xFFFFFFFFFFFFFFFF AS UNSIGNED)); +ALTER TABLE t1 MODIFY a BIT(64); +SELECT a+0 FROM t1; +DROP TABLE IF EXISTS t1; + + +--echo # +--echo # End of 10.2 tests +--echo # + diff --git a/mysql-test/t/type_int.test b/mysql-test/t/type_int.test index 271b4d5862a..76ed7eee824 100644 --- a/mysql-test/t/type_int.test +++ b/mysql-test/t/type_int.test @@ -73,6 +73,18 @@ ALTER TABLE t1 MODIFY a INT; SELECT * FROM t1; DROP TABLE t1; +--echo # +--echo # MDEV-9334 ALTER from DECIMAL to BIGINT UNSIGNED returns a wrong result +--echo # + +CREATE TABLE t1 (a DECIMAL(30,0)); +INSERT INTO t1 VALUES (CAST(0xFFFFFFFFFFFFFFFF AS UNSIGNED)); +SELECT * FROM t1; +ALTER TABLE t1 MODIFY a BIGINT UNSIGNED; +SELECT * FROM t1; +DROP TABLE t1; + + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/field.h b/sql/field.h index ca85a7c276f..e554f92031c 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1636,6 +1636,8 @@ public: bool eq_def(const Field *field) const; Copy_func *get_copy_func(const Field *from) const { + if (unsigned_flag && from->cmp_type() == DECIMAL_RESULT) + return do_field_decimal; return do_field_int; } int save_in_field(Field *to) @@ -3661,6 +3663,8 @@ public: } Copy_func *get_copy_func(const Field *from) const { + if (from->cmp_type() == DECIMAL_RESULT) + return do_field_decimal; return do_field_int; } int save_in_field(Field *to) { return to->store(val_int(), true); } From 93ab0effd37c3a96ab41cba9afc899d966048f42 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 15 Jun 2018 10:14:18 +0200 Subject: [PATCH 162/276] MDEV-16187 Ubuntu Bionic MariaDB has epoch version that makes 10.1 and 10.2 installs fail backport cb21e117bab from 10.2 --- debian/autobake-deb.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 9de37138ade..648c95cb416 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -57,9 +57,12 @@ sed -i -e "s/\\\${LIBSSL}/${LIBSSL}/g" debian/control # echo "Incrementing changelog and starting build scripts" -dch -b -D ${CODENAME} -v "${UPSTREAM}${PATCHLEVEL}-${RELEASE_NAME}${RELEASE_EXTRA:+-${RELEASE_EXTRA}}1~${CODENAME}" "Automatic build with ${LOGSTRING}." +if [[ "$CODENAME" == bionic ]]; then + EPOCH="1:" +fi +dch -b -D ${CODENAME} -v "${EPOCH}${UPSTREAM}${PATCHLEVEL}-${RELEASE_NAME}${RELEASE_EXTRA:+-${RELEASE_EXTRA}}1~${CODENAME}" "Automatic build with ${LOGSTRING}." -echo "Creating package version ${UPSTREAM}${PATCHLEVEL}-${RELEASE_NAME}${RELEASE_EXTRA:+-${RELEASE_EXTRA}}1~${CODENAME} ... " +echo "Creating package version ${EPOCH}${UPSTREAM}${PATCHLEVEL}-${RELEASE_NAME}${RELEASE_EXTRA:+-${RELEASE_EXTRA}}1~${CODENAME} ... " # Build the package. # Pass -I so that .git and other unnecessary temporary and source control files From 6b8802e8dd5467556a024d807a1df23940b00895 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Tue, 8 May 2018 15:26:26 +0200 Subject: [PATCH 163/276] MDEV-11071: Assertion `thd->transaction.stmt.is_empty()' failed in Locked_tables_list::unlock_locked_table fix_length_and_dec now return result (error/OK) --- mysql-test/r/alter_table.result | 11 +++ mysql-test/t/alter_table.test | 16 ++++ sql/item.h | 2 +- sql/item_cmpfunc.cc | 106 ++++++++++++--------- sql/item_cmpfunc.h | 60 +++++++----- sql/item_func.cc | 106 ++++++++++++--------- sql/item_func.h | 118 ++++++++++++----------- sql/item_geofunc.cc | 9 +- sql/item_geofunc.h | 72 ++++++++------ sql/item_inetfunc.h | 12 ++- sql/item_jsonfunc.cc | 57 +++++++---- sql/item_jsonfunc.h | 38 ++++---- sql/item_strfunc.cc | 164 ++++++++++++++++++++------------ sql/item_strfunc.h | 156 ++++++++++++++++++------------ sql/item_subselect.cc | 71 +++++++++----- sql/item_subselect.h | 22 ++--- sql/item_sum.cc | 28 +++--- sql/item_sum.h | 30 +++--- sql/item_timefunc.cc | 46 +++++---- sql/item_timefunc.h | 125 +++++++++++++----------- sql/item_windowfunc.cc | 6 +- sql/item_windowfunc.h | 9 +- sql/item_xmlfunc.cc | 13 +-- sql/item_xmlfunc.h | 2 +- sql/sql_select.cc | 3 +- sql/sql_table.cc | 14 ++- 26 files changed, 778 insertions(+), 518 deletions(-) diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index 89bf9ed638e..2bea055df4c 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -2369,5 +2369,16 @@ t1 CREATE TABLE `t1` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 drop table t1; # +# MDEV-11071: Assertion `thd->transaction.stmt.is_empty()' failed +# in Locked_tables_list::unlock_locked_tables +# +CREATE TABLE t1 (d DATETIME DEFAULT CURRENT_TIMESTAMP, i INT) ENGINE=InnoDB; +INSERT INTO t1 (i) VALUES (1),(1); +LOCK TABLE t1 WRITE; +ALTER TABLE t1 ADD UNIQUE(i); +ERROR 23000: Duplicate entry '1' for key 'i' +UNLOCK TABLES; +DROP TABLE t1; +# # End of 10.2 tests # diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index eddc9f704da..092569dcadb 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1931,6 +1931,22 @@ alter table t1 change b new_b int not null, add column b char(1), add constraint show create table t1; drop table t1; +--echo # +--echo # MDEV-11071: Assertion `thd->transaction.stmt.is_empty()' failed +--echo # in Locked_tables_list::unlock_locked_tables +--echo # + +CREATE TABLE t1 (d DATETIME DEFAULT CURRENT_TIMESTAMP, i INT) ENGINE=InnoDB; +INSERT INTO t1 (i) VALUES (1),(1); +LOCK TABLE t1 WRITE; +--error ER_DUP_ENTRY +ALTER TABLE t1 ADD UNIQUE(i); + +# Cleanup +UNLOCK TABLES; +DROP TABLE t1; + + --echo # --echo # End of 10.2 tests --echo # diff --git a/sql/item.h b/sql/item.h index 423ee4cc089..5ed45c69e96 100644 --- a/sql/item.h +++ b/sql/item.h @@ -4211,7 +4211,7 @@ public: also to make printing of items inherited from Item_sum uniform. */ virtual const char *func_name() const= 0; - virtual void fix_length_and_dec()= 0; + virtual bool fix_length_and_dec()= 0; bool const_item() const { return const_item_cache; } table_map used_tables() const { return used_tables_cache; } Item* build_clone(THD *thd, MEM_ROOT *mem_root); diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index f552a3eca58..b86c0079bce 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -526,7 +526,7 @@ bool Item_func::setup_args_and_comparator(THD *thd, Arg_comparator *cmp) } -void Item_bool_rowready_func2::fix_length_and_dec() +bool Item_bool_rowready_func2::fix_length_and_dec() { max_length= 1; // Function returns 0 or 1 @@ -535,8 +535,8 @@ void Item_bool_rowready_func2::fix_length_and_dec() we have to check for out of memory conditions here */ if (!args[0] || !args[1]) - return; - setup_args_and_comparator(current_thd, &cmp); + return FALSE; + return setup_args_and_comparator(current_thd, &cmp); } @@ -1156,12 +1156,13 @@ int Arg_comparator::compare_e_str_json() } -void Item_func_truth::fix_length_and_dec() +bool Item_func_truth::fix_length_and_dec() { maybe_null= 0; null_value= 0; decimals= 0; max_length= 1; + return FALSE; } @@ -1780,10 +1781,11 @@ longlong Item_func_eq::val_int() /** Same as Item_func_eq, but NULL = NULL. */ -void Item_func_equal::fix_length_and_dec() +bool Item_func_equal::fix_length_and_dec() { - Item_bool_rowready_func2::fix_length_and_dec(); + bool rc= Item_bool_rowready_func2::fix_length_and_dec(); maybe_null=null_value=0; + return rc; } longlong Item_func_equal::val_int() @@ -1880,7 +1882,7 @@ bool Item_func_interval::fix_fields(THD *thd, Item **ref) } -void Item_func_interval::fix_length_and_dec() +bool Item_func_interval::fix_length_and_dec() { uint rows= row->cols(); @@ -1898,10 +1900,13 @@ void Item_func_interval::fix_length_and_dec() not_null_consts&= el->const_item() && !el->is_null(); } - if (not_null_consts && - (intervals= (interval_range*) current_thd->alloc(sizeof(interval_range) * - (rows - 1)))) + if (not_null_consts) { + intervals= (interval_range*) current_thd->alloc(sizeof(interval_range) * + (rows - 1)); + if (!intervals) + return TRUE; + if (use_decimal_comparison) { for (uint i= 1; i < rows; i++) @@ -1942,6 +1947,7 @@ void Item_func_interval::fix_length_and_dec() with_sum_func= with_sum_func || row->with_sum_func; with_param= with_param || row->with_param; with_field= with_field || row->with_field; + return FALSE; } @@ -2102,7 +2108,7 @@ void Item_func_between::fix_after_pullout(st_select_lex *new_parent, eval_not_null_tables(NULL); } -void Item_func_between::fix_length_and_dec() +bool Item_func_between::fix_length_and_dec() { THD *thd= current_thd; max_length= 1; @@ -2113,13 +2119,13 @@ void Item_func_between::fix_length_and_dec() we have to check for out of memory conditions here */ if (!args[0] || !args[1] || !args[2]) - return; + return TRUE; if (agg_cmp_type(&m_compare_type, args, 3)) - return; + return TRUE; if (m_compare_type == STRING_RESULT && agg_arg_charsets_for_comparison(cmp_collation, args, 3)) - return; + return TRUE; /* When comparing as date/time, we need to convert non-temporal values @@ -2144,6 +2150,7 @@ void Item_func_between::fix_length_and_dec() m_compare_type= INT_RESULT; // Works for all types. } } + return FALSE; } @@ -2441,7 +2448,7 @@ void Item_func_if::cache_type_info(Item *source) } -void +bool Item_func_if::fix_length_and_dec() { // Let IF(cond, expr, NULL) and IF(cond, NULL, expr) inherit type from expr. @@ -2452,15 +2459,15 @@ Item_func_if::fix_length_and_dec() // If both arguments are NULL, make resulting type BINARY(0). if (args[2]->type() == NULL_ITEM) set_handler_by_field_type(MYSQL_TYPE_STRING); - return; + return FALSE; } if (args[2]->type() == NULL_ITEM) { cache_type_info(args[1]); maybe_null= true; - return; + return FALSE; } - Item_func_case_abbreviation2::fix_length_and_dec2(args + 1); + return Item_func_case_abbreviation2::fix_length_and_dec2(args + 1); } @@ -2574,7 +2581,7 @@ void Item_func_nullif::update_used_tables() -void +bool Item_func_nullif::fix_length_and_dec() { /* @@ -2724,6 +2731,8 @@ Item_func_nullif::fix_length_and_dec() m_cache= args[0]->cmp_type() == STRING_RESULT ? new (thd->mem_root) Item_cache_str_for_nullif(thd, args[0]) : Item_cache::get_cache(thd, args[0]); + if (!m_cache) + return TRUE; m_cache->setup(thd, args[0]); m_cache->store(args[0]); m_cache->set_used_tables(args[0]->used_tables()); @@ -2737,7 +2746,8 @@ Item_func_nullif::fix_length_and_dec() fix_char_length(args[2]->max_char_length()); maybe_null=1; m_arg0= args[0]; - setup_args_and_comparator(thd, &cmp); + if (setup_args_and_comparator(thd, &cmp)) + return TRUE; /* A special code for EXECUTE..PREPARE. @@ -2777,6 +2787,7 @@ Item_func_nullif::fix_length_and_dec() */ if (args[0] == m_arg0) m_arg0= NULL; + return FALSE; } @@ -3158,7 +3169,7 @@ static void change_item_tree_if_needed(THD *thd, Item **place, Item *new_value) } -void Item_func_case::fix_length_and_dec() +bool Item_func_case::fix_length_and_dec() { m_found_types= 0; if (else_expr_num == -1 || args[else_expr_num]->maybe_null) @@ -3175,7 +3186,7 @@ void Item_func_case::fix_length_and_dec() if (Item_func_case::result_type() == STRING_RESULT) { if (count_string_result_length(Item_func_case::field_type(), rets, nrets)) - return; + return TRUE; } else fix_attributes(rets, nrets); @@ -3189,7 +3200,7 @@ void Item_func_case::fix_length_and_dec() left_cmp_type= args[0]->cmp_type(); if (!(m_found_types= collect_cmp_types(args, nwhens + 1))) - return; + return TRUE; Item *date_arg= 0; if (m_found_types & (1U << TIME_RESULT)) @@ -3222,7 +3233,7 @@ void Item_func_case::fix_length_and_dec() CASE utf16_item WHEN CONVERT(latin1_item USING utf16) THEN ... END */ if (agg_arg_charsets_for_comparison(cmp_collation, args, nwhens + 1)) - return; + return TRUE; } for (uint i= 0; i <= (uint)TIME_RESULT; i++) @@ -3233,10 +3244,11 @@ void Item_func_case::fix_length_and_dec() if (!(cmp_items[i]= cmp_item::get_comparator((Item_result)i, date_arg, cmp_collation.collation))) - return; + return TRUE; } } } + return FALSE; } @@ -4117,7 +4129,6 @@ static int srtcmp_in(CHARSET_INFO *cs, const String *x,const String *y) (uchar *) y->ptr(),y->length()); } - /* Create 'array' for this IN predicate with the respect to its result type and put values from in 'array'. @@ -4178,7 +4189,7 @@ bool Item_func_in::create_array(THD *thd) } -void Item_func_in::fix_length_and_dec() +bool Item_func_in::fix_length_and_dec() { Item **arg, **arg_end; bool const_itm= 1; @@ -4190,7 +4201,7 @@ void Item_func_in::fix_length_and_dec() m_compare_type= STRING_RESULT; left_cmp_type= args[0]->cmp_type(); if (!(found_types= collect_cmp_types(args, arg_count, true))) - return; + return TRUE; for (arg= args + 1, arg_end= args + arg_count; arg != arg_end ; arg++) { @@ -4239,7 +4250,7 @@ void Item_func_in::fix_length_and_dec() { if (m_compare_type == STRING_RESULT && agg_arg_charsets_for_comparison(cmp_collation, args, arg_count)) - return; + return TRUE; arg_types_compatible= TRUE; if (m_compare_type == ROW_RESULT) @@ -4250,12 +4261,14 @@ void Item_func_in::fix_length_and_dec() if (bisection_possible) { array= new (thd->mem_root) in_row(thd, arg_count-1, 0); + if (!array) + return TRUE; cmp= &((in_row*)array)->tmp; } else { if (!(cmp= new (thd->mem_root) cmp_item_row)) - return; + return TRUE; cmp_items[ROW_RESULT]= cmp; } cmp->n= cols; @@ -4272,6 +4285,8 @@ void Item_func_in::fix_length_and_dec() else cmp= ((cmp_item_row*)cmp_items[ROW_RESULT])->comparators + col; *cmp= new (thd->mem_root) cmp_item_datetime(date_arg); + if (!(*cmp)) + return TRUE; } } } @@ -4306,7 +4321,7 @@ void Item_func_in::fix_length_and_dec() } } if (create_array(thd)) - return; + return TRUE; } else { @@ -4314,7 +4329,7 @@ void Item_func_in::fix_length_and_dec() date_arg= find_date_time_item(thd, args, arg_count, 0, true); if (found_types & (1U << STRING_RESULT) && agg_arg_charsets_for_comparison(cmp_collation, args, arg_count)) - return; + return TRUE; for (i= 0; i <= (uint) TIME_RESULT; i++) { if (found_types & (1U << i) && !cmp_items[i]) @@ -4322,11 +4337,12 @@ void Item_func_in::fix_length_and_dec() if (!cmp_items[i] && !(cmp_items[i]= cmp_item::get_comparator((Item_result)i, date_arg, cmp_collation.collation))) - return; + return TRUE; } } } max_length= 1; + return FALSE; } @@ -4617,7 +4633,8 @@ Item_cond::fix_fields(THD *thd, Item **ref) with_window_func|= item->with_window_func; maybe_null|= item->maybe_null; } - fix_length_and_dec(); + if (fix_length_and_dec()) + return TRUE; fixed= 1; return FALSE; } @@ -5656,16 +5673,16 @@ bool Item_func_regex::fix_fields(THD *thd, Item **ref) return Item_bool_func::fix_fields(thd, ref); } -void +bool Item_func_regex::fix_length_and_dec() { - Item_bool_func::fix_length_and_dec(); - - if (agg_arg_charsets_for_comparison(cmp_collation, args, 2)) - return; + if (Item_bool_func::fix_length_and_dec() || + agg_arg_charsets_for_comparison(cmp_collation, args, 2)) + return TRUE; re.init(cmp_collation.collation, 0); re.fix_owner(this, args[0], args[1]); + return FALSE; } @@ -5689,14 +5706,15 @@ bool Item_func_regexp_instr::fix_fields(THD *thd, Item **ref) } -void +bool Item_func_regexp_instr::fix_length_and_dec() { if (agg_arg_charsets_for_comparison(cmp_collation, args, 2)) - return; + return TRUE; re.init(cmp_collation.collation, 0); re.fix_owner(this, args[0], args[1]); + return FALSE; } @@ -6638,7 +6656,8 @@ bool Item_equal::fix_fields(THD *thd, Item **ref) } if (prev_equal_field && last_equal_field != first_equal_field) last_equal_field->next_equal_field= first_equal_field; - fix_length_and_dec(); + if (fix_length_and_dec()) + return TRUE; fixed= 1; return FALSE; } @@ -6724,11 +6743,12 @@ longlong Item_equal::val_int() } -void Item_equal::fix_length_and_dec() +bool Item_equal::fix_length_and_dec() { Item *item= get_first(NO_PARTICULAR_TAB, NULL); eval_item= cmp_item::get_comparator(item->cmp_type(), item, item->collation.collation); + return FALSE; } diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 78a9e384b55..a59432058eb 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -200,7 +200,7 @@ public: Item_bool_func(THD *thd, Item_bool_func *item) :Item_int_func(thd, item) {} bool is_bool_type() { return true; } virtual CHARSET_INFO *compare_collation() const { return NULL; } - void fix_length_and_dec() { decimals=0; max_length=1; } + bool fix_length_and_dec() { decimals=0; max_length=1; return FALSE; } uint decimal_precision() const { return 1; } bool need_parentheses_in_default() { return true; } }; @@ -216,7 +216,7 @@ class Item_func_truth : public Item_bool_func public: virtual bool val_bool(); virtual longlong val_int(); - virtual void fix_length_and_dec(); + virtual bool fix_length_and_dec(); virtual void print(String *str, enum_query_type query_type); enum precedence precedence() const { return CMP_PRECEDENCE; } @@ -501,7 +501,7 @@ public: cond); return this; } - void fix_length_and_dec(); + bool fix_length_and_dec(); int set_cmp_func() { return cmp.set_cmp_func(this, tmp_arg, tmp_arg + 1, true); @@ -711,7 +711,7 @@ public: Item_func_equal(THD *thd, Item *a, Item *b): Item_bool_rowready_func2(thd, a, b) {} longlong val_int(); - void fix_length_and_dec(); + bool fix_length_and_dec(); table_map not_null_tables() const { return 0; } enum Functype functype() const { return EQUAL_FUNC; } enum Functype rev_functype() const { return EQUAL_FUNC; } @@ -878,7 +878,7 @@ public: enum Functype functype() const { return BETWEEN; } const char *func_name() const { return "between"; } enum precedence precedence() const { return BETWEEN_PRECEDENCE; } - void fix_length_and_dec(); + bool fix_length_and_dec(); virtual void print(String *str, enum_query_type query_type); bool eval_not_null_tables(void *opt_arg); void fix_after_pullout(st_select_lex *new_parent, Item **ref, bool merge); @@ -911,10 +911,12 @@ public: longlong val_int(); uint decimal_precision() const { return 1; } const char *func_name() const { return "strcmp"; } - void fix_length_and_dec() + bool fix_length_and_dec() { - agg_arg_charsets_for_comparison(cmp_collation, args, 2); - fix_char_length(2); // returns "1" or "0" or "-1" + if (agg_arg_charsets_for_comparison(cmp_collation, args, 2)) + return TRUE; + fix_char_length(2); + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -941,7 +943,7 @@ public: } bool fix_fields(THD *, Item **); longlong val_int(); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "interval"; } uint decimal_precision() const { return 2; } void print(String *str, enum_query_type query_type) @@ -966,10 +968,11 @@ public: String *str_op(String *); my_decimal *decimal_op(my_decimal *); bool date_op(MYSQL_TIME *ltime,uint fuzzydate); - void fix_length_and_dec() + bool fix_length_and_dec() { set_handler_by_field_type(agg_field_type(args, arg_count, true)); fix_attributes(args, arg_count); + return FALSE; } const char *func_name() const { return "coalesce"; } table_map not_null_tables() const { return 0; } @@ -986,10 +989,11 @@ public: class Item_func_case_abbreviation2 :public Item_func_hybrid_field_type { protected: - void fix_length_and_dec2(Item **items) + bool fix_length_and_dec2(Item **items) { set_handler_by_field_type(agg_field_type(items, 2, true)); fix_attributes(items, 2); + return FALSE; } uint decimal_precision2(Item **args) const; public: @@ -1010,10 +1014,12 @@ public: String *str_op(String *str); my_decimal *decimal_op(my_decimal *); bool date_op(MYSQL_TIME *ltime,uint fuzzydate); - void fix_length_and_dec() + bool fix_length_and_dec() { - Item_func_case_abbreviation2::fix_length_and_dec2(args); + if (Item_func_case_abbreviation2::fix_length_and_dec2(args)) + return TRUE; maybe_null= args[1]->maybe_null; + return FALSE; } const char *func_name() const { return "ifnull"; } Field *create_field_for_create_select(TABLE *table) @@ -1041,7 +1047,7 @@ public: my_decimal *decimal_op(my_decimal *); String *str_op(String *); bool fix_fields(THD *, Item **); - void fix_length_and_dec(); + bool fix_length_and_dec(); uint decimal_precision() const { return Item_func_case_abbreviation2::decimal_precision2(args + 1); @@ -1108,7 +1114,7 @@ public: longlong int_op(); String *str_op(String *str); my_decimal *decimal_op(my_decimal *); - void fix_length_and_dec(); + bool fix_length_and_dec(); bool walk(Item_processor processor, bool walk_subquery, void *arg); uint decimal_precision() const { return args[2]->decimal_precision(); } const char *func_name() const { return "nullif"; } @@ -1570,7 +1576,7 @@ public: my_decimal *decimal_op(my_decimal *); bool date_op(MYSQL_TIME *ltime, uint fuzzydate); bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); uint decimal_precision() const; table_map not_null_tables() const { return 0; } const char *func_name() const { return "case"; } @@ -1649,7 +1655,7 @@ public: longlong val_int(); bool fix_fields(THD *, Item **); bool create_array(THD *thd); - void fix_length_and_dec(); + bool fix_length_and_dec(); void cleanup() { uint i; @@ -1710,7 +1716,7 @@ public: int compare(cmp_item *arg); cmp_item *make_same(); void store_value_by_template(THD *thd, cmp_item *tmpl, Item *); - friend void Item_func_in::fix_length_and_dec(); + friend bool Item_func_in::fix_length_and_dec(); cmp_item *get_comparator(uint i) { return comparators[i]; } }; @@ -1724,7 +1730,7 @@ public: void set(uint pos,Item *item); uchar *get_value(Item *item); friend bool Item_func_in::create_array(THD *thd); - friend void Item_func_in::fix_length_and_dec(); + friend bool Item_func_in::fix_length_and_dec(); Item_result result_type() { return ROW_RESULT; } cmp_item *get_cmp_item() { return &tmp; } }; @@ -1756,7 +1762,11 @@ public: } CHARSET_INFO *compare_collation() const { return args[0]->collation.collation; } - void fix_length_and_dec() { decimals=0; max_length=1; maybe_null=0; } + bool fix_length_and_dec() + { + decimals=0; max_length=1; maybe_null=0; + return FALSE; + } bool count_sargable_conds(void *arg); }; @@ -1980,10 +1990,10 @@ public: const char *func_name() const { return "like"; } enum precedence precedence() const { return CMP_PRECEDENCE; } bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec() + bool fix_length_and_dec() { max_length= 1; - agg_arg_charsets_for_comparison(cmp_collation, args, 2); + return agg_arg_charsets_for_comparison(cmp_collation, args, 2); } void cleanup(); @@ -2104,7 +2114,7 @@ public: } longlong val_int(); bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "regexp"; } enum precedence precedence() const { return CMP_PRECEDENCE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -2142,7 +2152,7 @@ public: } longlong val_int(); bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "regexp_instr"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -2378,7 +2388,7 @@ public: longlong val_int(); const char *func_name() const { return "multiple equal"; } void sort(Item_field_cmpfunc compare, void *arg); - void fix_length_and_dec(); + bool fix_length_and_dec(); bool fix_fields(THD *thd, Item **ref); void cleanup() { diff --git a/sql/item_func.cc b/sql/item_func.cc index cd23c0b64fe..4bdb18ad95a 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -237,8 +237,7 @@ Item_func::fix_fields(THD *thd, Item **ref) with_subselect|= item->has_subquery(); } } - fix_length_and_dec(); - if (thd->is_error()) // An error inside fix_length_and_dec occurred + if (fix_length_and_dec()) return TRUE; fixed= 1; return FALSE; @@ -733,10 +732,12 @@ String *Item_int_func::val_str(String *str) } -void Item_func_connection_id::fix_length_and_dec() +bool Item_func_connection_id::fix_length_and_dec() { - Item_int_func::fix_length_and_dec(); + if (Item_int_func::fix_length_and_dec()) + return TRUE; max_length= 10; + return FALSE; } @@ -755,7 +756,7 @@ bool Item_func_connection_id::fix_fields(THD *thd, Item **ref) function of two arguments. */ -void Item_num_op::fix_length_and_dec(void) +bool Item_num_op::fix_length_and_dec(void) { DBUG_ENTER("Item_num_op::fix_length_and_dec"); DBUG_PRINT("info", ("name %s", func_name())); @@ -791,7 +792,7 @@ void Item_num_op::fix_length_and_dec(void) result_type() == DECIMAL_RESULT ? "DECIMAL_RESULT" : result_type() == INT_RESULT ? "INT_RESULT" : "--ILLEGAL!!!--"))); - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } @@ -801,7 +802,7 @@ void Item_num_op::fix_length_and_dec(void) type depends only on the first argument) */ -void Item_func_num1::fix_length_and_dec() +bool Item_func_num1::fix_length_and_dec() { DBUG_ENTER("Item_func_num1::fix_length_and_dec"); DBUG_PRINT("info", ("name %s", func_name())); @@ -832,7 +833,7 @@ void Item_func_num1::fix_length_and_dec() result_type() == DECIMAL_RESULT ? "DECIMAL_RESULT" : result_type() == INT_RESULT ? "INT_RESULT" : "--ILLEGAL!!!--"))); - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } @@ -1417,12 +1418,14 @@ void Item_func_additive_op::result_precision() subtraction of UNSIGNED BIGINT to return negative values. */ -void Item_func_minus::fix_length_and_dec() +bool Item_func_minus::fix_length_and_dec() { - Item_num_op::fix_length_and_dec(); + if (Item_num_op::fix_length_and_dec()) + return TRUE; if (unsigned_flag && (current_thd->variables.sql_mode & MODE_NO_UNSIGNED_SUBTRACTION)) unsigned_flag=0; + return FALSE; } @@ -1720,11 +1723,12 @@ void Item_func_div::result_precision() } -void Item_func_div::fix_length_and_dec() +bool Item_func_div::fix_length_and_dec() { DBUG_ENTER("Item_func_div::fix_length_and_dec"); prec_increment= current_thd->variables.div_precincrement; - Item_num_op::fix_length_and_dec(); + if (Item_num_op::fix_length_and_dec()) + DBUG_RETURN(TRUE); switch (Item_func_div::result_type()) { case REAL_RESULT: { @@ -1755,7 +1759,7 @@ void Item_func_div::fix_length_and_dec() DBUG_ASSERT(0); } maybe_null= 1; // devision by zero - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } @@ -1831,7 +1835,7 @@ longlong Item_func_int_div::val_int() } -void Item_func_int_div::fix_length_and_dec() +bool Item_func_int_div::fix_length_and_dec() { Item_result argtype= args[0]->result_type(); /* use precision ony for the data type it is applicable for and valid */ @@ -1842,6 +1846,7 @@ void Item_func_int_div::fix_length_and_dec() MY_INT64_NUM_DECIMAL_DIGITS : char_length); maybe_null=1; unsigned_flag=args[0]->unsigned_flag | args[1]->unsigned_flag; + return false; } @@ -1925,11 +1930,13 @@ void Item_func_mod::result_precision() } -void Item_func_mod::fix_length_and_dec() +bool Item_func_mod::fix_length_and_dec() { - Item_num_op::fix_length_and_dec(); + if (Item_num_op::fix_length_and_dec()) + return true; maybe_null= 1; unsigned_flag= args[0]->unsigned_flag; + return false; } @@ -1976,10 +1983,11 @@ my_decimal *Item_func_neg::decimal_op(my_decimal *decimal_value) } -void Item_func_neg::fix_length_and_dec() +bool Item_func_neg::fix_length_and_dec() { DBUG_ENTER("Item_func_neg::fix_length_and_dec"); - Item_func_num1::fix_length_and_dec(); + if (Item_func_num1::fix_length_and_dec()) + DBUG_RETURN(TRUE); /* 1 add because sign can appear */ max_length= args[0]->max_length + 1; @@ -2005,7 +2013,7 @@ void Item_func_neg::fix_length_and_dec() } } unsigned_flag= 0; - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } @@ -2045,10 +2053,12 @@ my_decimal *Item_func_abs::decimal_op(my_decimal *decimal_value) } -void Item_func_abs::fix_length_and_dec() +bool Item_func_abs::fix_length_and_dec() { - Item_func_num1::fix_length_and_dec(); + if (Item_func_num1::fix_length_and_dec()) + return TRUE; unsigned_flag= args[0]->unsigned_flag; + return FALSE; } @@ -2280,7 +2290,7 @@ longlong Item_func_bit_neg::val_int() // Conversion functions -void Item_func_int_val::fix_length_and_dec() +bool Item_func_int_val::fix_length_and_dec() { DBUG_ENTER("Item_func_int_val::fix_length_and_dec"); DBUG_PRINT("info", ("name %s", func_name())); @@ -2328,7 +2338,7 @@ void Item_func_int_val::fix_length_and_dec() result_type() == INT_RESULT ? "INT_RESULT" : "--ILLEGAL!!!--"))); - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } @@ -2426,7 +2436,7 @@ my_decimal *Item_func_floor::decimal_op(my_decimal *decimal_value) } -void Item_func_round::fix_length_and_dec() +bool Item_func_round::fix_length_and_dec() { int decimals_to_set; longlong val1; @@ -2444,12 +2454,12 @@ void Item_func_round::fix_length_and_dec() } else set_handler_by_result_type(REAL_RESULT); - return; + return FALSE; } val1= args[1]->val_int(); if ((null_value= args[1]->null_value)) - return; + return FALSE; val1_unsigned= args[1]->unsigned_flag; if (val1 < 0) @@ -2462,7 +2472,7 @@ void Item_func_round::fix_length_and_dec() decimals= MY_MIN(decimals_to_set, NOT_FIXED_DEC); max_length= float_length(decimals); set_handler_by_result_type(REAL_RESULT); - return; + return FALSE; } switch (args[0]->result_type()) { @@ -2503,6 +2513,7 @@ void Item_func_round::fix_length_and_dec() case TIME_RESULT: DBUG_ASSERT(0); /* This result type isn't handled */ } + return FALSE; } double my_double_round(double value, longlong dec, bool dec_unsigned, @@ -2727,7 +2738,7 @@ double Item_func_units::val_real() } -void Item_func_min_max::fix_length_and_dec() +bool Item_func_min_max::fix_length_and_dec() { uint unsigned_count= 0; int max_int_part=0; @@ -2838,6 +2849,7 @@ void Item_func_min_max::fix_length_and_dec() set_handler_by_field_type(MYSQL_TYPE_DOUBLE); break; } + return FALSE; } @@ -3088,10 +3100,10 @@ longlong Item_func_coercibility::val_int() } -void Item_func_locate::fix_length_and_dec() +bool Item_func_locate::fix_length_and_dec() { max_length= MY_INT32_NUM_DECIMAL_DIGITS; - agg_arg_charsets_for_comparison(cmp_collation, args, 2); + return agg_arg_charsets_for_comparison(cmp_collation, args, 2); } @@ -3208,14 +3220,15 @@ longlong Item_func_field::val_int() } -void Item_func_field::fix_length_and_dec() +bool Item_func_field::fix_length_and_dec() { maybe_null=0; max_length=3; cmp_type= args[0]->result_type(); for (uint i=1; i < arg_count ; i++) cmp_type= item_cmp_type(cmp_type, args[i]->result_type()); if (cmp_type == STRING_RESULT) - agg_arg_charsets_for_comparison(cmp_collation, args, arg_count); + return agg_arg_charsets_for_comparison(cmp_collation, args, arg_count); + return FALSE; } @@ -3262,7 +3275,7 @@ longlong Item_func_ord::val_int() /* Returns number of found type >= 1 or 0 if not found */ /* This optimizes searching in enums to bit testing! */ -void Item_func_find_in_set::fix_length_and_dec() +bool Item_func_find_in_set::fix_length_and_dec() { decimals=0; max_length=3; // 1-999 @@ -3284,7 +3297,7 @@ void Item_func_find_in_set::fix_length_and_dec() } } } - agg_arg_charsets_for_comparison(cmp_collation, args, 2); + return agg_arg_charsets_for_comparison(cmp_collation, args, 2); } static const char separator=','; @@ -3490,7 +3503,8 @@ udf_handler::fix_fields(THD *thd, Item_func_or_sum *func, DBUG_RETURN(TRUE); } } - func->fix_length_and_dec(); + if (func->fix_length_and_dec()) + DBUG_RETURN(TRUE); initid.max_length=func->max_length; initid.maybe_null=func->maybe_null; initid.const_item=func->const_item_cache; @@ -3828,13 +3842,13 @@ String *Item_func_udf_decimal::val_str(String *str) /* Default max_length is max argument length */ -void Item_func_udf_str::fix_length_and_dec() +bool Item_func_udf_str::fix_length_and_dec() { DBUG_ENTER("Item_func_udf_str::fix_length_and_dec"); max_length=0; for (uint i = 0; i < arg_count; i++) set_if_bigger(max_length,args[i]->max_length); - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } String *Item_func_udf_str::val_str(String *str) @@ -4721,7 +4735,7 @@ bool Item_func_set_user_var::fix_fields(THD *thd, Item **ref) } -void +bool Item_func_set_user_var::fix_length_and_dec() { maybe_null=args[0]->maybe_null; @@ -4735,6 +4749,7 @@ Item_func_set_user_var::fix_length_and_dec() args[0]->collation.collation); } unsigned_flag= args[0]->unsigned_flag; + return FALSE; } @@ -5573,7 +5588,7 @@ err: return 1; } -void Item_func_get_user_var::fix_length_and_dec() +bool Item_func_get_user_var::fix_length_and_dec() { THD *thd=current_thd; int error; @@ -5623,6 +5638,7 @@ void Item_func_get_user_var::fix_length_and_dec() set_handler_by_field_type(MYSQL_TYPE_LONG_BLOB); max_length= MAX_BLOB_WIDTH; } + return false; } @@ -5765,7 +5781,7 @@ void Item_func_get_system_var::update_null_value() } -void Item_func_get_system_var::fix_length_and_dec() +bool Item_func_get_system_var::fix_length_and_dec() { char *cptr; maybe_null= TRUE; @@ -5777,7 +5793,7 @@ void Item_func_get_system_var::fix_length_and_dec() { my_error(ER_INCORRECT_GLOBAL_LOCAL_VAR, MYF(0), var->name.str, var_type == OPT_GLOBAL ? "SESSION" : "GLOBAL"); - return; + return TRUE; } /* As there was no local variable, return the global value */ var_type= OPT_GLOBAL; @@ -5841,6 +5857,7 @@ void Item_func_get_system_var::fix_length_and_dec() my_error(ER_VAR_CANT_BE_READ, MYF(0), var->name.str); break; } + return FALSE; } @@ -6617,7 +6634,7 @@ bool Item_func_sp::is_expensive() @note called from Item::fix_fields. */ -void Item_func_sp::fix_length_and_dec() +bool Item_func_sp::fix_length_and_dec() { DBUG_ENTER("Item_func_sp::fix_length_and_dec"); @@ -6625,7 +6642,7 @@ void Item_func_sp::fix_length_and_dec() Type_std_attributes::set(sp_result_field); maybe_null= 1; - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } @@ -6964,9 +6981,10 @@ my_decimal *Item_func_last_value::val_decimal(my_decimal *decimal_value) } -void Item_func_last_value::fix_length_and_dec() +bool Item_func_last_value::fix_length_and_dec() { last_value= args[arg_count -1]; Type_std_attributes::set(last_value); maybe_null= last_value->maybe_null; + return FALSE; } diff --git a/sql/item_func.h b/sql/item_func.h index 3787b4f4924..0969cf2e3f0 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -399,8 +399,8 @@ public: { DBUG_ASSERT(fixed == 1); return (longlong) rint(val_real()); } enum Item_result result_type () const { return REAL_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_DOUBLE; } - void fix_length_and_dec() - { decimals= NOT_FIXED_DEC; max_length= float_length(decimals); } + bool fix_length_and_dec() + { decimals= NOT_FIXED_DEC; max_length= float_length(decimals); return FALSE; } }; @@ -577,7 +577,7 @@ class Item_func_num1: public Item_func_numhybrid public: Item_func_num1(THD *thd, Item *a): Item_func_numhybrid(thd, a) {} Item_func_num1(THD *thd, Item *a, Item *b): Item_func_numhybrid(thd, a, b) {} - void fix_length_and_dec(); + bool fix_length_and_dec(); }; @@ -592,7 +592,7 @@ class Item_num_op :public Item_func_numhybrid { print_op(str, query_type); } - void fix_length_and_dec(); + bool fix_length_and_dec(); bool need_parentheses_in_default() { return true; } }; @@ -619,7 +619,7 @@ public: String *val_str(String*str); enum Item_result result_type () const { return INT_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_LONGLONG; } - void fix_length_and_dec() {} + bool fix_length_and_dec() { return FALSE; } }; @@ -630,7 +630,7 @@ class Item_func_connection_id :public Item_int_func public: Item_func_connection_id(THD *thd): Item_int_func(thd) {} const char *func_name() const { return "connection_id"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); bool fix_fields(THD *thd, Item **ref); longlong val_int() { DBUG_ASSERT(fixed == 1); return value; } bool check_vcol_func_processor(void *arg) @@ -656,7 +656,7 @@ public: null_value= args[0]->null_value; return value; } - void fix_length_and_dec() + bool fix_length_and_dec() { uint32 char_length= MY_MIN(args[0]->max_char_length(), MY_INT64_NUM_DECIMAL_DIGITS); @@ -667,6 +667,7 @@ public: */ set_if_bigger(char_length, 1U + (unsigned_flag ? 0 : 1)); fix_char_length(char_length); + return FALSE; } virtual void print(String *str, enum_query_type query_type); uint decimal_precision() const { return args[0]->decimal_precision(); } @@ -713,7 +714,7 @@ public: my_decimal *val_decimal(my_decimal*); enum Item_result result_type () const { return DECIMAL_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_NEWDECIMAL; } - void fix_length_and_dec() {} + bool fix_length_and_dec() { return FALSE; } const char *func_name() const { return "decimal_typecast"; } virtual void print(String *str, enum_query_type query_type); bool need_parentheses_in_default() { return true; } @@ -733,7 +734,7 @@ public: } double val_real(); enum_field_types field_type() const { return MYSQL_TYPE_DOUBLE; } - void fix_length_and_dec() { maybe_null= 1; } + bool fix_length_and_dec() { maybe_null= 1; return FALSE; } const char *func_name() const { return "double_typecast"; } virtual void print(String *str, enum_query_type query_type); bool need_parentheses_in_default() { return true; } @@ -777,7 +778,7 @@ public: longlong int_op(); double real_op(); my_decimal *decimal_op(my_decimal *); - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -811,7 +812,7 @@ public: my_decimal *decimal_op(my_decimal *); const char *func_name() const { return "/"; } enum precedence precedence() const { return MUL_PRECEDENCE; } - void fix_length_and_dec(); + bool fix_length_and_dec(); void result_precision(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -826,7 +827,7 @@ public: longlong val_int(); const char *func_name() const { return "DIV"; } enum precedence precedence() const { return MUL_PRECEDENCE; } - void fix_length_and_dec(); + bool fix_length_and_dec(); void print(String *str, enum_query_type query_type) { print_op(str, query_type); @@ -850,7 +851,7 @@ public: const char *func_name() const { return "%"; } enum precedence precedence() const { return MUL_PRECEDENCE; } void result_precision(); - void fix_length_and_dec(); + bool fix_length_and_dec(); bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -873,7 +874,7 @@ public: str->append(func_name()); args[0]->print_parenthesised(str, query_type, precedence()); } - void fix_length_and_dec(); + bool fix_length_and_dec(); uint decimal_precision() const { return args[0]->decimal_precision(); } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -891,7 +892,7 @@ public: longlong int_op(); my_decimal *decimal_op(my_decimal *); const char *func_name() const { return "abs"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -905,10 +906,11 @@ class Item_dec_func :public Item_real_func public: Item_dec_func(THD *thd, Item *a): Item_real_func(thd, a) {} Item_dec_func(THD *thd, Item *a, Item *b): Item_real_func(thd, a, b) {} - void fix_length_and_dec() + bool fix_length_and_dec() { decimals=NOT_FIXED_DEC; max_length=float_length(decimals); maybe_null=1; + return FALSE; } }; @@ -1066,7 +1068,7 @@ class Item_func_int_val :public Item_func_num1 { public: Item_func_int_val(THD *thd, Item *a): Item_func_num1(thd, a) {} - void fix_length_and_dec(); + bool fix_length_and_dec(); }; @@ -1111,7 +1113,7 @@ public: double real_op(); longlong int_op(); my_decimal *decimal_op(my_decimal *); - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1163,8 +1165,8 @@ public: Item_real_func(thd, a), name(name_arg), mul(mul_arg), add(add_arg) {} double val_real(); const char *func_name() const { return name; } - void fix_length_and_dec() - { decimals= NOT_FIXED_DEC; max_length= float_length(decimals); } + bool fix_length_and_dec() + { decimals= NOT_FIXED_DEC; max_length= float_length(decimals); return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1194,7 +1196,7 @@ public: String *val_str(String *); my_decimal *val_decimal(my_decimal *); bool get_date(MYSQL_TIME *res, ulonglong fuzzy_date); - void fix_length_and_dec(); + bool fix_length_and_dec(); }; class Item_func_min :public Item_func_min_max @@ -1237,13 +1239,14 @@ public: bool const_item() const { return 0; } Item_result result_type() const { return args[0]->result_type(); } enum_field_types field_type() const { return args[0]->field_type(); } - void fix_length_and_dec() + bool fix_length_and_dec() { collation= args[0]->collation; max_length= args[0]->max_length; decimals=args[0]->decimals; /* The item could be a NULL constant. */ null_value= args[0]->is_null(); + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1257,7 +1260,7 @@ public: Item_func_length(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "length"; } - void fix_length_and_dec() { max_length=10; } + bool fix_length_and_dec() { max_length=10; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1280,7 +1283,7 @@ public: Item_func_char_length(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "char_length"; } - void fix_length_and_dec() { max_length=10; } + bool fix_length_and_dec() { max_length=10; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1291,7 +1294,7 @@ public: Item_func_coercibility(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "coercibility"; } - void fix_length_and_dec() { max_length=10; maybe_null= 0; } + bool fix_length_and_dec() { max_length=10; maybe_null= 0; return FALSE; } bool eval_not_null_tables(void *) { not_null_tables_cache= 0; @@ -1313,7 +1316,7 @@ public: Item_func_locate(THD *thd, Item *a, Item *b, Item *c): Item_int_func(thd, a, b, c) {} const char *func_name() const { return "locate"; } longlong val_int(); - void fix_length_and_dec(); + bool fix_length_and_dec(); virtual void print(String *str, enum_query_type query_type); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1329,7 +1332,7 @@ public: Item_func_field(THD *thd, List &list): Item_int_func(thd, list) {} longlong val_int(); const char *func_name() const { return "field"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1342,7 +1345,7 @@ public: Item_func_ascii(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "ascii"; } - void fix_length_and_dec() { max_length=3; } + bool fix_length_and_dec() { max_length=3; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1369,7 +1372,7 @@ public: Item_int_func(thd, a, b), enum_value(0) {} longlong val_int(); const char *func_name() const { return "find_in_set"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1381,7 +1384,7 @@ class Item_func_bit: public Item_int_func public: Item_func_bit(THD *thd, Item *a, Item *b): Item_int_func(thd, a, b) {} Item_func_bit(THD *thd, Item *a): Item_int_func(thd, a) {} - void fix_length_and_dec() { unsigned_flag= 1; } + bool fix_length_and_dec() { unsigned_flag= 1; return FALSE; } virtual inline void print(String *str, enum_query_type query_type) { @@ -1418,7 +1421,7 @@ public: Item_func_bit_count(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "bit_count"; } - void fix_length_and_dec() { max_length=2; } + bool fix_length_and_dec() { max_length=2; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1469,12 +1472,13 @@ public: Item_func_last_insert_id(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "last_insert_id"; } - void fix_length_and_dec() + bool fix_length_and_dec() { unsigned_flag= TRUE; if (arg_count) max_length= args[0]->max_length; unsigned_flag=1; + return FALSE; } bool fix_fields(THD *thd, Item **ref); bool check_vcol_func_processor(void *arg) @@ -1494,7 +1498,7 @@ public: {} longlong val_int(); const char *func_name() const { return "benchmark"; } - void fix_length_and_dec() { max_length=1; maybe_null=0; } + bool fix_length_and_dec() { max_length=1; maybe_null=0; return FALSE; } virtual void print(String *str, enum_query_type query_type); bool check_vcol_func_processor(void *arg) { @@ -1652,7 +1656,7 @@ class Item_func_udf_float :public Item_udf_func double val_real(); String *val_str(String *str); enum_field_types field_type() const { return MYSQL_TYPE_DOUBLE; } - void fix_length_and_dec() { fix_num_length_and_dec(); } + bool fix_length_and_dec() { fix_num_length_and_dec(); return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1671,7 +1675,7 @@ public: String *val_str(String *str); enum Item_result result_type () const { return INT_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_LONGLONG; } - void fix_length_and_dec() { decimals= 0; max_length= 21; } + bool fix_length_and_dec() { decimals= 0; max_length= 21; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1690,7 +1694,7 @@ public: String *val_str(String *str); enum Item_result result_type () const { return DECIMAL_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_NEWDECIMAL; } - void fix_length_and_dec() { fix_num_length_and_dec(); } + bool fix_length_and_dec() { fix_num_length_and_dec(); return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1730,7 +1734,7 @@ public: } enum Item_result result_type () const { return STRING_RESULT; } enum_field_types field_type() const { return string_field_type(); } - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -1782,7 +1786,7 @@ public: double val_real() { DBUG_ASSERT(fixed == 1); null_value= 1; return 0.0; } longlong val_int() { DBUG_ASSERT(fixed == 1); null_value=1; return 0; } enum Item_result result_type () const { return STRING_RESULT; } - void fix_length_and_dec() { maybe_null=1; max_length=0; } + bool fix_length_and_dec() { maybe_null=1; max_length=0; return FALSE; } }; #endif /* HAVE_DLOPEN */ @@ -1797,7 +1801,7 @@ class Item_func_get_lock :public Item_int_func Item_func_get_lock(THD *thd, Item *a, Item *b) :Item_int_func(thd, a, b) {} longlong val_int(); const char *func_name() const { return "get_lock"; } - void fix_length_and_dec() { max_length=1; maybe_null=1;} + bool fix_length_and_dec() { max_length=1; maybe_null=1; return FALSE; } table_map used_tables() const { return used_tables_cache | RAND_TABLE_BIT; @@ -1819,7 +1823,7 @@ public: Item_func_release_lock(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "release_lock"; } - void fix_length_and_dec() { max_length= 1; maybe_null= 1;} + bool fix_length_and_dec() { max_length= 1; maybe_null= 1; return FALSE; } table_map used_tables() const { return used_tables_cache | RAND_TABLE_BIT; @@ -1847,7 +1851,7 @@ public: Item_int_func(thd, a, b, c, d) {} longlong val_int(); const char *func_name() const { return "master_pos_wait"; } - void fix_length_and_dec() { max_length=21; maybe_null=1;} + bool fix_length_and_dec() { max_length=21; maybe_null=1; return FALSE; } bool check_vcol_func_processor(void *arg) { return mark_unsupported_function(func_name(), "()", arg, VCOL_IMPOSSIBLE); @@ -1865,7 +1869,7 @@ public: Item_master_gtid_wait(THD *thd, Item *a, Item *b): Item_int_func(thd, a, b) {} longlong val_int(); const char *func_name() const { return "master_gtid_wait"; } - void fix_length_and_dec() { max_length=2; } + bool fix_length_and_dec() { max_length=2; return FALSE; } bool check_vcol_func_processor(void *arg) { return mark_unsupported_function(func_name(), "()", arg, VCOL_IMPOSSIBLE); @@ -1956,7 +1960,7 @@ public: void save_item_result(Item *item); bool update(); bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); Field *create_field_for_create_select(TABLE *table) { return result_type() != STRING_RESULT ? @@ -1997,7 +2001,7 @@ public: longlong val_int(); my_decimal *val_decimal(my_decimal*); String *val_str(String* str); - void fix_length_and_dec(); + bool fix_length_and_dec(); virtual void print(String *str, enum_query_type query_type); /* We must always return variables as strings to guard against selects of type @@ -2079,7 +2083,7 @@ public: size_t name_len_arg); enum Functype functype() const { return GSYSVAR_FUNC; } void update_null_value(); - void fix_length_and_dec(); + bool fix_length_and_dec(); void print(String *str, enum_query_type query_type); bool const_item() const { return true; } table_map used_tables() const { return 0; } @@ -2218,7 +2222,11 @@ public: Item_func_is_free_lock(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "is_free_lock"; } - void fix_length_and_dec() { decimals=0; max_length=1; maybe_null=1;} + bool fix_length_and_dec() + { + decimals=0; max_length=1; maybe_null=1; + return FALSE; + } bool check_vcol_func_processor(void *arg) { return mark_unsupported_function(func_name(), "()", arg, VCOL_IMPOSSIBLE); @@ -2234,7 +2242,11 @@ public: Item_func_is_used_lock(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "is_used_lock"; } - void fix_length_and_dec() { decimals=0; max_length=10; maybe_null=1;} + bool fix_length_and_dec() + { + decimals=0; max_length=10; maybe_null=1; + return FALSE; + } bool check_vcol_func_processor(void *arg) { return mark_unsupported_function(func_name(), "()", arg, VCOL_IMPOSSIBLE); @@ -2286,7 +2298,7 @@ public: Item_func_row_count(THD *thd): Item_int_func(thd) {} longlong val_int(); const char *func_name() const { return "row_count"; } - void fix_length_and_dec() { decimals= 0; maybe_null=0; } + bool fix_length_and_dec() { decimals= 0; maybe_null=0; return FALSE; } bool check_vcol_func_processor(void *arg) { return mark_unsupported_function(func_name(), "()", arg, VCOL_IMPOSSIBLE); @@ -2414,7 +2426,7 @@ public: virtual enum Functype functype() const { return FUNC_SP; } bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(void); + bool fix_length_and_dec(void); bool is_expensive(); inline Field *get_sp_result_field() @@ -2450,7 +2462,7 @@ public: Item_func_found_rows(THD *thd): Item_int_func(thd) {} longlong val_int(); const char *func_name() const { return "found_rows"; } - void fix_length_and_dec() { decimals= 0; maybe_null=0; } + bool fix_length_and_dec() { decimals= 0; maybe_null=0; return FALSE; } bool check_vcol_func_processor(void *arg) { return mark_unsupported_function(func_name(), "()", arg, VCOL_IMPOSSIBLE); @@ -2469,8 +2481,8 @@ public: const char *func_name() const { return "uuid_short"; } longlong val_int(); bool const_item() const { return false; } - void fix_length_and_dec() - { max_length= 21; unsigned_flag=1; } + bool fix_length_and_dec() + { max_length= 21; unsigned_flag=1; return FALSE; } table_map used_tables() const { return RAND_TABLE_BIT; } bool check_vcol_func_processor(void *arg) { @@ -2491,7 +2503,7 @@ public: longlong val_int(); String *val_str(String *); my_decimal *val_decimal(my_decimal *); - void fix_length_and_dec(); + bool fix_length_and_dec(); enum Item_result result_type () const { return last_value->result_type(); } const char *func_name() const { return "last_value"; } bool eval_not_null_tables(void *) diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc index 3119f5a577b..246be438e36 100644 --- a/sql/item_geofunc.cc +++ b/sql/item_geofunc.cc @@ -49,12 +49,13 @@ Field *Item_geometry_func::create_field_for_create_select(TABLE *t_arg) return result; } -void Item_geometry_func::fix_length_and_dec() +bool Item_geometry_func::fix_length_and_dec() { collation.set(&my_charset_bin); decimals=0; max_length= (uint32) 4294967295U; maybe_null= 1; + return FALSE; } @@ -223,11 +224,12 @@ String *Item_func_as_wkt::val_str_ascii(String *str) } -void Item_func_as_wkt::fix_length_and_dec() +bool Item_func_as_wkt::fix_length_and_dec() { collation.set(default_charset(), DERIVATION_COERCIBLE, MY_REPERTOIRE_ASCII); max_length=MAX_BLOB_WIDTH; maybe_null= 1; + return FALSE; } @@ -249,11 +251,12 @@ String *Item_func_as_wkb::val_str(String *str) } -void Item_func_as_geojson::fix_length_and_dec() +bool Item_func_as_geojson::fix_length_and_dec() { collation.set(default_charset(), DERIVATION_COERCIBLE, MY_REPERTOIRE_ASCII); max_length=MAX_BLOB_WIDTH; maybe_null= 1; + return FALSE; } diff --git a/sql/item_geofunc.h b/sql/item_geofunc.h index 199bc1f47de..acc94183d47 100644 --- a/sql/item_geofunc.h +++ b/sql/item_geofunc.h @@ -38,7 +38,7 @@ public: Item_geometry_func(THD *thd, Item *a, Item *b, Item *c): Item_str_func(thd, a, b, c) {} Item_geometry_func(THD *thd, List &list): Item_str_func(thd, list) {} - void fix_length_and_dec(); + bool fix_length_and_dec(); enum_field_types field_type() const { return MYSQL_TYPE_GEOMETRY; } Field *create_field_for_create_select(TABLE *table); }; @@ -90,7 +90,7 @@ public: Item_func_as_wkt(THD *thd, Item *a): Item_str_ascii_func(thd, a) {} const char *func_name() const { return "st_astext"; } String *val_str_ascii(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -116,7 +116,7 @@ public: Item_func_as_geojson(THD *thd, Item *js, Item *max_dec_digits, Item *opt): Item_str_ascii_func(thd, js, max_dec_digits, opt) {} const char *func_name() const { return "st_asgeojson"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str_ascii(String *); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -129,11 +129,12 @@ public: Item_func_geometry_type(THD *thd, Item *a): Item_str_ascii_func(thd, a) {} String *val_str_ascii(String *); const char *func_name() const { return "st_geometrytype"; } - void fix_length_and_dec() + bool fix_length_and_dec() { // "GeometryCollection" is the longest fix_length_and_charset(20, default_charset()); maybe_null= 1; + return FALSE; }; Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -308,9 +309,10 @@ public: item_type=it; } String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { - Item_geometry_func::fix_length_and_dec(); + if (Item_geometry_func::fix_length_and_dec()) + return TRUE; for (unsigned int i= 0; i < arg_count; ++i) { if (args[i]->fixed && args[i]->field_type() != MYSQL_TYPE_GEOMETRY) @@ -320,8 +322,10 @@ public: str.append('\0'); my_error(ER_ILLEGAL_VALUE_FOR_TYPE, MYF(0), "non geometric", str.ptr()); + return TRUE; } } + return FALSE; } const char *func_name() const { return "geometrycollection"; } @@ -510,7 +514,7 @@ public: Item_func_isempty(THD *thd, Item *a): Item_bool_func(thd, a) {} longlong val_int(); const char *func_name() const { return "st_isempty"; } - void fix_length_and_dec() { maybe_null= 1; } + bool fix_length_and_dec() { maybe_null= 1; return FALSE; } bool need_parentheses_in_default() { return false; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -526,7 +530,7 @@ public: Item_func_issimple(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "st_issimple"; } - void fix_length_and_dec() { decimals=0; max_length=2; } + bool fix_length_and_dec() { decimals=0; max_length=2; return FALSE; } uint decimal_precision() const { return 1; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -538,7 +542,7 @@ public: Item_func_isclosed(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "st_isclosed"; } - void fix_length_and_dec() { decimals=0; max_length=2; } + bool fix_length_and_dec() { decimals=0; max_length=2; return FALSE; } uint decimal_precision() const { return 1; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -561,7 +565,7 @@ public: Item_func_dimension(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "st_dimension"; } - void fix_length_and_dec() { max_length= 10; maybe_null= 1; } + bool fix_length_and_dec() { max_length= 10; maybe_null= 1; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -573,10 +577,12 @@ public: Item_func_x(THD *thd, Item *a): Item_real_func(thd, a) {} double val_real(); const char *func_name() const { return "st_x"; } - void fix_length_and_dec() - { - Item_real_func::fix_length_and_dec(); - maybe_null= 1; + bool fix_length_and_dec() + { + if (Item_real_func::fix_length_and_dec()) + return TRUE; + maybe_null= 1; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -590,10 +596,12 @@ public: Item_func_y(THD *thd, Item *a): Item_real_func(thd, a) {} double val_real(); const char *func_name() const { return "st_y"; } - void fix_length_and_dec() - { - Item_real_func::fix_length_and_dec(); - maybe_null= 1; + bool fix_length_and_dec() + { + if (Item_real_func::fix_length_and_dec()) + return TRUE; + maybe_null= 1; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -607,7 +615,7 @@ public: Item_func_numgeometries(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "st_numgeometries"; } - void fix_length_and_dec() { max_length= 10; maybe_null= 1; } + bool fix_length_and_dec() { max_length= 10; maybe_null= 1; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -620,7 +628,7 @@ public: Item_func_numinteriorring(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "st_numinteriorrings"; } - void fix_length_and_dec() { max_length= 10; maybe_null= 1; } + bool fix_length_and_dec() { max_length= 10; maybe_null= 1; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -633,7 +641,7 @@ public: Item_func_numpoints(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "st_numpoints"; } - void fix_length_and_dec() { max_length= 10; maybe_null= 1; } + bool fix_length_and_dec() { max_length= 10; maybe_null= 1; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -646,10 +654,12 @@ public: Item_func_area(THD *thd, Item *a): Item_real_func(thd, a) {} double val_real(); const char *func_name() const { return "st_area"; } - void fix_length_and_dec() - { - Item_real_func::fix_length_and_dec(); - maybe_null= 1; + bool fix_length_and_dec() + { + if (Item_real_func::fix_length_and_dec()) + return TRUE; + maybe_null= 1; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -663,10 +673,12 @@ public: Item_func_glength(THD *thd, Item *a): Item_real_func(thd, a) {} double val_real(); const char *func_name() const { return "st_length"; } - void fix_length_and_dec() - { - Item_real_func::fix_length_and_dec(); - maybe_null= 1; + bool fix_length_and_dec() + { + if (Item_real_func::fix_length_and_dec()) + return TRUE; + maybe_null= 1; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -680,7 +692,7 @@ public: Item_func_srid(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "srid"; } - void fix_length_and_dec() { max_length= 10; maybe_null= 1; } + bool fix_length_and_dec() { max_length= 10; maybe_null= 1; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; diff --git a/sql/item_inetfunc.h b/sql/item_inetfunc.h index f19749df0af..670dce3da9f 100644 --- a/sql/item_inetfunc.h +++ b/sql/item_inetfunc.h @@ -30,12 +30,13 @@ public: Item_func_inet_aton(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "inet_aton"; } - void fix_length_and_dec() + bool fix_length_and_dec() { decimals= 0; max_length= 21; maybe_null= 1; unsigned_flag= 1; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -53,11 +54,12 @@ public: { } String* val_str(String* str); const char *func_name() const { return "inet_ntoa"; } - void fix_length_and_dec() + bool fix_length_and_dec() { decimals= 0; fix_length_and_charset(3 * 8 + 7, default_charset()); maybe_null= 1; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -122,11 +124,12 @@ public: virtual const char *func_name() const { return "inet6_aton"; } - virtual void fix_length_and_dec() + virtual bool fix_length_and_dec() { decimals= 0; fix_length_and_charset(16, &my_charset_bin); maybe_null= 1; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -151,7 +154,7 @@ public: virtual const char *func_name() const { return "inet6_ntoa"; } - virtual void fix_length_and_dec() + virtual bool fix_length_and_dec() { decimals= 0; @@ -161,6 +164,7 @@ public: fix_length_and_charset(8 * 4 + 7, default_charset()); maybe_null= 1; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 794831ae1a5..72aeac5b930 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -388,11 +388,13 @@ longlong Item_func_json_valid::val_int() } -void Item_func_json_exists::fix_length_and_dec() +bool Item_func_json_exists::fix_length_and_dec() { - Item_int_func::fix_length_and_dec(); + if (Item_int_func::fix_length_and_dec()) + return TRUE; maybe_null= 1; path.set_constant_flag(args[1]->const_item()); + return FALSE; } @@ -439,12 +441,13 @@ err_return: } -void Item_func_json_value::fix_length_and_dec() +bool Item_func_json_value::fix_length_and_dec() { collation.set(args[0]->collation); max_length= args[0]->max_length; path.set_constant_flag(args[1]->const_item()); maybe_null= 1; + return FALSE; } @@ -546,7 +549,7 @@ bool Item_func_json_query::check_and_get_value(json_engine_t *je, String *res, } -void Item_func_json_quote::fix_length_and_dec() +bool Item_func_json_quote::fix_length_and_dec() { collation.set(&my_charset_utf8mb4_bin); /* @@ -554,6 +557,7 @@ void Item_func_json_quote::fix_length_and_dec() of the argument turn into '\uXXXX\uXXXX', which is 12. */ max_length= args[0]->max_length * 12 + 2; + return FALSE; } @@ -581,12 +585,13 @@ String *Item_func_json_quote::val_str(String *str) } -void Item_func_json_unquote::fix_length_and_dec() +bool Item_func_json_unquote::fix_length_and_dec() { collation.set(&my_charset_utf8_general_ci, DERIVATION_COERCIBLE, MY_REPERTOIRE_ASCII); max_length= args[0]->max_length; maybe_null= 1; + return FALSE; } @@ -702,13 +707,14 @@ void Item_json_str_multipath::cleanup() } -void Item_func_json_extract::fix_length_and_dec() +bool Item_func_json_extract::fix_length_and_dec() { collation.set(args[0]->collation); max_length= args[0]->max_length * (arg_count - 1); mark_constant_paths(paths, args+1, arg_count-1); maybe_null= 1; + return FALSE; } @@ -937,14 +943,14 @@ double Item_func_json_extract::val_real() } -void Item_func_json_contains::fix_length_and_dec() +bool Item_func_json_contains::fix_length_and_dec() { a2_constant= args[1]->const_item(); a2_parsed= FALSE; maybe_null= 1; if (arg_count > 2) path.set_constant_flag(args[2]->const_item()); - Item_int_func::fix_length_and_dec(); + return Item_int_func::fix_length_and_dec(); } @@ -1190,13 +1196,13 @@ bool Item_func_json_contains_path::fix_fields(THD *thd, Item **ref) } -void Item_func_json_contains_path::fix_length_and_dec() +bool Item_func_json_contains_path::fix_length_and_dec() { ooa_constant= args[1]->const_item(); ooa_parsed= FALSE; maybe_null= 1; mark_constant_paths(paths, args+2, arg_count-2); - Item_int_func::fix_length_and_dec(); + return Item_int_func::fix_length_and_dec(); } @@ -1454,7 +1460,7 @@ append_null: } -void Item_func_json_array::fix_length_and_dec() +bool Item_func_json_array::fix_length_and_dec() { ulonglong char_length= 2; uint n_arg; @@ -1467,17 +1473,18 @@ void Item_func_json_array::fix_length_and_dec() DERIVATION_COERCIBLE, MY_REPERTOIRE_ASCII); tmp_val.set_charset(&my_charset_utf8_general_ci); max_length= 2; - return; + return FALSE; } if (agg_arg_charsets_for_string_result(collation, args, arg_count)) - return; + return TRUE; for (n_arg=0 ; n_arg < arg_count ; n_arg++) char_length+= args[n_arg]->max_char_length() + 4; fix_char_length_ulonglong(char_length); tmp_val.set_charset(collation.collation); + return FALSE; } @@ -1521,7 +1528,7 @@ err_return: } -void Item_func_json_array_append::fix_length_and_dec() +bool Item_func_json_array_append::fix_length_and_dec() { uint n_arg; ulonglong char_length; @@ -1536,6 +1543,7 @@ void Item_func_json_array_append::fix_length_and_dec() } fix_char_length_ulonglong(char_length); + return FALSE; } @@ -2122,11 +2130,12 @@ null_return: } -void Item_func_json_length::fix_length_and_dec() +bool Item_func_json_length::fix_length_and_dec() { if (arg_count > 1) path.set_constant_flag(args[1]->const_item()); maybe_null= 1; + return FALSE; } @@ -2266,11 +2275,12 @@ longlong Item_func_json_depth::val_int() } -void Item_func_json_type::fix_length_and_dec() +bool Item_func_json_type::fix_length_and_dec() { collation.set(&my_charset_utf8_general_ci); max_length= 12; maybe_null= 1; + return FALSE; } @@ -2323,7 +2333,7 @@ error: } -void Item_func_json_insert::fix_length_and_dec() +bool Item_func_json_insert::fix_length_and_dec() { uint n_arg; ulonglong char_length; @@ -2339,6 +2349,7 @@ void Item_func_json_insert::fix_length_and_dec() fix_char_length_ulonglong(char_length); maybe_null= 1; + return FALSE; } @@ -2583,13 +2594,14 @@ return_null: } -void Item_func_json_remove::fix_length_and_dec() +bool Item_func_json_remove::fix_length_and_dec() { collation.set(args[0]->collation); max_length= args[0]->max_length; mark_constant_paths(paths, args+1, arg_count-1); maybe_null= 1; + return FALSE; } @@ -2769,13 +2781,14 @@ null_return: } -void Item_func_json_keys::fix_length_and_dec() +bool Item_func_json_keys::fix_length_and_dec() { collation.set(args[0]->collation); max_length= args[0]->max_length; maybe_null= 1; if (arg_count > 1) path.set_constant_flag(args[1]->const_item()); + return FALSE; } @@ -2936,7 +2949,7 @@ bool Item_func_json_search::fix_fields(THD *thd, Item **ref) static const uint SQR_MAX_BLOB_WIDTH= (uint) sqrt(MAX_BLOB_WIDTH); -void Item_func_json_search::fix_length_and_dec() +bool Item_func_json_search::fix_length_and_dec() { collation.set(args[0]->collation); @@ -2958,6 +2971,7 @@ void Item_func_json_search::fix_length_and_dec() if (arg_count > 4) mark_constant_paths(paths, args+4, arg_count-4); maybe_null= 1; + return FALSE; } @@ -3135,11 +3149,12 @@ const char *Item_func_json_format::func_name() const } -void Item_func_json_format::fix_length_and_dec() +bool Item_func_json_format::fix_length_and_dec() { decimals= 0; max_length= args[0]->max_length; maybe_null= 1; + return FALSE; } diff --git a/sql/item_jsonfunc.h b/sql/item_jsonfunc.h index 927b60015b8..f331ee3b582 100644 --- a/sql/item_jsonfunc.h +++ b/sql/item_jsonfunc.h @@ -49,10 +49,12 @@ public: Item_func_json_valid(THD *thd, Item *json) : Item_int_func(thd, json) {} longlong val_int(); const char *func_name() const { return "json_valid"; } - void fix_length_and_dec() + bool fix_length_and_dec() { - Item_int_func::fix_length_and_dec(); + if (Item_int_func::fix_length_and_dec()) + return TRUE; maybe_null= 1; + return FALSE; } bool is_bool_type() { return true; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -71,7 +73,7 @@ public: Item_int_func(thd, js, i_path) {} const char *func_name() const { return "json_exists"; } bool is_bool_type() { return true; } - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } longlong val_int(); @@ -88,7 +90,7 @@ public: Item_func_json_value(THD *thd, Item *js, Item *i_path): Item_str_func(thd, js, i_path) {} const char *func_name() const { return "json_value"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); virtual bool check_and_get_value(json_engine_t *je, String *res, int *error); Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -117,7 +119,7 @@ protected: public: Item_func_json_quote(THD *thd, Item *s): Item_str_func(thd, s) {} const char *func_name() const { return "json_quote"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -132,7 +134,7 @@ protected: public: Item_func_json_unquote(THD *thd, Item *s): Item_str_func(thd, s) {} const char *func_name() const { return "json_unquote"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -165,7 +167,7 @@ public: Item_json_str_multipath(thd, list) {} const char *func_name() const { return "json_extract"; } enum Functype functype() const { return JSON_EXTRACT_FUNC; } - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); longlong val_int(); double val_real(); @@ -187,7 +189,7 @@ public: Item_func_json_contains(THD *thd, List &list): Item_int_func(thd, list) {} const char *func_name() const { return "json_contains"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); longlong val_int(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -209,7 +211,7 @@ public: Item_int_func(thd, list), tmp_paths(0) {} const char *func_name() const { return "json_contains_path"; } bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); void cleanup(); longlong val_int(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -229,7 +231,7 @@ public: Item_str_func(thd, list) {} String *val_str(String *); bool is_json_type() { return true; } - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "json_array"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -244,7 +246,7 @@ protected: public: Item_func_json_array_append(THD *thd, List &list): Item_json_str_multipath(thd, list) {} - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); uint get_n_paths() const { return arg_count/2; } const char *func_name() const { return "json_array_append"; } @@ -305,7 +307,7 @@ public: Item_func_json_length(THD *thd, List &list): Item_int_func(thd, list) {} const char *func_name() const { return "json_length"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); longlong val_int(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -332,7 +334,7 @@ protected: public: Item_func_json_type(THD *thd, Item *js): Item_str_func(thd, js) {} const char *func_name() const { return "json_type"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -349,7 +351,7 @@ public: Item_func_json_insert(bool i_mode, bool r_mode, THD *thd, List &list): Item_json_str_multipath(thd, list), mode_insert(i_mode), mode_replace(r_mode) {} - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); uint get_n_paths() const { return arg_count/2; } const char *func_name() const @@ -369,7 +371,7 @@ protected: public: Item_func_json_remove(THD *thd, List &list): Item_json_str_multipath(thd, list) {} - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); uint get_n_paths() const { return arg_count - 1; } const char *func_name() const { return "json_remove"; } @@ -388,7 +390,7 @@ public: Item_func_json_keys(THD *thd, List &list): Item_str_func(thd, list) {} const char *func_name() const { return "json_keys"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -412,7 +414,7 @@ public: Item_json_str_multipath(thd, list) {} const char *func_name() const { return "json_search"; } bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *); uint get_n_paths() const { return arg_count > 4 ? arg_count - 4 : 0; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -440,7 +442,7 @@ public: Item_str_func(thd, list), fmt(DETAILED) {} const char *func_name() const; - void fix_length_and_dec(); + bool fix_length_and_dec(); String *val_str(String *str); String *val_json(String *str); bool is_json_type() { return true; } diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index 3d543c8c390..87c766340c7 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -188,10 +188,11 @@ String *Item_func_sha::val_str_ascii(String *str) return 0; } -void Item_func_sha::fix_length_and_dec() +bool Item_func_sha::fix_length_and_dec() { // size of hex representation of hash fix_length_and_charset(MY_SHA1_HASH_SIZE * 2, default_charset()); + return FALSE; } String *Item_func_sha2::val_str_ascii(String *str) @@ -267,7 +268,7 @@ String *Item_func_sha2::val_str_ascii(String *str) } -void Item_func_sha2::fix_length_and_dec() +bool Item_func_sha2::fix_length_and_dec() { maybe_null= 1; max_length = 0; @@ -292,6 +293,7 @@ void Item_func_sha2::fix_length_and_dec() ER_THD(thd, ER_WRONG_PARAMETERS_TO_NATIVE_FCT), "sha2"); } + return FALSE; } /* Implementation of AES encryption routines */ @@ -344,23 +346,25 @@ String *Item_aes_crypt::val_str(String *str2) return 0; } -void Item_func_aes_encrypt::fix_length_and_dec() +bool Item_func_aes_encrypt::fix_length_and_dec() { max_length=my_aes_get_size(MY_AES_ECB, args[0]->max_length); what= ENCRYPTION_FLAG_ENCRYPT; + return FALSE; } -void Item_func_aes_decrypt::fix_length_and_dec() +bool Item_func_aes_decrypt::fix_length_and_dec() { max_length=args[0]->max_length; maybe_null= 1; what= ENCRYPTION_FLAG_DECRYPT; + return FALSE; } -void Item_func_to_base64::fix_length_and_dec() +bool Item_func_to_base64::fix_length_and_dec() { maybe_null= args[0]->maybe_null; collation.set(default_charset(), DERIVATION_COERCIBLE, MY_REPERTOIRE_ASCII); @@ -375,6 +379,7 @@ void Item_func_to_base64::fix_length_and_dec() DBUG_ASSERT(length > 0); fix_char_length_ulonglong((ulonglong) length - 1); } + return FALSE; } @@ -410,7 +415,7 @@ String *Item_func_to_base64::val_str_ascii(String *str) } -void Item_func_from_base64::fix_length_and_dec() +bool Item_func_from_base64::fix_length_and_dec() { if (args[0]->max_length > (uint) my_base64_decode_max_arg_length()) { @@ -422,6 +427,7 @@ void Item_func_from_base64::fix_length_and_dec() fix_char_length_ulonglong((ulonglong) length); } maybe_null= 1; // Can be NULL, e.g. in case of badly formed input string + return FALSE; } @@ -630,17 +636,18 @@ null: } -void Item_func_concat::fix_length_and_dec() +bool Item_func_concat::fix_length_and_dec() { ulonglong char_length= 0; if (agg_arg_charsets_for_string_result(collation, args, arg_count)) - return; + return TRUE; for (uint i=0 ; i < arg_count ; i++) char_length+= args[i]->max_char_length(); fix_char_length_ulonglong(char_length); + return FALSE; } /** @@ -992,12 +999,12 @@ null: } -void Item_func_concat_ws::fix_length_and_dec() +bool Item_func_concat_ws::fix_length_and_dec() { ulonglong char_length; if (agg_arg_charsets_for_string_result(collation, args, arg_count)) - return; + return TRUE; /* arg_count cannot be less than 2, @@ -1009,6 +1016,7 @@ void Item_func_concat_ws::fix_length_and_dec() char_length+= args[i]->max_char_length(); fix_char_length_ulonglong(char_length); + return FALSE; } @@ -1062,11 +1070,12 @@ String *Item_func_reverse::val_str(String *str) } -void Item_func_reverse::fix_length_and_dec() +bool Item_func_reverse::fix_length_and_dec() { agg_arg_charsets_for_string_result(collation, args, 1); DBUG_ASSERT(collation.collation != NULL); fix_char_length(args[0]->max_char_length()); + return FALSE; } /** @@ -1208,7 +1217,7 @@ null: } -void Item_func_replace::fix_length_and_dec() +bool Item_func_replace::fix_length_and_dec() { ulonglong char_length= (ulonglong) args[0]->max_char_length(); int diff=(int) (args[2]->max_char_length() - args[1]->max_char_length()); @@ -1219,8 +1228,9 @@ void Item_func_replace::fix_length_and_dec() } if (agg_arg_charsets_for_string_result_with_comparison(collation, args, 3)) - return; + return TRUE; fix_char_length_ulonglong(char_length); + return FALSE; } @@ -1232,13 +1242,14 @@ bool Item_func_regexp_replace::fix_fields(THD *thd, Item **ref) } -void Item_func_regexp_replace::fix_length_and_dec() +bool Item_func_regexp_replace::fix_length_and_dec() { if (agg_arg_charsets_for_string_result_with_comparison(collation, args, 3)) - return; + return TRUE; max_length= MAX_BLOB_WIDTH; re.init(collation.collation, 0); re.fix_owner(this, args[0], args[1]); + return FALSE; } @@ -1374,13 +1385,14 @@ bool Item_func_regexp_substr::fix_fields(THD *thd, Item **ref) } -void Item_func_regexp_substr::fix_length_and_dec() +bool Item_func_regexp_substr::fix_length_and_dec() { if (agg_arg_charsets_for_string_result_with_comparison(collation, args, 2)) - return; + return TRUE; fix_char_length(args[0]->max_char_length()); re.init(collation.collation, 0); re.fix_owner(this, args[0], args[1]); + return FALSE; } @@ -1488,16 +1500,17 @@ null: } -void Item_func_insert::fix_length_and_dec() +bool Item_func_insert::fix_length_and_dec() { ulonglong char_length; // Handle character set for args[0] and args[3]. if (agg_arg_charsets_for_string_result(collation, args, 2, 3)) - return; + return TRUE; char_length= ((ulonglong) args[0]->max_char_length() + (ulonglong) args[3]->max_char_length()); fix_char_length_ulonglong(char_length); + return FALSE; } @@ -1534,22 +1547,26 @@ String *Item_str_conv::val_str(String *str) } -void Item_func_lcase::fix_length_and_dec() +bool Item_func_lcase::fix_length_and_dec() { - agg_arg_charsets_for_string_result(collation, args, 1); + if (agg_arg_charsets_for_string_result(collation, args, 1)) + return TRUE; DBUG_ASSERT(collation.collation != NULL); multiply= collation.collation->casedn_multiply; converter= collation.collation->cset->casedn; fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * multiply); + return FALSE; } -void Item_func_ucase::fix_length_and_dec() +bool Item_func_ucase::fix_length_and_dec() { - agg_arg_charsets_for_string_result(collation, args, 1); + if (agg_arg_charsets_for_string_result(collation, args, 1)) + return TRUE; DBUG_ASSERT(collation.collation != NULL); multiply= collation.collation->caseup_multiply; converter= collation.collation->cset->caseup; fix_char_length_ulonglong((ulonglong) args[0]->max_char_length() * multiply); + return FALSE; } @@ -1592,11 +1609,13 @@ void Item_str_func::left_right_max_length() } -void Item_func_left::fix_length_and_dec() +bool Item_func_left::fix_length_and_dec() { - agg_arg_charsets_for_string_result(collation, args, 1); + if (agg_arg_charsets_for_string_result(collation, args, 1)) + return TRUE; DBUG_ASSERT(collation.collation != NULL); left_right_max_length(); + return FALSE; } @@ -1626,11 +1645,13 @@ String *Item_func_right::val_str(String *str) } -void Item_func_right::fix_length_and_dec() +bool Item_func_right::fix_length_and_dec() { - agg_arg_charsets_for_string_result(collation, args, 1); + if (agg_arg_charsets_for_string_result(collation, args, 1)) + return TRUE; DBUG_ASSERT(collation.collation != NULL); left_right_max_length(); + return FALSE; } @@ -1681,11 +1702,12 @@ String *Item_func_substr::val_str(String *str) } -void Item_func_substr::fix_length_and_dec() +bool Item_func_substr::fix_length_and_dec() { max_length=args[0]->max_length; - agg_arg_charsets_for_string_result(collation, args, 1); + if (agg_arg_charsets_for_string_result(collation, args, 1)) + return TRUE; DBUG_ASSERT(collation.collation != NULL); if (args[1]->const_item()) { @@ -1706,14 +1728,16 @@ void Item_func_substr::fix_length_and_dec() set_if_smaller(max_length,(uint) length); } max_length*= collation.collation->mbmaxlen; + return FALSE; } -void Item_func_substr_index::fix_length_and_dec() -{ +bool Item_func_substr_index::fix_length_and_dec() +{ if (agg_arg_charsets_for_string_result_with_comparison(collation, args, 2)) - return; + return TRUE; fix_char_length(args[0]->max_char_length()); + return FALSE; } @@ -2037,11 +2061,12 @@ String *Item_func_trim::val_str(String *str) return trimmed_value(res, (uint32) (ptr - res->ptr()), (uint32) (end - ptr)); } -void Item_func_trim::fix_length_and_dec() +bool Item_func_trim::fix_length_and_dec() { if (arg_count == 1) { - agg_arg_charsets_for_string_result(collation, args, 1); + if (agg_arg_charsets_for_string_result(collation, args, 1)) + return TRUE; DBUG_ASSERT(collation.collation != NULL); remove.set_charset(collation.collation); remove.set_ascii(" ",1); @@ -2052,9 +2077,10 @@ void Item_func_trim::fix_length_and_dec() // Note that we pass args[1] as the first item, and args[0] as the second. if (agg_arg_charsets_for_string_result_with_comparison(collation, &args[1], 2, -1)) - return; + return TRUE; } fix_char_length(args[0]->max_char_length()); + return FALSE; } void Item_func_trim::print(String *str, enum_query_type query_type) @@ -2194,7 +2220,7 @@ bool Item_func_encode::seed() return FALSE; } -void Item_func_encode::fix_length_and_dec() +bool Item_func_encode::fix_length_and_dec() { max_length=args[0]->max_length; maybe_null=args[0]->maybe_null || args[1]->maybe_null; @@ -2202,6 +2228,7 @@ void Item_func_encode::fix_length_and_dec() /* Precompute the seed state if the item is constant. */ seeded= args[1]->const_item() && (args[1]->result_type() == STRING_RESULT) && !seed(); + return FALSE; } String *Item_func_encode::val_str(String *str) @@ -2347,13 +2374,14 @@ bool Item_func_current_role::fix_fields(THD *thd, Item **ref) return 0; } -void Item_func_soundex::fix_length_and_dec() +bool Item_func_soundex::fix_length_and_dec() { uint32 char_length= args[0]->max_char_length(); agg_arg_charsets_for_string_result(collation, args, 1); DBUG_ASSERT(collation.collation != NULL); set_if_bigger(char_length, 4); fix_char_length(char_length); + return FALSE; } @@ -2529,7 +2557,7 @@ MY_LOCALE *Item_func_format::get_locale(Item *item) return lc; } -void Item_func_format::fix_length_and_dec() +bool Item_func_format::fix_length_and_dec() { uint32 char_length= args[0]->max_char_length(); uint32 max_sep_count= (char_length / 3) + (decimals ? 1 : 0) + /*sign*/1; @@ -2539,6 +2567,7 @@ void Item_func_format::fix_length_and_dec() locale= args[2]->basic_const_item() ? get_locale(args[2]) : NULL; else locale= &my_locale_en_US; /* Two arguments */ + return FALSE; } @@ -2652,13 +2681,13 @@ String *Item_func_format::val_str_ascii(String *str) } -void Item_func_elt::fix_length_and_dec() +bool Item_func_elt::fix_length_and_dec() { uint32 char_length= 0; decimals=0; if (agg_arg_charsets_for_string_result(collation, args + 1, arg_count - 1)) - return; + return TRUE; for (uint i= 1 ; i < arg_count ; i++) { @@ -2667,6 +2696,7 @@ void Item_func_elt::fix_length_and_dec() } fix_char_length(char_length); maybe_null=1; // NULL if wrong first arg + return FALSE; } @@ -2713,16 +2743,17 @@ String *Item_func_elt::val_str(String *str) } -void Item_func_make_set::fix_length_and_dec() +bool Item_func_make_set::fix_length_and_dec() { uint32 char_length= arg_count - 2; /* Separators */ if (agg_arg_charsets_for_string_result(collation, args + 1, arg_count - 1)) - return; + return TRUE; for (uint i=1 ; i < arg_count ; i++) char_length+= args[i]->max_char_length(); fix_char_length(char_length); + return FALSE; } @@ -2853,9 +2884,10 @@ inline String* alloc_buffer(String *res,String *str,String *tmp_value, } -void Item_func_repeat::fix_length_and_dec() +bool Item_func_repeat::fix_length_and_dec() { - agg_arg_charsets_for_string_result(collation, args, 1); + if (agg_arg_charsets_for_string_result(collation, args, 1)) + return TRUE; DBUG_ASSERT(collation.collation != NULL); if (args[1]->const_item()) { @@ -2877,6 +2909,7 @@ void Item_func_repeat::fix_length_and_dec() max_length= MAX_BLOB_WIDTH; maybe_null= 1; } + return FALSE; } /** @@ -2938,7 +2971,7 @@ err: } -void Item_func_space::fix_length_and_dec() +bool Item_func_space::fix_length_and_dec() { collation.set(default_charset(), DERIVATION_COERCIBLE, MY_REPERTOIRE_ASCII); if (args[0]->const_item()) @@ -2954,12 +2987,13 @@ void Item_func_space::fix_length_and_dec() if (count > INT_MAX32) count= INT_MAX32; fix_char_length_ulonglong(count); - return; + return FALSE; } end: max_length= MAX_BLOB_WIDTH; maybe_null= 1; + return FALSE; } @@ -3009,11 +3043,12 @@ err: } -void Item_func_binlog_gtid_pos::fix_length_and_dec() +bool Item_func_binlog_gtid_pos::fix_length_and_dec() { collation.set(system_charset_info); max_length= MAX_BLOB_WIDTH; maybe_null= 1; + return FALSE; } @@ -3049,11 +3084,11 @@ err: } -void Item_func_rpad::fix_length_and_dec() +bool Item_func_rpad::fix_length_and_dec() { // Handle character set for args[0] and args[2]. if (agg_arg_charsets_for_string_result(collation, &args[0], 2, 2)) - return; + return TRUE; if (args[1]->const_item()) { ulonglong char_length= (ulonglong) args[1]->val_int(); @@ -3071,6 +3106,7 @@ void Item_func_rpad::fix_length_and_dec() max_length= MAX_BLOB_WIDTH; maybe_null= 1; } + return FALSE; } @@ -3157,12 +3193,12 @@ String *Item_func_rpad::val_str(String *str) } -void Item_func_lpad::fix_length_and_dec() +bool Item_func_lpad::fix_length_and_dec() { // Handle character set for args[0] and args[2]. if (agg_arg_charsets_for_string_result(collation, &args[0], 2, 2)) - return; - + return TRUE; + if (args[1]->const_item()) { ulonglong char_length= (ulonglong) args[1]->val_int(); @@ -3180,6 +3216,7 @@ void Item_func_lpad::fix_length_and_dec() max_length= MAX_BLOB_WIDTH; maybe_null= 1; } + return FALSE; } @@ -3329,10 +3366,11 @@ String *Item_func_conv_charset::val_str(String *str) 0 : str; } -void Item_func_conv_charset::fix_length_and_dec() +bool Item_func_conv_charset::fix_length_and_dec() { DBUG_ASSERT(collation.derivation == DERIVATION_IMPLICIT); fix_char_length(args[0]->max_char_length()); + return FALSE; } void Item_func_conv_charset::print(String *str, enum_query_type query_type) @@ -3354,7 +3392,7 @@ String *Item_func_set_collation::val_str(String *str) return str; } -void Item_func_set_collation::fix_length_and_dec() +bool Item_func_set_collation::fix_length_and_dec() { CHARSET_INFO *set_collation; const char *colname; @@ -3365,8 +3403,8 @@ void Item_func_set_collation::fix_length_and_dec() MY_CS_BINSORT,MYF(0)); else { - if (!(set_collation= mysqld_collation_get_by_name(colname))) - return; + if (!(set_collation= mysqld_collation_get_by_name(colname))) + return TRUE; } if (!set_collation || @@ -3374,11 +3412,12 @@ void Item_func_set_collation::fix_length_and_dec() { my_error(ER_COLLATION_CHARSET_MISMATCH, MYF(0), colname, args[0]->collation.collation->csname); - return; + return TRUE; } collation.set(set_collation, DERIVATION_EXPLICIT, args[0]->collation.repertoire); max_length= args[0]->max_length; + return FALSE; } @@ -3437,7 +3476,7 @@ String *Item_func_collation::val_str(String *str) } -void Item_func_weight_string::fix_length_and_dec() +bool Item_func_weight_string::fix_length_and_dec() { CHARSET_INFO *cs= args[0]->collation.collation; collation.set(&my_charset_bin, args[0]->collation.derivation); @@ -3455,6 +3494,7 @@ void Item_func_weight_string::fix_length_and_dec() max_length= cs->coll->strnxfrmlen(cs, char_length * cs->mbmaxlen); } maybe_null= 1; + return FALSE; } @@ -3838,15 +3878,16 @@ String* Item_func_export_set::val_str(String* str) return str; } -void Item_func_export_set::fix_length_and_dec() +bool Item_func_export_set::fix_length_and_dec() { uint32 length= MY_MAX(args[1]->max_char_length(), args[2]->max_char_length()); uint32 sep_length= (arg_count > 3 ? args[3]->max_char_length() : 1); if (agg_arg_charsets_for_string_result(collation, args + 1, MY_MIN(4, arg_count) - 1)) - return; + return TRUE; fix_char_length(length * 64 + sep_length * 63); + return FALSE; } @@ -4246,12 +4287,13 @@ bool Item_func_dyncol_create::fix_fields(THD *thd, Item **ref) } -void Item_func_dyncol_create::fix_length_and_dec() +bool Item_func_dyncol_create::fix_length_and_dec() { max_length= MAX_BLOB_WIDTH; maybe_null= TRUE; collation.set(&my_charset_bin); decimals= 0; + return FALSE; } bool Item_func_dyncol_create::prepare_arguments(THD *thd, bool force_names_arg) diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index c9956ab364e..a908aba2d2f 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -145,9 +145,10 @@ class Item_func_md5 :public Item_str_ascii_checksum_func public: Item_func_md5(THD *thd, Item *a): Item_str_ascii_checksum_func(thd, a) {} String *val_str_ascii(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { fix_length_and_charset(32, default_charset()); + return FALSE; } const char *func_name() const { return "md5"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -160,7 +161,7 @@ class Item_func_sha :public Item_str_ascii_checksum_func public: Item_func_sha(THD *thd, Item *a): Item_str_ascii_checksum_func(thd, a) {} String *val_str_ascii(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "sha"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -172,7 +173,7 @@ public: Item_func_sha2(THD *thd, Item *a, Item *b) :Item_str_ascii_checksum_func(thd, a, b) {} String *val_str_ascii(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "sha2"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -185,7 +186,7 @@ public: Item_func_to_base64(THD *thd, Item *a) :Item_str_ascii_checksum_func(thd, a) {} String *val_str_ascii(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "to_base64"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -198,7 +199,7 @@ public: Item_func_from_base64(THD *thd, Item *a) :Item_str_binary_checksum_func(thd, a) { } String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "from_base64"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -225,7 +226,7 @@ class Item_func_aes_encrypt :public Item_aes_crypt public: Item_func_aes_encrypt(THD *thd, Item *a, Item *b) :Item_aes_crypt(thd, a, b) {} - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "aes_encrypt"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -236,7 +237,7 @@ class Item_func_aes_decrypt :public Item_aes_crypt public: Item_func_aes_decrypt(THD *thd, Item *a, Item *b): Item_aes_crypt(thd, a, b) {} - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "aes_decrypt"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -251,7 +252,7 @@ public: Item_func_concat(THD *thd, List &list): Item_str_func(thd, list) {} Item_func_concat(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "concat"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -263,11 +264,12 @@ public: Item_func_decode_histogram(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(system_charset_info); max_length= MAX_BLOB_WIDTH; maybe_null= 1; + return FALSE; } const char *func_name() const { return "decode_histogram"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -280,7 +282,7 @@ class Item_func_concat_ws :public Item_str_func public: Item_func_concat_ws(THD *thd, List &list): Item_str_func(thd, list) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "concat_ws"; } table_map not_null_tables() const { return 0; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -293,7 +295,7 @@ class Item_func_reverse :public Item_str_func public: Item_func_reverse(THD *thd, Item *a): Item_str_func(thd, a) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "reverse"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -307,7 +309,7 @@ public: Item_func_replace(THD *thd, Item *org, Item *find, Item *replace): Item_str_func(thd, org, find, replace) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "replace"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -333,7 +335,7 @@ public: } String *val_str(String *str); bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "regexp_replace"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return 0;} }; @@ -355,7 +357,7 @@ public: } String *val_str(String *str); bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "regexp_substr"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return 0; } }; @@ -369,7 +371,7 @@ public: Item *new_str): Item_str_func(thd, org, start, length, new_str) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "insert"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -393,7 +395,7 @@ class Item_func_lcase :public Item_str_conv public: Item_func_lcase(THD *thd, Item *item): Item_str_conv(thd, item) {} const char *func_name() const { return "lcase"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -403,7 +405,7 @@ class Item_func_ucase :public Item_str_conv public: Item_func_ucase(THD *thd, Item *item): Item_str_conv(thd, item) {} const char *func_name() const { return "ucase"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -415,7 +417,7 @@ class Item_func_left :public Item_str_func public: Item_func_left(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "left"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -428,7 +430,7 @@ class Item_func_right :public Item_str_func public: Item_func_right(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "right"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -442,7 +444,7 @@ public: Item_func_substr(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} Item_func_substr(THD *thd, Item *a, Item *b, Item *c): Item_str_func(thd, a, b, c) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "substr"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -456,7 +458,7 @@ public: Item_func_substr_index(THD *thd, Item *a,Item *b,Item *c): Item_str_func(thd, a, b, c) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "substring_index"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -488,7 +490,7 @@ public: Item_func_trim(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} Item_func_trim(THD *thd, Item *a): Item_str_func(thd, a) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "trim"; } void print(String *str, enum_query_type query_type); virtual const char *mode_name() const { return "both"; } @@ -546,12 +548,13 @@ public: Item_str_ascii_checksum_func(thd, a), alg(al), deflt(0) {} String *val_str_ascii(String *str); bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec() + bool fix_length_and_dec() { fix_length_and_charset((alg == 1 ? SCRAMBLED_PASSWORD_CHAR_LENGTH : SCRAMBLED_PASSWORD_CHAR_LENGTH_323), default_charset()); + return FALSE; } const char *func_name() const { return ((deflt || alg == 1) ? "password" : "old_password"); } @@ -572,11 +575,12 @@ public: Item_func_des_encrypt(THD *thd, Item *a, Item *b) :Item_str_binary_checksum_func(thd, a, b) {} String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { maybe_null=1; /* 9 = MAX ((8- (arg_len % 8)) + 1) */ max_length = args[0]->max_length + 9; + return FALSE; } const char *func_name() const { return "des_encrypt"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -592,13 +596,14 @@ public: Item_func_des_decrypt(THD *thd, Item *a, Item *b) :Item_str_binary_checksum_func(thd, a, b) {} String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { maybe_null=1; /* 9 = MAX ((8- (arg_len % 8)) + 1) */ max_length= args[0]->max_length; if (max_length >= 9U) max_length-= 9U; + return FALSE; } const char *func_name() const { return "des_decrypt"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -631,7 +636,7 @@ public: constructor_helper(); } String *val_str(String *); - void fix_length_and_dec() { maybe_null=1; max_length = 13; } + bool fix_length_and_dec() { maybe_null=1; max_length = 13; return FALSE; } const char *func_name() const { return "encrypt"; } bool check_vcol_func_processor(void *arg) { @@ -655,7 +660,7 @@ public: Item_func_encode(THD *thd, Item *a, Item *seed_arg): Item_str_binary_checksum_func(thd, a, seed_arg) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "encode"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -705,10 +710,11 @@ class Item_func_database :public Item_func_sysconst public: Item_func_database(THD *thd): Item_func_sysconst(thd) {} String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { max_length= MAX_FIELD_NAME * system_charset_info->mbmaxlen; maybe_null=1; + return FALSE; } const char *func_name() const { return "database"; } const char *fully_qualified_func_name() const { return "database()"; } @@ -733,10 +739,11 @@ public: return (null_value ? 0 : &str_value); } bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec() + bool fix_length_and_dec() { max_length= (uint32) (username_char_length + HOSTNAME_LENGTH + 1) * SYSTEM_CHARSET_MBMAXLEN; + return FALSE; } const char *func_name() const { return "user"; } const char *fully_qualified_func_name() const { return "user()"; } @@ -776,8 +783,11 @@ public: Item_func_current_role(THD *thd, Name_resolution_context *context_arg): Item_func_sysconst(thd), context(context_arg) {} bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec() - { max_length= (uint32) username_char_length * SYSTEM_CHARSET_MBMAXLEN; } + bool fix_length_and_dec() + { + max_length= (uint32) username_char_length * SYSTEM_CHARSET_MBMAXLEN; + return FALSE; + } int save_in_field(Field *field, bool no_conversions) { return save_str_value_in_field(field, &str_value); } const char *func_name() const { return "current_role"; } @@ -805,7 +815,7 @@ class Item_func_soundex :public Item_str_func public: Item_func_soundex(THD *thd, Item *a): Item_str_func(thd, a) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "soundex"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -819,7 +829,7 @@ public: double val_real(); longlong val_int(); String *val_str(String *str); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "elt"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -833,7 +843,7 @@ class Item_func_make_set :public Item_str_func public: Item_func_make_set(THD *thd, List &list): Item_str_func(thd, list) {} String *val_str(String *str); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "make_set"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -851,7 +861,7 @@ public: MY_LOCALE *get_locale(Item *item); String *val_str_ascii(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "format"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -867,9 +877,10 @@ public: Item_str_func(thd, list) { collation.set(cs); } String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { max_length= arg_count * 4; + return FALSE; } const char *func_name() const { return "char"; } void print(String *str, enum_query_type query_type); @@ -885,7 +896,7 @@ public: Item_func_repeat(THD *thd, Item *arg1, Item *arg2): Item_str_func(thd, arg1, arg2) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "repeat"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -897,7 +908,7 @@ class Item_func_space :public Item_str_func public: Item_func_space(THD *thd, Item *arg1): Item_str_func(thd, arg1) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "space"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -910,7 +921,7 @@ public: Item_func_binlog_gtid_pos(THD *thd, Item *arg1, Item *arg2): Item_str_func(thd, arg1, arg2) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "binlog_gtid_pos"; } bool check_vcol_func_processor(void *arg) { @@ -928,7 +939,7 @@ public: Item_func_rpad(THD *thd, Item *arg1, Item *arg2, Item *arg3): Item_str_func(thd, arg1, arg2, arg3) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "rpad"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -942,7 +953,7 @@ public: Item_func_lpad(THD *thd, Item *arg1, Item *arg2, Item *arg3): Item_str_func(thd, arg1, arg2, arg3) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "lpad"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -956,11 +967,12 @@ public: Item_str_func(thd, a, b, c) {} const char *func_name() const { return "conv"; } String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(default_charset()); max_length=64; maybe_null= 1; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -975,11 +987,12 @@ public: Item_str_ascii_checksum_func(thd, a) {} const char *func_name() const { return "hex"; } String *val_str_ascii(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(default_charset()); decimals=0; fix_char_length(args[0]->max_length * 2); + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -996,11 +1009,12 @@ public: } const char *func_name() const { return "unhex"; } String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(&my_charset_bin); decimals=0; max_length=(1+args[0]->max_length)/2; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1019,11 +1033,12 @@ public: Item_str_func(thd, a, b), is_min(is_min_arg) { maybe_null= 1; } String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(args[0]->collation); decimals=0; max_length= MAX_BLOB_WIDTH; + return FALSE; } }; @@ -1064,10 +1079,11 @@ public: tmp->set_charset(&my_charset_bin); return tmp; } - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(&my_charset_bin); max_length=args[0]->max_length; + return FALSE; } void print(String *str, enum_query_type query_type); const char *func_name() const { return "cast_as_binary"; } @@ -1084,11 +1100,12 @@ public: Item_load_file(THD *thd, Item *a): Item_str_func(thd, a) {} String *val_str(String *); const char *func_name() const { return "load_file"; } - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(&my_charset_bin, DERIVATION_COERCIBLE); maybe_null=1; max_length=MAX_BLOB_WIDTH; + return FALSE; } bool check_vcol_func_processor(void *arg) { @@ -1109,7 +1126,7 @@ class Item_func_export_set: public Item_str_func Item_func_export_set(THD *thd, Item *a, Item *b, Item* c, Item* d, Item* e): Item_str_func(thd, a, b, c, d, e) {} String *val_str(String *str); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "export_set"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1123,12 +1140,13 @@ public: Item_func_quote(THD *thd, Item *a): Item_str_func(thd, a) {} const char *func_name() const { return "quote"; } String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(args[0]->collation); ulonglong max_result_length= (ulonglong) args[0]->max_length * 2 + 2 * collation.collation->mbmaxlen; max_length= (uint32) MY_MIN(max_result_length, MAX_BLOB_WIDTH); + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1211,7 +1229,7 @@ public: return 1; return res; } - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const { return "convert"; } void print(String *str, enum_query_type query_type); Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -1224,7 +1242,7 @@ public: Item_func_set_collation(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); bool eq(const Item *item, bool binary_cmp) const; const char *func_name() const { return "collate"; } enum precedence precedence() const { return COLLATE_PRECEDENCE; } @@ -1245,11 +1263,12 @@ class Item_func_expr_str_metadata :public Item_str_func { public: Item_func_expr_str_metadata(THD *thd, Item *a): Item_str_func(thd, a) { } - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(system_charset_info); max_length= 64 * collation.collation->mbmaxlen; // should be enough maybe_null= 0; + return FALSE; }; table_map not_null_tables() const { return 0; } Item* propagate_equal_fields(THD *thd, const Context &ctx, COND_EQUAL *cond) @@ -1299,7 +1318,7 @@ public: } const char *func_name() const { return "weight_string"; } String *val_str(String *); - void fix_length_and_dec(); + bool fix_length_and_dec(); bool eq(const Item *item, bool binary_cmp) const { if (!Item_str_func::eq(item, binary_cmp)) @@ -1323,7 +1342,7 @@ public: Item_func_crc32(THD *thd, Item *a): Item_int_func(thd, a) { unsigned_flag= 1; } const char *func_name() const { return "crc32"; } - void fix_length_and_dec() { max_length=10; } + bool fix_length_and_dec() { max_length=10; return FALSE; } longlong val_int(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1335,7 +1354,7 @@ class Item_func_uncompressed_length : public Item_int_func public: Item_func_uncompressed_length(THD *thd, Item *a): Item_int_func(thd, a) {} const char *func_name() const{return "uncompressed_length";} - void fix_length_and_dec() { max_length=10; maybe_null= true; } + bool fix_length_and_dec() { max_length=10; maybe_null= true; return FALSE; } longlong val_int(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1353,7 +1372,11 @@ class Item_func_compress: public Item_str_binary_checksum_func public: Item_func_compress(THD *thd, Item *a) :Item_str_binary_checksum_func(thd, a) {} - void fix_length_and_dec(){max_length= (args[0]->max_length*120)/100+12;} + bool fix_length_and_dec() + { + max_length= (args[0]->max_length * 120) / 100 + 12; + return FALSE; + } const char *func_name() const{return "compress";} String *val_str(String *) ZLIB_DEPENDED_FUNCTION Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -1366,7 +1389,11 @@ class Item_func_uncompress: public Item_str_binary_checksum_func public: Item_func_uncompress(THD *thd, Item *a) :Item_str_binary_checksum_func(thd, a) {} - void fix_length_and_dec(){ maybe_null= 1; max_length= MAX_BLOB_WIDTH; } + bool fix_length_and_dec() + { + maybe_null= 1; max_length= MAX_BLOB_WIDTH; + return FALSE; + } const char *func_name() const{return "uncompress";} String *val_str(String *) ZLIB_DEPENDED_FUNCTION Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -1378,11 +1405,12 @@ class Item_func_uuid: public Item_str_func { public: Item_func_uuid(THD *thd): Item_str_func(thd) {} - void fix_length_and_dec() + bool fix_length_and_dec() { collation.set(system_charset_info, DERIVATION_COERCIBLE, MY_REPERTOIRE_ASCII); fix_char_length(MY_UUID_STRING_LENGTH); + return FALSE; } bool const_item() const { return false; } table_map used_tables() const { return RAND_TABLE_BIT; } @@ -1410,7 +1438,7 @@ protected: public: Item_func_dyncol_create(THD *thd, List &args, DYNCALL_CREATE_DEF *dfs); bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); const char *func_name() const{ return "column_create"; } String *val_str(String *); void print(String *str, enum_query_type query_type); @@ -1440,11 +1468,12 @@ public: {collation.set(DYNCOL_UTF);} const char *func_name() const{ return "column_json"; } String *val_str(String *); - void fix_length_and_dec() + bool fix_length_and_dec() { max_length= MAX_BLOB_WIDTH; maybe_null= 1; decimals= 0; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1459,8 +1488,8 @@ class Item_dyncol_get: public Item_str_func public: Item_dyncol_get(THD *thd, Item *str, Item *num): Item_str_func(thd, str, num) {} - void fix_length_and_dec() - { maybe_null= 1;; max_length= MAX_BLOB_WIDTH; } + bool fix_length_and_dec() + { maybe_null= 1;; max_length= MAX_BLOB_WIDTH; return FALSE; } /* Mark that collation can change between calls */ bool dynamic_result() { return 1; } @@ -1498,7 +1527,8 @@ class Item_func_dyncol_list: public Item_str_func public: Item_func_dyncol_list(THD *thd, Item *str): Item_str_func(thd, str) {collation.set(DYNCOL_UTF);} - void fix_length_and_dec() { maybe_null= 1; max_length= MAX_BLOB_WIDTH; }; + bool fix_length_and_dec() + { maybe_null= 1; max_length= MAX_BLOB_WIDTH; return FALSE; }; const char *func_name() const{ return "column_list"; } String *val_str(String *); Item *get_copy(THD *thd, MEM_ROOT *mem_root) diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index a8e83d298f6..8a9dd083911 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -305,10 +305,14 @@ bool Item_subselect::fix_fields(THD *thd_param, Item **ref) if (engine->cols() > max_columns) { my_error(ER_OPERAND_COLUMNS, MYF(0), 1); - + res= TRUE; + goto end; + } + if (fix_length_and_dec()) + { + res= TRUE; goto end; } - fix_length_and_dec(); } else goto end; @@ -905,9 +909,11 @@ Item::Type Item_subselect::type() const } -void Item_subselect::fix_length_and_dec() +bool Item_subselect::fix_length_and_dec() { - engine->fix_length_and_dec(0); + if (engine->fix_length_and_dec(0)) + return TRUE; + return FALSE; } @@ -1195,18 +1201,19 @@ enum_field_types Item_singlerow_subselect::field_type() const return engine->field_type(); } -void Item_singlerow_subselect::fix_length_and_dec() +bool Item_singlerow_subselect::fix_length_and_dec() { if ((max_columns= engine->cols()) == 1) { - engine->fix_length_and_dec(row= &value); + if (engine->fix_length_and_dec(row= &value)) + return TRUE; } else { if (!(row= (Item_cache**) current_thd->alloc(sizeof(Item_cache*) * - max_columns))) - return; - engine->fix_length_and_dec(row); + max_columns)) || + engine->fix_length_and_dec(row)) + return TRUE; value= *row; } unsigned_flag= value->unsigned_flag; @@ -1222,6 +1229,7 @@ void Item_singlerow_subselect::fix_length_and_dec() for (uint i= 0; i < max_columns; i++) row[i]->maybe_null= TRUE; } + return FALSE; } @@ -1506,7 +1514,7 @@ void Item_exists_subselect::init_length_and_dec() } -void Item_exists_subselect::fix_length_and_dec() +bool Item_exists_subselect::fix_length_and_dec() { DBUG_ENTER("Item_exists_subselect::fix_length_and_dec"); init_length_and_dec(); @@ -1514,14 +1522,17 @@ void Item_exists_subselect::fix_length_and_dec() We need only 1 row to determine existence (i.e. any EXISTS that is not an IN always requires LIMIT 1) */ + Item *item= new (thd->mem_root) Item_int(thd, (int32) 1); + if (!item) + DBUG_RETURN(TRUE); thd->change_item_tree(&unit->global_parameters()->select_limit, - new (thd->mem_root) Item_int(thd, (int32) 1)); + item); DBUG_PRINT("info", ("Set limit to 1")); - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } -void Item_in_subselect::fix_length_and_dec() +bool Item_in_subselect::fix_length_and_dec() { DBUG_ENTER("Item_in_subselect::fix_length_and_dec"); init_length_and_dec(); @@ -1529,7 +1540,7 @@ void Item_in_subselect::fix_length_and_dec() Unlike Item_exists_subselect, LIMIT 1 is set later for Item_in_subselect, depending on the chosen strategy. */ - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } @@ -3712,11 +3723,11 @@ bool subselect_single_select_engine::no_rows() } -/* - makes storage for the output values for the subquery and calcuates +/** + Makes storage for the output values for the subquery and calcuates their data and column types and their nullability. -*/ -void subselect_engine::set_row(List &item_list, Item_cache **row) +*/ +bool subselect_engine::set_row(List &item_list, Item_cache **row) { Item *sel_item; List_iterator_fast li(item_list); @@ -3732,44 +3743,51 @@ void subselect_engine::set_row(List &item_list, Item_cache **row) item->unsigned_flag= sel_item->unsigned_flag; maybe_null= sel_item->maybe_null; if (!(row[i]= Item_cache::get_cache(thd, sel_item, sel_item->cmp_type()))) - return; + return TRUE; row[i]->setup(thd, sel_item); //psergey-backport-timours: row[i]->store(sel_item); } if (item_list.elements > 1) cmp_type= res_type= ROW_RESULT; + return FALSE; } -void subselect_single_select_engine::fix_length_and_dec(Item_cache **row) +bool subselect_single_select_engine::fix_length_and_dec(Item_cache **row) { DBUG_ASSERT(row || select_lex->item_list.elements==1); - set_row(select_lex->item_list, row); + if (set_row(select_lex->item_list, row)) + return TRUE; item->collation.set(row[0]->collation); if (cols() != 1) maybe_null= 0; + return FALSE; } -void subselect_union_engine::fix_length_and_dec(Item_cache **row) +bool subselect_union_engine::fix_length_and_dec(Item_cache **row) { DBUG_ASSERT(row || unit->first_select()->item_list.elements==1); if (unit->first_select()->item_list.elements == 1) { - set_row(unit->types, row); + if (set_row(unit->types, row)) + return TRUE; item->collation.set(row[0]->collation); } else { bool maybe_null_saved= maybe_null; - set_row(unit->types, row); + if (set_row(unit->types, row)) + return TRUE; maybe_null= maybe_null_saved; } + return FALSE; } -void subselect_uniquesubquery_engine::fix_length_and_dec(Item_cache **row) +bool subselect_uniquesubquery_engine::fix_length_and_dec(Item_cache **row) { //this never should be called DBUG_ASSERT(0); + return FALSE; } int read_first_record_seq(JOIN_TAB *tab); @@ -5602,9 +5620,10 @@ void subselect_hash_sj_engine::print(String *str, enum_query_type query_type) )); } -void subselect_hash_sj_engine::fix_length_and_dec(Item_cache** row) +bool subselect_hash_sj_engine::fix_length_and_dec(Item_cache** row) { DBUG_ASSERT(FALSE); + return FALSE; } void subselect_hash_sj_engine::exclude() diff --git a/sql/item_subselect.h b/sql/item_subselect.h index 7e99e2c3075..bd6a1bdc498 100644 --- a/sql/item_subselect.h +++ b/sql/item_subselect.h @@ -196,7 +196,7 @@ public: const_item_cache= 0; forced_const= TRUE; } - virtual void fix_length_and_dec(); + virtual bool fix_length_and_dec(); table_map used_tables() const; table_map not_null_tables() const { return 0; } bool const_item() const; @@ -306,7 +306,7 @@ public: enum Item_result result_type() const; enum Item_result cmp_type() const; enum_field_types field_type() const; - void fix_length_and_dec(); + bool fix_length_and_dec(); uint cols(); Item* element_index(uint i) { return reinterpret_cast(row[i]); } @@ -403,7 +403,7 @@ public: my_decimal *val_decimal(my_decimal *); bool val_bool(); bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); void print(String *str, enum_query_type query_type); bool select_transformer(JOIN *join); void top_level_item() { abort_on_null=1; } @@ -626,7 +626,7 @@ public: void print(String *str, enum_query_type query_type); enum precedence precedence() const { return CMP_PRECEDENCE; } bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); void fix_after_pullout(st_select_lex *new_parent, Item **ref, bool merge); bool const_item() const { @@ -810,7 +810,7 @@ public: void set_thd(THD *thd_arg); THD * get_thd() { return thd ? thd : current_thd; } virtual int prepare(THD *)= 0; - virtual void fix_length_and_dec(Item_cache** row)= 0; + virtual bool fix_length_and_dec(Item_cache** row)= 0; /* Execute the engine @@ -854,7 +854,7 @@ public: virtual int get_identifier() { DBUG_ASSERT(0); return 0; } virtual void force_reexecution() {} protected: - void set_row(List &item_list, Item_cache **row); + bool set_row(List &item_list, Item_cache **row); }; @@ -870,7 +870,7 @@ public: Item_subselect *item); void cleanup(); int prepare(THD *thd); - void fix_length_and_dec(Item_cache** row); + bool fix_length_and_dec(Item_cache** row); int exec(); uint cols(); uint8 uncacheable(); @@ -905,7 +905,7 @@ public: Item_subselect *item); void cleanup(); int prepare(THD *); - void fix_length_and_dec(Item_cache** row); + bool fix_length_and_dec(Item_cache** row); int exec(); uint cols(); uint8 uncacheable(); @@ -963,7 +963,7 @@ public: ~subselect_uniquesubquery_engine(); void cleanup(); int prepare(THD *); - void fix_length_and_dec(Item_cache** row); + bool fix_length_and_dec(Item_cache** row); int exec(); uint cols() { return 1; } uint8 uncacheable() { return UNCACHEABLE_DEPENDENT_INJECTED; } @@ -1112,7 +1112,7 @@ public: TODO: factor out all these methods in a base subselect_index_engine class because all of them have dummy implementations and should never be called. */ - void fix_length_and_dec(Item_cache** row);//=>base class + bool fix_length_and_dec(Item_cache** row);//=>base class void exclude(); //=>base class //=>base class bool change_result(Item_subselect *si, @@ -1385,7 +1385,7 @@ public: uint count_columns_with_nulls_arg); int prepare(THD *thd_arg) { set_thd(thd_arg); return 0; } int exec(); - void fix_length_and_dec(Item_cache**) {} + bool fix_length_and_dec(Item_cache**) { return FALSE; } uint cols() { /* TODO: what is the correct value? */ return 1; } uint8 uncacheable() { return UNCACHEABLE_DEPENDENT; } void exclude() {} diff --git a/sql/item_sum.cc b/sql/item_sum.cc index cb150db3031..9e59ec4e373 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -1132,9 +1132,8 @@ Item_sum_num::fix_fields(THD *thd, Item **ref) result_field=0; max_length=float_length(decimals); null_value=1; - fix_length_and_dec(); - - if (check_sum_func(thd, ref)) + if (fix_length_and_dec() || + check_sum_func(thd, ref)) return TRUE; memcpy (orig_args, args, sizeof (Item *) * arg_count); @@ -1189,9 +1188,8 @@ Item_sum_hybrid::fix_fields(THD *thd, Item **ref) maybe_null= 1; result_field=0; null_value=1; - fix_length_and_dec(); - - if (check_sum_func(thd, ref)) + if (fix_length_and_dec() || + check_sum_func(thd, ref)) return TRUE; orig_args[0]= args[0]; @@ -1329,7 +1327,7 @@ void Item_sum_sum::clear() } -void Item_sum_sum::fix_length_and_dec() +bool Item_sum_sum::fix_length_and_dec() { DBUG_ENTER("Item_sum_sum::fix_length_and_dec"); maybe_null=null_value=1; @@ -1364,7 +1362,7 @@ void Item_sum_sum::fix_length_and_dec() "--ILLEGAL!!!--"), max_length, (int)decimals)); - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } @@ -1664,9 +1662,10 @@ void Item_sum_count::cleanup() /* Avgerage */ -void Item_sum_avg::fix_length_and_dec() +bool Item_sum_avg::fix_length_and_dec() { - Item_sum_sum::fix_length_and_dec(); + if (Item_sum_sum::fix_length_and_dec()) + return TRUE; maybe_null=null_value=1; prec_increment= current_thd->variables.div_precincrement; if (Item_sum_avg::result_type() == DECIMAL_RESULT) @@ -1686,6 +1685,7 @@ void Item_sum_avg::fix_length_and_dec() FLOATING_POINT_DECIMALS); max_length= MY_MIN(args[0]->max_length + prec_increment, float_length(decimals)); } + return FALSE; } @@ -1884,7 +1884,7 @@ Item_sum_variance::Item_sum_variance(THD *thd, Item_sum_variance *item): } -void Item_sum_variance::fix_length_and_dec() +bool Item_sum_variance::fix_length_and_dec() { DBUG_ENTER("Item_sum_variance::fix_length_and_dec"); maybe_null= null_value= 1; @@ -1919,7 +1919,7 @@ void Item_sum_variance::fix_length_and_dec() DBUG_ASSERT(0); } DBUG_PRINT("info", ("Type: REAL_RESULT (%d, %d)", max_length, (int)decimals)); - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } @@ -2989,13 +2989,13 @@ my_decimal *Item_sum_udf_int::val_decimal(my_decimal *dec) /** Default max_length is max argument length. */ -void Item_sum_udf_str::fix_length_and_dec() +bool Item_sum_udf_str::fix_length_and_dec() { DBUG_ENTER("Item_sum_udf_str::fix_length_and_dec"); max_length=0; for (uint i = 0; i < arg_count; i++) set_if_bigger(max_length,args[i]->max_length); - DBUG_VOID_RETURN; + DBUG_RETURN(FALSE); } diff --git a/sql/item_sum.h b/sql/item_sum.h index 5c446e5779d..5c8ff520259 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -457,7 +457,8 @@ public: */ virtual void update_field()=0; virtual bool keep_field_type(void) const { return 0; } - virtual void fix_length_and_dec() { maybe_null=1; null_value=1; } + virtual bool fix_length_and_dec() + { maybe_null=1; null_value=1; return FALSE; } virtual Item *result_item(THD *thd, Field *field); void update_used_tables (); @@ -752,8 +753,8 @@ public: my_decimal *val_decimal(my_decimal *); enum Item_result result_type () const { return INT_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_LONGLONG; } - void fix_length_and_dec() - { decimals=0; max_length=21; maybe_null=null_value=0; } + bool fix_length_and_dec() + { decimals=0; max_length=21; maybe_null=null_value=0; return FALSE; } }; @@ -764,7 +765,7 @@ protected: double sum; my_decimal dec_buffs[2]; uint curr_dec_buff; - void fix_length_and_dec(); + bool fix_length_and_dec(); public: Item_sum_sum(THD *thd, Item *item_par, bool distinct): @@ -888,7 +889,7 @@ public: :Item_sum_sum(thd, item), count(item->count), prec_increment(item->prec_increment) {} - void fix_length_and_dec(); + bool fix_length_and_dec(); enum Sumfunctype sum_func () const { return has_with_distinct() ? AVG_DISTINCT_FUNC : AVG_FUNC; @@ -948,7 +949,7 @@ But, this falls prey to catastrophic cancellation. Instead, use the recurrence class Item_sum_variance : public Item_sum_num { - void fix_length_and_dec(); + bool fix_length_and_dec(); public: double recurrence_m, recurrence_s; /* Used in recurrence relation. */ @@ -1110,8 +1111,11 @@ public: longlong val_int(); void reset_field(); void update_field(); - void fix_length_and_dec() - { decimals= 0; max_length=21; unsigned_flag= 1; maybe_null= null_value= 0; } + bool fix_length_and_dec() + { + decimals= 0; max_length=21; unsigned_flag= 1; maybe_null= null_value= 0; + return FALSE; + } void cleanup() { bits= reset_bits; @@ -1401,7 +1405,7 @@ class Item_sum_udf_float :public Item_udf_sum enum Item_result result_type () const { return REAL_RESULT; } enum Item_result cmp_type () const { return REAL_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_DOUBLE; } - void fix_length_and_dec() { fix_num_length_and_dec(); } + bool fix_length_and_dec() { fix_num_length_and_dec(); return FALSE; } Item *copy_or_same(THD* thd); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1424,7 +1428,7 @@ public: my_decimal *val_decimal(my_decimal *); enum Item_result result_type () const { return INT_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_LONGLONG; } - void fix_length_and_dec() { decimals=0; max_length=21; } + bool fix_length_and_dec() { decimals=0; max_length=21; return FALSE; } Item *copy_or_same(THD* thd); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1466,7 +1470,7 @@ public: my_decimal *val_decimal(my_decimal *dec); enum Item_result result_type () const { return STRING_RESULT; } enum_field_types field_type() const { return string_field_type(); } - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *copy_or_same(THD* thd); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1488,7 +1492,7 @@ public: my_decimal *val_decimal(my_decimal *); enum Item_result result_type () const { return DECIMAL_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_NEWDECIMAL; } - void fix_length_and_dec() { fix_num_length_and_dec(); } + bool fix_length_and_dec() { fix_num_length_and_dec(); return FALSE; } Item *copy_or_same(THD* thd); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -1563,7 +1567,7 @@ public: double val_real() { DBUG_ASSERT(fixed == 1); null_value=1; return 0.0; } longlong val_int() { DBUG_ASSERT(fixed == 1); null_value=1; return 0; } enum Item_result result_type () const { return STRING_RESULT; } - void fix_length_and_dec() { maybe_null=1; max_length=0; } + bool fix_length_and_dec() { maybe_null=1; max_length=0; return FALSE; } enum Sumfunctype sum_func () const { return UDF_SUM_FUNC; } void clear() {} bool add() { return 0; } diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 87c6489189c..bb75ad7c28c 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -953,7 +953,7 @@ longlong Item_func_month::val_int() } -void Item_func_monthname::fix_length_and_dec() +bool Item_func_monthname::fix_length_and_dec() { THD* thd= current_thd; CHARSET_INFO *cs= thd->variables.collation_connection; @@ -961,7 +961,8 @@ void Item_func_monthname::fix_length_and_dec() collation.set(cs, DERIVATION_COERCIBLE, locale->repertoire()); decimals=0; max_length= locale->max_month_name_length * collation.collation->mbmaxlen; - maybe_null=1; + maybe_null=1; + return FALSE; } @@ -1101,7 +1102,7 @@ longlong Item_func_weekday::val_int() odbc_type) + MY_TEST(odbc_type); } -void Item_func_dayname::fix_length_and_dec() +bool Item_func_dayname::fix_length_and_dec() { THD* thd= current_thd; CHARSET_INFO *cs= thd->variables.collation_connection; @@ -1109,7 +1110,8 @@ void Item_func_dayname::fix_length_and_dec() collation.set(cs, DERIVATION_COERCIBLE, locale->repertoire()); decimals=0; max_length= locale->max_day_name_length * collation.collation->mbmaxlen; - maybe_null=1; + maybe_null=1; + return FALSE; } @@ -1476,7 +1478,7 @@ bool get_interval_value(Item *args,interval_type int_type, INTERVAL *interval) } -void Item_temporal_func::fix_length_and_dec() +bool Item_temporal_func::fix_length_and_dec() { uint char_length= mysql_temporal_int_part_length(field_type()); /* @@ -1502,6 +1504,7 @@ void Item_temporal_func::fix_length_and_dec() DERIVATION_COERCIBLE : DERIVATION_NUMERIC, MY_REPERTOIRE_ASCII); fix_char_length(char_length); + return FALSE; } String *Item_temporal_func::val_str(String *str) @@ -1873,7 +1876,7 @@ overflow: return 0; } -void Item_func_date_format::fix_length_and_dec() +bool Item_func_date_format::fix_length_and_dec() { THD* thd= current_thd; locale= thd->variables.lc_time_names; @@ -1904,6 +1907,7 @@ void Item_func_date_format::fix_length_and_dec() set_if_smaller(max_length,MAX_BLOB_WIDTH); } maybe_null=1; // If wrong date + return FALSE; } @@ -2050,13 +2054,13 @@ null_date: } -void Item_func_from_unixtime::fix_length_and_dec() -{ +bool Item_func_from_unixtime::fix_length_and_dec() +{ THD *thd= current_thd; thd->time_zone_used= 1; tz= thd->variables.time_zone; decimals= args[0]->decimals; - Item_temporal_func::fix_length_and_dec(); + return Item_temporal_func::fix_length_and_dec(); } @@ -2083,10 +2087,10 @@ bool Item_func_from_unixtime::get_date(MYSQL_TIME *ltime, } -void Item_func_convert_tz::fix_length_and_dec() +bool Item_func_convert_tz::fix_length_and_dec() { decimals= args[0]->temporal_precision(MYSQL_TYPE_DATETIME); - Item_temporal_func::fix_length_and_dec(); + return Item_temporal_func::fix_length_and_dec(); } @@ -2135,7 +2139,7 @@ void Item_func_convert_tz::cleanup() } -void Item_date_add_interval::fix_length_and_dec() +bool Item_date_add_interval::fix_length_and_dec() { enum_field_types arg0_field_type; @@ -2190,7 +2194,7 @@ void Item_date_add_interval::fix_length_and_dec() } else decimals= MY_MAX(args[0]->temporal_precision(MYSQL_TYPE_DATETIME), interval_dec); - Item_temporal_func::fix_length_and_dec(); + return Item_temporal_func::fix_length_and_dec(); } @@ -2261,7 +2265,7 @@ void Item_extract::print(String *str, enum_query_type query_type) str->append(')'); } -void Item_extract::fix_length_and_dec() +bool Item_extract::fix_length_and_dec() { maybe_null=1; // If wrong date switch (int_type) { @@ -2287,6 +2291,7 @@ void Item_extract::fix_length_and_dec() case INTERVAL_SECOND_MICROSECOND: set_time_length(8); break; // ssffffff case INTERVAL_LAST: DBUG_ASSERT(0); break; /* purecov: deadcode */ } + return FALSE; } @@ -2548,7 +2553,7 @@ end: } -void Item_char_typecast::fix_length_and_dec() +bool Item_char_typecast::fix_length_and_dec() { uint32 char_length; /* @@ -2594,6 +2599,7 @@ void Item_char_typecast::fix_length_and_dec() (cast_cs == &my_charset_bin ? 1 : args[0]->collation.collation->mbmaxlen)); max_length= char_length * cast_cs->mbmaxlen; + return FALSE; } @@ -2682,7 +2688,7 @@ err: } -void Item_func_add_time::fix_length_and_dec() +bool Item_func_add_time::fix_length_and_dec() { enum_field_types arg0_field_type; decimals= MY_MAX(args[0]->decimals, args[1]->decimals); @@ -2714,7 +2720,7 @@ void Item_func_add_time::fix_length_and_dec() decimals= MY_MAX(args[0]->temporal_precision(MYSQL_TYPE_TIME), args[1]->temporal_precision(MYSQL_TYPE_TIME)); } - Item_temporal_func::fix_length_and_dec(); + return Item_temporal_func::fix_length_and_dec(); } /** @@ -3203,10 +3209,10 @@ get_date_time_result_type(const char *format, uint length) } -void Item_func_str_to_date::fix_length_and_dec() +bool Item_func_str_to_date::fix_length_and_dec() { if (agg_arg_charsets(collation, args, 2, MY_COLL_ALLOW_CONV, 1)) - return; + return TRUE; if (collation.collation->mbminlen > 1) { #if MYSQL_VERSION_ID > 50500 @@ -3249,7 +3255,7 @@ void Item_func_str_to_date::fix_length_and_dec() } } cached_timestamp_type= mysql_type_to_time_type(field_type()); - Item_temporal_func::fix_length_and_dec(); + return Item_temporal_func::fix_length_and_dec(); } diff --git a/sql/item_timefunc.h b/sql/item_timefunc.h index 354a54a5c1a..c983e1a6f8a 100644 --- a/sql/item_timefunc.h +++ b/sql/item_timefunc.h @@ -49,9 +49,10 @@ public: Item_func_period_add(THD *thd, Item *a, Item *b): Item_int_func(thd, a, b) {} longlong val_int(); const char *func_name() const { return "period_add"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { max_length=6*MY_CHARSET_BIN_MB_MAXLEN; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -64,10 +65,11 @@ public: Item_func_period_diff(THD *thd, Item *a, Item *b): Item_int_func(thd, a, b) {} longlong val_int(); const char *func_name() const { return "period_diff"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; max_length=6*MY_CHARSET_BIN_MB_MAXLEN; + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -80,11 +82,12 @@ public: Item_func_to_days(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "to_days"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; max_length=6*MY_CHARSET_BIN_MB_MAXLEN; - maybe_null=1; + maybe_null=1; + return FALSE; } enum_monotonicity_info get_monotonicity_info() const; longlong val_int_endpoint(bool left_endp, bool *incl_endp); @@ -105,11 +108,12 @@ public: Item_func_to_seconds(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "to_seconds"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; max_length=6*MY_CHARSET_BIN_MB_MAXLEN; maybe_null= 1; + return FALSE; } enum_monotonicity_info get_monotonicity_info() const; longlong val_int_endpoint(bool left_endp, bool *incl_endp); @@ -131,11 +135,12 @@ public: Item_func_dayofmonth(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "dayofmonth"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; max_length=2*MY_CHARSET_BIN_MB_MAXLEN; - maybe_null=1; + maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -167,11 +172,12 @@ public: const char *func_name() const { return "month"; } enum Item_result result_type () const { return INT_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_LONGLONG; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals= 0; fix_char_length(2); maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -191,7 +197,7 @@ public: Item_func_monthname(THD *thd, Item *a): Item_str_func(thd, a) {} const char *func_name() const { return "monthname"; } String *val_str(String *str); - void fix_length_and_dec(); + bool fix_length_and_dec(); bool check_partition_func_processor(void *int_arg) {return TRUE;} bool check_valid_arguments_processor(void *int_arg) { @@ -212,11 +218,12 @@ public: Item_func_dayofyear(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "dayofyear"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals= 0; fix_char_length(3); maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -235,11 +242,12 @@ public: Item_func_hour(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "hour"; } - void fix_length_and_dec() + bool fix_length_and_dec() { decimals=0; max_length=2*MY_CHARSET_BIN_MB_MAXLEN; maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -258,11 +266,12 @@ public: Item_func_minute(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "minute"; } - void fix_length_and_dec() + bool fix_length_and_dec() { decimals=0; max_length=2*MY_CHARSET_BIN_MB_MAXLEN; maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -281,11 +290,12 @@ public: Item_func_quarter(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "quarter"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; max_length=1*MY_CHARSET_BIN_MB_MAXLEN; maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -304,11 +314,12 @@ public: Item_func_second(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "second"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; max_length=2*MY_CHARSET_BIN_MB_MAXLEN; maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -328,11 +339,12 @@ public: Item_func_week(THD *thd, Item *a, Item *b): Item_int_func(thd, a, b) {} longlong val_int(); const char *func_name() const { return "week"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; max_length=2*MY_CHARSET_BIN_MB_MAXLEN; maybe_null=1; + return FALSE; } bool check_vcol_func_processor(void *arg) { @@ -354,11 +366,12 @@ public: Item_func_yearweek(THD *thd, Item *a, Item *b): Item_int_func(thd, a, b) {} longlong val_int(); const char *func_name() const { return "yearweek"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; max_length=6*MY_CHARSET_BIN_MB_MAXLEN; maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -379,11 +392,12 @@ public: const char *func_name() const { return "year"; } enum_monotonicity_info get_monotonicity_info() const; longlong val_int_endpoint(bool left_endp, bool *incl_endp); - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; max_length=4*MY_CHARSET_BIN_MB_MAXLEN; maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -416,11 +430,12 @@ public: } enum Item_result result_type () const { return INT_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_LONGLONG; } - void fix_length_and_dec() + bool fix_length_and_dec() { decimals= 0; fix_char_length(1); maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -441,7 +456,7 @@ class Item_func_dayname :public Item_func_weekday String *val_str(String *str); enum Item_result result_type () const { return STRING_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_VARCHAR; } - void fix_length_and_dec(); + bool fix_length_and_dec(); bool check_partition_func_processor(void *int_arg) {return TRUE;} bool check_vcol_func_processor(void *arg) { @@ -457,7 +472,7 @@ protected: public: Item_func_seconds_hybrid(THD *thd): Item_func_numhybrid(thd) {} Item_func_seconds_hybrid(THD *thd, Item *a): Item_func_numhybrid(thd, a) {} - void fix_length_and_dec() + bool fix_length_and_dec() { if (arg_count) decimals= args[0]->temporal_precision(arg0_expected_type()); @@ -465,6 +480,7 @@ public: max_length=17 + (decimals ? decimals + 1 : 0); maybe_null= true; set_handler_by_result_type(decimals ? DECIMAL_RESULT : INT_RESULT); + return FALSE; } double real_op() { DBUG_ASSERT(0); return 0; } String *str_op(String *str) { DBUG_ASSERT(0); return 0; } @@ -556,7 +572,7 @@ public: save_date_in_field(field); } #endif - void fix_length_and_dec(); + bool fix_length_and_dec(); }; @@ -834,7 +850,7 @@ public: String *val_str(String *str); const char *func_name() const { return is_time_format ? "time_format" : "date_format"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); uint format_length(const String *format); bool eq(const Item *item, bool binary_cmp) const; bool check_vcol_func_processor(void *arg) @@ -854,7 +870,7 @@ class Item_func_from_unixtime :public Item_datetimefunc public: Item_func_from_unixtime(THD *thd, Item *a): Item_datetimefunc(thd, a) {} const char *func_name() const { return "from_unixtime"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); bool get_date(MYSQL_TIME *res, ulonglong fuzzy_date); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } @@ -889,7 +905,7 @@ class Item_func_convert_tz :public Item_datetimefunc Item_func_convert_tz(THD *thd, Item *a, Item *b, Item *c): Item_datetimefunc(thd, a, b, c), from_tz_cached(0), to_tz_cached(0) {} const char *func_name() const { return "convert_tz"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); bool get_date(MYSQL_TIME *res, ulonglong fuzzy_date); void cleanup(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -902,10 +918,10 @@ class Item_func_sec_to_time :public Item_timefunc public: Item_func_sec_to_time(THD *thd, Item *item): Item_timefunc(thd, item) {} bool get_date(MYSQL_TIME *res, ulonglong fuzzy_date); - void fix_length_and_dec() + bool fix_length_and_dec() { decimals= MY_MIN(args[0]->decimals, TIME_SECOND_PART_DIGITS); - Item_timefunc::fix_length_and_dec(); + return Item_timefunc::fix_length_and_dec(); } const char *func_name() const { return "sec_to_time"; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -923,7 +939,7 @@ public: Item_temporal_hybrid_func(thd, a, b),int_type(type_arg), date_sub_interval(neg_arg) {} const char *func_name() const { return "date_add_interval"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); bool get_date(MYSQL_TIME *res, ulonglong fuzzy_date); bool eq(const Item *item, bool binary_cmp) const; void print(String *str, enum_query_type query_type); @@ -991,7 +1007,7 @@ class Item_extract :public Item_int_func longlong val_int(); enum Functype functype() const { return EXTRACT_FUNC; } const char *func_name() const { return "extract"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); bool eq(const Item *item, bool binary_cmp) const; void print(String *str, enum_query_type query_type); bool check_partition_func_processor(void *int_arg) {return FALSE;} @@ -1063,7 +1079,7 @@ public: bool eq(const Item *item, bool binary_cmp) const; const char *func_name() const { return "cast_as_char"; } String *val_str(String *a); - void fix_length_and_dec(); + bool fix_length_and_dec(); void print(String *str, enum_query_type query_type); bool need_parentheses_in_default() { return true; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -1077,11 +1093,11 @@ public: Item_temporal_typecast(THD *thd, Item *a): Item_temporal_func(thd, a) {} virtual const char *cast_type() const = 0; void print(String *str, enum_query_type query_type); - void fix_length_and_dec() + bool fix_length_and_dec() { if (decimals == NOT_FIXED_DEC) decimals= args[0]->temporal_precision(field_type()); - Item_temporal_func::fix_length_and_dec(); + return Item_temporal_func::fix_length_and_dec(); } }; @@ -1148,7 +1164,7 @@ public: Item_func_add_time(THD *thd, Item *a, Item *b, bool type_arg, bool neg_arg): Item_temporal_hybrid_func(thd, a, b), is_date(type_arg) { sign= neg_arg ? -1 : 1; } - void fix_length_and_dec(); + bool fix_length_and_dec(); bool get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date); void print(String *str, enum_query_type query_type); const char *func_name() const { return "add_time"; } @@ -1161,11 +1177,11 @@ class Item_func_timediff :public Item_timefunc public: Item_func_timediff(THD *thd, Item *a, Item *b): Item_timefunc(thd, a, b) {} const char *func_name() const { return "timediff"; } - void fix_length_and_dec() + bool fix_length_and_dec() { decimals= MY_MAX(args[0]->temporal_precision(MYSQL_TYPE_TIME), args[1]->temporal_precision(MYSQL_TYPE_TIME)); - Item_timefunc::fix_length_and_dec(); + return Item_timefunc::fix_length_and_dec(); } bool get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date); Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -1178,10 +1194,10 @@ public: Item_func_maketime(THD *thd, Item *a, Item *b, Item *c): Item_timefunc(thd, a, b, c) {} - void fix_length_and_dec() + bool fix_length_and_dec() { decimals= MY_MIN(args[2]->decimals, TIME_SECOND_PART_DIGITS); - Item_timefunc::fix_length_and_dec(); + return Item_timefunc::fix_length_and_dec(); } const char *func_name() const { return "maketime"; } bool get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date); @@ -1196,10 +1212,11 @@ public: Item_func_microsecond(THD *thd, Item *a): Item_int_func(thd, a) {} longlong val_int(); const char *func_name() const { return "microsecond"; } - void fix_length_and_dec() - { + bool fix_length_and_dec() + { decimals=0; maybe_null=1; + return FALSE; } bool check_partition_func_processor(void *int_arg) {return FALSE;} bool check_vcol_func_processor(void *arg) { return FALSE;} @@ -1220,10 +1237,11 @@ public: Item_int_func(thd, a, b), int_type(type_arg) {} const char *func_name() const { return "timestampdiff"; } longlong val_int(); - void fix_length_and_dec() + bool fix_length_and_dec() { decimals=0; maybe_null=1; + return FALSE; } virtual void print(String *str, enum_query_type query_type); Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -1245,11 +1263,12 @@ public: {} String *val_str_ascii(String *str); const char *func_name() const { return "get_format"; } - void fix_length_and_dec() + bool fix_length_and_dec() { maybe_null= 1; decimals=0; fix_length_and_charset(17, default_charset()); + return FALSE; } virtual void print(String *str, enum_query_type query_type); Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -1271,7 +1290,7 @@ public: {} bool get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date); const char *func_name() const { return "str_to_date"; } - void fix_length_and_dec(); + bool fix_length_and_dec(); Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; diff --git a/sql/item_windowfunc.cc b/sql/item_windowfunc.cc index 52738bfab87..7cb07af440b 100644 --- a/sql/item_windowfunc.cc +++ b/sql/item_windowfunc.cc @@ -106,7 +106,8 @@ Item_window_func::fix_fields(THD *thd, Item **ref) with_window_func= true; with_sum_func= false; - fix_length_and_dec(); + if (fix_length_and_dec()) + return TRUE; max_length= window_func()->max_length; maybe_null= window_func()->maybe_null; @@ -268,7 +269,8 @@ bool Item_sum_hybrid_simple::fix_fields(THD *thd, Item **ref) maybe_null= 1; result_field=0; null_value=1; - fix_length_and_dec(); + if (fix_length_and_dec()) + return TRUE; if (check_sum_func(thd, ref)) return TRUE; diff --git a/sql/item_windowfunc.h b/sql/item_windowfunc.h index a1ef6854288..e5afb9e1555 100644 --- a/sql/item_windowfunc.h +++ b/sql/item_windowfunc.h @@ -516,10 +516,11 @@ class Item_sum_percent_rank: public Item_sum_window_with_row_count enum Item_result result_type () const { return REAL_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_DOUBLE; } - void fix_length_and_dec() + bool fix_length_and_dec() { decimals = 10; // TODO-cvicentiu find out how many decimals the standard // requires. + return FALSE; } void setup_window_func(THD *thd, Window_spec *window_spec); @@ -602,10 +603,11 @@ class Item_sum_cume_dist: public Item_sum_window_with_row_count enum Item_result result_type () const { return REAL_RESULT; } enum_field_types field_type() const { return MYSQL_TYPE_DOUBLE; } - void fix_length_and_dec() + bool fix_length_and_dec() { decimals = 10; // TODO-cvicentiu find out how many decimals the standard // requires. + return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) @@ -950,9 +952,10 @@ public: void split_sum_func(THD *thd, Ref_ptr_array ref_pointer_array, List &fields, uint flags); - void fix_length_and_dec() + bool fix_length_and_dec() { decimals = window_func()->decimals; + return FALSE; } const char* func_name() const { return "WF"; } diff --git a/sql/item_xmlfunc.cc b/sql/item_xmlfunc.cc index af62a94afd0..738047fea9b 100644 --- a/sql/item_xmlfunc.cc +++ b/sql/item_xmlfunc.cc @@ -222,13 +222,14 @@ public: return str; } enum Item_result result_type () const { return STRING_RESULT; } - void fix_length_and_dec() + bool fix_length_and_dec() { max_length= MAX_BLOB_WIDTH; collation.collation= pxml->charset(); // To avoid premature evaluation, mark all nodeset functions as non-const. used_tables_cache= RAND_TABLE_BIT; const_item_cache= false; + return FALSE; } const char *func_name() const { return "nodeset"; } bool check_vcol_func_processor(void *arg) @@ -456,7 +457,7 @@ public: Item_nodeset_func(thd, pxml), string_cache(str_arg) { } String *val_nodeset(String *res) { return string_cache; } - void fix_length_and_dec() { max_length= MAX_BLOB_WIDTH; } + bool fix_length_and_dec() { max_length= MAX_BLOB_WIDTH; return FALSE; } Item *get_copy(THD *thd, MEM_ROOT *mem_root) { return get_item_copy(thd, mem_root, this); } }; @@ -470,7 +471,7 @@ public: Item_func_xpath_position(THD *thd, Item *a, String *p): Item_int_func(thd, a), pxml(p) {} const char *func_name() const { return "xpath_position"; } - void fix_length_and_dec() { max_length=10; } + bool fix_length_and_dec() { max_length=10; return FALSE; } longlong val_int() { String *flt= args[0]->val_nodeset(&tmp_value); @@ -491,7 +492,7 @@ public: Item_func_xpath_count(THD *thd, Item *a, String *p): Item_int_func(thd, a), pxml(p) {} const char *func_name() const { return "xpath_count"; } - void fix_length_and_dec() { max_length=10; } + bool fix_length_and_dec() { max_length=10; return FALSE; } longlong val_int() { uint predicate_supplied_context_size; @@ -2709,10 +2710,10 @@ my_xpath_parse(MY_XPATH *xpath, const char *str, const char *strend) } -void Item_xml_str_func::fix_length_and_dec() +bool Item_xml_str_func::fix_length_and_dec() { max_length= MAX_BLOB_WIDTH; - agg_arg_charsets_for_comparison(collation, args, arg_count); + return agg_arg_charsets_for_comparison(collation, args, arg_count); } diff --git a/sql/item_xmlfunc.h b/sql/item_xmlfunc.h index d281e076ec8..c46365ee5f0 100644 --- a/sql/item_xmlfunc.h +++ b/sql/item_xmlfunc.h @@ -82,7 +82,7 @@ public: maybe_null= TRUE; } bool fix_fields(THD *thd, Item **ref); - void fix_length_and_dec(); + bool fix_length_and_dec(); bool const_item() const { return const_item_cache && (!nodeset_func || nodeset_func->const_item()); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 4ebd5ef4954..10e9e65c0c4 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -13527,7 +13527,8 @@ COND *Item_func_eq::build_equal_items(THD *thd, List_iterator_fast it(cond_equal.current_level); while ((item_equal= it++)) { - item_equal->fix_length_and_dec(); + if (item_equal->fix_length_and_dec()) + return NULL; item_equal->update_used_tables(); set_if_bigger(thd->lex->current_select->max_equal_elems, item_equal->n_field_items()); diff --git a/sql/sql_table.cc b/sql/sql_table.cc index dd21c8fb107..9a87b0e7461 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -4896,7 +4896,12 @@ int create_table_impl(THD *thd, file= mysql_create_frm_image(thd, orig_db, orig_table_name, create_info, alter_info, create_table_mode, key_info, key_count, frm); - if (!file) + /* + TODO: remove this check of thd->is_error() (now it intercept + errors in some val_*() methoids and bring some single place to + such error interception). + */ + if (!file || thd->is_error()) goto err; if (rea_create_table(thd, frm, path, db, table_name, create_info, file, frm_only)) @@ -7398,10 +7403,15 @@ static bool mysql_inplace_alter_table(THD *thd, /* Replace the old .FRM with the new .FRM, but keep the old name for now. Rename to the new name (if needed) will be handled separately below. + + TODO: remove this check of thd->is_error() (now it intercept + errors in some val_*() methoids and bring some single place to + such error interception). */ if (mysql_rename_table(db_type, alter_ctx->new_db, alter_ctx->tmp_name, alter_ctx->db, alter_ctx->alias, - FN_FROM_IS_TMP | NO_HA_TABLE)) + FN_FROM_IS_TMP | NO_HA_TABLE) || + thd->is_error()) { // Since changes were done in-place, we can't revert them. (void) quick_rm_table(thd, db_type, From ff317fe08ea4d0e65ea421ef20786be0edf5423e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 15 Jun 2018 13:31:43 +0300 Subject: [PATCH 164/276] Follow-up to MDEV-16367 mariabackup: error: failed to copy enough redo log Commit dc9c555415b1d37b713839b9f8143a053f402c0e moved the final phase of the redo log copying to the background thread. This would sometimes cause too little redo log to be copied at the end of the backup. We would only guarantee copying up to the latest redo log checkpoint. This would produce a consistent backup, but it could refer to a too old point of time. xtrabackup_copy_log(), xtrabackup_copy_logfile(): Add the parameter 'last'. xtrabackup_backup_low(): Copy any remaining part of the log after the backup threads have terminated. --- extra/mariabackup/xtrabackup.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index bd5e28a4f5f..7a4442e6860 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2469,9 +2469,10 @@ skip: /** Copy redo log blocks to the data sink. @param start_lsn buffer start LSN @param end_lsn buffer end LSN +@param last whether we are copying the final part of the log @return last scanned LSN @retval 0 on failure */ -static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn) +static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn, bool last) { lsn_t scanned_lsn = start_lsn; const byte* log_block = log_sys->buf; @@ -2530,7 +2531,7 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn) log_sys->log.scanned_lsn = scanned_lsn; - end_lsn = metadata_to_lsn + end_lsn = last ? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE) : scanned_lsn & ~lsn_t(OS_FILE_LOG_BLOCK_SIZE - 1); @@ -2550,8 +2551,9 @@ static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn) } /** Copy redo log until the current end of the log is reached +@param last whether we are copying the final part of the log @return whether the operation failed */ -static bool xtrabackup_copy_logfile() +static bool xtrabackup_copy_logfile(bool last = false) { ut_a(dst_log_file != NULL); ut_ad(recv_sys != NULL); @@ -2582,7 +2584,7 @@ static bool xtrabackup_copy_logfile() } start_lsn = (lsn == start_lsn) - ? 0 : xtrabackup_copy_log(start_lsn, lsn); + ? 0 : xtrabackup_copy_log(start_lsn, lsn, last); log_mutex_exit(); @@ -3726,6 +3728,11 @@ static bool xtrabackup_backup_low() stop_backup_threads(); + if (metadata_to_lsn && xtrabackup_copy_logfile(true)) { + ds_close(dst_log_file); + return false; + } + if (ds_close(dst_log_file) || !metadata_to_lsn) { dst_log_file = NULL; return false; From b8514c94f61f37757a24df1c6b9a7dd530b3de12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 15 Jun 2018 14:09:15 +0300 Subject: [PATCH 165/276] MDEV-16496 Mariabackup: Implement --verbose option to instrument InnoDB log apply srv_print_verbose_log: Introduce the value 2 to refer to mariabackup --verbose. recv_recover_page(), recv_parse_log_recs(): Add output for mariabackup --verbose. --- extra/mariabackup/xtrabackup.cc | 9 ++++++-- .../mariabackup/apply-log-only-incr.test | 4 ++-- mysql-test/suite/mariabackup/mdev-14447.test | 4 ++-- storage/innobase/log/log0recv.cc | 22 +++++++++++++++++++ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 7a4442e6860..843a93cd251 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4,7 +4,7 @@ MariaBackup: hot backup tool for InnoDB Originally Created 3/3/2009 Yasufumi Kinoshita Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko, Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz. -(c) 2017, MariaDB Corporation. +(c) 2017, 2018, MariaDB Corporation. Portions written by Marko Mäkelä. This program is free software; you can redistribute it and/or modify @@ -109,6 +109,7 @@ int sys_var_init(); char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/"; char *xtrabackup_target_dir= xtrabackup_real_target_dir; static my_bool xtrabackup_version; +static my_bool verbose; my_bool xtrabackup_backup; my_bool xtrabackup_prepare; my_bool xtrabackup_copy_back; @@ -696,6 +697,9 @@ enum options_xtrabackup struct my_option xb_client_options[] = { + {"verbose", 'V', "display verbose output", + (G_PTR*) &verbose, (G_PTR*) &verbose, 0, GET_BOOL, NO_ARG, + FALSE, 0, 0, 0, 0, 0}, {"version", 'v', "print xtrabackup version information", (G_PTR *) &xtrabackup_version, (G_PTR *) &xtrabackup_version, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -1756,7 +1760,7 @@ innodb_init_param(void) srv_max_n_open_files = (ulint) innobase_open_files; srv_innodb_status = (ibool) innobase_create_status_file; - srv_print_verbose_log = 1; + srv_print_verbose_log = verbose ? 2 : 1; /* Store the default charset-collation number of this MySQL installation */ @@ -3730,6 +3734,7 @@ static bool xtrabackup_backup_low() if (metadata_to_lsn && xtrabackup_copy_logfile(true)) { ds_close(dst_log_file); + dst_log_file = NULL; return false; } diff --git a/mysql-test/suite/mariabackup/apply-log-only-incr.test b/mysql-test/suite/mariabackup/apply-log-only-incr.test index 25cb3a82023..74ab680d840 100644 --- a/mysql-test/suite/mariabackup/apply-log-only-incr.test +++ b/mysql-test/suite/mariabackup/apply-log-only-incr.test @@ -36,7 +36,7 @@ connection default; --disable_result_log exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --ftwrl-wait-timeout=5 --ftwrl-wait-threshold=300 --ftwrl-wait-query-type=all --target-dir=$incremental_dir --incremental-basedir=$basedir ; -exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir ; +exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir ; --enable_result_log let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test; @@ -44,7 +44,7 @@ let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test; --source include/search_pattern_in_file.inc --echo # expect NOT FOUND -exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; +exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; --source include/search_pattern_in_file.inc --echo # expect NOT FOUND diff --git a/mysql-test/suite/mariabackup/mdev-14447.test b/mysql-test/suite/mariabackup/mdev-14447.test index 48f37646231..689b578f2ab 100644 --- a/mysql-test/suite/mariabackup/mdev-14447.test +++ b/mysql-test/suite/mariabackup/mdev-14447.test @@ -18,8 +18,8 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir --disable_result_log echo # Prepare full backup, apply incremental one; -exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir; -exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; +exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir; +exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; echo # Restore and check results; let $targetdir=$basedir; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index b7ab1311cb7..c8976b193e2 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1772,6 +1772,11 @@ recv_recover_page(bool just_read_in, buf_block_t* block) ut_ad(recv_needed_recovery); + if (UNIV_UNLIKELY(srv_print_verbose_log == 2)) { + fprintf(stderr, "Applying log to page %u:%u\n", + recv_addr->space, recv_addr->page_no); + } + DBUG_PRINT("ib_log", ("Applying log to page %u:%u", recv_addr->space, recv_addr->page_no)); @@ -1872,6 +1877,13 @@ recv_recover_page(bool just_read_in, buf_block_t* block) start_lsn = recv->start_lsn; } + if (UNIV_UNLIKELY(srv_print_verbose_log == 2)) { + fprintf(stderr, "apply " LSN_PF ":" + " %d len " ULINTPF " page %u:%u\n", + recv->start_lsn, recv->type, recv->len, + recv_addr->space, recv_addr->page_no); + } + DBUG_PRINT("ib_log", ("apply " LSN_PF ":" " %s len " ULINTPF " page %u:%u", @@ -2422,6 +2434,16 @@ loop: #endif lsn = mach_read_from_8(ptr + 1); + if (UNIV_UNLIKELY(srv_print_verbose_log == 2)) { + fprintf(stderr, + "MLOG_CHECKPOINT(" LSN_PF ") %s at " + LSN_PF "\n", lsn, + lsn != checkpoint_lsn ? "ignored" + : recv_sys->mlog_checkpoint_lsn + ? "reread" : "read", + recv_sys->recovered_lsn); + } + DBUG_PRINT("ib_log", ("MLOG_CHECKPOINT(" LSN_PF ") %s at " LSN_PF, From f2c418079deff5fc0b460961094d8b833b4e30b4 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 15 Jun 2018 17:13:31 +0300 Subject: [PATCH 166/276] Fix a typo in get_best_ror_intersect cpk_scan should not be used if using it increases the cost of the query plan. --- sql/opt_range.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 8422917065f..d3f8acd8350 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -6529,6 +6529,8 @@ TRP_ROR_INTERSECT *get_best_ror_intersect(const PARAM *param, SEL_TREE *tree, if (ror_intersect_add(intersect, cpk_scan, TRUE) && (intersect->total_cost < min_cost)) intersect_best= intersect; //just set pointer here + else + cpk_scan= 0; // Don't use cpk_scan } else cpk_scan= 0; // Don't use cpk_scan From 7fdb7d4058e5a41687057a4b56c5a7698521ba99 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 15 Jun 2018 18:20:36 +0200 Subject: [PATCH 167/276] more sst test failures * xtrabackup no longer support --compact * wsrep_sst_mysqldump wasn't always using --default-file * wsrep_sst.cc overquoted --default-file for wsrep_sst_mysqldump also remove redundant lines config and test lines, compiler warnings, and mark big tests as big. --- .../r/galera_sst_mysqldump_with_key.result | 4 +-- .../galera/t/galera_sst_mariabackup.test | 1 + .../suite/galera/t/galera_sst_mysqldump.test | 1 + .../t/galera_sst_mysqldump_with_key.cnf | 4 --- .../t/galera_sst_mysqldump_with_key.test | 14 ++++---- .../suite/galera/t/galera_sst_rsync.test | 1 + .../t/galera_sst_xtrabackup-v2-options.cnf | 2 +- .../galera/t/galera_sst_xtrabackup-v2.test | 1 + scripts/wsrep_sst_mysqldump.sh | 13 ++++--- sql/wsrep_sst.cc | 34 +++++++++---------- 10 files changed, 36 insertions(+), 39 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result index 227e1c15444..074481afc2d 100644 --- a/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result +++ b/mysql-test/suite/galera/r/galera_sst_mysqldump_with_key.result @@ -1,4 +1,6 @@ Setting SST method to mysqldump ... +call mtr.add_suppression("WSREP: wsrep_sst_method is set to 'mysqldump' yet mysqld bind_address is set to '127.0.0.1'"); +call mtr.add_suppression("Failed to load slave replication state from table mysql.gtid_slave_pos"); CREATE USER 'sst'; GRANT ALL PRIVILEGES ON *.* TO 'sst'; SET GLOBAL wsrep_sst_auth = 'sst:'; @@ -105,5 +107,3 @@ CALL mtr.add_suppression("Can't open and lock privilege tables"); CALL mtr.add_suppression("Info table is not ready to be used"); CALL mtr.add_suppression("Native table .* has the wrong structure"); DROP USER sslsst; -SET GLOBAL general_log = ON; -SET GLOBAL slow_query_log = ON; diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup.test b/mysql-test/suite/galera/t/galera_sst_mariabackup.test index 0e7ac487700..bcb9ade3a25 100644 --- a/mysql-test/suite/galera/t/galera_sst_mariabackup.test +++ b/mysql-test/suite/galera/t/galera_sst_mariabackup.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source include/have_innodb.inc --source include/have_mariabackup.inc diff --git a/mysql-test/suite/galera/t/galera_sst_mysqldump.test b/mysql-test/suite/galera/t/galera_sst_mysqldump.test index 390e9815b20..835fac94a68 100644 --- a/mysql-test/suite/galera/t/galera_sst_mysqldump.test +++ b/mysql-test/suite/galera/t/galera_sst_mysqldump.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source suite/galera/include/galera_sst_set_mysqldump.inc diff --git a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.cnf b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.cnf index e108484b248..44e5573b3e6 100644 --- a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.cnf +++ b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.cnf @@ -12,10 +12,6 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore [mysqld] wsrep_debug=ON -ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/galera-cert.pem -ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/galera-cert.pem -ssl-key=@ENV.MYSQL_TEST_DIR/std_data/galera-key.pem - [client] ssl-ca=@ENV.MYSQL_TEST_DIR/std_data/cacert.pem ssl-cert=@ENV.MYSQL_TEST_DIR/std_data/client-cert.pem diff --git a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test index c813e04169f..0dbc63b531c 100644 --- a/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test +++ b/mysql-test/suite/galera/t/galera_sst_mysqldump_with_key.test @@ -5,9 +5,12 @@ --source include/big_test.inc --source include/galera_cluster.inc --source include/have_innodb.inc ---source include/have_openssl.inc +--source include/have_ssl_communication.inc --source suite/galera/include/galera_sst_set_mysqldump.inc +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc --connection node_1 CREATE USER sslsst; @@ -18,12 +21,7 @@ SET GLOBAL wsrep_sst_auth = 'sslsst:'; --source suite/galera/include/galera_st_disconnect_slave.inc +--source include/auto_increment_offset_restore.inc --source suite/galera/include/galera_sst_restore.inc + DROP USER sslsst; - ---connection node_2 -# We have to manually restore global_log and slow_query_log due to mysql-wsrep#108 -# Otherwise MTR's check_testcases complains - -SET GLOBAL general_log = ON; -SET GLOBAL slow_query_log = ON; diff --git a/mysql-test/suite/galera/t/galera_sst_rsync.test b/mysql-test/suite/galera/t/galera_sst_rsync.test index f796356cac7..5c08707e870 100644 --- a/mysql-test/suite/galera/t/galera_sst_rsync.test +++ b/mysql-test/suite/galera/t/galera_sst_rsync.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --let $node_1=node_1 diff --git a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf index 1e29673c0ff..3abf2549aae 100644 --- a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf +++ b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf @@ -8,7 +8,7 @@ wsrep_debug=ON [xtrabackup] backup-locks close-files -compact +#compact - disabled in xtrabackup 2.4, https://bugs.launchpad.net/percona-xtrabackup/+bug/1192834/comments/29 # compression requires qpress from the Percona repositories # compress # compress-threads=2 diff --git a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test index f1fd0f3ddf3..c270e4d0b19 100644 --- a/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test +++ b/mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.test @@ -1,3 +1,4 @@ +--source include/big_test.inc --source include/galera_cluster.inc --source include/have_innodb.inc --source include/have_xtrabackup.inc diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index fe49279a680..faa3f10639b 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -108,11 +108,6 @@ then DROP PREPARE stmt;" fi -# Retrieve the donor's @@global.gtid_binlog_state. -GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" |\ -$MYSQL_CLIENT $AUTH -S$WSREP_SST_OPT_SOCKET --disable-reconnect --connect_timeout=10 |\ -tail -1 | awk -F ' ' '{ print $2 }') - MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\ "$AUTH -h${WSREP_SST_OPT_HOST_UNESCAPED} "\ "-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10" @@ -126,12 +121,16 @@ tail -1 | awk -F ' ' '{ print $2 }') SERVER_VERSION=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'version'" | $MYSQL |\ tail -1 | awk -F ' ' '{ print $2 }') +# Retrieve the donor's @@global.gtid_binlog_state. +GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" | $MYSQL |\ +tail -1 | awk -F ' ' '{ print $2 }') + RESET_MASTER="" SET_GTID_BINLOG_STATE="" SQL_LOG_BIN_OFF="" # Safety check -if [ ${SERVER_VERSION%%.*} != '5' ] +if [ "${SERVER_VERSION%%.*}" != '5' ] then # If binary logging is enabled on the joiner node, we need to copy donor's # gtid_binlog_state to joiner. In order to do that, a RESET MASTER must be @@ -146,7 +145,7 @@ then fi # NOTE: we don't use --routines here because we're dumping mysql.proc table -MYSQLDUMP="$MYSQLDUMP $AUTH -S$WSREP_SST_OPT_SOCKET \ +MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF $AUTH -S$WSREP_SST_OPT_SOCKET \ --add-drop-database --add-drop-table --skip-add-locks --create-options \ --disable-keys --extended-insert --skip-lock-tables --quick --set-charset \ --skip-comments --flush-privileges --all-databases --events" diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index 260755d08a8..4df969496bc 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -556,11 +556,11 @@ static ssize_t sst_prepare_other (const char* method, ret= snprintf (cmd_str(), cmd_len, "wsrep_sst_%s " - WSREP_SST_OPT_ROLE" 'joiner' " - WSREP_SST_OPT_ADDR" '%s' " - WSREP_SST_OPT_DATA" '%s' " + WSREP_SST_OPT_ROLE " 'joiner' " + WSREP_SST_OPT_ADDR " '%s' " + WSREP_SST_OPT_DATA " '%s' " " %s " - WSREP_SST_OPT_PARENT" '%d'" + WSREP_SST_OPT_PARENT " '%d'" " %s '%s' ", method, addr_in, mysql_real_data_home, wsrep_defaults_file, @@ -842,13 +842,13 @@ static int sst_donate_mysqldump (const char* addr, int ret= snprintf (cmd_str(), cmd_len, "wsrep_sst_mysqldump " - WSREP_SST_OPT_ADDR" '%s' " - WSREP_SST_OPT_PORT" '%d' " - WSREP_SST_OPT_LPORT" '%u' " - WSREP_SST_OPT_SOCKET" '%s' " - " '%s' " - WSREP_SST_OPT_GTID" '%s:%lld' " - WSREP_SST_OPT_GTID_DOMAIN_ID" '%d'" + WSREP_SST_OPT_ADDR " '%s' " + WSREP_SST_OPT_PORT " '%d' " + WSREP_SST_OPT_LPORT " '%u' " + WSREP_SST_OPT_SOCKET " '%s' " + " %s " + WSREP_SST_OPT_GTID " '%s:%lld' " + WSREP_SST_OPT_GTID_DOMAIN_ID " '%d'" "%s", addr, port, mysqld_port, mysqld_unix_port, wsrep_defaults_file, uuid_str, @@ -1205,14 +1205,14 @@ static int sst_donate_other (const char* method, ret= snprintf (cmd_str(), cmd_len, "wsrep_sst_%s " - WSREP_SST_OPT_ROLE" 'donor' " - WSREP_SST_OPT_ADDR" '%s' " - WSREP_SST_OPT_SOCKET" '%s' " - WSREP_SST_OPT_DATA" '%s' " + WSREP_SST_OPT_ROLE " 'donor' " + WSREP_SST_OPT_ADDR " '%s' " + WSREP_SST_OPT_SOCKET " '%s' " + WSREP_SST_OPT_DATA " '%s' " " %s " " %s '%s' " - WSREP_SST_OPT_GTID" '%s:%lld' " - WSREP_SST_OPT_GTID_DOMAIN_ID" '%d'" + WSREP_SST_OPT_GTID " '%s:%lld' " + WSREP_SST_OPT_GTID_DOMAIN_ID " '%d'" "%s", method, addr, mysqld_unix_port, mysql_real_data_home, wsrep_defaults_file, From 9f848da640dd6c3f44d56eae18204370ae7f835c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 16 Jun 2018 01:20:44 +0200 Subject: [PATCH 168/276] fix dependencies on bionic --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 84fe9e07f88..3048ec78d62 100644 --- a/debian/control +++ b/debian/control @@ -316,7 +316,7 @@ Depends: bsdutils, galera-3 (>=25.3), gawk, grep, - iproute, + iproute | iproute2, libdbi-perl, lsb-base (>= 3.0-10), lsof, From 63027a5763b2b9550979366f9e7488b2d9328cc0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 13 Jun 2018 16:45:11 +0200 Subject: [PATCH 169/276] .gitignore after 4461b0f9b30 --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 932c666b74a..ac6fea03b4b 100644 --- a/.gitignore +++ b/.gitignore @@ -166,9 +166,9 @@ sql/mysql_tzinfo_to_sql sql/mysqld sql/sql_builtin.cc sql/sql_yacc.cc -sql/sql_yacc.h +sql/sql_yacc.hh sql/sql_yacc_ora.cc -sql/sql_yacc_ora.h +sql/sql_yacc_ora.hh storage/heap/hp_test1 storage/heap/hp_test2 storage/maria/aria_chk From 352c7e0dfaa0f121c5b35e1d9fafb9ec8897e768 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Sun, 17 Jun 2018 17:15:21 +0400 Subject: [PATCH 170/276] MDEV-15905 select json_value('{"b":true}','$.b')=1 --> false with "Truncated incorrect DOUBLE value: 'true'". JSON_VALUE_TRUE and JSON_VALUE_FALSE should be handled specifically in Item_json_value. --- mysql-test/r/func_json.result | 3 +++ mysql-test/t/func_json.test | 7 +++++++ sql/item_jsonfunc.cc | 21 ++++++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/func_json.result b/mysql-test/r/func_json.result index 2ffeb83b0de..77a424171b1 100644 --- a/mysql-test/r/func_json.result +++ b/mysql-test/r/func_json.result @@ -739,3 +739,6 @@ drop table t1; select json_extract('{"test":8.437e-5}','$.test'); json_extract('{"test":8.437e-5}','$.test') 8.437e-5 +select json_value('{"b":true}','$.b')=1; +json_value('{"b":true}','$.b')=1 +1 diff --git a/mysql-test/t/func_json.test b/mysql-test/t/func_json.test index a6ae934f7b4..b74915cd064 100644 --- a/mysql-test/t/func_json.test +++ b/mysql-test/t/func_json.test @@ -398,3 +398,10 @@ drop table t1; select json_extract('{"test":8.437e-5}','$.test'); +# +# MDEV-15905 select json_value('{"b":true}','$.b')=1 --> false with +# "Truncated incorrect DOUBLE value: 'true'" +# +select json_value('{"b":true}','$.b')=1; + + diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 72aeac5b930..c8cda66580c 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -513,6 +513,10 @@ err_return: bool Item_func_json_value::check_and_get_value(json_engine_t *je, String *res, int *error) { + CHARSET_INFO *json_cs; + const uchar *js; + uint js_len; + if (!json_value_scalar(je)) { /* We only look for scalar values! */ @@ -521,7 +525,22 @@ bool Item_func_json_value::check_and_get_value(json_engine_t *je, String *res, return true; } - return st_append_json(res, je->s.cs, je->value, je->value_len); + if (je->value_type == JSON_VALUE_TRUE || + je->value_type == JSON_VALUE_FALSE) + { + json_cs= &my_charset_utf8mb4_bin; + js= (const uchar *) ((je->value_type == JSON_VALUE_TRUE) ? "1" : "0"); + js_len= 1; + } + else + { + json_cs= je->s.cs; + js= je->value; + js_len= je->value_len; + } + + + return st_append_json(res, json_cs, js, js_len); } From eb77f8cf8de44ecaa7155afa7f55ece73b2b0497 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 18 Jun 2018 14:26:37 +0530 Subject: [PATCH 171/276] MDEV-16087 Inconsistent SELECT results when query cache is enabled The following conditions will decide the query cache retrieval or storing inside innodb: (1) There should not be any locks on the table. (2) Some other trx shouldn't invalidated the cache before the transaction started. (3) Read view shouldn't exist. If exists then the view low_limit_id should be greater than or equal to the transaction that invalidates the cache for the particular table. For read-only transaction: should satisfy the above (1) and (3) For read-write transaction: should satisfy the above (1), (2), (3). - Changed the variable from query_cache_inv_id to query_cache_inv_trx_id. - Moved the function row_search_check_if_query_cache_permitted from row0sel.h and made it as static function in ha_innodb.cc --- .../suite/innodb/r/innodb_query_cache.result | 39 +++++++++ .../suite/innodb/t/innodb_query_cache.test | 47 +++++++++++ storage/innobase/handler/ha_innodb.cc | 79 ++++++++++++++++++- storage/innobase/include/dict0mem.h | 4 +- storage/innobase/include/row0sel.h | 10 --- storage/innobase/lock/lock0lock.cc | 2 +- storage/innobase/row/row0sel.cc | 54 ------------- 7 files changed, 167 insertions(+), 68 deletions(-) create mode 100644 mysql-test/suite/innodb/r/innodb_query_cache.result create mode 100644 mysql-test/suite/innodb/t/innodb_query_cache.test diff --git a/mysql-test/suite/innodb/r/innodb_query_cache.result b/mysql-test/suite/innodb/r/innodb_query_cache.result new file mode 100644 index 00000000000..bbacbdfd748 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb_query_cache.result @@ -0,0 +1,39 @@ +# +# MDEV-16087 Inconsistent SELECT results when query cache is enabled +# +set GLOBAL query_cache_type=ON; +set LOCAL query_cache_type=ON; +create table t1 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +create table t2 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +create table t3 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +connect con1,localhost,root; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +select * from t2; +id +connection default; +insert into t3 () values (); +connection con1; +insert into t1 () values (); +select * from t3; +id +connect con2,localhost,root; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +select * from t3; +id +1 +select * from t3; +id +1 +select sql_no_cache * from t3; +id +1 +rollback; +connection con1; +rollback; +disconnect con1; +disconnect con2; +connection default; +drop table t1; +drop table t2; +drop table t3; +set GLOBAL query_cache_type=default; diff --git a/mysql-test/suite/innodb/t/innodb_query_cache.test b/mysql-test/suite/innodb/t/innodb_query_cache.test new file mode 100644 index 00000000000..7dbdf986946 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb_query_cache.test @@ -0,0 +1,47 @@ +-- source include/have_innodb.inc +-- source include/have_query_cache.inc +-- source include/not_embedded.inc + +--echo # +--echo # MDEV-16087 Inconsistent SELECT results when query cache is enabled +--echo # + +set GLOBAL query_cache_type=ON; +set LOCAL query_cache_type=ON; + +create table t1 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +create table t2 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; +create table t3 (id bigint(20) auto_increment, primary key (id)) ENGINE=InnoDB; + +connect (con1,localhost,root); +START TRANSACTION WITH CONSISTENT SNAPSHOT; +select * from t2; + +connection default; +insert into t3 () values (); + +connection con1; +insert into t1 () values (); +select * from t3; + +connect (con2,localhost,root); +START TRANSACTION WITH CONSISTENT SNAPSHOT; +select * from t3; +select * from t3; +select sql_no_cache * from t3; + +rollback; + +connection con1; + +rollback; + +disconnect con1; +disconnect con2; +connection default; + +drop table t1; +drop table t2; +drop table t3; + +set GLOBAL query_cache_type=default; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index ba7a476c754..2c8ea81e286 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3167,6 +3167,83 @@ AUTOCOMMIT==0 or we are inside BEGIN ... COMMIT. Thus transactions no longer put restrictions on the use of the query cache. */ +/** Check if mysql can allow the transaction to read from/store to +the query cache. +@param[in] table table object +@param[in] trx transaction object +@return whether the storing or retrieving from the query cache is permitted */ +static bool innobase_query_caching_table_check_low( + const dict_table_t* table, + trx_t* trx) +{ + /* The following conditions will decide the query cache + retrieval or storing into: + + (1) There should not be any locks on the table. + (2) Someother trx shouldn't invalidate the cache before this + transaction started. + (3) Read view shouldn't exist. If exists then the view + low_limit_id should be greater than or equal to the transaction that + invalidates the cache for the particular table. + + For read-only transaction: should satisfy (1) and (3) + For read-write transaction: should satisfy (1), (2), (3) */ + + if (lock_table_get_n_locks(table)) { + return false; + } + + if (trx->id && trx->id < table->query_cache_inv_trx_id) { + return false; + } + + return !MVCC::is_view_active(trx->read_view) + || trx->read_view->low_limit_id() + >= table->query_cache_inv_trx_id; +} + +/** Checks if MySQL at the moment is allowed for this table to retrieve a +consistent read result, or store it to the query cache. +@param[in,out] trx transaction +@param[in] norm_name concatenation of database name, + '/' char, table name +@return whether storing or retrieving from the query cache is permitted */ +static bool innobase_query_caching_table_check( + trx_t* trx, + const char* norm_name) +{ + dict_table_t* table = dict_table_open_on_name( + norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); + + if (table == NULL) { + return false; + } + + /* Start the transaction if it is not started yet */ + trx_start_if_not_started(trx, false); + + bool allow = innobase_query_caching_table_check_low(table, trx); + + dict_table_close(table, FALSE, FALSE); + + if (allow) { + /* If the isolation level is high, assign a read view for the + transaction if it does not yet have one */ + + if (trx->isolation_level >= TRX_ISO_REPEATABLE_READ + && !srv_read_only_mode + && !MVCC::is_view_active(trx->read_view)) { + + /* Start the transaction if it is not started yet */ + trx_start_if_not_started(trx, false); + + trx_sys->mvcc->view_open(trx->read_view, trx); + } + } + + return allow; +} + /******************************************************************//** The MySQL query cache uses this to check from InnoDB if the query cache at the moment is allowed to operate on an InnoDB table. The SQL query must @@ -3242,7 +3319,7 @@ innobase_query_caching_of_table_permitted( innobase_register_trx(innodb_hton_ptr, thd, trx); - return(row_search_check_if_query_cache_permitted(trx, norm_name)); + return innobase_query_caching_table_check(trx, norm_name); } /*****************************************************************//** diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index a18d6c1abdd..56f695f305d 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1546,8 +1546,8 @@ struct dict_table_t { /** Transactions whose view low limit is greater than this number are not allowed to store to the MySQL query cache or retrieve from it. When a trx with undo logs commits, it sets this to the value of the - current time. */ - trx_id_t query_cache_inv_id; + transaction id. */ + trx_id_t query_cache_inv_trx_id; /** Transaction id that last touched the table definition. Either when loading the definition or CREATE TABLE, or ALTER TABLE (prepare, diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index d73c186b12e..552680b16d1 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -213,16 +213,6 @@ row_count_rtree_recs( ulint* n_rows); /*!< out: number of entries seen in the consistent read */ -/*******************************************************************//** -Checks if MySQL at the moment is allowed for this table to retrieve a -consistent read result, or store it to the query cache. -@return whether storing or retrieving from the query cache is permitted */ -bool -row_search_check_if_query_cache_permitted( -/*======================================*/ - trx_t* trx, /*!< in: transaction object */ - const char* norm_name); /*!< in: concatenation of database name, - '/' char, table name */ /** Read the max AUTOINC value from an index. @param[in] index index starting with an AUTO_INCREMENT column @return the largest AUTO_INCREMENT value diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 240510ffa59..ee0dc5a9000 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -4456,7 +4456,7 @@ lock_release( block the use of the MySQL query cache for all currently active transactions. */ - table->query_cache_inv_id = max_trx_id; + table->query_cache_inv_trx_id = max_trx_id; } lock_table_dequeue(lock); diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index b5e1bab352d..7383d3d9510 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -5905,60 +5905,6 @@ func_exit: goto loop; } -/*******************************************************************//** -Checks if MySQL at the moment is allowed for this table to retrieve a -consistent read result, or store it to the query cache. -@return whether storing or retrieving from the query cache is permitted */ -bool -row_search_check_if_query_cache_permitted( -/*======================================*/ - trx_t* trx, /*!< in: transaction object */ - const char* norm_name) /*!< in: concatenation of database name, - '/' char, table name */ -{ - dict_table_t* table = dict_table_open_on_name( - norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); - - if (table == NULL) { - - return(false); - } - - /* Start the transaction if it is not started yet */ - - trx_start_if_not_started(trx, false); - - /* If there are locks on the table or some trx has invalidated the - cache before this transaction started then this transaction cannot - read/write from/to the cache. - - If a read view has not been created for the transaction then it doesn't - really matter what this transaction sees. If a read view was created - then the view low_limit_id is the max trx id that this transaction - saw at the time of the read view creation. */ - - const bool ret = lock_table_get_n_locks(table) == 0 - && ((trx->id != 0 && trx->id >= table->query_cache_inv_id) - || !MVCC::is_view_active(trx->read_view) - || trx->read_view->low_limit_id() - >= table->query_cache_inv_id); - if (ret) { - /* If the isolation level is high, assign a read view for the - transaction if it does not yet have one */ - - if (trx->isolation_level >= TRX_ISO_REPEATABLE_READ - && !srv_read_only_mode - && !MVCC::is_view_active(trx->read_view)) { - - trx_sys->mvcc->view_open(trx->read_view, trx); - } - } - - dict_table_close(table, FALSE, FALSE); - - return(ret); -} - /*******************************************************************//** Read the AUTOINC column from the current row. If the value is less than 0 and the type is not unsigned then we reset the value to 0. From f7b1b2bc5d6e1f923aca0ee8c5232cf095075c7b Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Mon, 18 Jun 2018 07:40:58 -0400 Subject: [PATCH 172/276] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f9ccf7c6188..c169837c460 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=1 -MYSQL_VERSION_PATCH=34 +MYSQL_VERSION_PATCH=35 From c69efab3963703688486287918dda105942d24e7 Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Mon, 18 Jun 2018 17:54:30 +0200 Subject: [PATCH 173/276] - Fix MDEV-16167 Cannot insert unsigned values into a VEC table modified: storage/connect/filamvct.cpp modified: storage/connect/tabvct.cpp _ Typo modified: storage/connect/CMakeLists.txt --- storage/connect/CMakeLists.txt | 17 ----------------- storage/connect/filamvct.cpp | 22 ++++++++++++++-------- storage/connect/tabvct.cpp | 6 ++---- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 58d9ab1f1c3..408f6d223a1 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -346,23 +346,6 @@ IF(WIN32) DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) ENDIF(WIN32) -IF(NOT TARGET connect) - RETURN() -ENDIF() - -# Install some extra files that belong to connect engine -IF(WIN32) - # install ha_connect.lib - GET_TARGET_PROPERTY(CONNECT_LOCATION connect LOCATION) - STRING(REPLACE "dll" "lib" CONNECT_LIB ${CONNECT_LOCATION}) - IF(CMAKE_CONFIGURATION_TYPES) - STRING(REPLACE "${CMAKE_CFG_INTDIR}" "\${CMAKE_INSTALL_CONFIG_NAME}" - CONNECT_LIB ${CONNECT_LIB}) - ENDIF() - INSTALL(FILES ${CONNECT_LIB} - DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) -ENDIF(WIN32) - IF(CONNECT_WITH_JDBC AND JAVA_FOUND AND JNI_FOUND) # TODO: Find how to compile and install the java wrapper classes # Find required libraries and include directories diff --git a/storage/connect/filamvct.cpp b/storage/connect/filamvct.cpp index 244acfdc5c8..a660461e9ee 100755 --- a/storage/connect/filamvct.cpp +++ b/storage/connect/filamvct.cpp @@ -515,7 +515,8 @@ bool VCTFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) cp->Blk = AllocValBlock(g, NewBlock + Nrec * cp->Deplac, cp->Buf_Type, Nrec, cp->Format.Length, - cp->Format.Prec, chk); + cp->Format.Prec, chk, true, + cp->IsUnsigned()); return InitInsert(g); // Initialize inserting } else { @@ -549,7 +550,8 @@ bool VCTFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) // Not a pseudo column cp->Blk = AllocValBlock(g, NULL, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); } //endif mode @@ -1516,7 +1518,8 @@ bool VCMFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)Tdbp->GetColumns(); cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) { // Not a pseudo column cp->Blk = AllocValBlock(g, (void*)1, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); cp->AddStatus(BUF_MAPPED); } // endif IsSpecial @@ -2067,7 +2070,7 @@ bool VECFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)tdbp->Columns; cp; cp = (PVCTCOL)cp->Next) cp->Blk = AllocValBlock(g, To_Bufs[cp->Index - 1], cp->Buf_Type, Nrec, cp->Format.Length, - cp->Format.Prec, chk); + cp->Format.Prec, chk, true, cp->IsUnsigned()); return InitInsert(g); } else { @@ -2116,7 +2119,8 @@ bool VECFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)tdbp->Columns; cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) // Not a pseudo column cp->Blk = AllocValBlock(g, NULL, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); } // endif mode @@ -2887,7 +2891,8 @@ bool VMPFAM::AllocateBuffer(PGLOBAL g) for (cp = (PVCTCOL)Tdbp->GetColumns(); cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) { // Not a pseudo column cp->Blk = AllocValBlock(g, (void*)1, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); cp->AddStatus(BUF_MAPPED); } // endif IsSpecial @@ -3669,7 +3674,7 @@ bool BGVFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) cp->Blk = AllocValBlock(g, NewBlock + Nrec * cp->Deplac, cp->Buf_Type, Nrec, cp->Format.Length, - cp->Format.Prec, chk); + cp->Format.Prec, chk, true, cp->IsUnsigned()); InitInsert(g); // Initialize inserting @@ -3717,7 +3722,8 @@ bool BGVFAM::AllocateBuffer(PGLOBAL g) for (; cp; cp = (PVCTCOL)cp->Next) if (!cp->IsSpecial()) // Not a pseudo column cp->Blk = AllocValBlock(g, NULL, cp->Buf_Type, Nrec, - cp->Format.Length, cp->Format.Prec); + cp->Format.Length, cp->Format.Prec, + true, true, cp->IsUnsigned()); } //endif mode diff --git a/storage/connect/tabvct.cpp b/storage/connect/tabvct.cpp index 11b344ef652..40d020202ea 100644 --- a/storage/connect/tabvct.cpp +++ b/storage/connect/tabvct.cpp @@ -456,13 +456,11 @@ bool VCTCOL::SetBuffer(PGLOBAL g, PVAL value, bool ok, bool check) if (tdbp->Txfp->GetAmType() == TYPE_AM_VMP && ok) { Blk = AllocValBlock(g, (void*)1, Buf_Type, tdbp->Txfp->Nrec, - Format.Length, - Format.Prec, check); + Format.Length, Format.Prec, check, true, Unsigned); Status |= BUF_MAPPED; // Will point into mapped file } else Blk = AllocValBlock(g, NULL, Buf_Type, tdbp->Txfp->Nrec, - Format.Length, - Format.Prec, check); + Format.Length, Format.Prec, check, true, Unsigned); } // endif Mode return false; From 5ba6cee01255186ea77a69f122d3a15c07f44f6d Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 18 Jun 2018 23:00:34 +0400 Subject: [PATCH 174/276] MDEV-16209 JSON_EXTRACT in query crashes server. The optimizer can create various item's over the original one, so we can't count on the exact item's type inside the comparison. --- mysql-test/r/func_json.result | 5 +++ mysql-test/t/func_json.test | 8 +++++ sql/item_jsonfunc.cc | 58 ++++++++++++++++++++--------------- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/mysql-test/r/func_json.result b/mysql-test/r/func_json.result index 77a424171b1..564442a2880 100644 --- a/mysql-test/r/func_json.result +++ b/mysql-test/r/func_json.result @@ -742,3 +742,8 @@ json_extract('{"test":8.437e-5}','$.test') select json_value('{"b":true}','$.b')=1; json_value('{"b":true}','$.b')=1 1 +CREATE TABLE t1 (c VARCHAR(8)); +INSERT INTO t1 VALUES ('foo'),('bar'); +SELECT * FROM t1 WHERE c IN (JSON_EXTRACT('{"a":"b"}', '$.*')); +c +DROP TABLE t1; diff --git a/mysql-test/t/func_json.test b/mysql-test/t/func_json.test index b74915cd064..1b91061b9a3 100644 --- a/mysql-test/t/func_json.test +++ b/mysql-test/t/func_json.test @@ -404,4 +404,12 @@ select json_extract('{"test":8.437e-5}','$.test'); # select json_value('{"b":true}','$.b')=1; +# +# MDEV-16209 JSON_EXTRACT in query crashes server. +# + +CREATE TABLE t1 (c VARCHAR(8)); +INSERT INTO t1 VALUES ('foo'),('bar'); +SELECT * FROM t1 WHERE c IN (JSON_EXTRACT('{"a":"b"}', '$.*')); +DROP TABLE t1; diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index c8cda66580c..8103aa906ff 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -3229,34 +3229,44 @@ String *Item_func_json_format::val_json(String *str) int Arg_comparator::compare_json_str_basic(Item *j, Item *s) { - String *res1,*res2; - json_value_types type; - char *value; - int value_len, c_len; - Item_func_json_extract *e= (Item_func_json_extract *) j; + String *js,*str; + int c_len; + json_engine_t je; - if ((res1= e->read_json(&value1, &type, &value, &value_len))) + if ((js= j->val_str(&value1))) { - if ((res2= s->val_str(&value2))) - { - if (type == JSON_VALUE_STRING) - { - if (value1.realloc_with_extra_if_needed(value_len) || - (c_len= json_unescape(value1.charset(), (uchar *) value, - (uchar *) value+value_len, - &my_charset_utf8_general_ci, - (uchar *) value1.ptr(), - (uchar *) (value1.ptr() + value_len))) < 0) - goto error; - value1.length(c_len); - res1= &value1; - } + json_scan_start(&je, js->charset(), (const uchar *) js->ptr(), + (const uchar *) js->ptr()+js->length()); + if (json_read_value(&je)) + goto error; + if (je.value_type == JSON_VALUE_STRING) + { + if (value2.realloc_with_extra_if_needed(je.value_len) || + (c_len= json_unescape(js->charset(), je.value, + je.value + je.value_len, + &my_charset_utf8_general_ci, + (uchar *) value2.ptr(), + (uchar *) (value2.ptr() + je.value_len))) < 0) + goto error; - if (set_null) - owner->null_value= 0; - return sortcmp(res1, res2, compare_collation()); - } + value2.length(c_len); + js= &value2; + str= &value1; + } + else + { + str= &value2; + } + + + if ((str= s->val_str(str))) + { + if (set_null) + owner->null_value= 0; + return sortcmp(js, str, compare_collation()); + } } + error: if (set_null) owner->null_value= 1; From b27ec709350e13c5cdc08dcdcaeb31b1cc0f803e Mon Sep 17 00:00:00 2001 From: Jacob Mathew Date: Mon, 18 Jun 2018 15:29:50 -0700 Subject: [PATCH 175/276] MDEV-16319: Test for crash introduced by b4a2baffa8 fixed by 4968049799 Add a Spider test to ensure that a bug similar to MDEV-11084 is not re-introduced. Spider would crash if the first partition was not used first. Author: Eric Herman. First Reviewer: Jacob Mathew. Second Reviewer: Kentoku Shiba. --- .../spider/r/spider_fixes_part.result | 31 +++++ .../spider/t/spider_fixes_part.test | 109 ++++++++++++++++++ 2 files changed, 140 insertions(+) diff --git a/storage/spider/mysql-test/spider/r/spider_fixes_part.result b/storage/spider/mysql-test/spider/r/spider_fixes_part.result index faf0b6efbea..9668ef60adc 100644 --- a/storage/spider/mysql-test/spider/r/spider_fixes_part.result +++ b/storage/spider/mysql-test/spider/r/spider_fixes_part.result @@ -214,6 +214,37 @@ id 10000 connection master_1; +Crash from b4a2baffa82e5c07b96a1c752228560dcac1359b (MDEV-11084) +Fixed with 4968049799193394d442f26b4e3a8d95b185be72 +Spider crashed if the first partition was not used first +connection master_1; +CREATE TABLE ta_l2 ( +a INT, +b CHAR(1), +c DATETIME, +PRIMARY KEY(a) +) MASTER_1_ENGINE MASTER_1_COMMENT2_P_2_1 +INSERT INTO ta_l2 SELECT a, b, c FROM tb_l; +Query a Spider table only using the second partition +SELECT a,b,c FROM ta_l2 PARTITION (pt2); +a b c +4 d 2003-11-30 05:01:03 +5 c 2001-12-31 23:59:59 +Query a Spider table only using the first partition +SELECT a,b,c FROM ta_l2 PARTITION (pt1); +a b c +1 a 2008-08-01 10:21:39 +2 b 2000-01-01 00:00:00 +3 e 2007-06-04 20:03:11 +Query a Spider table by all paritions, then the second partition +SELECT min(a), max(a), min(b), max(b) FROM ta_l2; +min(a) max(a) min(b) max(b) +1 5 a e +SELECT a,b,c FROM ta_l2 PARTITION (pt2); +a b c +4 d 2003-11-30 05:01:03 +5 c 2001-12-31 23:59:59 + deinit connection master_1; DROP DATABASE IF EXISTS auto_test_local; diff --git a/storage/spider/mysql-test/spider/t/spider_fixes_part.test b/storage/spider/mysql-test/spider/t/spider_fixes_part.test index ef5a8026c02..c729131f338 100644 --- a/storage/spider/mysql-test/spider/t/spider_fixes_part.test +++ b/storage/spider/mysql-test/spider/t/spider_fixes_part.test @@ -616,6 +616,115 @@ if ($HAVE_PARTITION) } } +--echo +--echo Crash from b4a2baffa82e5c07b96a1c752228560dcac1359b (MDEV-11084) +--echo Fixed with 4968049799193394d442f26b4e3a8d95b185be72 +--echo Spider crashed if the first partition was not used first +if ($HAVE_PARTITION) +{ + if ($USE_CHILD_GROUP2) + { + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_2 + if ($OUTPUT_CHILD_GROUP2) + { + --disable_query_log + echo CHILD2_2_DROP_TABLES; + echo CHILD2_2_CREATE_TABLES; + } + --disable_warnings + eval $CHILD2_2_DROP_TABLES; + --enable_warnings + eval $CHILD2_2_CREATE_TABLES; + if ($OUTPUT_CHILD_GROUP2) + { + --enable_query_log + } + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + --connection child2_1 + if ($OUTPUT_CHILD_GROUP2) + { + --disable_query_log + echo CHILD2_1_DROP_TABLES2; + echo CHILD2_1_CREATE_TABLES2; + } + --disable_warnings + eval $CHILD2_1_DROP_TABLES2; + --enable_warnings + eval $CHILD2_1_CREATE_TABLES2; + if ($OUTPUT_CHILD_GROUP2) + { + --enable_query_log + } + if ($USE_GENERAL_LOG) + { + TRUNCATE TABLE mysql.general_log; + } + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } + } + --connection master_1 + --disable_query_log + --disable_warnings + DROP TABLE IF EXISTS ta_l2; + --enable_warnings + echo CREATE TABLE ta_l2 ( + a INT, + b CHAR(1), + c DATETIME, + PRIMARY KEY(a) + ) MASTER_1_ENGINE MASTER_1_COMMENT2_P_2_1; + eval CREATE TABLE ta_l2 ( + a INT, + b CHAR(1), + c DATETIME, + PRIMARY KEY(a) + ) $MASTER_1_ENGINE $MASTER_1_COMMENT2_P_2_1; + --enable_query_log + INSERT INTO ta_l2 SELECT a, b, c FROM tb_l; + --echo Query a Spider table only using the second partition + SELECT a,b,c FROM ta_l2 PARTITION (pt2); + --echo Query a Spider table only using the first partition + SELECT a,b,c FROM ta_l2 PARTITION (pt1); + --echo Query a Spider table by all paritions, then the second partition + SELECT min(a), max(a), min(b), max(b) FROM ta_l2; + SELECT a,b,c FROM ta_l2 PARTITION (pt2); + if ($USE_CHILD_GROUP2) + { + if (!$OUTPUT_CHILD_GROUP2) + { + --disable_query_log + --disable_result_log + } + --connection child2_2 + if ($USE_GENERAL_LOG) + { + SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'; + } + eval $CHILD2_2_SELECT_TABLES; + --connection child2_1 + if ($USE_GENERAL_LOG) + { + SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %'; + } + eval $CHILD2_1_SELECT_TABLES2; + if (!$OUTPUT_CHILD_GROUP2) + { + --enable_query_log + --enable_result_log + } + } +} --echo --echo deinit From 956b296248acbe35aa87f056ccf99dbb999eea02 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 14 Jun 2018 15:48:57 -0700 Subject: [PATCH 176/276] MDEV-16420 View stop working after upgrade from 10.1.15 to 10.3.7 This bug happened for queries that used a materialized view that renamed columns of the specifying query in an inner table of an outer join. For such a query name resolution for a column belonging the view could fail if the underlying column was non-nullable. When creating the defintion of the the temporary table for the materialized view used in the inner part of an outer join the definition of the non-nullable columns are created by the function create_tmp_field_from_item() that names the columns according to the names of the underlying columns. So these names should be changed for the view column names. This bug cannot be reproduced in 10.2 because there setup_fields() called when preparing joins in the view specification effectively renames the underlying columns in the function find_field_in_view(). In 10.3 this renaming was removed as improper (see Monty's commit b478276b04d4dd122e2ed4d4e2cd7eb69c0fb2d2). --- mysql-test/main/derived_view.result | 40 +++++++++++++++++++++++++++++ mysql-test/main/derived_view.test | 24 +++++++++++++++++ sql/sql_select.cc | 3 +++ 3 files changed, 67 insertions(+) diff --git a/mysql-test/main/derived_view.result b/mysql-test/main/derived_view.result index 6c4b3310e11..86dd73f5733 100644 --- a/mysql-test/main/derived_view.result +++ b/mysql-test/main/derived_view.result @@ -2977,5 +2977,45 @@ id select_type table type possible_keys key key_len ref rows filtered Extra Warnings: Note 1003 /* select#1 */ select straight_join `test`.`t1`.`c1` AS `c1` from `test`.`t1` where (`test`.`t1`.`c1`,(/* select#3 */ select `test`.`t2`.`c2` from `test`.`t2` where (`test`.`t1`.`c1`) = `test`.`t2`.`c2`)) DROP TABLE t1, t2; +# +# Bug mdev-16420: materialized view that renames columns +# in inner part of outer join +# +CREATE TABLE t1 (id int, PRIMARY KEY (id)); +INSERT INTO t1 VALUES (2), (3), (7), (1); +CREATE VIEW v1 AS SELECT * FROM t1; +CREATE VIEW v2 AS SELECT v1.id AS order_pk FROM v1 GROUP BY v1.id; +CREATE VIEW v3 AS +SELECT t.id AS order_pk FROM (SELECT * FROM t1) AS t GROUP BY t.id; +SELECT * FROM t1 LEFT JOIN v2 ON t1.id=v2.order_pk; +id order_pk +1 1 +2 2 +3 3 +7 7 +EXPLAIN EXTENDED +SELECT * FROM t1 LEFT JOIN v2 ON t1.id=v2.order_pk; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 index NULL PRIMARY 4 NULL 4 100.00 Using index +1 PRIMARY ref key0 key0 5 test.t1.id 2 100.00 +2 DERIVED t1 index PRIMARY PRIMARY 4 NULL 4 100.00 Using index; Using filesort +Warnings: +Note 1003 /* select#1 */ select `test`.`t1`.`id` AS `id`,`v2`.`order_pk` AS `order_pk` from `test`.`t1` left join `test`.`v2` on(`v2`.`order_pk` = `test`.`t1`.`id`) where 1 +SELECT * FROM t1 LEFT JOIN v3 ON t1.id=v3.order_pk; +id order_pk +1 1 +2 2 +3 3 +7 7 +EXPLAIN EXTENDED +SELECT * FROM t1 LEFT JOIN v3 ON t1.id=v3.order_pk; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 index NULL PRIMARY 4 NULL 4 100.00 Using index +1 PRIMARY ref key0 key0 5 test.t1.id 2 100.00 +2 DERIVED t1 index PRIMARY PRIMARY 4 NULL 4 100.00 Using index; Using filesort +Warnings: +Note 1003 /* select#1 */ select `test`.`t1`.`id` AS `id`,`v3`.`order_pk` AS `order_pk` from `test`.`t1` left join `test`.`v3` on(`v3`.`order_pk` = `test`.`t1`.`id`) where 1 +DROP VIEW v1,v2,v3; +DROP TABLE t1; set optimizer_switch=@exit_optimizer_switch; set join_cache_level=@exit_join_cache_level; diff --git a/mysql-test/main/derived_view.test b/mysql-test/main/derived_view.test index 9b0cf9dca7d..68f334e1ac4 100644 --- a/mysql-test/main/derived_view.test +++ b/mysql-test/main/derived_view.test @@ -1949,6 +1949,30 @@ eval EXPLAIN EXTENDED $q; DROP TABLE t1, t2; +--echo # +--echo # Bug mdev-16420: materialized view that renames columns +--echo # in inner part of outer join +--echo # + +CREATE TABLE t1 (id int, PRIMARY KEY (id)); +INSERT INTO t1 VALUES (2), (3), (7), (1); + +CREATE VIEW v1 AS SELECT * FROM t1; +CREATE VIEW v2 AS SELECT v1.id AS order_pk FROM v1 GROUP BY v1.id; +CREATE VIEW v3 AS +SELECT t.id AS order_pk FROM (SELECT * FROM t1) AS t GROUP BY t.id; + +SELECT * FROM t1 LEFT JOIN v2 ON t1.id=v2.order_pk; +EXPLAIN EXTENDED +SELECT * FROM t1 LEFT JOIN v2 ON t1.id=v2.order_pk; + +SELECT * FROM t1 LEFT JOIN v3 ON t1.id=v3.order_pk; +EXPLAIN EXTENDED +SELECT * FROM t1 LEFT JOIN v3 ON t1.id=v3.order_pk; + +DROP VIEW v1,v2,v3; +DROP TABLE t1; + # The following command must be the last one the file set optimizer_switch=@exit_optimizer_switch; set join_cache_level=@exit_join_cache_level; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 54bb877ea2f..bd8620624ac 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -16813,7 +16813,10 @@ Field *create_tmp_field(THD *thd, TABLE *table,Item *item, Item::Type type, if (result && modify_item) field->result_field= result; if (orig_item) + { item->maybe_null= save_maybe_null; + result->field_name= orig_item->name; + } } else if (table_cant_handle_bit_fields && field->field->type() == MYSQL_TYPE_BIT) From 15b92915ed93661a56f40430204d18bf7b7cf1fc Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 19 Jun 2018 13:02:02 +0400 Subject: [PATCH 177/276] MDEV-15834 The code in TABLE_SHARE::init_from_binary_frm_image() is not safe --- mysql-test/std_data/frm/t1.frm | Bin 0 -> 8584 bytes mysql-test/suite/vcol/r/vcol_misc.result | 10 ++++++++++ mysql-test/suite/vcol/t/vcol_misc.test | 16 ++++++++++++++++ sql/table.cc | 6 +++++- 4 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 mysql-test/std_data/frm/t1.frm diff --git a/mysql-test/std_data/frm/t1.frm b/mysql-test/std_data/frm/t1.frm new file mode 100644 index 0000000000000000000000000000000000000000..a998f54ec67d650e93535a513951727f378b10de GIT binary patch literal 8584 zcmeI&p$@_@5XSMZS0FSq2!g?22?DSLgGY2`^tUD!}Vsh{VbpE2@OU-E;mST3bmJ0T;000IagfB*srAb(Z<0m=97W1MYHjI_F Date: Sat, 16 Jun 2018 12:03:15 +0300 Subject: [PATCH 178/276] Add PART_INDIRECT_KEY_FLAG This is to mark that a field is indirectly part of a key, which simplifes checking if we need to have this field up to date to evaluate a key. For example: CREATE TABLE t1 (a int, b int as (a) virtual, c int as (b) virtual, index(c)) would mark a and b with PART_INDIRECT_KEY_FLAG. c is marked with PART_KEY_FLAG as before. --- include/mysql_com.h | 1 + mysql-test/suite/vcol/r/index.result | 38 ++++++++++++++ mysql-test/suite/vcol/t/index.test | 25 ++++++++++ sql/sql_union.cc | 6 +-- sql/table.cc | 75 +++++++++++++++++++++------- sql/table.h | 2 +- 6 files changed, 126 insertions(+), 21 deletions(-) create mode 100644 mysql-test/suite/vcol/r/index.result create mode 100644 mysql-test/suite/vcol/t/index.test diff --git a/include/mysql_com.h b/include/mysql_com.h index 06c934bf9bd..17366e22da8 100644 --- a/include/mysql_com.h +++ b/include/mysql_com.h @@ -179,6 +179,7 @@ enum enum_indicator_type #define BINCMP_FLAG 131072U /* Intern: Used by sql_yacc */ #define GET_FIXED_FIELDS_FLAG (1U << 18) /* Used to get fields in item tree */ #define FIELD_IN_PART_FUNC_FLAG (1U << 19)/* Field part of partition func */ +#define PART_INDIRECT_KEY_FLAG (1U << 20) /** Intern: Field in TABLE object for new version of altered table, diff --git a/mysql-test/suite/vcol/r/index.result b/mysql-test/suite/vcol/r/index.result new file mode 100644 index 00000000000..cd4ebc96024 --- /dev/null +++ b/mysql-test/suite/vcol/r/index.result @@ -0,0 +1,38 @@ +CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=myisam; +insert into t1 (a) values (1),(2),(3); +update t1 set a=5 where a=3; +delete from t1 where a=1; +select * from t1; +a b c +2 3 4 +5 6 7 +select * from t1 where c=7; +a b c +5 6 7 +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select * from t1; +a b c +2 3 4 +5 6 7 +drop table t1; +CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=innodb; +insert into t1 (a) values (1),(2),(3); +update t1 set a=5 where a=3; +delete from t1 where a=1; +select * from t1; +a b c +2 3 4 +5 6 7 +select * from t1 where c=7; +a b c +5 6 7 +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select * from t1; +a b c +2 3 4 +5 6 7 +drop table t1; diff --git a/mysql-test/suite/vcol/t/index.test b/mysql-test/suite/vcol/t/index.test new file mode 100644 index 00000000000..55d5b68f26b --- /dev/null +++ b/mysql-test/suite/vcol/t/index.test @@ -0,0 +1,25 @@ +--source include/have_innodb.inc + +# +# Test creating table with a key that consists of indirect virtual fields +# + +CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=myisam; +insert into t1 (a) values (1),(2),(3); +update t1 set a=5 where a=3; +delete from t1 where a=1; +select * from t1; +select * from t1 where c=7; +check table t1; +select * from t1; +drop table t1; + +CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=innodb; +insert into t1 (a) values (1),(2),(3); +update t1 set a=5 where a=3; +delete from t1 where a=1; +select * from t1; +select * from t1 where c=7; +check table t1; +select * from t1; +drop table t1; diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 178d7393878..b409790c044 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -185,7 +185,7 @@ select_union::create_result_table(THD *thd_arg, List *column_types, table->keys_in_use_for_query.clear_all(); for (uint i=0; i < table->s->fields; i++) - table->field[i]->flags &= ~PART_KEY_FLAG; + table->field[i]->flags &= ~(PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG); if (create_table) { @@ -219,7 +219,7 @@ select_union_recursive::create_result_table(THD *thd_arg, incr_table->keys_in_use_for_query.clear_all(); for (uint i=0; i < table->s->fields; i++) - incr_table->field[i]->flags &= ~PART_KEY_FLAG; + incr_table->field[i]->flags &= ~(PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG); TABLE *rec_table= 0; if (! (rec_table= create_tmp_table(thd_arg, &tmp_table_param, *column_types, @@ -230,7 +230,7 @@ select_union_recursive::create_result_table(THD *thd_arg, rec_table->keys_in_use_for_query.clear_all(); for (uint i=0; i < table->s->fields; i++) - rec_table->field[i]->flags &= ~PART_KEY_FLAG; + rec_table->field[i]->flags &= ~(PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG); if (rec_tables.push_back(rec_table)) return true; diff --git a/sql/table.cc b/sql/table.cc index 12fc10c3259..198e77f1872 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3357,7 +3357,7 @@ partititon_err: } } - outparam->mark_columns_used_by_check_constraints(); + outparam->mark_columns_used_by_virtual_fields(); if (share->table_category == TABLE_CATEGORY_LOG) { @@ -6293,11 +6293,12 @@ void TABLE::mark_columns_needed_for_delete() Field **reg_field; for (reg_field= field ; *reg_field ; reg_field++) { - if ((*reg_field)->flags & PART_KEY_FLAG) + Field *cur_field= *reg_field; + if (cur_field->flags & (PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG)) { - bitmap_set_bit(read_set, (*reg_field)->field_index); - if ((*reg_field)->vcol_info) - mark_virtual_col(*reg_field); + bitmap_set_bit(read_set, cur_field->field_index); + if (cur_field->vcol_info) + bitmap_set_bit(vcol_set, cur_field->field_index); } } need_signal= true; @@ -6655,7 +6656,8 @@ bool TABLE::mark_virtual_columns_for_write(bool insert_fl if (bitmap_is_set(write_set, tmp_vfield->field_index)) bitmap_updated|= mark_virtual_col(tmp_vfield); else if (tmp_vfield->vcol_info->stored_in_db || - (tmp_vfield->flags & (PART_KEY_FLAG | FIELD_IN_PART_FUNC_FLAG))) + (tmp_vfield->flags & (PART_KEY_FLAG | FIELD_IN_PART_FUNC_FLAG | + PART_INDIRECT_KEY_FLAG))) { bitmap_set_bit(write_set, tmp_vfield->field_index); mark_virtual_col(tmp_vfield); @@ -6668,27 +6670,64 @@ bool TABLE::mark_virtual_columns_for_write(bool insert_fl } /* - Mark fields used by check constraints. + Mark fields used by check constraints into s->check_set. + Mark all fields used in an expression that is part of an index + with PART_INDIRECT_KEY_FLAG + This is done once for the TABLE_SHARE the first time the table is opened. The marking must be done non-destructively to handle the case when this could be run in parallely by two threads */ -void TABLE::mark_columns_used_by_check_constraints(void) +void TABLE::mark_columns_used_by_virtual_fields(void) { MY_BITMAP *save_read_set; - /* If there is no check constraints or if check_set is already initialized */ - if (!s->check_set || s->check_set_initialized) + Field **vfield_ptr; + + /* If there is virtual fields are already initialized */ + if (s->check_set_initialized) return; - save_read_set= read_set; - read_set= s->check_set; + if (s->tmp_table == NO_TMP_TABLE) + mysql_mutex_lock(&s->LOCK_share); + if (s->check_set) + { + /* Mark fields used by check constraint */ + save_read_set= read_set; + read_set= s->check_set; - for (Virtual_column_info **chk= check_constraints ; *chk ; chk++) - (*chk)->expr->walk(&Item::register_field_in_read_map, 1, 0); + for (Virtual_column_info **chk= check_constraints ; *chk ; chk++) + (*chk)->expr->walk(&Item::register_field_in_read_map, 1, 0); + read_set= save_read_set; + } - read_set= save_read_set; + /* + mark all fields that part of a virtual indexed field with + PART_INDIRECT_KEY_FLAG. This is used to ensure that all fields + that are part of an index exits before write/delete/update. + + As this code is only executed once per open share, it's reusing + existing functionality instead of adding an extra argument to + add_field_to_set_processor or adding another processor. + */ + if (vfield) + { + for (vfield_ptr= vfield; *vfield_ptr; vfield_ptr++) + { + if ((*vfield_ptr)->flags & PART_KEY_FLAG) + (*vfield_ptr)->vcol_info->expr->walk(&Item::add_field_to_set_processor, + 1, this); + } + for (uint i= 0 ; i < s->fields ; i++) + { + if (bitmap_is_set(&tmp_set, i)) + field[i]->flags|= PART_INDIRECT_KEY_FLAG; + } + bitmap_clear_all(&tmp_set); + } s->check_set_initialized= 1; + if (s->tmp_table == NO_TMP_TABLE) + mysql_mutex_unlock(&s->LOCK_share); } /* Add fields used by CHECK CONSTRAINT to read map */ @@ -7465,7 +7504,8 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) update= bitmap_is_set(vcol_set, vf->field_index); break; case VCOL_UPDATE_FOR_REPLACE: - update= ((!vcol_info->stored_in_db && (vf->flags & PART_KEY_FLAG) && + update= ((!vcol_info->stored_in_db && + (vf->flags & (PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG)) && bitmap_is_set(vcol_set, vf->field_index)) || update_all_columns); if (update && (vf->flags & BLOB_FLAG)) @@ -7484,7 +7524,8 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) case VCOL_UPDATE_INDEXED: case VCOL_UPDATE_INDEXED_FOR_UPDATE: /* Read indexed fields that was not updated in VCOL_UPDATE_FOR_READ */ - update= (!vcol_info->stored_in_db && (vf->flags & PART_KEY_FLAG) && + update= (!vcol_info->stored_in_db && + (vf->flags & (PART_KEY_FLAG | PART_INDIRECT_KEY_FLAG)) && !bitmap_is_set(vcol_set, vf->field_index)); swap_values= 1; break; diff --git a/sql/table.h b/sql/table.h index 6febeb555f9..4c409342c27 100644 --- a/sql/table.h +++ b/sql/table.h @@ -1323,7 +1323,7 @@ public: bool mark_virtual_col(Field *field); bool mark_virtual_columns_for_write(bool insert_fl); void mark_default_fields_for_write(bool insert_fl); - void mark_columns_used_by_check_constraints(void); + void mark_columns_used_by_virtual_fields(void); void mark_check_constraint_columns_for_read(void); int verify_constraints(bool ignore_failure); inline void column_bitmaps_set(MY_BITMAP *read_set_arg) From ab194666564acab29a4bcb7ca763e0ae0e691d1f Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 17 Jun 2018 14:19:51 +0300 Subject: [PATCH 179/276] MDEV-15114 ASAN heap-use-after-free in mem_heap_dup or dfield_data_is_binary_equal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bug was that innobase_get_computed_value() trashed record[0] and data in Field_blob::value Fixed by using a record on the heap for innobase_get_computed_value() Reviewer: Marko Mäkelä --- mysql-test/suite/vcol/r/index.result | 53 +++++++++++++++++ mysql-test/suite/vcol/t/index.test | 62 +++++++++++++++++++- sql/field.h | 4 ++ sql/table.cc | 48 ++++++++++++++++ sql/table.h | 7 ++- storage/innobase/handler/ha_innodb.cc | 82 ++++++++++++++++++++++++--- storage/innobase/include/row0mysql.h | 39 +++++++++++++ storage/innobase/row/row0ins.cc | 20 ++++++- storage/innobase/row/row0merge.cc | 27 +++++++-- storage/innobase/row/row0sel.cc | 19 ++++++- storage/innobase/row/row0upd.cc | 23 +++++++- storage/innobase/row/row0vers.cc | 18 +++++- 12 files changed, 376 insertions(+), 26 deletions(-) diff --git a/mysql-test/suite/vcol/r/index.result b/mysql-test/suite/vcol/r/index.result index cd4ebc96024..8860a728bd1 100644 --- a/mysql-test/suite/vcol/r/index.result +++ b/mysql-test/suite/vcol/r/index.result @@ -1,3 +1,6 @@ +# +# Test table with a key that consists of indirect virtual fields +# CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=myisam; insert into t1 (a) values (1),(2),(3); update t1 set a=5 where a=3; @@ -36,3 +39,53 @@ a b c 2 3 4 5 6 7 drop table t1; +# +# MDEV-15114 +# ASAN heap-use-after-free in mem_heap_dup or +# dfield_data_is_binary_equal +# +CREATE TABLE t1 ( +pk INT, +extra tinyint, +c TEXT, +vc LONGTEXT AS (c) VIRTUAL, +i INT, +PRIMARY KEY(pk), +UNIQUE(i), +INDEX(vc(64)) +) ENGINE=InnoDB; +INSERT INTO t1 (pk,extra, c, i) VALUES (1, 10, REPEAT('foo ',15000),0); +REPLACE INTO t1 (pk,extra, c,i) SELECT pk,extra+10, c,i FROM t1; +select pk, extra, left(c, 10), length(c), left(vc,10), length(vc), extra from t1; +pk extra left(c, 10) length(c) left(vc,10) length(vc) extra +1 20 foo foo fo 60000 foo foo fo 60000 20 +DROP TABLE t1; +# +# Update of deleted row with binary logging enabled +# +SET BINLOG_FORMAT=row; +CREATE TABLE t1 ( +pk INT, +c TEXT, +vc LONGTEXT AS (c) VIRTUAL, +i INT, +PRIMARY KEY(pk), +UNIQUE(i), +INDEX(vc(64)) +) ENGINE=InnoDB; +INSERT INTO t1 (pk,c,i) VALUES (1,REPEAT('foo ',15000),10); +INSERT INTO t1 (pk,c,i) VALUES (2,REPEAT('bar ',15000),11); +connect c1,localhost,root,,; +connection c1; +begin; +DELETE from t1 WHERE pk=1; +connection default; +update t1 set pk=1 where pk=2; +connection c1; +commit; +connection default; +select pk, left(c, 10), length(c), i from t1; +pk left(c, 10) length(c) i +1 bar bar ba 60000 11 +drop table t1; +disconnect c1; diff --git a/mysql-test/suite/vcol/t/index.test b/mysql-test/suite/vcol/t/index.test index 55d5b68f26b..72eed0a8a40 100644 --- a/mysql-test/suite/vcol/t/index.test +++ b/mysql-test/suite/vcol/t/index.test @@ -1,8 +1,9 @@ --source include/have_innodb.inc +--source include/have_log_bin.inc -# -# Test creating table with a key that consists of indirect virtual fields -# +--echo # +--echo # Test table with a key that consists of indirect virtual fields +--echo # CREATE TABLE t1 (a int, b int as (a+1) virtual, c int as (b+1) virtual, index(c)) engine=myisam; insert into t1 (a) values (1),(2),(3); @@ -23,3 +24,58 @@ select * from t1 where c=7; check table t1; select * from t1; drop table t1; + +--echo # +--echo # MDEV-15114 +--echo # ASAN heap-use-after-free in mem_heap_dup or +--echo # dfield_data_is_binary_equal +--echo # + +CREATE TABLE t1 ( + pk INT, + extra tinyint, + c TEXT, + vc LONGTEXT AS (c) VIRTUAL, + i INT, + PRIMARY KEY(pk), + UNIQUE(i), + INDEX(vc(64)) +) ENGINE=InnoDB; + +INSERT INTO t1 (pk,extra, c, i) VALUES (1, 10, REPEAT('foo ',15000),0); +REPLACE INTO t1 (pk,extra, c,i) SELECT pk,extra+10, c,i FROM t1; +select pk, extra, left(c, 10), length(c), left(vc,10), length(vc), extra from t1; +DROP TABLE t1; + +--echo # +--echo # Update of deleted row with binary logging enabled +--echo # + +SET BINLOG_FORMAT=row; + +CREATE TABLE t1 ( + pk INT, + c TEXT, + vc LONGTEXT AS (c) VIRTUAL, + i INT, + PRIMARY KEY(pk), + UNIQUE(i), + INDEX(vc(64)) +) ENGINE=InnoDB; + +INSERT INTO t1 (pk,c,i) VALUES (1,REPEAT('foo ',15000),10); +INSERT INTO t1 (pk,c,i) VALUES (2,REPEAT('bar ',15000),11); + +--connect (c1,localhost,root,,) +--connection c1 +begin; +DELETE from t1 WHERE pk=1; +--connection default +--send update t1 set pk=1 where pk=2 +--connection c1 +commit; +--connection default +--reap +select pk, left(c, 10), length(c), i from t1; +drop table t1; +disconnect c1; diff --git a/sql/field.h b/sql/field.h index e554f92031c..820cc5f3a7a 100644 --- a/sql/field.h +++ b/sql/field.h @@ -3433,6 +3433,10 @@ public: uint32 max_display_length(); uint32 char_length() const; uint is_equal(Create_field *new_field); + + friend void TABLE::remember_blob_values(String *blob_storage); + friend void TABLE::restore_blob_values(String *blob_storage); + private: int do_save_field_metadata(uchar *first_byte); }; diff --git a/sql/table.cc b/sql/table.cc index 198e77f1872..f7aee3eae7d 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -2434,6 +2434,8 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, reg_field->vcol_info= vcol_info; share->virtual_fields++; share->stored_fields--; + if (reg_field->flags & BLOB_FLAG) + share->virtual_not_stored_blob_fields++; /* Correct stored_rec_length as non stored fields are last */ recpos= (uint) (reg_field->ptr - record); if (share->stored_rec_length >= recpos) @@ -6776,6 +6778,52 @@ void TABLE::move_fields(Field **ptr, const uchar *to, const uchar *from) } +/* + Store all allocated virtual fields blob values + Used by InnoDB when calculating virtual fields for it's own internal + records +*/ + +void TABLE::remember_blob_values(String *blob_storage) +{ + Field **vfield_ptr; + for (vfield_ptr= vfield; *vfield_ptr; vfield_ptr++) + { + if ((*vfield_ptr)->type() == MYSQL_TYPE_BLOB && + !(*vfield_ptr)->vcol_info->stored_in_db) + { + Field_blob *blob= ((Field_blob*) *vfield_ptr); + memcpy((void*) blob_storage, (void*) &blob->value, sizeof(blob->value)); + blob_storage++; + blob->value.release(); + } + } +} + + +/* + Restore all allocated virtual fields blob values + Used by InnoDB when calculating virtual fields for it's own internal + records +*/ + +void TABLE::restore_blob_values(String *blob_storage) +{ + Field **vfield_ptr; + for (vfield_ptr= vfield; *vfield_ptr; vfield_ptr++) + { + if ((*vfield_ptr)->type() == MYSQL_TYPE_BLOB && + !(*vfield_ptr)->vcol_info->stored_in_db) + { + Field_blob *blob= ((Field_blob*) *vfield_ptr); + blob->value.free(); + memcpy((void*) &blob->value, (void*) blob_storage, sizeof(blob->value)); + blob_storage++; + } + } +} + + /** @brief Allocate space for keys diff --git a/sql/table.h b/sql/table.h index 4c409342c27..c0cca1026ea 100644 --- a/sql/table.h +++ b/sql/table.h @@ -663,8 +663,11 @@ struct TABLE_SHARE */ uint null_bytes_for_compare; uint fields; /* number of fields */ - uint stored_fields; /* number of stored fields, purely virtual not included */ + /* number of stored fields, purely virtual not included */ + uint stored_fields; uint virtual_fields; /* number of purely virtual fields */ + /* number of purely virtual not stored blobs */ + uint virtual_not_stored_blob_fields; uint null_fields; /* number of null fields */ uint blob_fields; /* number of blob fields */ uint varchar_fields; /* number of varchar fields */ @@ -1423,6 +1426,8 @@ public: { return (my_ptrdiff_t) (s->default_values - record[0]); } void move_fields(Field **ptr, const uchar *to, const uchar *from); + void remember_blob_values(String *blob_storage); + void restore_blob_values(String *blob_storage); uint actual_n_key_parts(KEY *keyinfo); ulong actual_key_flags(KEY *keyinfo); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 2c8ea81e286..7ac7495e221 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -21760,6 +21760,77 @@ innobase_get_field_from_update_vector( return (NULL); } + +/** + Allocate a heap and record for calculating virtual fields + Used mainly for virtual fields in indexes + +@param[in] thd MariaDB THD +@param[in] index Index in use +@param[out] heap Heap that holds temporary row +@param[in,out] mysql_table MariaDB table +@param[out] rec Pointer to allocated MariaDB record +@param[out] storage Internal storage for blobs etc + +@return FALSE ok +@return TRUE malloc failure +*/ + +bool innobase_allocate_row_for_vcol( + THD * thd, + dict_index_t* index, + mem_heap_t** heap, + TABLE** table, + byte** record, + VCOL_STORAGE** storage) +{ + TABLE *maria_table; + String *blob_value_storage; + if (!*table) + *table= innobase_find_mysql_table_for_vc(thd, index->table); + maria_table= *table; + if (!*heap && !(*heap= mem_heap_create(srv_page_size))) + { + *storage= 0; + return TRUE; + } + *record= static_cast(mem_heap_alloc(*heap, + maria_table->s->reclength)); + *storage= static_cast + (mem_heap_alloc(*heap, sizeof(**storage))); + blob_value_storage= static_cast + (mem_heap_alloc(*heap, + maria_table->s->virtual_not_stored_blob_fields * + sizeof(String))); + if (!*record || !*storage || !blob_value_storage) + { + *storage= 0; + return TRUE; + } + (*storage)->maria_table= maria_table; + (*storage)->innobase_record= *record; + (*storage)->maria_record= maria_table->field[0]->record_ptr(); + (*storage)->blob_value_storage= blob_value_storage; + + maria_table->move_fields(maria_table->field, *record, + (*storage)->maria_record); + maria_table->remember_blob_values(blob_value_storage); + + return FALSE; +} + + +/** Free memory allocated by innobase_allocate_row_for_vcol() */ + +void innobase_free_row_for_vcol(VCOL_STORAGE *storage) +{ + TABLE *maria_table= storage->maria_table; + maria_table->move_fields(maria_table->field, storage->maria_record, + storage->innobase_record); + maria_table->restore_blob_values(storage->blob_value_storage); +} + + /** Get the computed value by supplying the base column values. @param[in,out] row the data row @param[in] col virtual column @@ -21785,12 +21856,12 @@ innobase_get_computed_value( const dict_field_t* ifield, THD* thd, TABLE* mysql_table, + byte* mysql_rec, const dict_table_t* old_table, upd_t* parent_update, dict_foreign_t* foreign) { byte rec_buf2[REC_VERSION_56_MAX_INDEX_COL_LEN]; - byte* mysql_rec; byte* buf; dfield_t* field; ulint len; @@ -21803,6 +21874,7 @@ innobase_get_computed_value( ut_ad(index->table->vc_templ); ut_ad(thd != NULL); + ut_ad(mysql_table); const mysql_row_templ_t* vctempl = index->table->vc_templ->vtempl[ @@ -21820,14 +21892,6 @@ innobase_get_computed_value( buf = rec_buf2; } - if (!mysql_table) { - mysql_table = innobase_find_mysql_table_for_vc(thd, index->table); - } - - ut_ad(mysql_table); - - mysql_rec = mysql_table->record[0]; - for (ulint i = 0; i < col->num_base; i++) { dict_col_t* base_col = col->base_col[i]; const dfield_t* row_field = NULL; diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 7a34c025dab..8a573a23652 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -846,6 +846,44 @@ struct SysIndexCallback { virtual void operator()(mtr_t* mtr, btr_pcur_t* pcur) throw() = 0; }; + +/** Storage for calculating virtual columns */ + +class String; +struct VCOL_STORAGE +{ + TABLE *maria_table; + byte *innobase_record; + byte *maria_record; + String *blob_value_storage; +}; + +/** + Allocate a heap and record for calculating virtual fields + Used mainly for virtual fields in indexes + +@param[in] thd MariaDB THD +@param[in] index Index in use +@param[out] heap Heap that holds temporary row +@param[in,out] mysql_table MariaDB table +@param[out] rec Pointer to allocated MariaDB record +@param[out] storage Internal storage for blobs etc + +@return FALSE ok +@return TRUE malloc failure +*/ + +bool innobase_allocate_row_for_vcol( + THD * thd, + dict_index_t* index, + mem_heap_t** heap, + TABLE** table, + byte** record, + VCOL_STORAGE** storage); + +/** Free memory allocated by innobase_allocate_row_for_vcol() */ +void innobase_free_row_for_vcol(VCOL_STORAGE *storage); + /** Get the computed value by supplying the base column values. @param[in,out] row the data row @param[in] col virtual column @@ -870,6 +908,7 @@ innobase_get_computed_value( const dict_field_t* ifield, THD* thd, TABLE* mysql_table, + byte* mysql_rec, const dict_table_t* old_table, upd_t* parent_update, dict_foreign_t* foreign); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 1aa97bc34c5..57796939be1 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -944,6 +944,9 @@ row_ins_foreign_fill_virtual( rec_get_offsets(rec, index, offsets_, true, ULINT_UNDEFINED, &cascade->heap); mem_heap_t* v_heap = NULL; + TABLE* mysql_table= NULL; + VCOL_STORAGE* vcol_storage= NULL; + byte* record; upd_t* update = cascade->update; ulint n_v_fld = index->table->n_v_def; ulint n_diff; @@ -963,6 +966,14 @@ row_ins_foreign_fill_virtual( innobase_init_vc_templ(index->table); } + if (innobase_allocate_row_for_vcol(thd, index, &v_heap, + &mysql_table, + &record, &vcol_storage)) + { + *err = DB_OUT_OF_MEMORY; + goto func_exit; + } + for (ulint i = 0; i < n_v_fld; i++) { dict_v_col_t* col = dict_table_get_nth_v_col( @@ -976,8 +987,8 @@ row_ins_foreign_fill_virtual( dfield_t* vfield = innobase_get_computed_value( update->old_vrow, col, index, - &v_heap, update->heap, NULL, thd, NULL, - NULL, NULL, NULL); + &v_heap, update->heap, NULL, thd, mysql_table, + record, NULL, NULL, NULL); if (vfield == NULL) { *err = DB_COMPUTE_VALUE_FAILED; @@ -1007,7 +1018,8 @@ row_ins_foreign_fill_virtual( dfield_t* new_vfield = innobase_get_computed_value( update->old_vrow, col, index, &v_heap, update->heap, NULL, thd, - NULL, NULL, node->update, foreign); + mysql_table, record, NULL, + node->update, foreign); if (new_vfield == NULL) { *err = DB_COMPUTE_VALUE_FAILED; @@ -1025,6 +1037,8 @@ row_ins_foreign_fill_virtual( func_exit: if (v_heap) { + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(v_heap); } } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index bed15941638..66f3b3936af 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -534,6 +534,8 @@ row_merge_buf_add( ulint bucket = 0; doc_id_t write_doc_id; ulint n_row_added = 0; + VCOL_STORAGE* vcol_storage= 0; + byte* record; DBUG_ENTER("row_merge_buf_add"); if (buf->n_tuples >= buf->max_tuples) { @@ -606,14 +608,21 @@ row_merge_buf_add( dict_index_t* clust_index = dict_table_get_first_index(new_table); + if (!vcol_storage && + innobase_allocate_row_for_vcol(trx->mysql_thd, clust_index, v_heap, &my_table, &record, &vcol_storage)) { + *err = DB_OUT_OF_MEMORY; + goto error; + } + row_field = innobase_get_computed_value( row, v_col, clust_index, v_heap, NULL, ifield, trx->mysql_thd, - my_table, old_table, NULL, NULL); + my_table, record, old_table, NULL, + NULL); if (row_field == NULL) { *err = DB_COMPUTE_VALUE_FAILED; - DBUG_RETURN(0); + goto error; } dfield_copy(field, row_field); } else { @@ -649,7 +658,7 @@ row_merge_buf_add( ib::warn() << "FTS Doc ID is" " zero. Record" " skipped"; - DBUG_RETURN(0); + goto error; } } @@ -797,7 +806,7 @@ row_merge_buf_add( /* If this is FTS index, we already populated the sort buffer, return here */ if (index->type & DICT_FTS) { - DBUG_RETURN(n_row_added); + goto end; } #ifdef UNIV_DEBUG @@ -831,7 +840,7 @@ row_merge_buf_add( /* Reserve bytes for the end marker of row_merge_block_t. */ if (buf->total_size + data_size >= srv_sort_buf_size) { - DBUG_RETURN(0); + goto error; } buf->total_size += data_size; @@ -850,7 +859,15 @@ row_merge_buf_add( mem_heap_empty(conv_heap); } +end: + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); DBUG_RETURN(n_row_added); + +error: + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); + DBUG_RETURN(0); } /*************************************************************//** diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 7383d3d9510..68a81615e88 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -179,6 +179,8 @@ row_sel_sec_rec_is_for_clust_rec( ulint* clust_offs = clust_offsets_; ulint* sec_offs = sec_offsets_; ibool is_equal = TRUE; + VCOL_STORAGE* vcol_storage= 0; + byte* record; rec_offs_init(clust_offsets_); rec_offs_init(sec_offsets_); @@ -226,6 +228,17 @@ row_sel_sec_rec_is_for_clust_rec( dfield_t* vfield; row_ext_t* ext; + if (!vcol_storage) + { + TABLE *mysql_table= thr->prebuilt->m_mysql_table; + innobase_allocate_row_for_vcol(thr_get_trx(thr)->mysql_thd, + clust_index, + &heap, + &mysql_table, + &record, + &vcol_storage); + } + v_col = reinterpret_cast(col); row = row_build(ROW_COPY_POINTERS, @@ -237,8 +250,8 @@ row_sel_sec_rec_is_for_clust_rec( row, v_col, clust_index, &heap, NULL, NULL, thr_get_trx(thr)->mysql_thd, - thr->prebuilt->m_mysql_table, NULL, - NULL, NULL); + thr->prebuilt->m_mysql_table, + record, NULL, NULL, NULL); clust_len = vfield->len; clust_field = static_cast(vfield->data); @@ -326,6 +339,8 @@ inequal: func_exit: if (UNIV_LIKELY_NULL(heap)) { + if (UNIV_LIKELY_NULL(vcol_storage)) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(heap); } return(is_equal); diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 18a6a26f3ce..171e06894ca 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1023,6 +1023,7 @@ row_upd_build_sec_rec_difference_binary( return(update); } + /** Builds an update vector from those fields, excluding the roll ptr and trx id fields, which in an index entry differ from a record that has the equal ordering fields. NOTE: we compare the fields as binary strings! @@ -1123,6 +1124,9 @@ row_upd_build_difference_binary( if (n_v_fld > 0) { row_ext_t* ext; mem_heap_t* v_heap = NULL; + byte* record; + VCOL_STORAGE* vcol_storage; + THD* thd; if (trx == NULL) { @@ -1133,6 +1137,10 @@ row_upd_build_difference_binary( ut_ad(!update->old_vrow); + innobase_allocate_row_for_vcol(thd, index, &v_heap, + &mysql_table, + &record, &vcol_storage); + for (i = 0; i < n_v_fld; i++) { const dict_v_col_t* col = dict_table_get_nth_v_col(index->table, i); @@ -1151,7 +1159,7 @@ row_upd_build_difference_binary( dfield_t* vfield = innobase_get_computed_value( update->old_vrow, col, index, - &v_heap, heap, NULL, thd, mysql_table, + &v_heap, heap, NULL, thd, mysql_table, record, NULL, NULL, NULL); if (!dfield_data_is_binary_equal( @@ -1177,6 +1185,8 @@ row_upd_build_difference_binary( } if (v_heap) { + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(v_heap); } } @@ -2116,6 +2126,12 @@ row_upd_store_v_row( { mem_heap_t* heap = NULL; dict_index_t* index = dict_table_get_first_index(node->table); + byte* record= 0; + VCOL_STORAGE *vcol_storage= 0; + + if (!update) + innobase_allocate_row_for_vcol(thd, index, &heap, &mysql_table, + &record, &vcol_storage); for (ulint col_no = 0; col_no < dict_table_get_n_v_cols(node->table); col_no++) { @@ -2168,7 +2184,7 @@ row_upd_store_v_row( innobase_get_computed_value( node->row, col, index, &heap, node->heap, NULL, - thd, mysql_table, NULL, + thd, mysql_table, record, NULL, NULL, NULL); } } @@ -2176,8 +2192,11 @@ row_upd_store_v_row( } if (heap) { + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(heap); } + } /** Stores to the heap the row on which the node->pcur is positioned. diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 31cef0bad2d..aebcf70715f 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -446,6 +446,19 @@ row_vers_build_clust_v_col( mem_heap_t* heap) { mem_heap_t* local_heap = NULL; + VCOL_STORAGE *vcol_storage= NULL; + THD* thd= current_thd; + TABLE* maria_table= 0; + byte* record= 0; + + ut_ad(dict_index_has_virtual(index)); + + innobase_allocate_row_for_vcol(thd, index, + &local_heap, + &maria_table, + &record, + &vcol_storage); + for (ulint i = 0; i < dict_index_get_n_fields(index); i++) { const dict_field_t* ind_field = dict_index_get_nth_field( index, i); @@ -458,15 +471,18 @@ row_vers_build_clust_v_col( innobase_get_computed_value( row, col, clust_index, &local_heap, - heap, NULL, current_thd, NULL, NULL, + heap, NULL, thd, maria_table, record, NULL, NULL, NULL); } } if (local_heap) { + if (vcol_storage) + innobase_free_row_for_vcol(vcol_storage); mem_heap_free(local_heap); } } + /** Build latest virtual column data from undo log @param[in] in_purge whether this is the purge thread @param[in] rec clustered index record From 10d09a57f88cafaabcb6ba8475c1951fe329756e Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 18 Jun 2018 22:44:58 +0300 Subject: [PATCH 180/276] Fixed failing test acl_load_mutex-5170 Added flush tables to ensure that MyISAM tables are properly flushed before reboot --- mysql-test/suite/roles/acl_load_mutex-5170.result | 1 + mysql-test/suite/roles/acl_load_mutex-5170.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/roles/acl_load_mutex-5170.result b/mysql-test/suite/roles/acl_load_mutex-5170.result index 6f6688f446e..775541b94f7 100644 --- a/mysql-test/suite/roles/acl_load_mutex-5170.result +++ b/mysql-test/suite/roles/acl_load_mutex-5170.result @@ -1,6 +1,7 @@ create user user1@localhost; create role r1 with admin user1@localhost; grant all on test.* to r1; +flush tables; select 1; 1 1 diff --git a/mysql-test/suite/roles/acl_load_mutex-5170.test b/mysql-test/suite/roles/acl_load_mutex-5170.test index e77d191ea16..22b9dffb5fd 100644 --- a/mysql-test/suite/roles/acl_load_mutex-5170.test +++ b/mysql-test/suite/roles/acl_load_mutex-5170.test @@ -5,6 +5,7 @@ create user user1@localhost; create role r1 with admin user1@localhost; grant all on test.* to r1; +flush tables; --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect wait From 778df04661c72544baed4363a55b07362d5020ed Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Tue, 19 Jun 2018 19:19:40 +0200 Subject: [PATCH 181/276] MDEV-16517: Server crash in Item_func_in::val_int() when IN predicate defined with non-constant values is pushed down The problem appears because of wrong changes made in MDEV-16090 in the Item_func_in::build_clone() method. For the clone of the IN predicate it copied 'cmp_fields' array values that become dirty after Item::cleanup_excluding_const_fields_processor has worked in pushdown. That causes crash. There is no need to copy 'cmp_fields' field, the array values should be NULLs in order to fix_fields() for the cloned IN predicate can set them correctly. fix_fields() computes values for 'cmp_fields' array only if they were not set earlier. --- mysql-test/r/derived_cond_pushdown.result | 139 ++++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 39 ++++++ sql/item_cmpfunc.cc | 2 +- 3 files changed, 179 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 83e70dd634c..a2e84a1e419 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -9669,3 +9669,142 @@ EXPLAIN } } DROP TABLE t1; +# +# MDEV-16517: pushdown condition with the IN predicate defined +# with non-constant values +# +CREATE TABLE t1 (a INT, b INT); +INSERT INTO t1 VALUES (1,2),(1,3); +SELECT * FROM +( +SELECT t1.a +FROM t1 +WHERE 1 IN (0,t1.a) +GROUP BY t1.a +) AS dt1 +JOIN +( +SELECT t1.a +FROM t1 +WHERE 1 IN (0,t1.a) +) AS dt2 +ON dt1.a = dt2.a; +a a +1 1 +1 1 +EXPLAIN FORMAT=JSON SELECT * FROM +( +SELECT t1.a +FROM t1 +WHERE 1 IN (0,t1.a) +GROUP BY t1.a +) AS dt1 +JOIN +( +SELECT t1.a +FROM t1 +WHERE 1 IN (0,t1.a) +) AS dt2 +ON dt1.a = dt2.a; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 2, + "filtered": 100, + "attached_condition": "1 in (0,dt1.a)", + "materialized": { + "query_block": { + "select_id": 2, + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 2, + "filtered": 100, + "attached_condition": "1 in (0,t1.a) and 1 in (0,t1.a)" + } + } + } + } + } + }, + "block-nl-join": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 2, + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "t1.a = dt1.a" + } + } +} +SELECT * FROM +( +SELECT t1.a,MAX(t1.b) +FROM t1 +GROUP BY t1.a +) AS dt, t1 +WHERE dt.a=t1.a AND dt.a IN (1,t1.a); +a MAX(t1.b) a b +1 3 1 2 +1 3 1 3 +EXPLAIN FORMAT=JSON SELECT * FROM +( +SELECT t1.a,MAX(t1.b) +FROM t1 +GROUP BY t1.a +) AS dt, t1 +WHERE dt.a=t1.a AND dt.a IN (1,t1.a); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 2, + "filtered": 100, + "attached_condition": "dt.a in (1,dt.a)", + "materialized": { + "query_block": { + "select_id": 2, + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 2, + "filtered": 100, + "attached_condition": "t1.a in (1,t1.a)" + } + } + } + } + } + }, + "block-nl-join": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 2, + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "t1.a = dt.a" + } + } +} +DROP TABLE t1; diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index 718140d3a77..3103ddb5c55 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -1801,3 +1801,42 @@ EVAL $query; EVAL EXPLAIN FORMAT=JSON $query; DROP TABLE t1; + +--echo # +--echo # MDEV-16517: pushdown condition with the IN predicate defined +--echo # with non-constant values +--echo # + +CREATE TABLE t1 (a INT, b INT); +INSERT INTO t1 VALUES (1,2),(1,3); + +LET $query= +SELECT * FROM +( + SELECT t1.a + FROM t1 + WHERE 1 IN (0,t1.a) + GROUP BY t1.a +) AS dt1 +JOIN +( + SELECT t1.a + FROM t1 + WHERE 1 IN (0,t1.a) +) AS dt2 +ON dt1.a = dt2.a; +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +LET $query= +SELECT * FROM +( + SELECT t1.a,MAX(t1.b) + FROM t1 + GROUP BY t1.a +) AS dt, t1 +WHERE dt.a=t1.a AND dt.a IN (1,t1.a); +EVAL $query; +EVAL EXPLAIN FORMAT=JSON $query; + +DROP TABLE t1; diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index b86c0079bce..f176a0a8193 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -4438,7 +4438,7 @@ Item *Item_func_in::build_clone(THD *thd, MEM_ROOT *mem_root) { if (array && clone->create_array(thd)) return NULL; - memcpy(&clone->cmp_items, &cmp_items, sizeof(cmp_items)); + bzero(&clone->cmp_items, sizeof(cmp_items)); } return clone; } From 5f2a67a6c35fd0d833024652ddc33eab8bcb1ed4 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Wed, 20 Jun 2018 02:36:00 +0530 Subject: [PATCH 182/276] MDEV-15247: Crash when SET NAMES 'utf8' is set In this case we are accessing incorrect memory when we have mergeable semi-joins. In the case when we have mergeable semi joins parent select will have a table count of all the tables in that select plus all the tables involved in the IN-subquery. But this table count does not include the "sjm table" (only includes the inner and outer tables) denotes as in explain. --- mysql-test/r/subselect_sj2_mat.result | 90 +++++++++++++++++++++++++++ mysql-test/t/subselect_sj2_mat.test | 87 ++++++++++++++++++++++++++ sql/sql_select.cc | 4 +- 3 files changed, 180 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/subselect_sj2_mat.result b/mysql-test/r/subselect_sj2_mat.result index c629c8196c7..c27beb295b8 100644 --- a/mysql-test/r/subselect_sj2_mat.result +++ b/mysql-test/r/subselect_sj2_mat.result @@ -1691,3 +1691,93 @@ id 12 13 drop table t1; +# +# MDEV-15247: Crash when SET NAMES 'utf8' is set +# +CREATE TABLE t1 ( +id_category int unsigned, +id_product int unsigned, +PRIMARY KEY (id_category,id_product) +) ENGINE=MyISAM; +INSERT INTO `t1` VALUES (31,216), (31,215), (31,214), (31,213), (31,212), (32,211), (32,210), (32,209), (32,208), (29,207), (30,315372), (2,161), (2,132), (33,315380), (31,315371), (29,315370), (29,315373), (29,315369), (29,315374), (29,315368), (29,315375), (29,315367), (29,183), (29,182), (30,177), (29,315376), (13,315365), (2,167), (2,315357), (2,164), (2,159), (2,131), (2,127), (14,315364), (27,315363), (29,205), (29,204), (29,203), (29,202), (29,201), (29,200), (29,199), (29,198), (29,197), (29,196), (29,195), (29,194), (29,193), (29,192), (29,191), (29,190), (29,189), (14,188), (29,187), (29,186), (29,185), (29,184), (29,315377), (29,315378), (29,181), (33,315379), (29,179), (30,178), (29,180), (30,176), (30,175), (30,174), (30,173), (30,172), (11,171), (27,315357), (23,108), (23,102); +CREATE TABLE t2 ( +id_product int, +id_t2 int, +KEY id_t2 (id_t2), +KEY id_product (id_product) +) ENGINE=MyISAM; +INSERT INTO `t2` VALUES (11,31), (11,31), (11,31), (11,32), (11,32), +(11,32), (10,26), (11,32), (10,28), (11,32), (10,29), (11,33), (10,26), +(11,33), (10,27), (9,23), (11,32), (10,26), (8,18), (7,15), (11,32), +(10,28), (11,32), (10,28), (11,32), (10,29), (11,32), (10,29), (8,19), +(7,16), (8,18), (7,16), (8,20), (7,16), (11,32), (10,28), (8,19), +(7,16), (8,20), (7,16), (11,32), (10,29), (8,19), (7,16), (8,20), +(7,16), (10,27), (9,23), (10,27), (9,23), (10,27), (9,23), (11,32), +(10,27), (11,32), (10,27), (8,18), (7,15), (10,26), (9,24), (8,19), +(7,16), (10,26), (9,23), (8,19), (7,16), (8,18), (7,16), (8,18), (7,16), +(9,23), (8,18), (9,23), (8,19), (7,16), (7,16), (8,19), (7,16), (11,31), +(10,27), (9,24), (11,31), (10,27), (9,23), (8,19), (11,31), (10,26), (9,24), +(8,19), (11,31), (10,26), (9,25), (8,18), (11,31), (10,26), (9,23), (8,19), +(11,31), (10,26), (9,23), (8,18), (11,31), (10,30), (9,23), (8,18), (11,31), +(10,30), (9,23), (8,19), (11,31), (10,26), (9,25), (8,19), (8,21), (11,32), +(10,26), (9,22), (8,19), (11,32), (10,26), (9,22), (8,18), (11,32), (10,26), +(9,22), (8,20), (11,33), (10,26), (9,22), (8,19), (11,33), (10,26), (9,22), +(8,18), (11,33), (10,26), (9,22), (8,20), (11,32), (10,26), (9,24), (8,19), +(11,32), (10,26), (9,25), (8,19), (11,32), (10,26), (9,25), (8,18), (11,32), +(10,26), (9,23), (8,18), (11,32), (10,30), (9,23), (8,18), (11,32), (10,30), +(9,23), (8,19), (11,32), (10,26), (9,23), (8,19), (11,32), (10,27), (9,23), +(11,32), (10,27), (9,23), (11,32), (10,27), (9,23), (10,26), (9,22), (8,19), +(7,15), (10,26), (9,22), (8,20), (7,15), (10,26), (9,22), (8,18), (7,15), +(8,19), (10,26), (10,26), (11,33), (10,26), (11,33), (10,26), (11,33), +(10,27), (11,33), (10,27), (11,31), (10,26), (11,31), (10,26), (8,18), +(7,15), (9,23), (9,23), (9,24), (8,21), (7,15), (7,15), (7,15), (7,15), +(7,15), (7,15), (7,15), (7,15), (7,15), (8,18), (7,17), (8,18), (7,17), (8,19), (8,19); +CREATE TABLE t3 ( +id_product int unsigned, +PRIMARY KEY (id_product) +) ENGINE=MyISAM; +INSERT INTO t3 VALUES +(102),(103),(104),(105),(106),(107),(108),(109),(110), +(315371),(315373),(315374),(315375),(315376),(315377), +(315378),(315379),(315380); +CREATE TABLE t4 ( +id_product int not null, +id_shop int, +PRIMARY KEY (id_product,id_shop) +) ENGINE=MyISAM ; +INSERT INTO t4 VALUES +(202,1),(201,1),(200,1),(199,1),(198,1),(197,1),(196,1),(195,1), +(194,1),(193,1),(192,1),(191,1),(190,1),(189,1),(188,1),(187,1), +(186,1),(185,1),(184,1),(183,1),(182,1),(181,1),(179,1),(178,1), +(177,1),(176,1),(126,1),(315380,1); +CREATE TABLE t5 (id_product int) ENGINE=MyISAM; +INSERT INTO `t5` VALUES +(652),(668),(669),(670),(671),(673),(674),(675),(676), +(677),(679),(680),(681),(682),(683),(684),(685),(686); +explain +SELECT * FROM t3 +JOIN t4 ON (t4.id_product = t3.id_product AND t4.id_shop = 1) +JOIN t1 ON (t1.id_product = t3.id_product) +LEFT JOIN t5 ON (t5.id_product = t3.id_product) +WHERE 1=1 +AND t3.id_product IN (SELECT id_product FROM t2 t2_1 WHERE t2_1.id_t2 = 32) +AND t3.id_product IN (SELECT id_product FROM t2 t2_2 WHERE t2_2.id_t2 = 15) +AND t3.id_product IN (SELECT id_product FROM t2 t2_3 WHERE t2_3.id_t2 = 18 OR t2_3.id_t2 = 19) +AND t3.id_product IN (SELECT id_product FROM t2 t2_4 WHERE t2_4.id_t2 = 34 OR t2_4.id_t2 = 23) +AND t3.id_product IN (SELECT id_product FROM t2 t2_5 WHERE t2_5.id_t2 = 29 OR t2_5.id_t2 = 28 OR t2_5.id_t2 = 26); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 index NULL PRIMARY 8 NULL 73 Using index +1 PRIMARY t3 eq_ref PRIMARY PRIMARY 4 test.t1.id_product 1 Using index +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY t4 eq_ref PRIMARY PRIMARY 8 test.t1.id_product,const 1 Using where; Using index +1 PRIMARY eq_ref distinct_key distinct_key 4 func 1 Using where +1 PRIMARY t5 ALL NULL NULL NULL NULL 18 Using where; Using join buffer (flat, BNL join) +5 MATERIALIZED t2_4 range id_t2,id_product id_t2 5 NULL 18 Using index condition; Using where +4 MATERIALIZED t2_3 range id_t2,id_product id_t2 5 NULL 32 Using index condition; Using where +3 MATERIALIZED t2_2 ref id_t2,id_product id_t2 5 const 12 +2 MATERIALIZED t2_1 ref id_t2,id_product id_t2 5 const 50 +6 MATERIALIZED t2_5 range id_t2,id_product id_t2 5 NULL 30 Using index condition; Using where +drop table t1,t2,t3,t4,t5; diff --git a/mysql-test/t/subselect_sj2_mat.test b/mysql-test/t/subselect_sj2_mat.test index 0665cdf68fe..68a888012f2 100644 --- a/mysql-test/t/subselect_sj2_mat.test +++ b/mysql-test/t/subselect_sj2_mat.test @@ -345,3 +345,90 @@ WHERE ( (t.id IN (0,4,12,13,1,10,3,11)) ); drop table t1; + +--echo # +--echo # MDEV-15247: Crash when SET NAMES 'utf8' is set +--echo # + +CREATE TABLE t1 ( + id_category int unsigned, + id_product int unsigned, + PRIMARY KEY (id_category,id_product) +) ENGINE=MyISAM; + +INSERT INTO `t1` VALUES (31,216), (31,215), (31,214), (31,213), (31,212), (32,211), (32,210), (32,209), (32,208), (29,207), (30,315372), (2,161), (2,132), (33,315380), (31,315371), (29,315370), (29,315373), (29,315369), (29,315374), (29,315368), (29,315375), (29,315367), (29,183), (29,182), (30,177), (29,315376), (13,315365), (2,167), (2,315357), (2,164), (2,159), (2,131), (2,127), (14,315364), (27,315363), (29,205), (29,204), (29,203), (29,202), (29,201), (29,200), (29,199), (29,198), (29,197), (29,196), (29,195), (29,194), (29,193), (29,192), (29,191), (29,190), (29,189), (14,188), (29,187), (29,186), (29,185), (29,184), (29,315377), (29,315378), (29,181), (33,315379), (29,179), (30,178), (29,180), (30,176), (30,175), (30,174), (30,173), (30,172), (11,171), (27,315357), (23,108), (23,102); + +CREATE TABLE t2 ( + id_product int, + id_t2 int, + KEY id_t2 (id_t2), + KEY id_product (id_product) +) ENGINE=MyISAM; + +INSERT INTO `t2` VALUES (11,31), (11,31), (11,31), (11,32), (11,32), +(11,32), (10,26), (11,32), (10,28), (11,32), (10,29), (11,33), (10,26), +(11,33), (10,27), (9,23), (11,32), (10,26), (8,18), (7,15), (11,32), +(10,28), (11,32), (10,28), (11,32), (10,29), (11,32), (10,29), (8,19), +(7,16), (8,18), (7,16), (8,20), (7,16), (11,32), (10,28), (8,19), +(7,16), (8,20), (7,16), (11,32), (10,29), (8,19), (7,16), (8,20), +(7,16), (10,27), (9,23), (10,27), (9,23), (10,27), (9,23), (11,32), +(10,27), (11,32), (10,27), (8,18), (7,15), (10,26), (9,24), (8,19), +(7,16), (10,26), (9,23), (8,19), (7,16), (8,18), (7,16), (8,18), (7,16), +(9,23), (8,18), (9,23), (8,19), (7,16), (7,16), (8,19), (7,16), (11,31), +(10,27), (9,24), (11,31), (10,27), (9,23), (8,19), (11,31), (10,26), (9,24), +(8,19), (11,31), (10,26), (9,25), (8,18), (11,31), (10,26), (9,23), (8,19), +(11,31), (10,26), (9,23), (8,18), (11,31), (10,30), (9,23), (8,18), (11,31), +(10,30), (9,23), (8,19), (11,31), (10,26), (9,25), (8,19), (8,21), (11,32), +(10,26), (9,22), (8,19), (11,32), (10,26), (9,22), (8,18), (11,32), (10,26), +(9,22), (8,20), (11,33), (10,26), (9,22), (8,19), (11,33), (10,26), (9,22), +(8,18), (11,33), (10,26), (9,22), (8,20), (11,32), (10,26), (9,24), (8,19), +(11,32), (10,26), (9,25), (8,19), (11,32), (10,26), (9,25), (8,18), (11,32), +(10,26), (9,23), (8,18), (11,32), (10,30), (9,23), (8,18), (11,32), (10,30), +(9,23), (8,19), (11,32), (10,26), (9,23), (8,19), (11,32), (10,27), (9,23), +(11,32), (10,27), (9,23), (11,32), (10,27), (9,23), (10,26), (9,22), (8,19), +(7,15), (10,26), (9,22), (8,20), (7,15), (10,26), (9,22), (8,18), (7,15), +(8,19), (10,26), (10,26), (11,33), (10,26), (11,33), (10,26), (11,33), +(10,27), (11,33), (10,27), (11,31), (10,26), (11,31), (10,26), (8,18), +(7,15), (9,23), (9,23), (9,24), (8,21), (7,15), (7,15), (7,15), (7,15), +(7,15), (7,15), (7,15), (7,15), (7,15), (8,18), (7,17), (8,18), (7,17), (8,19), (8,19); + +CREATE TABLE t3 ( + id_product int unsigned, + PRIMARY KEY (id_product) +) ENGINE=MyISAM; + +INSERT INTO t3 VALUES +(102),(103),(104),(105),(106),(107),(108),(109),(110), +(315371),(315373),(315374),(315375),(315376),(315377), +(315378),(315379),(315380); + +CREATE TABLE t4 ( + id_product int not null, + id_shop int, + PRIMARY KEY (id_product,id_shop) +) ENGINE=MyISAM ; + +INSERT INTO t4 VALUES +(202,1),(201,1),(200,1),(199,1),(198,1),(197,1),(196,1),(195,1), +(194,1),(193,1),(192,1),(191,1),(190,1),(189,1),(188,1),(187,1), +(186,1),(185,1),(184,1),(183,1),(182,1),(181,1),(179,1),(178,1), +(177,1),(176,1),(126,1),(315380,1); + +CREATE TABLE t5 (id_product int) ENGINE=MyISAM; +INSERT INTO `t5` VALUES +(652),(668),(669),(670),(671),(673),(674),(675),(676), +(677),(679),(680),(681),(682),(683),(684),(685),(686); + +explain +SELECT * FROM t3 + JOIN t4 ON (t4.id_product = t3.id_product AND t4.id_shop = 1) + JOIN t1 ON (t1.id_product = t3.id_product) +LEFT JOIN t5 ON (t5.id_product = t3.id_product) +WHERE 1=1 +AND t3.id_product IN (SELECT id_product FROM t2 t2_1 WHERE t2_1.id_t2 = 32) +AND t3.id_product IN (SELECT id_product FROM t2 t2_2 WHERE t2_2.id_t2 = 15) +AND t3.id_product IN (SELECT id_product FROM t2 t2_3 WHERE t2_3.id_t2 = 18 OR t2_3.id_t2 = 19) +AND t3.id_product IN (SELECT id_product FROM t2 t2_4 WHERE t2_4.id_t2 = 34 OR t2_4.id_t2 = 23) +AND t3.id_product IN (SELECT id_product FROM t2 t2_5 WHERE t2_5.id_t2 = 29 OR t2_5.id_t2 = 28 OR t2_5.id_t2 = 26); + +drop table t1,t2,t3,t4,t5; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 39209f04c9f..1ec6cca7c5b 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -9592,7 +9592,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) table_map current_map; i= join->const_tables; for (tab= first_depth_first_tab(join); tab; - tab= next_depth_first_tab(join, tab), i++) + tab= next_depth_first_tab(join, tab)) { bool is_hj; /* @@ -10063,6 +10063,8 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond) } first_inner_tab= first_inner_tab->first_upper; } + if (!tab->bush_children) + i++; } } DBUG_RETURN(0); From d79bf0009a17f0020203003a97ce7e83449aeb3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 20 Jun 2018 01:23:07 +0300 Subject: [PATCH 183/276] MDEV-16525: MyRocks linking fails with: Undefined reference to `ZDICT_trainFromBuffer' RocksDB will only build with libzstd support if libzstd version is >=1.1.13. Unfortunately CMake's FindPackage claims it found version 1.1.13 when we have 1.1.12-1 installed, so a workaround with CheckFunctionExists is used to properly check for correct libzstd support. --- storage/rocksdb/build_rocksdb.cmake | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index c76f711463e..8f01024be63 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -64,10 +64,20 @@ if(SNAPPY_FOUND AND (NOT WITH_ROCKSDB_SNAPPY STREQUAL "OFF")) list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES}) endif() +include(CheckFunctionExists) if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_ZSTD STREQUAL "OFF")) - add_definitions(-DZSTD) - include_directories(${ZSTD_INCLUDE_DIR}) - list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARY}) + SET(CMAKE_REQUIRED_LIBRARIES zstd) + CHECK_FUNCTION_EXISTS(ZDICT_trainFromBuffer ZSTD_VALID) + UNSET(CMAKE_REQUIRED_LIBRARIES) + if (WITH_ROCKSDB_ZSTD STREQUAL "ON" AND NOT ZSTD_VALID) + MESSAGE(FATAL_ERROR + "WITH_ROCKSDB_ZSTD is ON and ZSTD library was found, but the version needs to be >= 1.1.3") + endif() + if (ZSTD_VALID) + add_definitions(-DZSTD) + include_directories(${ZSTD_INCLUDE_DIR}) + list(APPEND THIRDPARTY_LIBS ${ZSTD_LIBRARY}) + endif() endif() add_definitions(-DZLIB) @@ -119,7 +129,6 @@ int main() { endif() endif() -include(CheckFunctionExists) CHECK_FUNCTION_EXISTS(malloc_usable_size HAVE_MALLOC_USABLE_SIZE) if(HAVE_MALLOC_USABLE_SIZE) add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE) From 04c47454781533408deb10bf770744327415aa59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Wed, 20 Jun 2018 01:28:59 +0300 Subject: [PATCH 184/276] Fix double WSREP_ISOLATION_BEGIN merge error --- sql/sql_plugin.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 7e4be9aed16..270466fa0f7 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2122,8 +2122,6 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, DBUG_RETURN(TRUE); WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) - /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table = open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT))) From 44682962e3fe591d16c6fa966c39e9c738427712 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Jun 2018 11:10:15 +0200 Subject: [PATCH 185/276] Fix another double WSREP_ISOLATION_BEGIN merge error --- sql/sql_plugin.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index 270466fa0f7..8d055fd612f 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2263,7 +2263,6 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, if (!opt_noacl && check_table_access(thd, DELETE_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) From bb24663f5ad955a615172512c04779d219bb6645 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 20 Jun 2018 10:45:57 +0200 Subject: [PATCH 186/276] MDEV-13577 slave_parallel_mode=optimistic should not report the mode's specific temporary errors Revert 7bbe324fc17 Fix the bug differently (in log_event.cc) Fix the test case to actually fail without the bug fix --- include/my_sys.h | 1 - .../suite/rpl/r/rpl_parallel_optimistic.result | 4 ++-- .../suite/rpl/t/rpl_parallel_optimistic.test | 7 ++----- sql/handler.cc | 3 --- sql/log_event.cc | 18 ++++++++++++++---- sql/mysqld.cc | 10 ---------- sql/sql_class.cc | 7 ------- sql/sql_class.h | 6 ------ 8 files changed, 18 insertions(+), 38 deletions(-) diff --git a/include/my_sys.h b/include/my_sys.h index 1c5649812d1..110a2ee9af3 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -112,7 +112,6 @@ typedef struct my_aio_result { #define ME_JUST_INFO 1024 /**< not error but just info */ #define ME_JUST_WARNING 2048 /**< not error but just warning */ #define ME_FATALERROR 4096 /* Fatal statement error */ -#define ME_LOG_AS_WARN 8192 /* is error but error-logged as warning */ /* Bits in last argument to fn_format */ #define MY_REPLACE_DIR 1 /* replace dir in name with 'dir' */ diff --git a/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result b/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result index a6da3399fab..3cd4f8231bf 100644 --- a/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result +++ b/mysql-test/suite/rpl/r/rpl_parallel_optimistic.result @@ -1,6 +1,4 @@ include/rpl_init.inc [topology=1->2] -call mtr.add_suppression("Warning.*Deadlock found when trying to get lock; try restarting transaction"); -call mtr.add_suppression("Warning.*mysqld: Can't find record in 't2'"); ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; @@ -549,6 +547,7 @@ DELETE FROM t1; DELETE FROM t2; include/save_master_gtid.inc include/sync_with_master_gtid.inc +set global log_warnings=2; BEGIN; INSERT INTO t1 SET a=1; SET @save.binlog_format=@@session.binlog_format; @@ -567,6 +566,7 @@ DELETE FROM t2; include/save_master_gtid.inc include/sync_with_master_gtid.inc include/stop_slave.inc +set global log_warnings=default; SET GLOBAL slave_parallel_mode=@old_parallel_mode; SET GLOBAL slave_parallel_threads=@old_parallel_threads; include/start_slave.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test b/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test index 28bf4a77fd4..9f6669279db 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_optimistic.test @@ -4,11 +4,6 @@ --let $rpl_topology=1->2 --source include/rpl_init.inc ---connection server_2 -call mtr.add_suppression("Warning.*Deadlock found when trying to get lock; try restarting transaction"); -# The following instruction is a part of the proof of MDEV-13577 fixes, below. -call mtr.add_suppression("Warning.*mysqld: Can't find record in 't2'"); - --connection server_1 ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB; @@ -500,6 +495,7 @@ DELETE FROM t2; # The 1st of the following two trx:s a blocker on slave --connection server_2 +set global log_warnings=2; BEGIN; INSERT INTO t1 SET a=1; @@ -546,6 +542,7 @@ DELETE FROM t2; # --connection server_2 --source include/stop_slave.inc +set global log_warnings=default; SET GLOBAL slave_parallel_mode=@old_parallel_mode; SET GLOBAL slave_parallel_threads=@old_parallel_threads; --source include/start_slave.inc diff --git a/sql/handler.cc b/sql/handler.cc index 1027a8b102d..35b0814ef79 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -3639,13 +3639,10 @@ void handler::print_error(int error, myf errflag) if ((debug_assert_if_crashed_table || global_system_variables.log_warnings > 1)) { - THD *thd= ha_thd(); /* Log error to log before we crash or if extended warnings are requested */ errflag|= ME_NOREFRESH; - if (thd && thd->is_optimistic_slave_worker()) - errflag|= ME_LOG_AS_WARN; } } diff --git a/sql/log_event.cc b/sql/log_event.cc index e70d97c54c0..3ac7ac5a20f 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -12080,6 +12080,16 @@ void issue_long_find_row_warning(Log_event_type type, } +/* + HA_ERR_KEY_NOT_FOUND is a fatal error normally, but it's an expected + error in speculate optimistic mode, so use something non-fatal instead +*/ +static int row_not_found_error(rpl_group_info *rgi) +{ + return rgi->speculation != rpl_group_info::SPECULATE_OPTIMISTIC + ? HA_ERR_KEY_NOT_FOUND : HA_ERR_RECORD_CHANGED; +} + /** Locate the current row in event's table. @@ -12167,8 +12177,8 @@ int Rows_log_event::find_row(rpl_group_info *rgi) if (error) { DBUG_PRINT("info",("rnd_pos returns error %d",error)); - if (error == HA_ERR_RECORD_DELETED) - error= HA_ERR_KEY_NOT_FOUND; + if (error == HA_ERR_RECORD_DELETED || error == HA_ERR_KEY_NOT_FOUND) + error= row_not_found_error(rgi); table->file->print_error(error, MYF(0)); } DBUG_RETURN(error); @@ -12233,8 +12243,8 @@ int Rows_log_event::find_row(rpl_group_info *rgi) HA_READ_KEY_EXACT))) { DBUG_PRINT("info",("no record matching the key found in the table")); - if (error == HA_ERR_RECORD_DELETED) - error= HA_ERR_KEY_NOT_FOUND; + if (error == HA_ERR_RECORD_DELETED || error == HA_ERR_KEY_NOT_FOUND) + error= row_not_found_error(rgi); table->file->print_error(error, MYF(0)); table->file->ha_index_end(); goto end; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5bf5e3f73fc..5f954f7576d 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3517,16 +3517,6 @@ void my_message_sql(uint error, const char *str, myf MyFlags) level= Sql_condition::WARN_LEVEL_WARN; func= sql_print_warning; } - else if (MyFlags & ME_LOG_AS_WARN) - { - /* - Typical use case is optimistic parallel slave where DA needs to hold - an error condition caused by the current error, but the error-log - level is relaxed to the warning one. - */ - level= Sql_condition::WARN_LEVEL_ERROR; - func= sql_print_warning; - } else { level= Sql_condition::WARN_LEVEL_ERROR; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 0a95d2e2cc5..0f629a2c995 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -7003,13 +7003,6 @@ bool THD::rgi_have_temporary_tables() return rgi_slave->rli->save_temporary_tables != 0; } -bool THD::is_optimistic_slave_worker() -{ - DBUG_ASSERT(system_thread != SYSTEM_THREAD_SLAVE_SQL || rgi_slave); - - return system_thread == SYSTEM_THREAD_SLAVE_SQL && rgi_slave && - rgi_slave->speculation == rpl_group_info::SPECULATE_OPTIMISTIC; -} void wait_for_commit::reinit() diff --git a/sql/sql_class.h b/sql/sql_class.h index fb3604b899b..6d3f0965df0 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -4168,12 +4168,6 @@ public: (THD_TRANS::DID_WAIT | THD_TRANS::CREATED_TEMP_TABLE | THD_TRANS::DROPPED_TEMP_TABLE | THD_TRANS::DID_DDL)); } - - /* - Returns true when the thread handle belongs to a slave worker thread - running in the optimistic execution mode. - */ - bool is_optimistic_slave_worker(); }; From 9c53cbdd8887f0f8bcdade24a4be183a3b354bc9 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 20 Jun 2018 13:29:11 +0400 Subject: [PATCH 187/276] MDEV-15941 Explicit cursor FOR loop does not close the cursor --- mysql-test/main/sp-code.result | 58 ++++++------ mysql-test/main/sp-cursor.result | 73 +++++++++++++++ mysql-test/main/sp-cursor.test | 82 +++++++++++++++++ .../suite/compat/oracle/r/sp-code.result | 58 ++++++------ .../compat/oracle/r/sp-cursor-rowtype.result | 87 ++++++++++++++++++ .../compat/oracle/t/sp-cursor-rowtype.test | 92 +++++++++++++++++++ sql/sql_lex.cc | 20 ++++ sql/sql_lex.h | 1 + sql/sql_yacc.yy | 8 +- sql/sql_yacc_ora.yy | 8 +- sql/structs.h | 4 + 11 files changed, 427 insertions(+), 64 deletions(-) diff --git a/mysql-test/main/sp-code.result b/mysql-test/main/sp-code.result index 001b03a0e3b..c3af01d19e8 100644 --- a/mysql-test/main/sp-code.result +++ b/mysql-test/main/sp-code.result @@ -1113,23 +1113,26 @@ Pos Instruction 10 stmt 0 "SELECT rec1.a, rec1.b" 11 cfetch cur1@1 rec1@0 12 jump 6 -13 cursor_copy_struct cur0 rec0@1 -14 copen cur0@0 -15 cfetch cur0@0 rec0@1 -16 jump_if_not 21(21) `cur0`%FOUND -17 set rec0.a@1["a"] 10 -18 set rec0.b@1["b"] 'b0' -19 cfetch cur0@0 rec0@1 -20 jump 16 -21 cursor_copy_struct cur2 rec2@2 -22 copen cur2@2 -23 cfetch cur2@2 rec2@2 -24 jump_if_not 29(29) `cur2`%FOUND -25 set rec2.a@2["a"] 10 -26 set rec2.b@2["b"] 'b0' -27 cfetch cur2@2 rec2@2 -28 jump 24 -29 cpop 3 +13 cclose cur1@1 +14 cursor_copy_struct cur0 rec0@1 +15 copen cur0@0 +16 cfetch cur0@0 rec0@1 +17 jump_if_not 22(22) `cur0`%FOUND +18 set rec0.a@1["a"] 10 +19 set rec0.b@1["b"] 'b0' +20 cfetch cur0@0 rec0@1 +21 jump 17 +22 cclose cur0@0 +23 cursor_copy_struct cur2 rec2@2 +24 copen cur2@2 +25 cfetch cur2@2 rec2@2 +26 jump_if_not 31(31) `cur2`%FOUND +27 set rec2.a@2["a"] 10 +28 set rec2.b@2["b"] 'b0' +29 cfetch cur2@2 rec2@2 +30 jump 26 +31 cclose cur2@2 +32 cpop 3 DROP PROCEDURE p1; # Nested explicit cursor FOR loops CREATE PROCEDURE p1() @@ -1164,14 +1167,14 @@ Pos Instruction 1 cursor_copy_struct cur0 rec0@0 2 copen cur0@0 3 cfetch cur0@0 rec0@0 -4 jump_if_not 29(29) `cur0`%FOUND +4 jump_if_not 31(31) `cur0`%FOUND 5 cpush cur1@1 6 set rec0.a@0["a"] 11 7 set rec0.b@0["b"] 'b0' 8 cursor_copy_struct cur1 rec1@1 9 copen cur1@1 10 cfetch cur1@1 rec1@1 -11 jump_if_not 26(26) `cur1`%FOUND +11 jump_if_not 27(27) `cur1`%FOUND 12 set rec1.a@1["a"] 11 13 set rec1.b@1["b"] 'b1' 14 cpush cur2@2 @@ -1183,13 +1186,16 @@ Pos Instruction 20 set rec2.b@2["b"] 'b2' 21 cfetch cur2@2 rec2@2 22 jump 18 -23 cpop 1 -24 cfetch cur1@1 rec1@1 -25 jump 11 -26 cpop 1 -27 cfetch cur0@0 rec0@0 -28 jump 4 -29 cpop 1 +23 cclose cur2@2 +24 cpop 1 +25 cfetch cur1@1 rec1@1 +26 jump 11 +27 cclose cur1@1 +28 cpop 1 +29 cfetch cur0@0 rec0@0 +30 jump 4 +31 cclose cur0@0 +32 cpop 1 DROP PROCEDURE p1; # Implicit cursor FOR loops CREATE PROCEDURE p1() diff --git a/mysql-test/main/sp-cursor.result b/mysql-test/main/sp-cursor.result index 1f8cb7f0635..42d6e455109 100644 --- a/mysql-test/main/sp-cursor.result +++ b/mysql-test/main/sp-cursor.result @@ -611,3 +611,76 @@ a b a b 2 b2 DROP TABLE t1; +# +# MDEV-15941 Explicit cursor FOR loop does not close the cursor +# +BEGIN NOT ATOMIC +DECLARE v INT; +DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; +FOR rec IN cur +DO +SELECT rec.a; +END FOR; +FETCH cur INTO v; +END; +$$ +rec.a +1 +ERROR 24000: Cursor is not open +BEGIN NOT ATOMIC +DECLARE v INT; +DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; +label: +FOR rec IN cur +DO +SELECT rec.a; +END FOR; +FETCH cur INTO v; +END; +$$ +rec.a +1 +ERROR 24000: Cursor is not open +BEGIN NOT ATOMIC +DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; +OPEN cur; +FOR rec IN cur DO +SELECT rec.a; +END FOR; +END; +$$ +ERROR 24000: Cursor is already open +BEGIN NOT ATOMIC +DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; +FOR rec IN cur +DO +SELECT rec.a; +END FOR; +FOR rec IN cur +DO +SELECT rec.a; +END FOR; +END; +$$ +rec.a +1 +rec.a +1 +BEGIN NOT ATOMIC +DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; +label1: +FOR rec IN cur +DO +SELECT rec.a; +END FOR; +label2: +FOR rec IN cur +DO +SELECT rec.a; +END FOR; +END; +$$ +rec.a +1 +rec.a +1 diff --git a/mysql-test/main/sp-cursor.test b/mysql-test/main/sp-cursor.test index 2e7a72cf8d0..bc352872075 100644 --- a/mysql-test/main/sp-cursor.test +++ b/mysql-test/main/sp-cursor.test @@ -607,3 +607,85 @@ END; $$ DELIMITER ;$$ DROP TABLE t1; + + +--echo # +--echo # MDEV-15941 Explicit cursor FOR loop does not close the cursor +--echo # + +DELIMITER $$; +--error ER_SP_CURSOR_NOT_OPEN +BEGIN NOT ATOMIC + DECLARE v INT; + DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; + FOR rec IN cur + DO + SELECT rec.a; + END FOR; + FETCH cur INTO v; +END; +$$ +DELIMITER ;$$ + + +DELIMITER $$; +--error ER_SP_CURSOR_NOT_OPEN +BEGIN NOT ATOMIC + DECLARE v INT; + DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; +label: + FOR rec IN cur + DO + SELECT rec.a; + END FOR; + FETCH cur INTO v; +END; +$$ +DELIMITER ;$$ + + +DELIMITER $$; +--error ER_SP_CURSOR_ALREADY_OPEN +BEGIN NOT ATOMIC + DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; + OPEN cur; + FOR rec IN cur DO + SELECT rec.a; + END FOR; +END; +$$ +DELIMITER ;$$ + + +DELIMITER $$; +BEGIN NOT ATOMIC + DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; + FOR rec IN cur + DO + SELECT rec.a; + END FOR; + FOR rec IN cur + DO + SELECT rec.a; + END FOR; +END; +$$ +DELIMITER ;$$ + + +DELIMITER $$; +BEGIN NOT ATOMIC + DECLARE cur CURSOR FOR SELECT 1 AS a FROM DUAL; +label1: + FOR rec IN cur + DO + SELECT rec.a; + END FOR; +label2: + FOR rec IN cur + DO + SELECT rec.a; + END FOR; +END; +$$ +DELIMITER ;$$ diff --git a/mysql-test/suite/compat/oracle/r/sp-code.result b/mysql-test/suite/compat/oracle/r/sp-code.result index f1dd4180854..1049563511c 100644 --- a/mysql-test/suite/compat/oracle/r/sp-code.result +++ b/mysql-test/suite/compat/oracle/r/sp-code.result @@ -1084,23 +1084,26 @@ Pos Instruction 10 stmt 0 "SELECT rec1.a, rec1.b" 11 cfetch cur1@1 rec1@0 12 jump 6 -13 cursor_copy_struct cur0 rec0@1 -14 copen cur0@0 -15 cfetch cur0@0 rec0@1 -16 jump_if_not 21(21) "cur0"%FOUND -17 set rec0.a@1["a"] 10 -18 set rec0.b@1["b"] 'b0' -19 cfetch cur0@0 rec0@1 -20 jump 16 -21 cursor_copy_struct cur2 rec2@2 -22 copen cur2@2 -23 cfetch cur2@2 rec2@2 -24 jump_if_not 29(29) "cur2"%FOUND -25 set rec2.a@2["a"] 10 -26 set rec2.b@2["b"] 'b0' -27 cfetch cur2@2 rec2@2 -28 jump 24 -29 cpop 3 +13 cclose cur1@1 +14 cursor_copy_struct cur0 rec0@1 +15 copen cur0@0 +16 cfetch cur0@0 rec0@1 +17 jump_if_not 22(22) "cur0"%FOUND +18 set rec0.a@1["a"] 10 +19 set rec0.b@1["b"] 'b0' +20 cfetch cur0@0 rec0@1 +21 jump 17 +22 cclose cur0@0 +23 cursor_copy_struct cur2 rec2@2 +24 copen cur2@2 +25 cfetch cur2@2 rec2@2 +26 jump_if_not 31(31) "cur2"%FOUND +27 set rec2.a@2["a"] 10 +28 set rec2.b@2["b"] 'b0' +29 cfetch cur2@2 rec2@2 +30 jump 26 +31 cclose cur2@2 +32 cpop 3 DROP PROCEDURE p1; CREATE PROCEDURE p1 AS @@ -1137,14 +1140,14 @@ Pos Instruction 1 cursor_copy_struct cur0 rec0@0 2 copen cur0@0 3 cfetch cur0@0 rec0@0 -4 jump_if_not 29(29) "cur0"%FOUND +4 jump_if_not 31(31) "cur0"%FOUND 5 cpush cur1@1 6 set rec0.a@0["a"] 11 7 set rec0.b@0["b"] 'b0' 8 cursor_copy_struct cur1 rec1@1 9 copen cur1@1 10 cfetch cur1@1 rec1@1 -11 jump_if_not 26(26) "cur1"%FOUND +11 jump_if_not 27(27) "cur1"%FOUND 12 set rec1.a@1["a"] 11 13 set rec1.b@1["b"] 'b1' 14 cpush cur2@2 @@ -1156,13 +1159,16 @@ Pos Instruction 20 set rec2.b@2["b"] 'b2' 21 cfetch cur2@2 rec2@2 22 jump 18 -23 cpop 1 -24 cfetch cur1@1 rec1@1 -25 jump 11 -26 cpop 1 -27 cfetch cur0@0 rec0@0 -28 jump 4 -29 cpop 1 +23 cclose cur2@2 +24 cpop 1 +25 cfetch cur1@1 rec1@1 +26 jump 11 +27 cclose cur1@1 +28 cpop 1 +29 cfetch cur0@0 rec0@0 +30 jump 4 +31 cclose cur0@0 +32 cpop 1 DROP PROCEDURE p1; # # MDEV-12098 sql_mode=ORACLE: Implicit cursor FOR loop diff --git a/mysql-test/suite/compat/oracle/r/sp-cursor-rowtype.result b/mysql-test/suite/compat/oracle/r/sp-cursor-rowtype.result index a46daf30a8f..093d52ba4e3 100644 --- a/mysql-test/suite/compat/oracle/r/sp-cursor-rowtype.result +++ b/mysql-test/suite/compat/oracle/r/sp-cursor-rowtype.result @@ -1331,6 +1331,93 @@ rec2.a rec2.b DROP PROCEDURE p1; DROP TABLE t1; # +# MDEV-15941 Explicit cursor FOR loop does not close the cursor +# +DECLARE +CURSOR cur IS SELECT 1 AS a FROM DUAL; +v INT; +BEGIN +FOR rec IN cur +LOOP +NULL; +END LOOP; +FETCH cur INTO v; +END; +$$ +ERROR 24000: Cursor is not open +DECLARE +CURSOR cur IS SELECT 1 AS a FROM DUAL; +v INT; +BEGIN +<