From f2d549d8dbda1906b3e5ae0c2fa5589f2b9de662 Mon Sep 17 00:00:00 2001 From: Sujatha Sivakumar Date: Wed, 27 Mar 2019 12:34:03 +0530 Subject: [PATCH 01/20] MDEV-14784: Slave crashes in show_status_array upon running a trigger with select from I_S Problem: ======== When applier thread tries to access 'variable_name' of INFORMATION_SCHEMA.SESSION_VARIABLES table through triggers, it results in an abnormal exit of slave server. Analysis: ======== At the time of replication of stored routines and triggers, their associated security context will be sent by the master. The applier thread on the slave server will use this information to set the required security context for the execution of stored routines and triggers. This is achieved as follows. ->The stored routine object has a member named 'm_security_ctx' which holds the security context received from master. ->The applier thread's security_ctx is stored into a 'backup' object. ->Set the applier thread's security_ctx to 'm_security_ctx'. ->Upon the completion of stored routine execution restore the original security context of applier thread from the backup. During the above process the 'm_security_ctx' object is not initialized properly. Hence the 'external_user' of 'm_security_ctx' has invalid value for this variable and accessing this variable results in abnormal exit of server. Fix: === Invoke the Security_context::init() call from the constructor of stored routine so that 'm_security_ctx' gets initialized properly. --- .../r/rpl_slave_invalid_external_user.result | 15 +++++++ .../t/rpl_slave_invalid_external_user.test | 42 +++++++++++++++++++ sql/sp_head.cc | 1 + 3 files changed, 58 insertions(+) create mode 100644 mysql-test/suite/rpl/r/rpl_slave_invalid_external_user.result create mode 100644 mysql-test/suite/rpl/t/rpl_slave_invalid_external_user.test diff --git a/mysql-test/suite/rpl/r/rpl_slave_invalid_external_user.result b/mysql-test/suite/rpl/r/rpl_slave_invalid_external_user.result new file mode 100644 index 00000000000..29b815420ba --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_slave_invalid_external_user.result @@ -0,0 +1,15 @@ +include/master-slave.inc +[connection master] +CREATE USER test_user@localhost; +SET PASSWORD FOR test_user@localhost = password('PWD'); +GRANT ALL ON *.* TO test_user@localhost WITH GRANT OPTION; +connect conn_test,localhost,test_user,PWD,test,$MASTER_MYPORT,$MASTER_MYSOCK; +connection conn_test; +CREATE TABLE t1 (f1 INT); +CREATE TABLE t2 (f2 VARCHAR(64)); +CREATE TRIGGER tr_before BEFORE INSERT ON t1 FOR EACH ROW INSERT INTO t2 SELECT variable_name FROM INFORMATION_SCHEMA.SESSION_VARIABLES; +CREATE DEFINER='root'@'localhost' TRIGGER tr_after AFTER INSERT ON t1 FOR EACH ROW INSERT INTO t2 SELECT variable_name FROM INFORMATION_SCHEMA.SESSION_VARIABLES; +INSERT INTO t1 VALUES (1); +DROP USER 'test_user'@'localhost'; +DROP TABLE t1, t2; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_slave_invalid_external_user.test b/mysql-test/suite/rpl/t/rpl_slave_invalid_external_user.test new file mode 100644 index 00000000000..5099d7ee49e --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_slave_invalid_external_user.test @@ -0,0 +1,42 @@ +# ==== Purpose ==== +# +# Test verifies that when applier thread tries to access 'variable_name' of +# INFORMATION_SCHEMA.SESSION_VARIABLES table through triggers it successfully +# retrieves all the session variables. +# +# ==== Implementation ==== +# +# Steps: +# 0 - Create two tables t1 and t2. +# 1 - Create a trigger such that it reads the names of all session variables +# from INFORMATION_SCHEMA.SESSION_VARIABLES table and populates one of the +# tables. +# 2 - Do a DML on master and wait for it to be replicated and ensure that +# slave is in sync with master and it is up and running. +# +# ==== References ==== +# +# MDEV-14784: Slave crashes in show_status_array upon running a trigger with +# select from I_S + +--source include/master-slave.inc +--source include/have_binlog_format_mixed.inc +--enable_connect_log +CREATE USER test_user@localhost; +SET PASSWORD FOR test_user@localhost = password('PWD'); +GRANT ALL ON *.* TO test_user@localhost WITH GRANT OPTION; +connect(conn_test,localhost,test_user,PWD,test,$MASTER_MYPORT,$MASTER_MYSOCK); + +--connection conn_test +CREATE TABLE t1 (f1 INT); +CREATE TABLE t2 (f2 VARCHAR(64)); +CREATE TRIGGER tr_before BEFORE INSERT ON t1 FOR EACH ROW INSERT INTO t2 SELECT variable_name FROM INFORMATION_SCHEMA.SESSION_VARIABLES; +CREATE DEFINER='root'@'localhost' TRIGGER tr_after AFTER INSERT ON t1 FOR EACH ROW INSERT INTO t2 SELECT variable_name FROM INFORMATION_SCHEMA.SESSION_VARIABLES; + +INSERT INTO t1 VALUES (1); +--disable_connect_log +# Cleanup +--connection master +DROP USER 'test_user'@'localhost'; +DROP TABLE t1, t2; +--source include/rpl_end.inc diff --git a/sql/sp_head.cc b/sql/sp_head.cc index 14a57914560..fec7f51eaf0 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -563,6 +563,7 @@ sp_head::sp_head() DBUG_ENTER("sp_head::sp_head"); + m_security_ctx.init(); m_backpatch.empty(); m_cont_backpatch.empty(); m_lex.empty(); From 9a8b8ea66b041719c2e95ed6beaf286628c935b4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 27 Mar 2019 14:37:14 +0100 Subject: [PATCH 02/20] MDEV-19060 : mariabackup continues, despite failing to open a tablespace Fix mariabackup to crash if opening tablespace fails, insitead of continuing after an error. --- storage/xtradb/fil/fil0fil.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index d6d2a6f887c..01ebd98929f 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -4827,12 +4827,6 @@ will_not_choose: srv_force_recovery); return; } - - /* In mariabackup lets not crash. */ - if (IS_XTRABACKUP()) { - return; - } - abort(); } From f97d879bf890fe1d93c459f46410e2be8d26d3b6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 16 Mar 2019 19:24:49 +0100 Subject: [PATCH 03/20] cmake: re-enable -Werror in the maintainer mode now we can afford it. Fix -Werror errors. Note: * old gcc is bad at detecting uninit variables, disable it. * time_t is int or long, cast it for printf's --- cmake/maintainer.cmake | 9 ++++++-- include/mysql/client_plugin.h | 26 +++++++++++++----------- sql-common/client.c | 6 +++--- sql/log.cc | 6 +++--- sql/sql_admin.cc | 4 ++-- sql/wsrep_sst.cc | 9 ++++---- storage/connect/CMakeLists.txt | 21 +++++++++++-------- storage/connect/ha_connect.h | 4 ++++ storage/connect/mycat.cc | 4 ---- storage/innobase/btr/btr0scrub.cc | 4 ++-- storage/innobase/fil/fil0crypt.cc | 4 ++-- storage/innobase/handler/ha_innodb.cc | 2 +- storage/innobase/include/univ.i | 2 -- storage/innobase/lock/lock0lock.cc | 2 +- storage/innobase/os/os0file.cc | 2 +- storage/innobase/row/row0mysql.cc | 8 ++++---- storage/mroonga/lib/mrn_context_pool.cpp | 2 +- storage/spider/spd_table.cc | 6 +++--- storage/xtradb/btr/btr0scrub.cc | 4 ++-- storage/xtradb/fil/fil0crypt.cc | 4 ++-- storage/xtradb/handler/ha_innodb.cc | 2 +- storage/xtradb/include/univ.i | 2 -- storage/xtradb/lock/lock0lock.cc | 2 +- storage/xtradb/row/row0mysql.cc | 8 ++++---- unittest/sql/mf_iocache-t.cc | 2 +- 25 files changed, 76 insertions(+), 69 deletions(-) diff --git a/cmake/maintainer.cmake b/cmake/maintainer.cmake index 4e902d7fed8..caba370c4ae 100644 --- a/cmake/maintainer.cmake +++ b/cmake/maintainer.cmake @@ -28,10 +28,15 @@ SET(MY_WARNING_FLAGS -Woverloaded-virtual -Wvla -Wwrite-strings + -Werror ) -IF(MYSQL_MAINTAINER_MODE MATCHES "ON") - SET(WHERE) +IF(CMAKE_COMPILER_IS_GNUCC AND CMAKE_C_COMPILER_VERSION VERSION_LESS "6.0.0") + SET(MY_WARNING_FLAGS ${MY_WARNING_FLAGS} -Wno-error=maybe-uninitialized) +ENDIF() + +IF(MYSQL_MAINTAINER_MODE MATCHES "OFF") + RETURN() ELSEIF(MYSQL_MAINTAINER_MODE MATCHES "AUTO") SET(WHERE DEBUG) ENDIF() diff --git a/include/mysql/client_plugin.h b/include/mysql/client_plugin.h index b2df0019dfe..84fb3c6798d 100644 --- a/include/mysql/client_plugin.h +++ b/include/mysql/client_plugin.h @@ -32,17 +32,18 @@ */ #undef MYSQL_PLUGIN_EXPORT #if defined(_MSC_VER) - #ifdef __cplusplus - #define MYSQL_PLUGIN_EXPORT extern "C" __declspec(dllexport) - #else - #define MYSQL_PLUGIN_EXPORT __declspec(dllexport) - #endif + #define MYSQL_PLUGIN_EXPORT_C __declspec(dllexport) #else /*_MSC_VER */ - #ifdef __cplusplus - #define MYSQL_PLUGIN_EXPORT extern "C" - #else - #define MYSQL_PLUGIN_EXPORT - #endif + #define MYSQL_PLUGIN_EXPORT_C +#endif +#ifdef __cplusplus +#define MYSQL_PLUGIN_EXPORT extern "C" MYSQL_PLUGIN_EXPORT_C +#define C_MODE_START extern "C" { +#define C_MODE_END } +#else +#define MYSQL_PLUGIN_EXPORT MYSQL_PLUGIN_EXPORT_C +#define C_MODE_START +#define C_MODE_END #endif #ifndef MYSQL_ABI_CHECK @@ -60,11 +61,12 @@ #define MYSQL_CLIENT_MAX_PLUGINS 3 #define mysql_declare_client_plugin(X) \ - MYSQL_PLUGIN_EXPORT struct st_mysql_client_plugin_ ## X \ + C_MODE_START MYSQL_PLUGIN_EXPORT_C \ + struct st_mysql_client_plugin_ ## X \ _mysql_client_plugin_declaration_ = { \ MYSQL_CLIENT_ ## X ## _PLUGIN, \ MYSQL_CLIENT_ ## X ## _PLUGIN_INTERFACE_VERSION, -#define mysql_end_client_plugin } +#define mysql_end_client_plugin }; C_MODE_END /* generic plugin header structure */ #define MYSQL_CLIENT_PLUGIN_HEADER \ diff --git a/sql-common/client.c b/sql-common/client.c index 4fa167a21f1..60aa558dfb3 100644 --- a/sql-common/client.c +++ b/sql-common/client.c @@ -1354,7 +1354,7 @@ unpack_fields(MYSQL *mysql, MYSQL_DATA *data,MEM_ROOT *alloc,uint fields, { uchar *pos; /* fields count may be wrong */ - if (field - result >= fields) + if (field - result >= (my_ptrdiff_t)fields) goto err; cli_fetch_lengths(&lengths[0], row->data, default_value ? 8 : 7); @@ -1401,7 +1401,7 @@ unpack_fields(MYSQL *mysql, MYSQL_DATA *data,MEM_ROOT *alloc,uint fields, /* old protocol, for backward compatibility */ for (row=data->data; row ; row = row->next,field++) { - if (field - result >= fields) + if (field - result >= (my_ptrdiff_t)fields) goto err; cli_fetch_lengths(&lengths[0], row->data, default_value ? 6 : 5); field->org_table= field->table= strdup_root(alloc,(char*) row->data[0]); @@ -1439,7 +1439,7 @@ unpack_fields(MYSQL *mysql, MYSQL_DATA *data,MEM_ROOT *alloc,uint fields, } } #endif /* DELETE_SUPPORT_OF_4_0_PROTOCOL */ - if (field - result < fields) + if (field - result < (my_ptrdiff_t)fields) goto err; free_rows(data); /* Free old data */ DBUG_RETURN(result); diff --git a/sql/log.cc b/sql/log.cc index a1e7e5b218f..bab250223f3 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -9593,9 +9593,9 @@ TC_LOG_BINLOG::log_and_order(THD *thd, my_xid xid, bool all, */ if (!xid || !need_unlog) DBUG_RETURN(BINLOG_COOKIE_DUMMY(cache_mngr->delayed_error)); - else - DBUG_RETURN(BINLOG_COOKIE_MAKE(cache_mngr->binlog_id, - cache_mngr->delayed_error)); + + DBUG_RETURN(BINLOG_COOKIE_MAKE(cache_mngr->binlog_id, + cache_mngr->delayed_error)); } /* diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index cbc9780d507..0ec6719037c 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -528,7 +528,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, if (!table->table->part_info) { my_error(ER_PARTITION_MGMT_ON_NONPARTITIONED, MYF(0)); - goto err2; + thd->resume_subsequent_commits(suspended_wfc); + DBUG_RETURN(TRUE); } if (set_part_state(alter_info, table->table->part_info, PART_ADMIN)) { @@ -1159,7 +1160,6 @@ err: } close_thread_tables(thd); // Shouldn't be needed thd->mdl_context.release_transactional_locks(); -err2: thd->resume_subsequent_commits(suspended_wfc); DBUG_RETURN(TRUE); } diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index e2c55583594..addbfeb99f6 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -231,7 +231,7 @@ bool wsrep_sst_wait () total_wtime += difftime(end_time, start_time); WSREP_DEBUG("Waiting for SST to complete. current seqno: %" PRId64 " waited %f secs.", local_seqno, total_wtime); service_manager_extend_timeout(WSREP_EXTEND_TIMEOUT_INTERVAL, - "WSREP state transfer ongoing, current seqno: %ld waited %f secs", local_seqno, total_wtime); + "WSREP state transfer ongoing, current seqno: %" PRId64 " waited %f secs", local_seqno, total_wtime); } } @@ -294,9 +294,8 @@ void wsrep_sst_received (wsrep_t* const wsrep, } else if (local_seqno > seqno) { - WSREP_WARN("SST postion is in the past: %lld, current: %lld. " - "Can't continue.", - (long long)seqno, (long long)local_seqno); + WSREP_WARN("SST postion is in the past: %" PRId64 ", current: %" PRId64 + ". Can't continue.", seqno, local_seqno); unireg_abort(1); } @@ -1416,7 +1415,7 @@ void wsrep_SE_init_wait() total_wtime += difftime(end_time, start_time); WSREP_DEBUG("Waiting for SST to complete. current seqno: %" PRId64 " waited %f secs.", local_seqno, total_wtime); service_manager_extend_timeout(WSREP_EXTEND_TIMEOUT_INTERVAL, - "WSREP state transfer ongoing, current seqno: %ld waited %f secs", local_seqno, total_wtime); + "WSREP state transfer ongoing, current seqno: %" PRId64 " waited %f secs", local_seqno, total_wtime); } } diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 0ca794df230..519e075aaaa 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -40,6 +40,10 @@ user_connect.h valblk.h value.h xindex.h xobject.h xtable.h) add_definitions( -DMARIADB -DFORCE_INIT_OF_VARS -Dconnect_EXPORTS) add_definitions( -DHUGE_SUPPORT -DGZ_SUPPORT ) +macro(DISABLE_WARNING W) + MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-error=${W}") + MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-${W}" DEBUG) +endmacro() # # OS specific C flags, definitions and source files. @@ -47,14 +51,15 @@ add_definitions( -DHUGE_SUPPORT -DGZ_SUPPORT ) IF(UNIX) MY_CHECK_AND_SET_COMPILER_FLAG("-Wall -Wmissing-declarations") if(NOT WITH_WARNINGS) - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-unused-function") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-unused-variable") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-unused-value") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-parentheses") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-strict-aliasing") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-misleading-indentation") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-format-truncation") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wno-implicit-fallthrough") + DISABLE_WARNING("unused-function") + DISABLE_WARNING("unused-variable") + DISABLE_WARNING("unused-value") + DISABLE_WARNING("parentheses") + DISABLE_WARNING("strict-aliasing") + DISABLE_WARNING("misleading-indentation") + DISABLE_WARNING("format-truncation") + DISABLE_WARNING("implicit-fallthrough") + DISABLE_WARNING("type-limits") endif(NOT WITH_WARNINGS) add_definitions( -DUNIX -DLINUX -DUBUNTU ) diff --git a/storage/connect/ha_connect.h b/storage/connect/ha_connect.h index 07b1c2d2e7d..13c9aa4835b 100644 --- a/storage/connect/ha_connect.h +++ b/storage/connect/ha_connect.h @@ -32,6 +32,10 @@ /****************************************************************************/ #include "mycat.h" +#if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) +bool MongoEnabled(void); +#endif // JAVA_SUPPORT || CMGO_SUPPORT + /****************************************************************************/ /* Structures used to pass info between CONNECT and ha_connect. */ /****************************************************************************/ diff --git a/storage/connect/mycat.cc b/storage/connect/mycat.cc index cc5675db5c8..9fb8fc8367d 100644 --- a/storage/connect/mycat.cc +++ b/storage/connect/mycat.cc @@ -102,10 +102,6 @@ extern "C" HINSTANCE s_hModule; // Saved module handle #endif // !__WIN__ -#if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT) -bool MongoEnabled(void); -#endif // JAVA_SUPPORT || CMGO_SUPPORT - PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char *tab, char *db, bool info); /***********************************************************************/ diff --git a/storage/innobase/btr/btr0scrub.cc b/storage/innobase/btr/btr0scrub.cc index 2e667e64d0b..0e7c0d5b061 100644 --- a/storage/innobase/btr/btr0scrub.cc +++ b/storage/innobase/btr/btr0scrub.cc @@ -145,10 +145,10 @@ btr_scrub_lock_dict_func(ulint space_id, bool lock_to_close_table, time_t now = time(0); if (now >= last + 30) { fprintf(stderr, - "WARNING: %s:%u waited " TIMETPF " seconds for" + "WARNING: %s:%u waited %ld seconds for" " dict_sys lock, space: %lu" " lock_to_close_table: %d\n", - file, line, now - start, space_id, + file, line, (long)(now - start), space_id, lock_to_close_table); last = now; diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index fb84b5703bf..296096e44d0 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2453,9 +2453,9 @@ fil_space_crypt_close_tablespace( if (now >= last + 30) { ib_logf(IB_LOG_LEVEL_WARN, - "Waited " TIMETPF " seconds to drop space: %s (" ULINTPF + "Waited %ld seconds to drop space: %s (" ULINTPF ") active threads %u flushing=%d.", - now - start, space->name, space->id, cnt, flushing); + (long)(now - start), space->name, space->id, cnt, flushing); last = now; } } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index c670839b5cd..cd605b6b791 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -10520,7 +10520,7 @@ wsrep_append_foreign_key( key_type, copy); if (rcode) { - DBUG_PRINT("wsrep", ("row key failed: %zu", rcode)); + DBUG_PRINT("wsrep", ("row key failed: %lu", rcode)); WSREP_ERROR("Appending cascaded fk row key failed: %s, %lu", (wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void", rcode); diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 8807e2bf9e9..58ff9e1b5e4 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -454,7 +454,6 @@ macro ULINTPF. */ # define INT64PF "%lld" # define UINT64PF "%llu" # define UINT64PFx "%016llx" -# define TIMETPF "%ld" typedef __int64 ib_int64_t; typedef unsigned __int64 ib_uint64_t; typedef unsigned __int32 ib_uint32_t; @@ -464,7 +463,6 @@ typedef unsigned __int32 ib_uint32_t; # define INT64PF "%" PRId64 # define UINT64PF "%" PRIu64 # define UINT64PFx "%016" PRIx64 -# define TIMETPF "%" PRIdFAST32 typedef int64_t ib_int64_t; typedef uint64_t ib_uint64_t; typedef uint32_t ib_uint32_t; diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 3970a559a4c..f06fcd6c4d8 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -5350,7 +5350,7 @@ lock_table( lock_mutex_enter(); DBUG_EXECUTE_IF("fatal-semaphore-timeout", - { os_thread_sleep(3600000000); }); + { os_thread_sleep(3600000000U); }); /* We have to check if the new lock is compatible with any locks other transactions have in the table lock queue. */ diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 5aa9b6040f8..a417de1261f 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -6183,7 +6183,7 @@ os_file_trim( size_t len = slot->len; size_t trim_len = slot->page_size - len; - os_offset_t off = slot->offset + len; + os_offset_t off __attribute__((unused)) = slot->offset + len; size_t bsize = slot->file_block_size; #ifdef UNIV_TRIM_DEBUG diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 5d0d2d3ea2e..542e2570cb1 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -3316,17 +3316,17 @@ fil_wait_crypt_bg_threads( time_t now = time(0); if (now >= last + 30) { fprintf(stderr, - "WARNING: waited " TIMETPF " seconds " + "WARNING: waited %ld seconds " "for ref-count on table: %s space: %u\n", - now - start, table->name, table->space); + (long)(now - start), table->name, table->space); last = now; } if (now >= start + 300) { fprintf(stderr, - "WARNING: after " TIMETPF " seconds, gave up waiting " + "WARNING: after %ld seconds, gave up waiting " "for ref-count on table: %s space: %u\n", - now - start, table->name, table->space); + (long)(now - start), table->name, table->space); break; } } diff --git a/storage/mroonga/lib/mrn_context_pool.cpp b/storage/mroonga/lib/mrn_context_pool.cpp index d297ee9a338..a26005b16c0 100644 --- a/storage/mroonga/lib/mrn_context_pool.cpp +++ b/storage/mroonga/lib/mrn_context_pool.cpp @@ -77,7 +77,7 @@ namespace mrn { } private: - static const unsigned int CLEAR_THREATHOLD_IN_SECONDS = 60 * 5; + static const int CLEAR_THREATHOLD_IN_SECONDS = 60 * 5; mysql_mutex_t *mutex_; LIST *pool_; diff --git a/storage/spider/spd_table.cc b/storage/spider/spd_table.cc index 130edc5ee8a..5f7e59b47da 100644 --- a/storage/spider/spd_table.cc +++ b/storage/spider/spd_table.cc @@ -7027,7 +7027,7 @@ int spider_get_sts( int sts_sync_level, uint flag ) { - int get_type; + int get_type __attribute__ ((unused)); int error_num = 0; DBUG_ENTER("spider_get_sts"); @@ -7149,7 +7149,7 @@ int spider_get_crd( #endif int crd_sync_level ) { - int get_type; + int get_type __attribute__ ((unused)); int error_num = 0; DBUG_ENTER("spider_get_crd"); @@ -8379,7 +8379,7 @@ int spider_discover_table_structure( #ifdef WITH_PARTITION_STORAGE_ENGINE partition_info *part_info = thd->work_part_info; #endif - uint str_len; + uint str_len __attribute__ ((unused)); char buf[MAX_FIELD_WIDTH]; spider_string str(buf, sizeof(buf), system_charset_info); DBUG_ENTER("spider_discover_table_structure"); diff --git a/storage/xtradb/btr/btr0scrub.cc b/storage/xtradb/btr/btr0scrub.cc index 2e667e64d0b..0e7c0d5b061 100644 --- a/storage/xtradb/btr/btr0scrub.cc +++ b/storage/xtradb/btr/btr0scrub.cc @@ -145,10 +145,10 @@ btr_scrub_lock_dict_func(ulint space_id, bool lock_to_close_table, time_t now = time(0); if (now >= last + 30) { fprintf(stderr, - "WARNING: %s:%u waited " TIMETPF " seconds for" + "WARNING: %s:%u waited %ld seconds for" " dict_sys lock, space: %lu" " lock_to_close_table: %d\n", - file, line, now - start, space_id, + file, line, (long)(now - start), space_id, lock_to_close_table); last = now; diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index fb84b5703bf..296096e44d0 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -2453,9 +2453,9 @@ fil_space_crypt_close_tablespace( if (now >= last + 30) { ib_logf(IB_LOG_LEVEL_WARN, - "Waited " TIMETPF " seconds to drop space: %s (" ULINTPF + "Waited %ld seconds to drop space: %s (" ULINTPF ") active threads %u flushing=%d.", - now - start, space->name, space->id, cnt, flushing); + (long)(now - start), space->name, space->id, cnt, flushing); last = now; } } diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index d943a87ab78..a0df23b60d4 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -11088,7 +11088,7 @@ wsrep_append_foreign_key( key_type, copy); if (rcode) { - DBUG_PRINT("wsrep", ("row key failed: %zu", rcode)); + DBUG_PRINT("wsrep", ("row key failed: %lu", rcode)); WSREP_ERROR("Appending cascaded fk row key failed: %s, %lu", (wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void", rcode); diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 6d1520bc6a3..c35ec40e381 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -471,7 +471,6 @@ macro ULINTPF. */ # define INT64PF "%lld" # define UINT64PF "%llu" # define UINT64PFx "%016llx" -# define TIMETPF "%ld" typedef __int64 ib_int64_t; typedef unsigned __int64 ib_uint64_t; typedef unsigned __int32 ib_uint32_t; @@ -481,7 +480,6 @@ typedef unsigned __int32 ib_uint32_t; # define INT64PF "%" PRId64 # define UINT64PF "%" PRIu64 # define UINT64PFx "%016" PRIx64 -# define TIMETPF "%" PRIdFAST32 typedef int64_t ib_int64_t; typedef uint64_t ib_uint64_t; typedef uint32_t ib_uint32_t; diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index 2183d281b78..9daa2cc906f 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -5401,7 +5401,7 @@ lock_table( lock_mutex_enter(); DBUG_EXECUTE_IF("fatal-semaphore-timeout", - { os_thread_sleep(3600000000); }); + { os_thread_sleep(3600000000U); }); /* We have to check if the new lock is compatible with any locks other transactions have in the table lock queue. */ diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 47075c6cbc8..5e7ff6e910b 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -3320,17 +3320,17 @@ fil_wait_crypt_bg_threads( time_t now = time(0); if (now >= last + 30) { fprintf(stderr, - "WARNING: waited " TIMETPF " seconds " + "WARNING: waited %ld seconds " "for ref-count on table: %s space: %u\n", - now - start, table->name, table->space); + (long)(now - start), table->name, table->space); last = now; } if (now >= start + 300) { fprintf(stderr, - "WARNING: after " TIMETPF " seconds, gave up waiting " + "WARNING: after %ld seconds, gave up waiting " "for ref-count on table: %s space: %u\n", - now - start, table->name, table->space); + (long)(now - start), table->name, table->space); break; } } diff --git a/unittest/sql/mf_iocache-t.cc b/unittest/sql/mf_iocache-t.cc index 97b2ef0f80f..51656c02e57 100644 --- a/unittest/sql/mf_iocache-t.cc +++ b/unittest/sql/mf_iocache-t.cc @@ -356,7 +356,7 @@ void mdev17133() // random size 2nd read res= my_b_read(&info, buf_i + total + MY_MIN(19, curr_read_size), 19 >= curr_read_size ? 0 : curr_read_size - 19); - ok(res == 0, "rest of read %lu", curr_read_size - 19); + ok(res == 0, "rest of read %zu", curr_read_size - 19); // mark read bytes in the used part of the cache buffer memset(info.buffer, 0, info.read_pos - info.buffer); From 6417297180c7bc9242037894e21b769b7c22ead3 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 16 Mar 2019 19:27:51 +0100 Subject: [PATCH 04/20] cmake: remove workarounds for cmake bugs #13248 and #12864 since long we use a different workaround, our own CPackRPM wrapper --- cmake/plugin.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index e38439f2a21..4396a0a6c25 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -236,9 +236,6 @@ MACRO(MYSQL_ADD_PLUGIN) IF (NOT ARG_CLIENT) SET(CPACK_RPM_${ARG_COMPONENT}_PACKAGE_REQUIRES "MariaDB${ver}" PARENT_SCOPE) ENDIF() - # workarounds for cmake issues #13248 and #12864: - SET(CPACK_RPM_${ARG_COMPONENT}_PACKAGE_PROVIDES "cmake_bug_13248" PARENT_SCOPE) - SET(CPACK_RPM_${ARG_COMPONENT}_PACKAGE_OBSOLETES "cmake_bug_13248" PARENT_SCOPE) SET(CPACK_RPM_${ARG_COMPONENT}_USER_FILELIST ${ignored} PARENT_SCOPE) IF(NOT ARG_CLIENT AND NOT ARG_CONFIG AND UNIX) SET(ARG_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/${target}.cnf") From 39cea72e7beeb46f8b9959c198dfb9bb19d90b26 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 17 Mar 2019 15:05:56 +0100 Subject: [PATCH 05/20] cmake: fix cpack_source_ignore_files add defensive $ for filenames, don't include .gitattributes and *.rpm, correct rules for *.gz and *.zip --- cmake/cpack_source_ignore_files.cmake | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cmake/cpack_source_ignore_files.cmake b/cmake/cpack_source_ignore_files.cmake index 0654fa38388..8e1b7bd5eda 100644 --- a/cmake/cpack_source_ignore_files.cmake +++ b/cmake/cpack_source_ignore_files.cmake @@ -15,18 +15,19 @@ SET(CPACK_SOURCE_IGNORE_FILES \\\\.git/ -\\\\.gitignore -CMakeCache\\\\.txt -cmake_dist\\\\.cmake -CPackSourceConfig\\\\.cmake -CPackConfig.cmake -/cmake_install\\\\.cmake -/CTestTestfile\\\\.cmake +\\\\.gitignore$ +\\\\.gitattributes$ +CMakeCache\\\\.txt$ +cmake_dist\\\\.cmake$ +CPackSourceConfig\\\\.cmake$ +CPackConfig.cmake$ +/cmake_install\\\\.cmake$ +/CTestTestfile\\\\.cmake$ /CMakeFiles/ /version_resources/ /_CPack_Packages/ -$\\\\.gz -$\\\\.zip +\\\\.gz$ +\\\\.zip$ /CMakeFiles/ /version_resources/ /_CPack_Packages/ @@ -50,5 +51,6 @@ include/config\\\\.h$ include/my_config\\\\.h$ /autom4te\\\\.cache/ errmsg\\\\.sys$ +\\\\.rpm$ # ) From 86e80f944fede44afc1c236a841d5417eebe04cb Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 16 Mar 2019 19:30:51 +0100 Subject: [PATCH 06/20] cmake: don't do fake rpm Provides instead remove internal modules from Requires/Provides --- cmake/cpack_rpm.cmake | 36 ++++++------------------------------ 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 962c9f1f56d..6321b125f84 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -67,6 +67,12 @@ SET(CPACK_RPM_SPEC_MORE_DEFINE " %define _bindir ${INSTALL_BINDIRABS} %define _sbindir ${INSTALL_SBINDIRABS} %define _sysconfdir ${INSTALL_SYSCONFDIR} + +%{?filter_setup: +%filter_from_provides /perl(\\\\(mtr\\\\|My::\\\\)/d +%filter_from_requires /\\\\(lib\\\\(ft\\\\|lzma\\\\|tokuportability\\\\)\\\\)\\\\|\\\\(perl(\\\\(.*mtr\\\\|My::\\\\|.*HandlerSocket\\\\|Mysql\\\\)\\\\)/d +%filter_setup +} ") # this creative hack is described here: http://www.cmake.org/pipermail/cmake/2012-January/048416.html @@ -225,36 +231,6 @@ ELSEIF(RPM MATCHES "fedora" OR RPM MATCHES "(rhel|centos)7") SET(CPACK_RPM_common_PACKAGE_CONFLICTS "mariadb-libs < 1:%{version}-%{release}") ENDIF() -# workaround for lots of perl dependencies added by rpmbuild -SETA(CPACK_RPM_test_PACKAGE_PROVIDES - "perl(lib::mtr_gcov.pl)" - "perl(lib::mtr_gprof.pl)" - "perl(lib::mtr_io.pl)" - "perl(lib::mtr_misc.pl)" - "perl(lib::mtr_process.pl)" - "perl(lib::v1/mtr_cases.pl)" - "perl(lib::v1/mtr_gcov.pl)" - "perl(lib::v1/mtr_gprof.pl)" - "perl(lib::v1/mtr_im.pl)" - "perl(lib::v1/mtr_io.pl)" - "perl(lib::v1/mtr_match.pl)" - "perl(lib::v1/mtr_misc.pl)" - "perl(lib::v1/mtr_process.pl)" - "perl(lib::v1/mtr_report.pl)" - "perl(lib::v1/mtr_stress.pl)" - "perl(lib::v1/mtr_timer.pl)" - "perl(lib::v1/mtr_unique.pl)" - "perl(mtr_cases)" - "perl(mtr_io.pl)" - "perl(mtr_match)" - "perl(mtr_misc.pl)" - "perl(mtr_gcov.pl)" - "perl(mtr_gprof.pl)" - "perl(mtr_process.pl)" - "perl(mtr_report)" - "perl(mtr_results)" - "perl(mtr_unique)") - # If we want to build build MariaDB-shared-compat, # extract compat libraries from MariaDB-shared-5.3 rpm FILE(GLOB compat_rpm RELATIVE ${CMAKE_SOURCE_DIR} From ecc2711328e2e6e77758b9fe6c2a9892a2fe9192 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 16 Mar 2019 19:33:07 +0100 Subject: [PATCH 07/20] MDEV-7066 No Source RPMs ... (and so no "yum-builddep MariaDB-server" either) create source RPM cpack-way when building binary packages, this source rpm will use same BUILD_CONFIG and WITH_SSL values that were used when creating the source RPM. Only do it for a reasonably new cmake, where source rpms are known to work (3.10.2 is ok, 3.5.2 is not). And force a shorter CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX so that a source rpm could be built from a standard location in /usr/src --- cmake/cpack_rpm.cmake | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 6321b125f84..bfc06d52efe 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -260,5 +260,24 @@ ENDIF(compat_rpm) SET(CPACK_RPM_compat_PACKAGE_PROVIDES "mysql-libs = 5.3.5") # exact version doesn't matter as long as it greater than 5.1 SET(CPACK_RPM_compat_PACKAGE_OBSOLETES "mysql-libs < 5.3.5") -ENDIF(RPM) +################ +IF(CMAKE_VERSION VERSION_GREATER "3.9.99") +SET(CPACK_SOURCE_GENERATOR "RPM") +SETA(CPACK_RPM_SOURCE_PKG_BUILD_PARAMS + "-DBUILD_CONFIG=mysql_release" + "-DRPM=${RPM}" + "-DCPACK_RPM_BUILD_SOURCE_DIRS_PREFIX=/usr/src/debug/${CPACK_RPM_PACKAGE_NAME}-${VERSION}" + ) + +MACRO(ADDIF var) + IF(DEFINED ${var}) + SETA(CPACK_RPM_SOURCE_PKG_BUILD_PARAMS "-D${var}=${${var}}") + ENDIF() +ENDMACRO() + +ADDIF(BUILD_CONFIG) +ADDIF(WITH_SSL) + +ENDIF() +ENDIF(RPM) From b12f14965db9586e70109fe4cbf5dbd18bf2158d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 16 Mar 2019 19:37:44 +0100 Subject: [PATCH 08/20] MDEV-7066 No Source RPMs ... (and so no "yum-builddep MariaDB-server" either) automatic BuildRequires for source RPM: for every FILEPATH and "Have library XXX" cached variable, detect what rpm package it comes from and add it to the list of dependencies. That is, the source RPM will BuildRequire all those packages that were found by cmake when the source RPM was built. Presumably, our CMakeLists.txt won't check for libraries that aren't needed for a build. It supports libraries/executables/files found with FIND_LIBRARY FIND_FILE FIND_PROGRAM CHECK_LIBRARY_EXISTS --- CMakeLists.txt | 2 + .../build_configurations/mysql_release.cmake | 4 +- cmake/build_depends.cmake | 39 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 cmake/build_depends.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 3bfbcc4995d..54645e0b5c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -499,6 +499,8 @@ IF(UNIX) INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY Docs/README-wsrep COMPONENT Readme) ENDIF() +INCLUDE(build_depends) + INCLUDE(CPack) IF(WIN32 AND SIGNCODE) diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake index b73b44cddc3..709d97ec693 100644 --- a/cmake/build_configurations/mysql_release.cmake +++ b/cmake/build_configurations/mysql_release.cmake @@ -89,8 +89,8 @@ IF(WIN32) ELSEIF(RPM) SET(WITH_SSL system CACHE STRING "") SET(WITH_ZLIB system CACHE STRING "") - SET(CHECKMODULE /usr/bin/checkmodule CACHE STRING "") - SET(SEMODULE_PACKAGE /usr/bin/semodule_package CACHE STRING "") + SET(CHECKMODULE /usr/bin/checkmodule CACHE FILEPATH "") + SET(SEMODULE_PACKAGE /usr/bin/semodule_package CACHE FILEPATH "") SET(WITH_LIBARCHIVE ON CACHE STRING "") ELSEIF(DEB) SET(WITH_SSL system CACHE STRING "") diff --git a/cmake/build_depends.cmake b/cmake/build_depends.cmake new file mode 100644 index 00000000000..0d17c22cf98 --- /dev/null +++ b/cmake/build_depends.cmake @@ -0,0 +1,39 @@ +IF(RPM) + MACRO(FIND_DEP V) + SET(out ${V}_DEP) + IF (NOT DEFINED ${out}) + IF(EXISTS ${${V}} AND NOT IS_DIRECTORY ${${V}}) + EXECUTE_PROCESS(COMMAND ${ARGN} RESULT_VARIABLE res OUTPUT_VARIABLE O OUTPUT_STRIP_TRAILING_WHITESPACE) + ELSE() + SET(res 1) + ENDIF() + IF (res) + SET(O) + ELSE() + MESSAGE(STATUS "Need ${O} for ${${V}}") + ENDIF() + SET(${out} ${O} CACHE INTERNAL "Package that contains ${${V}}" FORCE) + ENDIF() + ENDMACRO() + + GET_CMAKE_PROPERTY(ALL_VARS CACHE_VARIABLES) + FOREACH (V ${ALL_VARS}) + GET_PROPERTY(H CACHE ${V} PROPERTY HELPSTRING) + IF (H MATCHES "^Have library [^/]" AND ${V}) + STRING(REGEX REPLACE "^Have library " "" L ${H}) + SET(V ${L}_LIBRARY) + FIND_LIBRARY(${V} ${L}) + ENDIF() + GET_PROPERTY(T CACHE ${V} PROPERTY TYPE) + IF ((T STREQUAL FILEPATH OR V MATCHES "^CMAKE_COMMAND$") AND ${V} MATCHES "^/") + IF (RPM) + FIND_DEP(${V} rpm -q --qf "%{NAME}" -f ${${V}}) + ELSE() # must be DEB + MESSAGE(FATAL_ERROR "Not implemented") + ENDIF () + SET(BUILD_DEPS ${BUILD_DEPS} ${${V}_DEP}) + ENDIF() + ENDFOREACH() + LIST(REMOVE_DUPLICATES BUILD_DEPS) + STRING(REPLACE ";" " " CPACK_RPM_BUILDREQUIRES "${BUILD_DEPS}") +ENDIF(RPM) From d8084116b54d3d3f1d655a4e051a58ebdfb82570 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 16 Mar 2019 19:44:40 +0100 Subject: [PATCH 09/20] MDEV-7066 No Source RPMs ... (and so no "yum-builddep MariaDB-server" either) special cases: * change systemd detection to use CHECK_LIBRARY_EXISTS at least once, to have it detected by build_depends.cmake * similarly, use find_library for pam * unixODBC is weird, libodbc.so is in the unixODBC package, not in the unixODBC-devel, where normally all .so files belong. Packaging bug? As a workaround, use find_file(sql.h) instead of find_path(sql.h) to make sure that /usr/include/sql.h (not /usr/include) is cached by cmake, and later build_depends.cmake will select unixODBC-devel, as a package owning /usr/include/sql.h file. --- cmake/systemd.cmake | 14 +++----------- plugin/auth_pam/CMakeLists.txt | 1 + storage/connect/CMakeLists.txt | 10 ++++++---- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/cmake/systemd.cmake b/cmake/systemd.cmake index 692d4df9f26..478930c9f3d 100644 --- a/cmake/systemd.cmake +++ b/cmake/systemd.cmake @@ -39,22 +39,14 @@ MACRO(CHECK_SYSTEMD) SET(LIBSYSTEMD systemd) ENDIF() SET(CMAKE_REQUIRED_LIBRARIES ${LIBSYSTEMD}) - CHECK_C_SOURCE_COMPILES( - " - #include - int main() - { - sd_listen_fds(0); - }" - HAVE_SYSTEMD) + CHECK_LIBRARY_EXISTS(systemd sd_listen_fds "" HAVE_SYSTEMD_SD_LISTEN_FDS) CHECK_INCLUDE_FILES(systemd/sd-daemon.h HAVE_SYSTEMD_SD_DAEMON_H) - CHECK_FUNCTION_EXISTS(sd_listen_fds HAVE_SYSTEMD_SD_LISTEN_FDS) CHECK_FUNCTION_EXISTS(sd_notify HAVE_SYSTEMD_SD_NOTIFY) CHECK_FUNCTION_EXISTS(sd_notifyf HAVE_SYSTEMD_SD_NOTIFYF) SET(CMAKE_REQUIRED_LIBRARIES) - IF(HAVE_SYSTEMD AND HAVE_SYSTEMD_SD_DAEMON_H AND HAVE_SYSTEMD_SD_LISTEN_FDS + IF(HAVE_SYSTEMD_SD_DAEMON_H AND HAVE_SYSTEMD_SD_LISTEN_FDS AND HAVE_SYSTEMD_SD_NOTIFY AND HAVE_SYSTEMD_SD_NOTIFYF) - ADD_DEFINITIONS(-DHAVE_SYSTEMD) + SET(HAVE_SYSTEMD TRUE) SET(SYSTEMD_SCRIPTS mariadb-service-convert galera_new_cluster galera_recovery) SET(SYSTEMD_DEB_FILES "usr/bin/mariadb-service-convert usr/bin/galera_new_cluster diff --git a/plugin/auth_pam/CMakeLists.txt b/plugin/auth_pam/CMakeLists.txt index 51317527c77..606fef002e7 100644 --- a/plugin/auth_pam/CMakeLists.txt +++ b/plugin/auth_pam/CMakeLists.txt @@ -8,6 +8,7 @@ IF(HAVE_PAM_APPL_H) IF(HAVE_STRNDUP) ADD_DEFINITIONS(-DHAVE_STRNDUP) ENDIF(HAVE_STRNDUP) + FIND_LIBRARY(PAM_LIBRARY pam) MYSQL_ADD_PLUGIN(auth_pam auth_pam.c LINK_LIBRARIES pam MODULE_ONLY) ENDIF(HAVE_PAM_APPL_H) diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 519e075aaaa..e0d05808a38 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -170,7 +170,8 @@ IF(CONNECT_WITH_ODBC) # the library 'libiodbc' gets compiled with 'sql'h. # This will also need changes in the sources (e.g. #include ). - find_path(ODBC_INCLUDE_DIR sql.h + find_file(ODBC_INCLUDES sql.h + PATHS /usr/include /usr/include/odbc /usr/local/include @@ -180,7 +181,7 @@ IF(CONNECT_WITH_ODBC) #"C:/Program Files/Microsoft SDKs/Windows/v7.0A/include" #"C:/Program Files/Microsoft SDKs/Windows/v6.0a/include" #"C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/include" - DOC "Specify the directory containing sql.h." + DOC "Specify the path to sql.h." ) find_library(ODBC_LIBRARY @@ -199,9 +200,10 @@ IF(CONNECT_WITH_ODBC) DOC "Specify the ODBC driver manager library here." ) - mark_as_advanced(ODBC_LIBRARY ODBC_INCLUDE_DIR) + mark_as_advanced(ODBC_LIBRARY ODBC_INCLUDES) - IF(ODBC_INCLUDE_DIR AND ODBC_LIBRARY) + IF(ODBC_INCLUDES AND ODBC_LIBRARY) + get_filename_component(ODBC_INCLUDE_DIR "${ODBC_INCLUDES}" PATH) set(CMAKE_REQUIRED_LIBRARIES ${ODBC_LIBRARY}) set(CMAKE_REQUIRED_INCLUDES ${ODBC_INCLUDE_DIR}) CHECK_CXX_SOURCE_COMPILES( From deff3f757289bd5b98b1b44d857e7508c4be07eb Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 22 Mar 2019 19:28:59 +0100 Subject: [PATCH 10/20] MDEV-18466 Unsafe to log updates on tables referenced by foreign keys with triggers in statement format ignore FK-prelocked tables when looking for write-prelocked tables with auto-increment to complain about "Statement is unsafe because it invokes a trigger or a stored function that inserts into an AUTO_INCREMENT column" --- .../suite/binlog/r/binlog_innodb_stm.result | 17 ++++++++++++ .../suite/binlog/t/binlog_innodb_stm.test | 26 +++++++++++++++++++ sql/sql_class.cc | 4 +++ 3 files changed, 47 insertions(+) create mode 100644 mysql-test/suite/binlog/r/binlog_innodb_stm.result create mode 100644 mysql-test/suite/binlog/t/binlog_innodb_stm.test diff --git a/mysql-test/suite/binlog/r/binlog_innodb_stm.result b/mysql-test/suite/binlog/r/binlog_innodb_stm.result new file mode 100644 index 00000000000..829ed7d3c61 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_innodb_stm.result @@ -0,0 +1,17 @@ +create table categories( +cat_id int not null primary key, +cat_name varchar(255) not null, +cat_description text +) engine=innodb; +create table products( +prd_id int not null auto_increment primary key, +prd_name varchar(355) not null, +prd_price decimal, +cat_id int not null, +foreign key fk_cat(cat_id) +references categories(cat_id) +on update cascade +) engine=innodb; +insert into categories values (1, 'drinks', 'drinks'); +update categories set cat_description=2 where cat_id=1; +drop table products, categories; diff --git a/mysql-test/suite/binlog/t/binlog_innodb_stm.test b/mysql-test/suite/binlog/t/binlog_innodb_stm.test new file mode 100644 index 00000000000..4dfa7a76584 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_innodb_stm.test @@ -0,0 +1,26 @@ +source include/have_innodb.inc; +source include/have_binlog_format_statement.inc; + +# +# MDEV-18466 Unsafe to log updates on tables referenced by foreign keys with triggers in statement format +# + +create table categories( + cat_id int not null primary key, + cat_name varchar(255) not null, + cat_description text +) engine=innodb; + +create table products( + prd_id int not null auto_increment primary key, + prd_name varchar(355) not null, + prd_price decimal, + cat_id int not null, + foreign key fk_cat(cat_id) + references categories(cat_id) + on update cascade +) engine=innodb; + +insert into categories values (1, 'drinks', 'drinks'); +update categories set cat_description=2 where cat_id=1; +drop table products, categories; diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 02e76520cb4..639c7c1784a 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -5542,6 +5542,9 @@ int xid_cache_iterate(THD *thd, my_hash_walk_action action, void *arg) Call this function only when you have established the list of all tables which you'll want to update (including stored functions, triggers, views inside your statement). + + Ignore tables prelocked for foreign key cascading actions, as these + actions cannot generate new auto_increment values. */ static bool @@ -5551,6 +5554,7 @@ has_write_table_with_auto_increment(TABLE_LIST *tables) { /* we must do preliminary checks as table->table may be NULL */ if (!table->placeholder() && + table->prelocking_placeholder != TABLE_LIST::FK && table->table->found_next_number_field && (table->lock_type >= TL_WRITE_ALLOW_WRITE)) return 1; From 21b2fada7ab7f35c898c02d2f918461409cc9c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 27 Mar 2019 09:28:49 +0200 Subject: [PATCH 11/20] MDEV-18464: Port kill_one_trx fixes from 10.4 to 10.1 Pushed the decision for innodb transaction and system locking down to lock0lock.cc level. With this, we can avoid releasing these mutexes for executions where these mutexes were acquired upfront. This patch will also fix BF aborting of native threads, e.g. threads which have declared wsrep_on=OFF. Earlier, we have used, for innodb trx locks, was_chosen_as_deadlock_victim flag, for marking inodb transactions, which are victims for wsrep BF abort. With native threads (wsrep_on==OFF), re-using was_chosen_as_deadlock_victim flag may lead to inteference with real deadlock, and to deal with this, the patch has added new flag for marking wsrep BF aborts only: victim=true Similar way if replication decides to abort one of the threads we mark victim by: victim=true innobase_kill_query Remove lock sys and trx mutex handling. wsrep_innobase_kill_one_trx Mark victim trx with victim=true trx0trx.h Remove trx_abort_t type and abort type variable from trx struct. Add victim variable to trx. wsrep_kill_victim Remove abort_type lock_report_waiters_to_mysql Take also trx mutex and mark trx as a victim for replication abort. lock_trx_handle_wait_low New low level function to check whether the transaction has already been rolled back because it was selected as a deadlock victim, or if it has to wait then cancel the wait lock. lock_trx_handle_wait If transaction is not marked as victim take lock sys and trx mutex before calling lock_trx_handle_wait_low and release them after that. row_search_for_mysql Remove lock sys and trx mutex taking and releasing. trx_rollback_to_savepoint_for_mysql_low trx_commit_in_memory Clean up victim variable. --- storage/innobase/handler/ha_innodb.cc | 38 ++++---------------- storage/innobase/include/trx0trx.h | 19 ++++------ storage/innobase/lock/lock0lock.cc | 50 +++++++++++++++++---------- storage/innobase/row/row0sel.cc | 4 --- storage/innobase/trx/trx0roll.cc | 3 +- storage/innobase/trx/trx0trx.cc | 8 ++--- storage/xtradb/handler/ha_innodb.cc | 36 ++++--------------- storage/xtradb/include/trx0trx.h | 19 ++++------ storage/xtradb/lock/lock0lock.cc | 50 +++++++++++++++++---------- storage/xtradb/row/row0sel.cc | 4 --- storage/xtradb/trx/trx0roll.cc | 21 ++--------- storage/xtradb/trx/trx0trx.cc | 17 ++------- 12 files changed, 99 insertions(+), 170 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index cd605b6b791..c55eaf59879 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4929,8 +4929,6 @@ static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels) /* if victim has been signaled by BF thread and/or aborting is already progressing, following query aborting is not necessary any more. - Also, BF thread should own trx mutex for the victim, which would - conflict with trx_mutex_enter() below */ DBUG_VOID_RETURN; } @@ -4939,34 +4937,8 @@ static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels) if (trx_t* trx = thd_to_trx(thd)) { ut_ad(trx->mysql_thd == thd); - switch (trx->abort_type) { -#ifdef WITH_WSREP - case TRX_WSREP_ABORT: - break; -#endif - case TRX_SERVER_ABORT: - if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { - lock_mutex_enter(); - } - /* fall through */ - case TRX_REPLICATION_ABORT: - trx_mutex_enter(trx); - } /* Cancel a pending lock request if there are any */ lock_trx_handle_wait(trx); - switch (trx->abort_type) { -#ifdef WITH_WSREP - case TRX_WSREP_ABORT: - break; -#endif - case TRX_SERVER_ABORT: - if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { - lock_mutex_exit(); - } - /* fall through */ - case TRX_REPLICATION_ABORT: - trx_mutex_exit(trx); - } } DBUG_VOID_RETURN; @@ -18683,6 +18655,12 @@ wsrep_innobase_kill_one_trx( wsrep_thd_ws_handle(thd)->trx_id); wsrep_thd_LOCK(thd); + + /* We mark this as victim transaction, which is already marked + as BF victim. Both trx mutex and lock_sys mutex is held until + this victim has aborted. */ + victim_trx->victim = true; + DBUG_EXECUTE_IF("sync.wsrep_after_BF_victim_lock", { const char act[]= @@ -18866,7 +18844,7 @@ wsrep_abort_transaction( my_bool signal) { DBUG_ENTER("wsrep_innobase_abort_thd"); - + trx_t* victim_trx = thd_to_trx(victim_thd); trx_t* bf_trx = (bf_thd) ? thd_to_trx(bf_thd) : NULL; @@ -18878,12 +18856,10 @@ wsrep_abort_transaction( if (victim_trx) { lock_mutex_enter(); trx_mutex_enter(victim_trx); - victim_trx->abort_type = TRX_WSREP_ABORT; int rcode = wsrep_innobase_kill_one_trx(bf_thd, bf_trx, victim_trx, signal); trx_mutex_exit(victim_trx); lock_mutex_exit(); - victim_trx->abort_type = TRX_SERVER_ABORT; wsrep_srv_conc_cancel_wait(victim_trx); DBUG_RETURN(rcode); } else { diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index fe16b8272b8..b40c6ae4667 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2018, MariaDB Corporation. +Copyright (c) 2015, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -623,7 +623,6 @@ struct trx_lock_t { lock_sys->mutex. Otherwise, this may only be modified by the thread that is serving the running transaction. */ - mem_heap_t* lock_heap; /*!< memory heap for trx_locks; protected by lock_sys->mutex */ @@ -695,14 +694,6 @@ lock_rec_convert_impl_to_expl()) will access transactions associated to other connections. The locks of transactions are protected by lock_sys->mutex and sometimes by trx->mutex. */ -enum trx_abort_t { - TRX_SERVER_ABORT = 0, -#ifdef WITH_WSREP - TRX_WSREP_ABORT, -#endif - TRX_REPLICATION_ABORT -}; - struct trx_t{ ulint magic_n; @@ -880,8 +871,12 @@ struct trx_t{ /*------------------------------*/ THD* mysql_thd; /*!< MySQL thread handle corresponding to this trx, or NULL */ - trx_abort_t abort_type; /*!< Transaction abort type*/ - + bool victim; /*!< This transaction is + selected as victim for abort + either by replication or + high priority wsrep thread. This + field is protected by trx and + lock sys mutex. */ const char* mysql_log_file_name; /*!< if MySQL binlog is used, this field contains a pointer to the latest file diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index f06fcd6c4d8..33563a03f1a 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2018, MariaDB Corporation. +Copyright (c) 2014, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1793,10 +1793,8 @@ wsrep_kill_victim( } } - lock->trx->abort_type = TRX_WSREP_ABORT; wsrep_innobase_kill_one_trx(trx->mysql_thd, (const trx_t*) trx, lock->trx, TRUE); - lock->trx->abort_type = TRX_SERVER_ABORT; } } } @@ -4782,12 +4780,11 @@ lock_report_waiters_to_mysql( if (w_trx->id != victim_trx_id) { /* If thd_report_wait_for() decides to kill the transaction, then we will get a call back into - innobase_kill_query. We mark this by setting - current_lock_mutex_owner, so we can avoid trying - to recursively take lock_sys->mutex. */ - w_trx->abort_type = TRX_REPLICATION_ABORT; + innobase_kill_query.*/ + trx_mutex_enter(w_trx); + w_trx->victim = true; thd_report_wait_for(mysql_thd, w_trx->mysql_thd); - w_trx->abort_type = TRX_SERVER_ABORT; + trx_mutex_exit(w_trx); } ++i; } @@ -7967,16 +7964,7 @@ lock_trx_release_locks( lock_mutex_exit(); } -/*********************************************************************//** -Check whether the transaction has already been rolled back because it -was selected as a deadlock victim, or if it has to wait then cancel -the wait lock. -@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */ -UNIV_INTERN -dberr_t -lock_trx_handle_wait( -/*=================*/ - trx_t* trx) /*!< in/out: trx lock state */ +inline dberr_t lock_trx_handle_wait_low(trx_t* trx) { ut_ad(lock_mutex_own()); ut_ad(trx_mutex_own(trx)); @@ -7993,6 +7981,32 @@ lock_trx_handle_wait( return DB_LOCK_WAIT; } +/*********************************************************************//** +Check whether the transaction has already been rolled back because it +was selected as a deadlock victim, or if it has to wait then cancel +the wait lock. +@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */ +UNIV_INTERN +dberr_t +lock_trx_handle_wait( +/*=================*/ + trx_t* trx) /*!< in/out: trx lock state */ +{ + if (!trx->victim) { + lock_mutex_enter(); + trx_mutex_enter(trx); + } + + dberr_t err = lock_trx_handle_wait_low(trx); + + if (!trx->victim) { + lock_mutex_exit(); + trx_mutex_exit(trx); + } + + return err; +} + /*********************************************************************//** Get the number of locks on a table. @return number of locks */ diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 06bf4cc30c0..855266686e6 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -4746,11 +4746,7 @@ no_gap_lock: a deadlock and the transaction had to wait then release the lock it is waiting on. */ - lock_mutex_enter(); - trx_mutex_enter(trx); err = lock_trx_handle_wait(trx); - lock_mutex_exit(); - trx_mutex_exit(trx); switch (err) { case DB_SUCCESS: diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index 3fd71aff23a..127e834335d 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2018, MariaDB Corporation. +Copyright (c) 2016, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -370,6 +370,7 @@ trx_rollback_to_savepoint_for_mysql_low( trx_mark_sql_stat_end(trx); trx->op_info = ""; + trx->victim = false; return(err); } diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index f36aabba8b4..98ac08a00e7 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2018, MariaDB Corporation. +Copyright (c) 2015, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1339,11 +1339,7 @@ trx_commit_in_memory( ut_ad(!trx->in_ro_trx_list); ut_ad(!trx->in_rw_trx_list); -#ifdef WITH_WSREP - if (trx->mysql_thd && wsrep_on(trx->mysql_thd)) { - trx->lock.was_chosen_as_deadlock_victim = FALSE; - } -#endif + trx->victim = false; trx->dict_operation = TRX_DICT_OP_NONE; trx->error_state = DB_SUCCESS; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index a0df23b60d4..55c49711f19 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -5534,8 +5534,6 @@ static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels) /* if victim has been signaled by BF thread and/or aborting is already progressing, following query aborting is not necessary any more. - Also, BF thread should own trx mutex for the victim, which would - conflict with trx_mutex_enter() below */ DBUG_VOID_RETURN; } @@ -5543,34 +5541,8 @@ static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels) if (trx_t* trx = thd_to_trx(thd)) { ut_ad(trx->mysql_thd == thd); - switch (trx->abort_type) { -#ifdef WITH_WSREP - case TRX_WSREP_ABORT: - break; -#endif - case TRX_SERVER_ABORT: - if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { - lock_mutex_enter(); - } - /* fall through */ - case TRX_REPLICATION_ABORT: - trx_mutex_enter(trx); - } /* Cancel a pending lock request if there are any */ lock_trx_handle_wait(trx); - switch (trx->abort_type) { -#ifdef WITH_WSREP - case TRX_WSREP_ABORT: - break; -#endif - case TRX_SERVER_ABORT: - if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { - lock_mutex_exit(); - } - /* fall through */ - case TRX_REPLICATION_ABORT: - trx_mutex_exit(trx); - } } DBUG_VOID_RETURN; @@ -19723,6 +19695,12 @@ wsrep_innobase_kill_one_trx( (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void"); wsrep_thd_LOCK(thd); + + /* We mark this as victim transaction, which is already marked + as BF victim. Both trx mutex and lock_sys mutex is held until + this victim has aborted. */ + victim_trx->victim = true; + DBUG_EXECUTE_IF("sync.wsrep_after_BF_victim_lock", { const char act[]= @@ -19911,12 +19889,10 @@ wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd, if (victim_trx) { lock_mutex_enter(); trx_mutex_enter(victim_trx); - victim_trx->abort_type = TRX_WSREP_ABORT; int rcode = wsrep_innobase_kill_one_trx(bf_thd, bf_trx, victim_trx, signal); trx_mutex_exit(victim_trx); lock_mutex_exit(); - victim_trx->abort_type = TRX_SERVER_ABORT; wsrep_srv_conc_cancel_wait(victim_trx); DBUG_RETURN(rcode); } else { diff --git a/storage/xtradb/include/trx0trx.h b/storage/xtradb/include/trx0trx.h index 77afde4c35c..df01284af05 100644 --- a/storage/xtradb/include/trx0trx.h +++ b/storage/xtradb/include/trx0trx.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2018, MariaDB Corporation. +Copyright (c) 2015, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -672,7 +672,6 @@ struct trx_lock_t { lock_sys->mutex. Otherwise, this may only be modified by the thread that is serving the running transaction. */ - mem_heap_t* lock_heap; /*!< memory heap for trx_locks; protected by lock_sys->mutex */ @@ -744,14 +743,6 @@ lock_rec_convert_impl_to_expl()) will access transactions associated to other connections. The locks of transactions are protected by lock_sys->mutex and sometimes by trx->mutex. */ -enum trx_abort_t { - TRX_SERVER_ABORT = 0, -#ifdef WITH_WSREP - TRX_WSREP_ABORT, -#endif - TRX_REPLICATION_ABORT -}; - struct trx_t{ ulint magic_n; @@ -930,8 +921,12 @@ struct trx_t{ /*------------------------------*/ THD* mysql_thd; /*!< MySQL thread handle corresponding to this trx, or NULL */ - trx_abort_t abort_type; /*!< Transaction abort type */ - + bool victim; /*!< This transaction is + selected as victim for abort + either by replication or + high priority wsrep thread. This + field is protected by trx and + lock sys mutex. */ const char* mysql_log_file_name; /*!< if MySQL binlog is used, this field contains a pointer to the latest file diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index 9daa2cc906f..2999c339457 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2018, MariaDB Corporation. +Copyright (c) 2014, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1804,10 +1804,8 @@ wsrep_kill_victim( } } - lock->trx->abort_type = TRX_WSREP_ABORT; wsrep_innobase_kill_one_trx(trx->mysql_thd, (const trx_t*) trx, lock->trx, TRUE); - lock->trx->abort_type = TRX_SERVER_ABORT; } } } @@ -4821,12 +4819,11 @@ lock_report_waiters_to_mysql( if (w_trx->id != victim_trx_id) { /* If thd_report_wait_for() decides to kill the transaction, then we will get a call back into - innobase_kill_query. We mark this by setting - current_lock_mutex_owner, so we can avoid trying - to recursively take lock_sys->mutex. */ - w_trx->abort_type = TRX_REPLICATION_ABORT; + innobase_kill_query.*/ + trx_mutex_enter(w_trx); + w_trx->victim = true; thd_report_wait_for(mysql_thd, w_trx->mysql_thd); - w_trx->abort_type = TRX_SERVER_ABORT; + trx_mutex_exit(w_trx); } ++i; } @@ -8077,16 +8074,7 @@ lock_trx_release_locks( lock_mutex_exit(); } -/*********************************************************************//** -Check whether the transaction has already been rolled back because it -was selected as a deadlock victim, or if it has to wait then cancel -the wait lock. -@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */ -UNIV_INTERN -dberr_t -lock_trx_handle_wait( -/*=================*/ - trx_t* trx) /*!< in/out: trx lock state */ +inline dberr_t lock_trx_handle_wait_low(trx_t* trx) { ut_ad(lock_mutex_own()); ut_ad(trx_mutex_own(trx)); @@ -8103,6 +8091,32 @@ lock_trx_handle_wait( return DB_LOCK_WAIT; } +/*********************************************************************//** +Check whether the transaction has already been rolled back because it +was selected as a deadlock victim, or if it has to wait then cancel +the wait lock. +@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */ +UNIV_INTERN +dberr_t +lock_trx_handle_wait( +/*=================*/ + trx_t* trx) /*!< in/out: trx lock state */ +{ + if (!trx->victim) { + lock_mutex_enter(); + trx_mutex_enter(trx); + } + + dberr_t err = lock_trx_handle_wait_low(trx); + + if (!trx->victim) { + lock_mutex_exit(); + trx_mutex_exit(trx); + } + + return err; +} + /*********************************************************************//** Get the number of locks on a table. @return number of locks */ diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index b6b5d107885..87bc2aa2875 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -4755,11 +4755,7 @@ no_gap_lock: a deadlock and the transaction had to wait then release the lock it is waiting on. */ - lock_mutex_enter(); - trx_mutex_enter(trx); err = lock_trx_handle_wait(trx); - lock_mutex_exit(); - trx_mutex_exit(trx); switch (err) { case DB_SUCCESS: diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc index 56b7120fa34..127e834335d 100644 --- a/storage/xtradb/trx/trx0roll.cc +++ b/storage/xtradb/trx/trx0roll.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2018, MariaDB Corporation. +Copyright (c) 2016, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -33,8 +33,6 @@ Created 3/26/1996 Heikki Tuuri #include "trx0roll.ic" #endif -#include - #include "fsp0fsp.h" #include "mach0data.h" #include "trx0rseg.h" @@ -51,9 +49,6 @@ Created 3/26/1996 Heikki Tuuri #include "pars0pars.h" #include "srv0mon.h" #include "trx0sys.h" -#ifdef WITH_WSREP -#include "ha_prototypes.h" -#endif /* WITH_WSREP */ /** This many pages must be undone before a truncate is tried within rollback */ @@ -375,13 +370,7 @@ trx_rollback_to_savepoint_for_mysql_low( trx_mark_sql_stat_end(trx); trx->op_info = ""; - -#ifdef WITH_WSREP - if (wsrep_on(trx->mysql_thd) && - trx->lock.was_chosen_as_deadlock_victim) { - trx->lock.was_chosen_as_deadlock_victim = FALSE; - } -#endif + trx->victim = false; return(err); } @@ -1079,12 +1068,6 @@ trx_roll_try_truncate( if (trx->update_undo) { trx_undo_truncate_end(trx, trx->update_undo, limit); } - -#ifdef WITH_WSREP_OUT - if (wsrep_on(trx->mysql_thd)) { - trx->lock.was_chosen_as_deadlock_victim = FALSE; - } -#endif /* WITH_WSREP */ } /***********************************************************************//** diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index 17cba81daf3..8ca247fadc8 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2018, MariaDB Corporation. +Copyright (c) 2015, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1563,11 +1563,7 @@ trx_commit_in_memory( ut_ad(!trx->in_ro_trx_list); ut_ad(!trx->in_rw_trx_list); -#ifdef WITH_WSREP - if (trx->mysql_thd && wsrep_on(trx->mysql_thd)) { - trx->lock.was_chosen_as_deadlock_victim = FALSE; - } -#endif + trx->victim = false; trx->dict_operation = TRX_DICT_OP_NONE; trx->error_state = DB_SUCCESS; @@ -2668,10 +2664,6 @@ trx_start_if_not_started_low( { switch (trx->state) { case TRX_STATE_NOT_STARTED: -#ifdef WITH_WSREP - ut_d(trx->start_file = __FILE__); - ut_d(trx->start_line = __LINE__); -#endif /* WITH_WSREP */ trx_start_low(trx); /* fall through */ case TRX_STATE_ACTIVE: @@ -2705,11 +2697,6 @@ trx_start_for_ddl_low( trx->will_lock = 1; trx->ddl = true; - -#ifdef WITH_WSREP - ut_d(trx->start_file = __FILE__); - ut_d(trx->start_line = __LINE__); -#endif /* WITH_WSREP */ trx_start_low(trx); return; From 81d71ee6b21870772c336bff15b71904914f146a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 5 Feb 2019 15:41:53 +0200 Subject: [PATCH 12/20] MDEV-12009: Allow to force kill user threads/query which are flagged as high priority by Galera As noted on kill_one_thread SUPER should be able to kill even system threads i.e. threads/query flagged as high priority or wsrep applier thread. Normal user, should not able to kill threads/query flagged as high priority (BF) or wsrep applier thread. --- include/mysql/service_wsrep.h | 3 ++ .../suite/galera/r/galera_kill_applier.result | 4 ++ .../suite/galera/t/galera_kill_applier.cnf | 10 ++++ .../suite/galera/t/galera_kill_applier.test | 54 +++++++++++++++++-- sql/sql_parse.cc | 14 +++-- sql/sql_plugin_services.ic | 3 +- sql/wsrep_dummy.cc | 3 ++ sql/wsrep_mysqld.cc | 1 - sql/wsrep_thd.cc | 9 ++++ sql/wsrep_thd.h | 2 + 10 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/galera/t/galera_kill_applier.cnf diff --git a/include/mysql/service_wsrep.h b/include/mysql/service_wsrep.h index 267c8cb4e90..59df7a8b561 100644 --- a/include/mysql/service_wsrep.h +++ b/include/mysql/service_wsrep.h @@ -112,6 +112,7 @@ extern struct wsrep_service_st { int (*wsrep_trx_order_before_func)(MYSQL_THD, MYSQL_THD); void (*wsrep_unlock_rollback_func)(); void (*wsrep_set_data_home_dir_func)(const char *data_dir); + my_bool (*wsrep_thd_is_applier_func)(THD *thd); } *wsrep_service; #ifdef MYSQL_DYNAMIC_PLUGIN @@ -155,6 +156,7 @@ extern struct wsrep_service_st { #define wsrep_trx_order_before(T1,T2) wsrep_service->wsrep_trx_order_before_func(T1,T2) #define wsrep_unlock_rollback() wsrep_service->wsrep_unlock_rollback_func() #define wsrep_set_data_home_dir(A) wsrep_service->wsrep_set_data_home_dir_func(A) +#define wsrep_thd_is_applier(T) wsrep_service->wsrep_thd_is_applier_func(T) #define wsrep_debug get_wsrep_debug() #define wsrep_log_conflicts get_wsrep_log_conflicts() @@ -214,6 +216,7 @@ void wsrep_thd_set_conflict_state(THD *thd, enum wsrep_conflict_state state); bool wsrep_thd_ignore_table(THD *thd); void wsrep_unlock_rollback(); void wsrep_set_data_home_dir(const char *data_dir); +my_bool wsrep_thd_is_applier(THD *thd); #endif diff --git a/mysql-test/suite/galera/r/galera_kill_applier.result b/mysql-test/suite/galera/r/galera_kill_applier.result index fe4911639ed..89f8ead0b65 100644 --- a/mysql-test/suite/galera/r/galera_kill_applier.result +++ b/mysql-test/suite/galera/r/galera_kill_applier.result @@ -1,4 +1,8 @@ +CREATE USER foo@localhost; +GRANT SELECT on test.* TO foo@localhost; +# Open connection to the 1st node using 'test_user1' user. Got one of the listed errors Got one of the listed errors Got one of the listed errors Got one of the listed errors +DROP USER foo@localhost; diff --git a/mysql-test/suite/galera/t/galera_kill_applier.cnf b/mysql-test/suite/galera/t/galera_kill_applier.cnf new file mode 100644 index 00000000000..f2563e66f2e --- /dev/null +++ b/mysql-test/suite/galera/t/galera_kill_applier.cnf @@ -0,0 +1,10 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true' +auto_increment_offset=1 + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true' +auto_increment_offset=2 + diff --git a/mysql-test/suite/galera/t/galera_kill_applier.test b/mysql-test/suite/galera/t/galera_kill_applier.test index e14a8b9af23..5e4a587fe57 100644 --- a/mysql-test/suite/galera/t/galera_kill_applier.test +++ b/mysql-test/suite/galera/t/galera_kill_applier.test @@ -1,14 +1,25 @@ # # This test checks that applier threads are immune to KILL QUERY and KILL STATEMENT +# when USER is not SUPER # --source include/galera_cluster.inc ---source include/have_innodb.inc --connection node_1 +CREATE USER foo@localhost; +GRANT SELECT on test.* TO foo@localhost; + --let $applier_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE != 'wsrep aborter idle' OR STATE IS NULL LIMIT 1` +--let $aborter_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep aborter idle' LIMIT 1` + +--echo # Open connection to the 1st node using 'test_user1' user. +--let $port_1= \$NODE_MYPORT_1 +--connect(foo_node_1,localhost,foo,,test,$port_1,) + +--connection foo_node_1 + --disable_query_log --error ER_KILL_DENIED_ERROR,ER_KILL_DENIED_ERROR --eval KILL $applier_thread @@ -16,11 +27,48 @@ --error ER_KILL_DENIED_ERROR,ER_KILL_DENIED_ERROR --eval KILL QUERY $applier_thread ---let $aborter_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep aborter idle' LIMIT 1` - --error ER_KILL_DENIED_ERROR,ER_KILL_DENIED_ERROR --eval KILL $aborter_thread --error ER_KILL_DENIED_ERROR,ER_KILL_DENIED_ERROR --eval KILL QUERY $aborter_thread --enable_query_log + +# +# SUPER can kill applier threads +# +--connection node_2 + +--let $applier_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE != 'wsrep aborter idle' OR STATE IS NULL LIMIT 1` + +--disable_query_log +--eval KILL $applier_thread +--enable_query_log + +--let $aborter_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep aborter idle' LIMIT 1` + +--disable_query_log +--eval KILL $aborter_thread +--enable_query_log + +--source include/restart_mysqld.inc + +--connection node_2 +--let $applier_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE != 'wsrep aborter idle' OR STATE IS NULL LIMIT 1` + +--disable_query_log +--eval KILL QUERY $applier_thread +--enable_query_log + +--let $aborter_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep aborter idle' LIMIT 1` + +--disable_query_log +--eval KILL QUERY $aborter_thread +--enable_query_log + +--source include/restart_mysqld.inc + +--connection node_1 +--disconnect foo_node_1 +DROP USER foo@localhost; + diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 1f060305d4f..920f7ac0329 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -8292,11 +8292,19 @@ kill_one_thread(THD *thd, longlong id, killed_state kill_signal, killed_type typ It's ok to also kill DELAYED threads with KILL_CONNECTION instead of KILL_SYSTEM_THREAD; The difference is that KILL_CONNECTION may be faster and do a harder kill than KILL_SYSTEM_THREAD; + + Note that if thread is wsrep Brute Force or applier thread we + allow killing it only when we're SUPER. */ - if (((thd->security_ctx->master_access & SUPER_ACL) || - thd->security_ctx->user_matches(tmp->security_ctx)) && - !wsrep_thd_is_BF(tmp, false)) + if ((thd->security_ctx->master_access & SUPER_ACL) || + (thd->security_ctx->user_matches(tmp->security_ctx) +#ifdef WITH_WSREP + && + !tmp->wsrep_applier && + !wsrep_thd_is_BF(tmp, false) +#endif + )) { tmp->awake(kill_signal); error=0; diff --git a/sql/sql_plugin_services.ic b/sql/sql_plugin_services.ic index 427d8937c57..d70c76701fd 100644 --- a/sql/sql_plugin_services.ic +++ b/sql/sql_plugin_services.ic @@ -181,7 +181,8 @@ static struct wsrep_service_st wsrep_handler = { wsrep_trx_is_aborting, wsrep_trx_order_before, wsrep_unlock_rollback, - wsrep_set_data_home_dir + wsrep_set_data_home_dir, + wsrep_thd_is_applier, }; static struct thd_specifics_service_st thd_specifics_handler= diff --git a/sql/wsrep_dummy.cc b/sql/wsrep_dummy.cc index 795e2d19252..e2a4dcf3bad 100644 --- a/sql/wsrep_dummy.cc +++ b/sql/wsrep_dummy.cc @@ -20,6 +20,9 @@ my_bool wsrep_thd_is_BF(THD *, my_bool) { return 0; } +my_bool wsrep_thd_is_applier(THD *) +{ return 0; } + int wsrep_trx_order_before(THD *, THD *) { return 0; } diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index ee8509e3fa2..82415691b3f 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -2470,7 +2470,6 @@ extern "C" void wsrep_thd_set_exec_mode(THD *thd, enum wsrep_exec_mode mode) thd->wsrep_exec_mode= mode; } - extern "C" void wsrep_thd_set_query_state( THD *thd, enum wsrep_query_state state) { diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc index 551e710cfeb..91f0e1e8d8a 100644 --- a/sql/wsrep_thd.cc +++ b/sql/wsrep_thd.cc @@ -596,6 +596,15 @@ my_bool wsrep_thd_is_BF(THD *thd, my_bool sync) return status; } +my_bool wsrep_thd_is_applier(THD *thd) +{ + my_bool ret = FALSE; + if (thd) { + ret = thd->wsrep_applier; + } + return ret; +} + extern "C" my_bool wsrep_thd_is_BF_or_commit(void *thd_ptr, my_bool sync) { diff --git a/sql/wsrep_thd.h b/sql/wsrep_thd.h index 5900668f3fb..f5fcb50280c 100644 --- a/sql/wsrep_thd.h +++ b/sql/wsrep_thd.h @@ -37,6 +37,7 @@ int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, */ extern void wsrep_thd_set_PA_safe(void *thd_ptr, my_bool safe); extern my_bool wsrep_thd_is_BF(THD *thd, my_bool sync); +extern my_bool wsrep_thd_is_applier(THD *thd); extern my_bool wsrep_thd_is_wsrep(void *thd_ptr); enum wsrep_conflict_state wsrep_thd_conflict_state(void *thd_ptr, my_bool sync); @@ -47,6 +48,7 @@ extern "C" int wsrep_thd_in_locking_session(void *thd_ptr); #else /* WITH_WSREP */ #define wsrep_thd_is_BF(T, S) (0) +#define wsrep_thd_is_applier(T) (0) #define wsrep_abort_thd(X,Y,Z) do { } while(0) #define wsrep_create_appliers(T) do { } while(0) From 38cad6875f2c85f5190eaae7b9da7c43e591eff2 Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Thu, 28 Mar 2019 13:14:49 +0530 Subject: [PATCH 13/20] MDEV-18899: Server crashes in Field::set_warning_truncated_wrong_value To fix the crash there we need to make sure that the server while storing the statistical values in statistical tables should do it in a multi-byte safe way. Also there is no need to throw warnings if there is truncation while storing values from statistical fields. --- mysql-test/r/stat_tables.result | 57 ++++++++++++++++++++++++++ mysql-test/r/stat_tables_innodb.result | 57 ++++++++++++++++++++++++++ mysql-test/t/stat_tables.test | 40 ++++++++++++++++++ sql/field.cc | 7 +++- sql/sql_statistics.cc | 10 +++-- 5 files changed, 166 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/stat_tables.result b/mysql-test/r/stat_tables.result index 3ebc3b47833..be868e55e84 100644 --- a/mysql-test/r/stat_tables.result +++ b/mysql-test/r/stat_tables.result @@ -624,4 +624,61 @@ SELECT MAX(pk) FROM t1; MAX(pk) NULL DROP TABLE t1; +# +# MDEV-18899: Server crashes in Field::set_warning_truncated_wrong_value +# +set names utf8; +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set optimizer_use_condition_selectivity=4; +set use_stat_tables=preferably; +set @save_histogram_size= @@histogram_size; +set histogram_size=255; +create table t1 ( a varchar(255) character set utf8); +insert into t1 values (REPEAT('ӥ',255)), (REPEAT('ç',255)); +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats where db_name='test' and table_name='t1'; +HEX(RIGHT(min_value, 1)) length(min_value) +A7 254 +select HEX(RIGHT(max_value, 1)), length(max_value) from mysql.column_stats where db_name='test' and table_name='t1'; +HEX(RIGHT(max_value, 1)) length(max_value) +A5 254 +analyze select * from t1 where a >= 'ӥ'; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 2.00 50.00 50.00 Using where +set @save_sql_mode= @@sql_mode; +set sql_mode='ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'; +update mysql.column_stats set min_value= REPEAT('ӥ',255) where db_name='test' and table_name='t1'; +Warnings: +Warning 1265 Data truncated for column 'min_value' at row 1 +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats where db_name='test' and table_name='t1'; +HEX(RIGHT(min_value, 1)) length(min_value) +D3 255 +analyze select * from t1 where a >= 'ӥ'; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 2.00 50.00 50.00 Using where +set names latin1; +drop table t1; +CREATE TABLE t1 (col1 date); +INSERT INTO t1 VALUES('2004-01-01'),('2004-02-29'); +INSERT INTO t1 VALUES('0000-10-31'); +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +update mysql.column_stats set min_value='2004-0-31123' where db_name='test' and table_name='t1'; +select min_value from mysql.column_stats where db_name='test' and table_name='t1'; +min_value +2004-0-31123 +select * from t1; +col1 +2004-01-01 +2004-02-29 +0000-10-31 +drop table t1; +set @@sql_mode= @save_sql_mode; set use_stat_tables=@save_use_stat_tables; +set @@histogram_size= @save_histogram_size; +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; diff --git a/mysql-test/r/stat_tables_innodb.result b/mysql-test/r/stat_tables_innodb.result index a6c5525a0d3..86088490871 100644 --- a/mysql-test/r/stat_tables_innodb.result +++ b/mysql-test/r/stat_tables_innodb.result @@ -651,6 +651,63 @@ SELECT MAX(pk) FROM t1; MAX(pk) NULL DROP TABLE t1; +# +# MDEV-18899: Server crashes in Field::set_warning_truncated_wrong_value +# +set names utf8; +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set optimizer_use_condition_selectivity=4; +set use_stat_tables=preferably; +set @save_histogram_size= @@histogram_size; +set histogram_size=255; +create table t1 ( a varchar(255) character set utf8); +insert into t1 values (REPEAT('ӥ',255)), (REPEAT('ç',255)); +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats where db_name='test' and table_name='t1'; +HEX(RIGHT(min_value, 1)) length(min_value) +A7 254 +select HEX(RIGHT(max_value, 1)), length(max_value) from mysql.column_stats where db_name='test' and table_name='t1'; +HEX(RIGHT(max_value, 1)) length(max_value) +A5 254 +analyze select * from t1 where a >= 'ӥ'; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 2.00 50.00 50.00 Using where +set @save_sql_mode= @@sql_mode; +set sql_mode='ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'; +update mysql.column_stats set min_value= REPEAT('ӥ',255) where db_name='test' and table_name='t1'; +Warnings: +Warning 1265 Data truncated for column 'min_value' at row 1 +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats where db_name='test' and table_name='t1'; +HEX(RIGHT(min_value, 1)) length(min_value) +D3 255 +analyze select * from t1 where a >= 'ӥ'; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 2.00 50.00 50.00 Using where +set names latin1; +drop table t1; +CREATE TABLE t1 (col1 date); +INSERT INTO t1 VALUES('2004-01-01'),('2004-02-29'); +INSERT INTO t1 VALUES('0000-10-31'); +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +update mysql.column_stats set min_value='2004-0-31123' where db_name='test' and table_name='t1'; +select min_value from mysql.column_stats where db_name='test' and table_name='t1'; +min_value +2004-0-31123 +select * from t1; +col1 +2004-01-01 +2004-02-29 +0000-10-31 +drop table t1; +set @@sql_mode= @save_sql_mode; set use_stat_tables=@save_use_stat_tables; +set @@histogram_size= @save_histogram_size; +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; set optimizer_switch=@save_optimizer_switch_for_stat_tables_test; SET SESSION STORAGE_ENGINE=DEFAULT; diff --git a/mysql-test/t/stat_tables.test b/mysql-test/t/stat_tables.test index b89ab2bbd2d..89c11ed4acf 100644 --- a/mysql-test/t/stat_tables.test +++ b/mysql-test/t/stat_tables.test @@ -401,4 +401,44 @@ SELECT MAX(pk) FROM t1; DROP TABLE t1; +--echo # +--echo # MDEV-18899: Server crashes in Field::set_warning_truncated_wrong_value +--echo # + +set names utf8; +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; +set optimizer_use_condition_selectivity=4; +set use_stat_tables=preferably; +set @save_histogram_size= @@histogram_size; +set histogram_size=255; + +create table t1 ( a varchar(255) character set utf8); +insert into t1 values (REPEAT('ӥ',255)), (REPEAT('ç',255)); + +analyze table t1; +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats where db_name='test' and table_name='t1'; +select HEX(RIGHT(max_value, 1)), length(max_value) from mysql.column_stats where db_name='test' and table_name='t1'; +analyze select * from t1 where a >= 'ӥ'; + +set @save_sql_mode= @@sql_mode; +set sql_mode='ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'; +update mysql.column_stats set min_value= REPEAT('ӥ',255) where db_name='test' and table_name='t1'; +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats where db_name='test' and table_name='t1'; +analyze select * from t1 where a >= 'ӥ'; + +set names latin1; +drop table t1; + +CREATE TABLE t1 (col1 date); +INSERT INTO t1 VALUES('2004-01-01'),('2004-02-29'); +INSERT INTO t1 VALUES('0000-10-31'); +analyze table t1; +update mysql.column_stats set min_value='2004-0-31123' where db_name='test' and table_name='t1'; +select min_value from mysql.column_stats where db_name='test' and table_name='t1'; +select * from t1; +drop table t1; + +set @@sql_mode= @save_sql_mode; set use_stat_tables=@save_use_stat_tables; +set @@histogram_size= @save_histogram_size; +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; diff --git a/sql/field.cc b/sql/field.cc index 080cf34c76d..0621015c0e4 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7027,8 +7027,11 @@ Field_longstr::check_string_copy_error(const String_copier *copier, if (!(pos= copier->most_important_error_pos())) return FALSE; - convert_to_printable(tmp, sizeof(tmp), pos, (end - pos), cs, 6); - set_warning_truncated_wrong_value("string", tmp); + if (!is_stat_field) + { + convert_to_printable(tmp, sizeof(tmp), pos, (end - pos), cs, 6); + set_warning_truncated_wrong_value("string", tmp); + } return TRUE; } diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index b5811c683e8..0a51346adb2 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -1060,7 +1060,9 @@ public: else { table_field->collected_stats->min_value->val_str(&val); - stat_field->store(val.ptr(), val.length(), &my_charset_bin); + uint32 length= Well_formed_prefix(val.charset(), val.ptr(), + MY_MIN(val.length(), stat_field->field_length)).length(); + stat_field->store(val.ptr(), length, &my_charset_bin); } break; case COLUMN_STAT_MAX_VALUE: @@ -1069,7 +1071,9 @@ public: else { table_field->collected_stats->max_value->val_str(&val); - stat_field->store(val.ptr(), val.length(), &my_charset_bin); + uint32 length= Well_formed_prefix(val.charset(), val.ptr(), + MY_MIN(val.length(), stat_field->field_length)).length(); + stat_field->store(val.ptr(), length, &my_charset_bin); } break; case COLUMN_STAT_NULLS_RATIO: @@ -3059,7 +3063,7 @@ int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) } } } - + table->stats_is_read= TRUE; DBUG_RETURN(0); From 0623cc7c16c3280d1f81b9049e1561d1b4b6c1d0 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Wed, 27 Mar 2019 18:58:43 +0530 Subject: [PATCH 14/20] MDEV-19051 Avoid unnecessary writing MLOG_INDEX_LOAD 1) Avoid writing of MLOG_INDEX_LOAD redo log record during inplace alter table when the table is empty and also for spatial index. 2) Avoid creation of temporary merge file for spatial index during index creation process. --- storage/innobase/include/btr0bulk.h | 2 + storage/innobase/include/trx0trx.h | 36 ++++++----- storage/innobase/row/row0ftsort.cc | 2 +- storage/innobase/row/row0merge.cc | 93 +++++++++++++---------------- storage/innobase/trx/trx0trx.cc | 30 ++++++---- 5 files changed, 84 insertions(+), 79 deletions(-) diff --git a/storage/innobase/include/btr0bulk.h b/storage/innobase/include/btr0bulk.h index a63ab9ded0d..7e277adfbc3 100644 --- a/storage/innobase/include/btr0bulk.h +++ b/storage/innobase/include/btr0bulk.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2014, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -284,6 +285,7 @@ public: m_trx(trx), m_flush_observer(observer) { + ut_ad(!dict_index_is_spatial(index)); #ifdef UNIV_DEBUG if (m_flush_observer) fil_space_inc_redo_skipped_count(m_index->space); diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 8fde5689e40..159dcca2581 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2018, MariaDB Corporation. +Copyright (c) 2015, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -40,20 +40,9 @@ Created 3/26/1996 Heikki Tuuri // Forward declaration struct mtr_t; - -// Forward declaration class ReadView; - -// Forward declaration class FlushObserver; - -/** Set flush observer for the transaction -@param[in/out] trx transaction struct -@param[in] observer flush observer */ -void -trx_set_flush_observer( - trx_t* trx, - FlushObserver* observer); +class ut_stage_alter_t; /******************************************************************//** Set detailed error message for the transaction. */ @@ -1132,8 +1121,11 @@ struct trx_t { /*------------------------------*/ char* detailed_error; /*!< detailed error message for last error, or empty. */ - FlushObserver* flush_observer; /*!< flush observer */ - +private: + /** flush observer used to track flushing of non-redo logged pages + during bulk create index */ + FlushObserver* flush_observer; +public: /* Lock wait statistics */ ulint n_rec_lock_waits; /*!< Number of record lock waits, @@ -1177,6 +1169,20 @@ struct trx_t { return(assign_temp_rseg()); } + /** Set the innodb_log_optimize_ddl page flush observer + @param[in] space_id tablespace id + @param[in,out] stage performance_schema accounting */ + void set_flush_observer(ulint space_id, ut_stage_alter_t* stage); + + /** Remove the flush observer */ + void remove_flush_observer(); + + /** @return the flush observer */ + FlushObserver* get_flush_observer() const + { + return flush_observer; + } + private: /** Assign a rollback segment for modifying temporary tables. @return the assigned rollback segment */ diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index de622527766..2e321411a9a 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -1673,7 +1673,7 @@ row_fts_merge_insert( /* Create bulk load instance */ ins_ctx.btr_bulk = UT_NEW_NOKEY( BtrBulk(aux_index, trx, psort_info[0].psort_common->trx - ->flush_observer)); + ->get_flush_observer())); /* Create tuple for insert */ ins_ctx.tuple = dtuple_create(heap, dict_index_get_n_fields(aux_index)); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 72e0a055af3..a12a2a73ace 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1736,7 +1736,6 @@ row_merge_read_clustered_index( mem_heap_t* mtuple_heap = NULL; mtuple_t prev_mtuple; mem_heap_t* conv_heap = NULL; - FlushObserver* observer = trx->flush_observer; double curr_progress = 0.0; ib_uint64_t read_rows = 0; ib_uint64_t table_total_rows = 0; @@ -2211,9 +2210,8 @@ write_buffers: bool skip_sort = skip_pk_sort && dict_index_is_clust(merge_buf[0]->index); - for (ulint i = 0; i < n_index; i++, skip_sort = false) { + for (ulint k = 0, i = 0; i < n_index; i++, skip_sort = false) { row_merge_buf_t* buf = merge_buf[i]; - merge_file_t* file = &files[i]; ulint rows_added = 0; if (dict_index_is_spatial(buf->index)) { @@ -2237,6 +2235,8 @@ write_buffers: continue; } + merge_file_t* file = &files[k++]; + if (UNIV_LIKELY (row && (rows_added = row_merge_buf_add( buf, fts_index, old_table, new_table, @@ -2244,6 +2244,15 @@ write_buffers: conv_heap, &err, &v_heap, eval_table, trx)))) { + /* Set the page flush observer for the + transaction when buffering the very first + record for a non-redo-logged operation. */ + if (file->n_rec == 0 && i == 0 + && innodb_log_optimize_ddl) { + trx->set_flush_observer( + new_table->space, stage); + } + /* If we are creating FTS index, a single row can generate more records for tokenized word */ @@ -2383,7 +2392,7 @@ write_buffers: clust_btr_bulk = UT_NEW_NOKEY( BtrBulk(index[i], trx, - observer/**/)); + trx->get_flush_observer())); } else { clust_btr_bulk->latch(); } @@ -2495,8 +2504,9 @@ write_buffers: trx->error_key_num = i; goto all_done;); - BtrBulk btr_bulk(index[i], trx, - observer); + BtrBulk btr_bulk( + index[i], trx, + trx->get_flush_observer()); err = row_merge_insert_index_tuples( index[i], old_table, @@ -4597,47 +4607,26 @@ row_merge_build_indexes( } trx_start_if_not_started_xa(trx, true); + ulint n_merge_files = 0; - /* Check if we need a flush observer to flush dirty pages. - Since we disable redo logging in bulk load, so we should flush - dirty pages before online log apply, because online log apply enables - redo logging(we can do further optimization here). - 1. online add index: flush dirty pages right before row_log_apply(). - 2. table rebuild: flush dirty pages before row_log_table_apply(). - - we use bulk load to create all types of indexes except spatial index, - for which redo logging is enabled. If we create only spatial indexes, - we don't need to flush dirty pages at all. */ - bool need_flush_observer = bool(innodb_log_optimize_ddl); - - if (need_flush_observer) { - need_flush_observer = old_table != new_table; - - for (i = 0; i < n_indexes; i++) { - if (!dict_index_is_spatial(indexes[i])) { - need_flush_observer = true; - } + for (ulint i = 0; i < n_indexes; i++) + { + if (!dict_index_is_spatial(indexes[i])) { + n_merge_files++; } } - FlushObserver* flush_observer = NULL; - if (need_flush_observer) { - flush_observer = UT_NEW_NOKEY( - FlushObserver(new_table->space, trx, stage)); - - trx_set_flush_observer(trx, flush_observer); - } - merge_files = static_cast( - ut_malloc_nokey(n_indexes * sizeof *merge_files)); + ut_malloc_nokey(n_merge_files * sizeof *merge_files)); /* Initialize all the merge file descriptors, so that we don't call row_merge_file_destroy() on uninitialized merge file descriptor */ - for (i = 0; i < n_indexes; i++) { + for (i = 0; i < n_merge_files; i++) { merge_files[i].fd = -1; merge_files[i].offset = 0; + merge_files[i].n_rec = 0; } total_static_cost = COST_BUILD_INDEX_STATIC * n_indexes + COST_READ_CLUSTERED_INDEX; @@ -4720,7 +4709,7 @@ row_merge_build_indexes( " and create temporary files"); } - for (i = 0; i < n_indexes; i++) { + for (i = 0; i < n_merge_files; i++) { total_index_blocks += merge_files[i].offset; } @@ -4733,7 +4722,7 @@ row_merge_build_indexes( /* Now we have files containing index entries ready for sorting and inserting. */ - for (i = 0; i < n_indexes; i++) { + for (ulint k = 0, i = 0; i < n_indexes; i++) { dict_index_t* sort_idx = indexes[i]; if (dict_index_is_spatial(sort_idx)) { @@ -4812,13 +4801,13 @@ wait_again: #ifdef FTS_INTERNAL_DIAG_PRINT DEBUG_FTS_SORT_PRINT("FTS_SORT: Complete Insert\n"); #endif - } else if (merge_files[i].fd >= 0) { + } else if (merge_files[k].fd >= 0) { char buf[NAME_LEN + 1]; row_merge_dup_t dup = { sort_idx, table, col_map, 0}; pct_cost = (COST_BUILD_INDEX_STATIC + - (total_dynamic_cost * merge_files[i].offset / + (total_dynamic_cost * merge_files[k].offset / total_index_blocks)) / (total_static_cost + total_dynamic_cost) * PCT_COST_MERGESORT_INDEX * 100; @@ -4842,7 +4831,7 @@ wait_again: } error = row_merge_sort( - trx, &dup, &merge_files[i], + trx, &dup, &merge_files[k], block, &tmpfd, true, pct_progress, pct_cost, crypt_block, new_table->space, stage); @@ -4864,10 +4853,10 @@ wait_again: if (error == DB_SUCCESS) { BtrBulk btr_bulk(sort_idx, trx, - flush_observer); + trx->get_flush_observer()); pct_cost = (COST_BUILD_INDEX_STATIC + - (total_dynamic_cost * merge_files[i].offset / + (total_dynamic_cost * merge_files[k].offset / total_index_blocks)) / (total_static_cost + total_dynamic_cost) * PCT_COST_INSERT_INDEX * 100; @@ -4884,9 +4873,9 @@ wait_again: error = row_merge_insert_index_tuples( sort_idx, old_table, - merge_files[i].fd, block, NULL, + merge_files[k].fd, block, NULL, &btr_bulk, - merge_files[i].n_rec, pct_progress, pct_cost, + merge_files[k].n_rec, pct_progress, pct_cost, crypt_block, new_table->space, stage); error = btr_bulk.finish(error); @@ -4904,7 +4893,7 @@ wait_again: } /* Close the temporary file to free up space. */ - row_merge_file_destroy(&merge_files[i]); + row_merge_file_destroy(&merge_files[k++]); if (indexes[i]->type & DICT_FTS) { row_fts_psort_info_destroy(psort_info, merge_info); @@ -4916,7 +4905,12 @@ wait_again: ut_ad(sort_idx->online_status == ONLINE_INDEX_COMPLETE); } else { - if (flush_observer) { + if (dict_index_is_spatial(indexes[i])) { + /* We never disable redo logging for + creating SPATIAL INDEX. Avoid writing any + unnecessary MLOG_INDEX_LOAD record. */ + } else if (FlushObserver* flush_observer = + trx->get_flush_observer()) { flush_observer->flush(); row_merge_write_redo(indexes[i]); } @@ -4958,7 +4952,7 @@ func_exit: row_merge_file_destroy_low(tmpfd); - for (i = 0; i < n_indexes; i++) { + for (i = 0; i < n_merge_files; i++) { row_merge_file_destroy(&merge_files[i]); } @@ -5015,8 +5009,7 @@ func_exit: DBUG_EXECUTE_IF("ib_index_crash_after_bulk_load", DBUG_SUICIDE();); - if (flush_observer != NULL) { - ut_ad(need_flush_observer); + if (FlushObserver* flush_observer = trx->get_flush_observer()) { DBUG_EXECUTE_IF("ib_index_build_fail_before_flush", error = DB_INTERRUPTED; @@ -5028,7 +5021,7 @@ func_exit: flush_observer->flush(); - UT_DELETE(flush_observer); + trx->remove_flush_observer(); if (trx_is_interrupted(trx)) { error = DB_INTERRUPTED; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 704c52ff9af..e548ffbe62b 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2018, MariaDB Corporation. +Copyright (c) 2015, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -64,17 +64,6 @@ typedef std::set< std::less, ut_allocator > table_id_set; -/** Set flush observer for the transaction -@param[in/out] trx transaction struct -@param[in] observer flush observer */ -void -trx_set_flush_observer( - trx_t* trx, - FlushObserver* observer) -{ - trx->flush_observer = observer; -} - /*************************************************************//** Set detailed error message for the transaction. */ void @@ -165,7 +154,7 @@ trx_init( trx->lock.table_cached = 0; - trx->flush_observer = NULL; + ut_ad(trx->get_flush_observer() == NULL); } /** For managing the life-cycle of the trx_t instance that we get @@ -1088,6 +1077,21 @@ static trx_rseg_t* trx_assign_rseg_low() return(rseg); } +/** Set the innodb_log_optimize_ddl page flush observer +@param[in] space_id tablespace id +@param[in,out] stage performance_schema accounting */ +void trx_t::set_flush_observer(ulint space_id, ut_stage_alter_t* stage) +{ + flush_observer = UT_NEW_NOKEY(FlushObserver(space_id, this, stage)); +} + +/** Remove the flush observer */ +void trx_t::remove_flush_observer() +{ + UT_DELETE(flush_observer); + flush_observer = NULL; +} + /** Assign a rollback segment for modifying temporary tables. @return the assigned rollback segment */ trx_rseg_t* From d0116e10a5da52503a89a413e481996ce3f65e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 28 Mar 2019 12:27:06 +0200 Subject: [PATCH 15/20] Revert MDEV-18464 and MDEV-12009 This reverts commit 21b2fada7ab7f35c898c02d2f918461409cc9c8e and commit 81d71ee6b21870772c336bff15b71904914f146a. The MDEV-18464 change introduces a few data race issues. Contrary to the documentation, the field trx_t::victim is not always being protected by lock_sys_t::mutex and trx_t::mutex. Most importantly, it seems that KILL QUERY could wrongly avoid acquiring both mutexes when invoking lock_trx_handle_wait_low(), in case another thread had already set trx->victim=true. We also revert MDEV-12009, because it should depend on the MDEV-18464 fix being present. --- include/mysql/service_wsrep.h | 3 -- .../suite/galera/r/galera_kill_applier.result | 4 -- .../suite/galera/t/galera_kill_applier.cnf | 10 ---- .../suite/galera/t/galera_kill_applier.test | 54 ++----------------- sql/sql_parse.cc | 14 ++--- sql/sql_plugin_services.ic | 3 +- sql/wsrep_dummy.cc | 3 -- sql/wsrep_mysqld.cc | 1 + sql/wsrep_thd.cc | 9 ---- sql/wsrep_thd.h | 2 - storage/innobase/handler/ha_innodb.cc | 38 ++++++++++--- storage/innobase/include/trx0trx.h | 19 ++++--- storage/innobase/lock/lock0lock.cc | 50 +++++++---------- storage/innobase/row/row0sel.cc | 4 ++ storage/innobase/trx/trx0roll.cc | 3 +- storage/innobase/trx/trx0trx.cc | 8 ++- storage/xtradb/handler/ha_innodb.cc | 36 ++++++++++--- storage/xtradb/include/trx0trx.h | 19 ++++--- storage/xtradb/lock/lock0lock.cc | 50 +++++++---------- storage/xtradb/row/row0sel.cc | 4 ++ storage/xtradb/trx/trx0roll.cc | 21 +++++++- storage/xtradb/trx/trx0trx.cc | 17 +++++- 22 files changed, 178 insertions(+), 194 deletions(-) delete mode 100644 mysql-test/suite/galera/t/galera_kill_applier.cnf diff --git a/include/mysql/service_wsrep.h b/include/mysql/service_wsrep.h index 59df7a8b561..267c8cb4e90 100644 --- a/include/mysql/service_wsrep.h +++ b/include/mysql/service_wsrep.h @@ -112,7 +112,6 @@ extern struct wsrep_service_st { int (*wsrep_trx_order_before_func)(MYSQL_THD, MYSQL_THD); void (*wsrep_unlock_rollback_func)(); void (*wsrep_set_data_home_dir_func)(const char *data_dir); - my_bool (*wsrep_thd_is_applier_func)(THD *thd); } *wsrep_service; #ifdef MYSQL_DYNAMIC_PLUGIN @@ -156,7 +155,6 @@ extern struct wsrep_service_st { #define wsrep_trx_order_before(T1,T2) wsrep_service->wsrep_trx_order_before_func(T1,T2) #define wsrep_unlock_rollback() wsrep_service->wsrep_unlock_rollback_func() #define wsrep_set_data_home_dir(A) wsrep_service->wsrep_set_data_home_dir_func(A) -#define wsrep_thd_is_applier(T) wsrep_service->wsrep_thd_is_applier_func(T) #define wsrep_debug get_wsrep_debug() #define wsrep_log_conflicts get_wsrep_log_conflicts() @@ -216,7 +214,6 @@ void wsrep_thd_set_conflict_state(THD *thd, enum wsrep_conflict_state state); bool wsrep_thd_ignore_table(THD *thd); void wsrep_unlock_rollback(); void wsrep_set_data_home_dir(const char *data_dir); -my_bool wsrep_thd_is_applier(THD *thd); #endif diff --git a/mysql-test/suite/galera/r/galera_kill_applier.result b/mysql-test/suite/galera/r/galera_kill_applier.result index 89f8ead0b65..fe4911639ed 100644 --- a/mysql-test/suite/galera/r/galera_kill_applier.result +++ b/mysql-test/suite/galera/r/galera_kill_applier.result @@ -1,8 +1,4 @@ -CREATE USER foo@localhost; -GRANT SELECT on test.* TO foo@localhost; -# Open connection to the 1st node using 'test_user1' user. Got one of the listed errors Got one of the listed errors Got one of the listed errors Got one of the listed errors -DROP USER foo@localhost; diff --git a/mysql-test/suite/galera/t/galera_kill_applier.cnf b/mysql-test/suite/galera/t/galera_kill_applier.cnf deleted file mode 100644 index f2563e66f2e..00000000000 --- a/mysql-test/suite/galera/t/galera_kill_applier.cnf +++ /dev/null @@ -1,10 +0,0 @@ -!include ../galera_2nodes.cnf - -[mysqld.1] -wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true' -auto_increment_offset=1 - -[mysqld.2] -wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true' -auto_increment_offset=2 - diff --git a/mysql-test/suite/galera/t/galera_kill_applier.test b/mysql-test/suite/galera/t/galera_kill_applier.test index 5e4a587fe57..e14a8b9af23 100644 --- a/mysql-test/suite/galera/t/galera_kill_applier.test +++ b/mysql-test/suite/galera/t/galera_kill_applier.test @@ -1,25 +1,14 @@ # # This test checks that applier threads are immune to KILL QUERY and KILL STATEMENT -# when USER is not SUPER # --source include/galera_cluster.inc +--source include/have_innodb.inc --connection node_1 -CREATE USER foo@localhost; -GRANT SELECT on test.* TO foo@localhost; - --let $applier_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE != 'wsrep aborter idle' OR STATE IS NULL LIMIT 1` ---let $aborter_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep aborter idle' LIMIT 1` - ---echo # Open connection to the 1st node using 'test_user1' user. ---let $port_1= \$NODE_MYPORT_1 ---connect(foo_node_1,localhost,foo,,test,$port_1,) - ---connection foo_node_1 - --disable_query_log --error ER_KILL_DENIED_ERROR,ER_KILL_DENIED_ERROR --eval KILL $applier_thread @@ -27,48 +16,11 @@ GRANT SELECT on test.* TO foo@localhost; --error ER_KILL_DENIED_ERROR,ER_KILL_DENIED_ERROR --eval KILL QUERY $applier_thread +--let $aborter_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep aborter idle' LIMIT 1` + --error ER_KILL_DENIED_ERROR,ER_KILL_DENIED_ERROR --eval KILL $aborter_thread --error ER_KILL_DENIED_ERROR,ER_KILL_DENIED_ERROR --eval KILL QUERY $aborter_thread --enable_query_log - -# -# SUPER can kill applier threads -# ---connection node_2 - ---let $applier_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE != 'wsrep aborter idle' OR STATE IS NULL LIMIT 1` - ---disable_query_log ---eval KILL $applier_thread ---enable_query_log - ---let $aborter_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep aborter idle' LIMIT 1` - ---disable_query_log ---eval KILL $aborter_thread ---enable_query_log - ---source include/restart_mysqld.inc - ---connection node_2 ---let $applier_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE != 'wsrep aborter idle' OR STATE IS NULL LIMIT 1` - ---disable_query_log ---eval KILL QUERY $applier_thread ---enable_query_log - ---let $aborter_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep aborter idle' LIMIT 1` - ---disable_query_log ---eval KILL QUERY $aborter_thread ---enable_query_log - ---source include/restart_mysqld.inc - ---connection node_1 ---disconnect foo_node_1 -DROP USER foo@localhost; - diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 920f7ac0329..1f060305d4f 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -8292,19 +8292,11 @@ kill_one_thread(THD *thd, longlong id, killed_state kill_signal, killed_type typ It's ok to also kill DELAYED threads with KILL_CONNECTION instead of KILL_SYSTEM_THREAD; The difference is that KILL_CONNECTION may be faster and do a harder kill than KILL_SYSTEM_THREAD; - - Note that if thread is wsrep Brute Force or applier thread we - allow killing it only when we're SUPER. */ - if ((thd->security_ctx->master_access & SUPER_ACL) || - (thd->security_ctx->user_matches(tmp->security_ctx) -#ifdef WITH_WSREP - && - !tmp->wsrep_applier && - !wsrep_thd_is_BF(tmp, false) -#endif - )) + if (((thd->security_ctx->master_access & SUPER_ACL) || + thd->security_ctx->user_matches(tmp->security_ctx)) && + !wsrep_thd_is_BF(tmp, false)) { tmp->awake(kill_signal); error=0; diff --git a/sql/sql_plugin_services.ic b/sql/sql_plugin_services.ic index d70c76701fd..427d8937c57 100644 --- a/sql/sql_plugin_services.ic +++ b/sql/sql_plugin_services.ic @@ -181,8 +181,7 @@ static struct wsrep_service_st wsrep_handler = { wsrep_trx_is_aborting, wsrep_trx_order_before, wsrep_unlock_rollback, - wsrep_set_data_home_dir, - wsrep_thd_is_applier, + wsrep_set_data_home_dir }; static struct thd_specifics_service_st thd_specifics_handler= diff --git a/sql/wsrep_dummy.cc b/sql/wsrep_dummy.cc index e2a4dcf3bad..795e2d19252 100644 --- a/sql/wsrep_dummy.cc +++ b/sql/wsrep_dummy.cc @@ -20,9 +20,6 @@ my_bool wsrep_thd_is_BF(THD *, my_bool) { return 0; } -my_bool wsrep_thd_is_applier(THD *) -{ return 0; } - int wsrep_trx_order_before(THD *, THD *) { return 0; } diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 82415691b3f..ee8509e3fa2 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -2470,6 +2470,7 @@ extern "C" void wsrep_thd_set_exec_mode(THD *thd, enum wsrep_exec_mode mode) thd->wsrep_exec_mode= mode; } + extern "C" void wsrep_thd_set_query_state( THD *thd, enum wsrep_query_state state) { diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc index 91f0e1e8d8a..551e710cfeb 100644 --- a/sql/wsrep_thd.cc +++ b/sql/wsrep_thd.cc @@ -596,15 +596,6 @@ my_bool wsrep_thd_is_BF(THD *thd, my_bool sync) return status; } -my_bool wsrep_thd_is_applier(THD *thd) -{ - my_bool ret = FALSE; - if (thd) { - ret = thd->wsrep_applier; - } - return ret; -} - extern "C" my_bool wsrep_thd_is_BF_or_commit(void *thd_ptr, my_bool sync) { diff --git a/sql/wsrep_thd.h b/sql/wsrep_thd.h index f5fcb50280c..5900668f3fb 100644 --- a/sql/wsrep_thd.h +++ b/sql/wsrep_thd.h @@ -37,7 +37,6 @@ int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, */ extern void wsrep_thd_set_PA_safe(void *thd_ptr, my_bool safe); extern my_bool wsrep_thd_is_BF(THD *thd, my_bool sync); -extern my_bool wsrep_thd_is_applier(THD *thd); extern my_bool wsrep_thd_is_wsrep(void *thd_ptr); enum wsrep_conflict_state wsrep_thd_conflict_state(void *thd_ptr, my_bool sync); @@ -48,7 +47,6 @@ extern "C" int wsrep_thd_in_locking_session(void *thd_ptr); #else /* WITH_WSREP */ #define wsrep_thd_is_BF(T, S) (0) -#define wsrep_thd_is_applier(T) (0) #define wsrep_abort_thd(X,Y,Z) do { } while(0) #define wsrep_create_appliers(T) do { } while(0) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index c55eaf59879..cd605b6b791 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4929,6 +4929,8 @@ static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels) /* if victim has been signaled by BF thread and/or aborting is already progressing, following query aborting is not necessary any more. + Also, BF thread should own trx mutex for the victim, which would + conflict with trx_mutex_enter() below */ DBUG_VOID_RETURN; } @@ -4937,8 +4939,34 @@ static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels) if (trx_t* trx = thd_to_trx(thd)) { ut_ad(trx->mysql_thd == thd); + switch (trx->abort_type) { +#ifdef WITH_WSREP + case TRX_WSREP_ABORT: + break; +#endif + case TRX_SERVER_ABORT: + if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + lock_mutex_enter(); + } + /* fall through */ + case TRX_REPLICATION_ABORT: + trx_mutex_enter(trx); + } /* Cancel a pending lock request if there are any */ lock_trx_handle_wait(trx); + switch (trx->abort_type) { +#ifdef WITH_WSREP + case TRX_WSREP_ABORT: + break; +#endif + case TRX_SERVER_ABORT: + if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + lock_mutex_exit(); + } + /* fall through */ + case TRX_REPLICATION_ABORT: + trx_mutex_exit(trx); + } } DBUG_VOID_RETURN; @@ -18655,12 +18683,6 @@ wsrep_innobase_kill_one_trx( wsrep_thd_ws_handle(thd)->trx_id); wsrep_thd_LOCK(thd); - - /* We mark this as victim transaction, which is already marked - as BF victim. Both trx mutex and lock_sys mutex is held until - this victim has aborted. */ - victim_trx->victim = true; - DBUG_EXECUTE_IF("sync.wsrep_after_BF_victim_lock", { const char act[]= @@ -18844,7 +18866,7 @@ wsrep_abort_transaction( my_bool signal) { DBUG_ENTER("wsrep_innobase_abort_thd"); - + trx_t* victim_trx = thd_to_trx(victim_thd); trx_t* bf_trx = (bf_thd) ? thd_to_trx(bf_thd) : NULL; @@ -18856,10 +18878,12 @@ wsrep_abort_transaction( if (victim_trx) { lock_mutex_enter(); trx_mutex_enter(victim_trx); + victim_trx->abort_type = TRX_WSREP_ABORT; int rcode = wsrep_innobase_kill_one_trx(bf_thd, bf_trx, victim_trx, signal); trx_mutex_exit(victim_trx); lock_mutex_exit(); + victim_trx->abort_type = TRX_SERVER_ABORT; wsrep_srv_conc_cancel_wait(victim_trx); DBUG_RETURN(rcode); } else { diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index b40c6ae4667..fe16b8272b8 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2019, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -623,6 +623,7 @@ struct trx_lock_t { lock_sys->mutex. Otherwise, this may only be modified by the thread that is serving the running transaction. */ + mem_heap_t* lock_heap; /*!< memory heap for trx_locks; protected by lock_sys->mutex */ @@ -694,6 +695,14 @@ lock_rec_convert_impl_to_expl()) will access transactions associated to other connections. The locks of transactions are protected by lock_sys->mutex and sometimes by trx->mutex. */ +enum trx_abort_t { + TRX_SERVER_ABORT = 0, +#ifdef WITH_WSREP + TRX_WSREP_ABORT, +#endif + TRX_REPLICATION_ABORT +}; + struct trx_t{ ulint magic_n; @@ -871,12 +880,8 @@ struct trx_t{ /*------------------------------*/ THD* mysql_thd; /*!< MySQL thread handle corresponding to this trx, or NULL */ - bool victim; /*!< This transaction is - selected as victim for abort - either by replication or - high priority wsrep thread. This - field is protected by trx and - lock sys mutex. */ + trx_abort_t abort_type; /*!< Transaction abort type*/ + const char* mysql_log_file_name; /*!< if MySQL binlog is used, this field contains a pointer to the latest file diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 33563a03f1a..f06fcd6c4d8 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2019, MariaDB Corporation. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1793,8 +1793,10 @@ wsrep_kill_victim( } } + lock->trx->abort_type = TRX_WSREP_ABORT; wsrep_innobase_kill_one_trx(trx->mysql_thd, (const trx_t*) trx, lock->trx, TRUE); + lock->trx->abort_type = TRX_SERVER_ABORT; } } } @@ -4780,11 +4782,12 @@ lock_report_waiters_to_mysql( if (w_trx->id != victim_trx_id) { /* If thd_report_wait_for() decides to kill the transaction, then we will get a call back into - innobase_kill_query.*/ - trx_mutex_enter(w_trx); - w_trx->victim = true; + innobase_kill_query. We mark this by setting + current_lock_mutex_owner, so we can avoid trying + to recursively take lock_sys->mutex. */ + w_trx->abort_type = TRX_REPLICATION_ABORT; thd_report_wait_for(mysql_thd, w_trx->mysql_thd); - trx_mutex_exit(w_trx); + w_trx->abort_type = TRX_SERVER_ABORT; } ++i; } @@ -7964,7 +7967,16 @@ lock_trx_release_locks( lock_mutex_exit(); } -inline dberr_t lock_trx_handle_wait_low(trx_t* trx) +/*********************************************************************//** +Check whether the transaction has already been rolled back because it +was selected as a deadlock victim, or if it has to wait then cancel +the wait lock. +@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */ +UNIV_INTERN +dberr_t +lock_trx_handle_wait( +/*=================*/ + trx_t* trx) /*!< in/out: trx lock state */ { ut_ad(lock_mutex_own()); ut_ad(trx_mutex_own(trx)); @@ -7981,32 +7993,6 @@ inline dberr_t lock_trx_handle_wait_low(trx_t* trx) return DB_LOCK_WAIT; } -/*********************************************************************//** -Check whether the transaction has already been rolled back because it -was selected as a deadlock victim, or if it has to wait then cancel -the wait lock. -@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */ -UNIV_INTERN -dberr_t -lock_trx_handle_wait( -/*=================*/ - trx_t* trx) /*!< in/out: trx lock state */ -{ - if (!trx->victim) { - lock_mutex_enter(); - trx_mutex_enter(trx); - } - - dberr_t err = lock_trx_handle_wait_low(trx); - - if (!trx->victim) { - lock_mutex_exit(); - trx_mutex_exit(trx); - } - - return err; -} - /*********************************************************************//** Get the number of locks on a table. @return number of locks */ diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 855266686e6..06bf4cc30c0 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -4746,7 +4746,11 @@ no_gap_lock: a deadlock and the transaction had to wait then release the lock it is waiting on. */ + lock_mutex_enter(); + trx_mutex_enter(trx); err = lock_trx_handle_wait(trx); + lock_mutex_exit(); + trx_mutex_exit(trx); switch (err) { case DB_SUCCESS: diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index 127e834335d..3fd71aff23a 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2019, MariaDB Corporation. +Copyright (c) 2016, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -370,7 +370,6 @@ trx_rollback_to_savepoint_for_mysql_low( trx_mark_sql_stat_end(trx); trx->op_info = ""; - trx->victim = false; return(err); } diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 98ac08a00e7..f36aabba8b4 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2019, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1339,7 +1339,11 @@ trx_commit_in_memory( ut_ad(!trx->in_ro_trx_list); ut_ad(!trx->in_rw_trx_list); - trx->victim = false; +#ifdef WITH_WSREP + if (trx->mysql_thd && wsrep_on(trx->mysql_thd)) { + trx->lock.was_chosen_as_deadlock_victim = FALSE; + } +#endif trx->dict_operation = TRX_DICT_OP_NONE; trx->error_state = DB_SUCCESS; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 55c49711f19..a0df23b60d4 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -5534,6 +5534,8 @@ static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels) /* if victim has been signaled by BF thread and/or aborting is already progressing, following query aborting is not necessary any more. + Also, BF thread should own trx mutex for the victim, which would + conflict with trx_mutex_enter() below */ DBUG_VOID_RETURN; } @@ -5541,8 +5543,34 @@ static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels) if (trx_t* trx = thd_to_trx(thd)) { ut_ad(trx->mysql_thd == thd); + switch (trx->abort_type) { +#ifdef WITH_WSREP + case TRX_WSREP_ABORT: + break; +#endif + case TRX_SERVER_ABORT: + if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + lock_mutex_enter(); + } + /* fall through */ + case TRX_REPLICATION_ABORT: + trx_mutex_enter(trx); + } /* Cancel a pending lock request if there are any */ lock_trx_handle_wait(trx); + switch (trx->abort_type) { +#ifdef WITH_WSREP + case TRX_WSREP_ABORT: + break; +#endif + case TRX_SERVER_ABORT: + if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { + lock_mutex_exit(); + } + /* fall through */ + case TRX_REPLICATION_ABORT: + trx_mutex_exit(trx); + } } DBUG_VOID_RETURN; @@ -19695,12 +19723,6 @@ wsrep_innobase_kill_one_trx( (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void"); wsrep_thd_LOCK(thd); - - /* We mark this as victim transaction, which is already marked - as BF victim. Both trx mutex and lock_sys mutex is held until - this victim has aborted. */ - victim_trx->victim = true; - DBUG_EXECUTE_IF("sync.wsrep_after_BF_victim_lock", { const char act[]= @@ -19889,10 +19911,12 @@ wsrep_abort_transaction(handlerton* hton, THD *bf_thd, THD *victim_thd, if (victim_trx) { lock_mutex_enter(); trx_mutex_enter(victim_trx); + victim_trx->abort_type = TRX_WSREP_ABORT; int rcode = wsrep_innobase_kill_one_trx(bf_thd, bf_trx, victim_trx, signal); trx_mutex_exit(victim_trx); lock_mutex_exit(); + victim_trx->abort_type = TRX_SERVER_ABORT; wsrep_srv_conc_cancel_wait(victim_trx); DBUG_RETURN(rcode); } else { diff --git a/storage/xtradb/include/trx0trx.h b/storage/xtradb/include/trx0trx.h index df01284af05..77afde4c35c 100644 --- a/storage/xtradb/include/trx0trx.h +++ b/storage/xtradb/include/trx0trx.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2019, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -672,6 +672,7 @@ struct trx_lock_t { lock_sys->mutex. Otherwise, this may only be modified by the thread that is serving the running transaction. */ + mem_heap_t* lock_heap; /*!< memory heap for trx_locks; protected by lock_sys->mutex */ @@ -743,6 +744,14 @@ lock_rec_convert_impl_to_expl()) will access transactions associated to other connections. The locks of transactions are protected by lock_sys->mutex and sometimes by trx->mutex. */ +enum trx_abort_t { + TRX_SERVER_ABORT = 0, +#ifdef WITH_WSREP + TRX_WSREP_ABORT, +#endif + TRX_REPLICATION_ABORT +}; + struct trx_t{ ulint magic_n; @@ -921,12 +930,8 @@ struct trx_t{ /*------------------------------*/ THD* mysql_thd; /*!< MySQL thread handle corresponding to this trx, or NULL */ - bool victim; /*!< This transaction is - selected as victim for abort - either by replication or - high priority wsrep thread. This - field is protected by trx and - lock sys mutex. */ + trx_abort_t abort_type; /*!< Transaction abort type */ + const char* mysql_log_file_name; /*!< if MySQL binlog is used, this field contains a pointer to the latest file diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index 2999c339457..9daa2cc906f 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2019, MariaDB Corporation. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1804,8 +1804,10 @@ wsrep_kill_victim( } } + lock->trx->abort_type = TRX_WSREP_ABORT; wsrep_innobase_kill_one_trx(trx->mysql_thd, (const trx_t*) trx, lock->trx, TRUE); + lock->trx->abort_type = TRX_SERVER_ABORT; } } } @@ -4819,11 +4821,12 @@ lock_report_waiters_to_mysql( if (w_trx->id != victim_trx_id) { /* If thd_report_wait_for() decides to kill the transaction, then we will get a call back into - innobase_kill_query.*/ - trx_mutex_enter(w_trx); - w_trx->victim = true; + innobase_kill_query. We mark this by setting + current_lock_mutex_owner, so we can avoid trying + to recursively take lock_sys->mutex. */ + w_trx->abort_type = TRX_REPLICATION_ABORT; thd_report_wait_for(mysql_thd, w_trx->mysql_thd); - trx_mutex_exit(w_trx); + w_trx->abort_type = TRX_SERVER_ABORT; } ++i; } @@ -8074,7 +8077,16 @@ lock_trx_release_locks( lock_mutex_exit(); } -inline dberr_t lock_trx_handle_wait_low(trx_t* trx) +/*********************************************************************//** +Check whether the transaction has already been rolled back because it +was selected as a deadlock victim, or if it has to wait then cancel +the wait lock. +@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */ +UNIV_INTERN +dberr_t +lock_trx_handle_wait( +/*=================*/ + trx_t* trx) /*!< in/out: trx lock state */ { ut_ad(lock_mutex_own()); ut_ad(trx_mutex_own(trx)); @@ -8091,32 +8103,6 @@ inline dberr_t lock_trx_handle_wait_low(trx_t* trx) return DB_LOCK_WAIT; } -/*********************************************************************//** -Check whether the transaction has already been rolled back because it -was selected as a deadlock victim, or if it has to wait then cancel -the wait lock. -@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */ -UNIV_INTERN -dberr_t -lock_trx_handle_wait( -/*=================*/ - trx_t* trx) /*!< in/out: trx lock state */ -{ - if (!trx->victim) { - lock_mutex_enter(); - trx_mutex_enter(trx); - } - - dberr_t err = lock_trx_handle_wait_low(trx); - - if (!trx->victim) { - lock_mutex_exit(); - trx_mutex_exit(trx); - } - - return err; -} - /*********************************************************************//** Get the number of locks on a table. @return number of locks */ diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index 87bc2aa2875..b6b5d107885 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -4755,7 +4755,11 @@ no_gap_lock: a deadlock and the transaction had to wait then release the lock it is waiting on. */ + lock_mutex_enter(); + trx_mutex_enter(trx); err = lock_trx_handle_wait(trx); + lock_mutex_exit(); + trx_mutex_exit(trx); switch (err) { case DB_SUCCESS: diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc index 127e834335d..56b7120fa34 100644 --- a/storage/xtradb/trx/trx0roll.cc +++ b/storage/xtradb/trx/trx0roll.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2019, MariaDB Corporation. +Copyright (c) 2016, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -33,6 +33,8 @@ Created 3/26/1996 Heikki Tuuri #include "trx0roll.ic" #endif +#include + #include "fsp0fsp.h" #include "mach0data.h" #include "trx0rseg.h" @@ -49,6 +51,9 @@ Created 3/26/1996 Heikki Tuuri #include "pars0pars.h" #include "srv0mon.h" #include "trx0sys.h" +#ifdef WITH_WSREP +#include "ha_prototypes.h" +#endif /* WITH_WSREP */ /** This many pages must be undone before a truncate is tried within rollback */ @@ -370,7 +375,13 @@ trx_rollback_to_savepoint_for_mysql_low( trx_mark_sql_stat_end(trx); trx->op_info = ""; - trx->victim = false; + +#ifdef WITH_WSREP + if (wsrep_on(trx->mysql_thd) && + trx->lock.was_chosen_as_deadlock_victim) { + trx->lock.was_chosen_as_deadlock_victim = FALSE; + } +#endif return(err); } @@ -1068,6 +1079,12 @@ trx_roll_try_truncate( if (trx->update_undo) { trx_undo_truncate_end(trx, trx->update_undo, limit); } + +#ifdef WITH_WSREP_OUT + if (wsrep_on(trx->mysql_thd)) { + trx->lock.was_chosen_as_deadlock_victim = FALSE; + } +#endif /* WITH_WSREP */ } /***********************************************************************//** diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index 8ca247fadc8..17cba81daf3 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2019, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1563,7 +1563,11 @@ trx_commit_in_memory( ut_ad(!trx->in_ro_trx_list); ut_ad(!trx->in_rw_trx_list); - trx->victim = false; +#ifdef WITH_WSREP + if (trx->mysql_thd && wsrep_on(trx->mysql_thd)) { + trx->lock.was_chosen_as_deadlock_victim = FALSE; + } +#endif trx->dict_operation = TRX_DICT_OP_NONE; trx->error_state = DB_SUCCESS; @@ -2664,6 +2668,10 @@ trx_start_if_not_started_low( { switch (trx->state) { case TRX_STATE_NOT_STARTED: +#ifdef WITH_WSREP + ut_d(trx->start_file = __FILE__); + ut_d(trx->start_line = __LINE__); +#endif /* WITH_WSREP */ trx_start_low(trx); /* fall through */ case TRX_STATE_ACTIVE: @@ -2697,6 +2705,11 @@ trx_start_for_ddl_low( trx->will_lock = 1; trx->ddl = true; + +#ifdef WITH_WSREP + ut_d(trx->start_file = __FILE__); + ut_d(trx->start_line = __LINE__); +#endif /* WITH_WSREP */ trx_start_low(trx); return; From 05ad7fc3ed886351e23b61af1856d666d979146f Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 28 Mar 2019 11:42:21 +0100 Subject: [PATCH 16/20] MDEV-19054 : mysql_upgrade_service now allows MySQL 5.7 to MariaDB 10.2 upgrade. --- sql/winservice.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/winservice.c b/sql/winservice.c index efbbb527c9b..f817ab2b142 100644 --- a/sql/winservice.c +++ b/sql/winservice.c @@ -108,7 +108,7 @@ BOOL exclude_service(mysqld_service_properties *props) } if ((props->version_major == 0) || (props->version_major > 5 && props->version_major < 10) || - (props->version_major == 5 && props->version_minor > 6)) + (props->version_major == 5 && props->version_minor > 7)) { return TRUE; } From e42192d7b3821640bcf18c58dc303a2338c6d1aa Mon Sep 17 00:00:00 2001 From: Sujatha Sivakumar Date: Thu, 28 Mar 2019 20:35:39 +0530 Subject: [PATCH 17/20] MDEV-13895: GTID and Master_Delay causes excessive initial delay Problem: ======== When attempting to delay a Slave attached with GTID, there appears to be an extra delay applied initially. For example, this output reflects a Slave that is already delayed by 43200 seconds. When switching to GTID replication, replication is paused until SQL_Remaining_Delay counts down to 0: CHANGE MASTER TO master_use_gtid=current_pos; CHANGE MASTER TO MASTER_DELAY=43200; Seconds_Behind_Master: 44847 Using_Gtid: Current_Pos SQL_Delay: 43200 SQL_Remaining_Delay: 43089 Slave_SQL_Running_State: Waiting until MASTER_DELAY seconds after master executed event Analysis: ========= When slave initiates a GTID based connection request to master, the master sends two GTID_LIST events. The first one is actual GTID_LIST event and the second one is a fake GTID_LIST event. This is sent by master to provide its current binlary log file position. The fake GTID_LIST events will have their ev->when=0. 'when' (the timestamp) is set to 0 so that slave could distinguish between real and fake Rotate events. On slave side when MASTER_DELAY is configured to "X" the applier will ensure that there is a time delay of "X" seconds before the event is applied. General behaviour of MASTER_DELAY example:- Master timestamp of event e1=10 timestamp of event e2=11 On slave MASTER_DELAY=5 Event e1 will be applied at = 15 e2 will be applied at =16 In bug scenario:- On Master: With GTIDs timestamp of event e1=10 timestamp of event e2=0 On Slave: e1 will be applied at = 10 + 5 =15 For e2, since "e2->when=0" e2->when is set to current timestamp. i.e since the e2->when and current timestamp on slave is the same applier waits for additional master_delay=5 seconds. the ev->when contributes to "rli->last_master_timestamp". rli->last_master_timestamp= ev->when + (time_t) ev->exec_time; Fake events should not update the "ev->when" to "current timestamp" on slave. Fix: === Remove the assignment of current timestamp to "ev->when" when "ev->when=0". --- .../r/rpl_gtid_excess_initial_delay.result | 23 ++++++++ .../rpl/t/rpl_gtid_excess_initial_delay.test | 58 +++++++++++++++++++ sql/slave.cc | 6 -- 3 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_gtid_excess_initial_delay.result create mode 100644 mysql-test/suite/rpl/t/rpl_gtid_excess_initial_delay.test diff --git a/mysql-test/suite/rpl/r/rpl_gtid_excess_initial_delay.result b/mysql-test/suite/rpl/r/rpl_gtid_excess_initial_delay.result new file mode 100644 index 00000000000..641d186ab78 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_gtid_excess_initial_delay.result @@ -0,0 +1,23 @@ +include/master-slave.inc +[connection master] +CREATE TABLE t1 (i INT); +connection slave; +include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID= current_pos, MASTER_DELAY= 10; +include/start_slave.inc +connection master; +INSERT INTO t1 VALUES (1); +include/sync_slave_io_with_master.inc +connection slave; +"Sleeping for 15" +# Asserted this: Seconds_Behind_Master should be less than MASTER_DELAY +# Asserted this: One row shoule be found in table t1. +"======= Clean up ========" +STOP SLAVE; +CHANGE MASTER TO MASTER_USE_GTID=no, MASTER_DELAY=0; +START SLAVE; +connection master; +DROP TABLE t1; +connection slave; +connection master; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_gtid_excess_initial_delay.test b/mysql-test/suite/rpl/t/rpl_gtid_excess_initial_delay.test new file mode 100644 index 00000000000..d840b67e9e8 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_gtid_excess_initial_delay.test @@ -0,0 +1,58 @@ +# ==== Purpose ==== +# +# Test verifies that when "Master_Delay" is specified on slave with GTIDS there +# will not be any extra delay initially. +# +# ==== Implementation ==== +# +# Steps: +# 0 - Stop the slave and execute CHANGE MASTER command with +# master_use_gtid= curren_pos and master_delay= 10 +# 1 - On slave introduce a sleep of 15 seconds and check that the +# Seconds_Behind_Master is within specified master_delay limit. It should +# not be more that "10" seconds. +# +# ==== References ==== +# +# MDEV-13895: GTID and Master_Delay causes excessive initial delay + +--source include/have_binlog_format_mixed.inc +--source include/master-slave.inc + +CREATE TABLE t1 (i INT); +--sync_slave_with_master + +--source include/stop_slave.inc +CHANGE MASTER TO MASTER_USE_GTID= current_pos, MASTER_DELAY= 10; +--source include/start_slave.inc + +--connection master +INSERT INTO t1 VALUES (1); +--source include/sync_slave_io_with_master.inc + +--connection slave +--let $actual_delay= query_get_value(SHOW SLAVE STATUS, SQL_Delay, 1) +--let $sleep_time= `SELECT 5 + $actual_delay` +--echo "Sleeping for $sleep_time" +--sleep $sleep_time + +--let $assert_cond= [SHOW SLAVE STATUS, Seconds_Behind_Master, 1] <= 10 +--let $assert_text= Seconds_Behind_Master should be less than MASTER_DELAY +--source include/rpl_assert.inc + +# The row should be available in table after master_delay=20 seconds. +--let $assert_text= One row shoule be found in table t1. +--let $assert_cond= COUNT(*) = 1 FROM t1 +--source include/rpl_assert.inc + +--echo "======= Clean up ========" +STOP SLAVE; +CHANGE MASTER TO MASTER_USE_GTID=no, MASTER_DELAY=0; +START SLAVE; + +--connection master +DROP TABLE t1; +--sync_slave_with_master + +--connection master +--source include/rpl_end.inc diff --git a/sql/slave.cc b/sql/slave.cc index f3e08f14213..6b234697f09 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3532,12 +3532,6 @@ apply_event_and_update_pos_setup(Log_event* ev, THD* thd, rpl_group_info *rgi) thd->variables.server_id = ev->server_id; thd->set_time(); // time the query thd->lex->current_select= 0; - if (!ev->when) - { - my_hrtime_t hrtime= my_hrtime(); - ev->when= hrtime_to_my_time(hrtime); - ev->when_sec_part= hrtime_sec_part(hrtime); - } thd->variables.option_bits= (thd->variables.option_bits & ~OPTION_SKIP_REPLICATION) | (ev->flags & LOG_EVENT_SKIP_REPLICATION_F ? OPTION_SKIP_REPLICATION : 0); From 8fcd9478cc6fdb1d4cdb84a552d7dcf8e6f596d3 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Thu, 28 Mar 2019 15:54:42 +0300 Subject: [PATCH 18/20] MDEV-18080, part#1: MyRocks is slow with log-bin=off The cause for this was fix MDEV-15372, which was trying to speed up the parallel slave. Part#1: Do not attempt the "optimization" for transactions that are not replication slave workers. --- storage/rocksdb/ha_rocksdb.cc | 38 +++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 6ce00cb4d1c..022511320a6 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -3750,20 +3750,32 @@ static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx) - For a COMMIT statement that finishes a multi-statement transaction - For a statement that has its own transaction */ - - // First, commit without syncing. This establishes the commit order - tx->set_sync(false); - bool tx_had_writes = tx->get_write_count()? true : false ; - if (tx->commit()) { - DBUG_RETURN(HA_ERR_ROCKSDB_COMMIT_FAILED); - } - thd_wakeup_subsequent_commits(thd, 0); - - if (tx_had_writes && rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC) + if (thd->slave_thread) { - rocksdb::Status s= rdb->FlushWAL(true); - if (!s.ok()) - DBUG_RETURN(HA_ERR_INTERNAL_ERROR); + // An attempt to make parallel slave performant (not fully successful, + // see MDEV-15372): + + // First, commit without syncing. This establishes the commit order + tx->set_sync(false); + bool tx_had_writes = tx->get_write_count()? true : false ; + if (tx->commit()) { + DBUG_RETURN(HA_ERR_ROCKSDB_COMMIT_FAILED); + } + thd_wakeup_subsequent_commits(thd, 0); + + if (tx_had_writes && rocksdb_flush_log_at_trx_commit == FLUSH_LOG_SYNC) + { + rocksdb::Status s= rdb->FlushWAL(true); + if (!s.ok()) + DBUG_RETURN(HA_ERR_INTERNAL_ERROR); + } + } + else + { + /* Not a slave thread */ + if (tx->commit()) { + DBUG_RETURN(HA_ERR_ROCKSDB_COMMIT_FAILED); + } } } else { /* From fc168c3a5e58d8b364a2e87e0d876a261ec7fced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 29 Mar 2019 11:38:45 +0200 Subject: [PATCH 19/20] MDEV-15587 AES test fails, segfaults in EVP_CipherInit_ex When HAVE_YASSL is defined (due to cmake -DWITH_SSL=bundled or otherwise), mysys_ssl/my_crypt.cc will #include "yassl.cc" from the same directory. When MariaDB 10.2 or later is compiled with GCC 8 and optimizations are enabled, then the check if (iv) in EVP_CipherInit_ex() can be wrongly optimized away. The reason appears to be that __attribute__((nonnull)) is attached to the variable iv, because there is a (no-op) call memcpy(oiv, iv, ivlen=0) earlier in the code path. It is possible that this started failing after the code was refactored in MDEV-10332 (MariaDB 10.2.6). In MariaDB 10.1, there is a similar memcpy() call in MyCTX_nopad::init(), but the code appears to work fine. --- mysys_ssl/my_crypt.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mysys_ssl/my_crypt.cc b/mysys_ssl/my_crypt.cc index db303f37b0e..2d6f5188034 100644 --- a/mysys_ssl/my_crypt.cc +++ b/mysys_ssl/my_crypt.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2014 Google Inc. - Copyright (c) 2014, 2017 MariaDB Corporation + Copyright (c) 2014, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -93,7 +93,8 @@ public: this->key= key; this->klen= klen; this->buf_len= 0; - memcpy(oiv, iv, ivlen); + if (ivlen) + memcpy(oiv, iv, ivlen); DBUG_ASSERT(ivlen == 0 || ivlen == sizeof(oiv)); int res= MyCTX::init(cipher, encrypt, key, klen, iv, ivlen); From cc71e7501cb623b2d600c47dbe10a31dcf6e8fcf Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 28 Mar 2019 12:07:20 +0100 Subject: [PATCH 20/20] post-merge: -Werror fixes in 10.2 --- cmake/mariadb_connector_c.cmake | 5 +++++ sql/spatial.cc | 12 ++++++------ storage/innobase/buf/buf0dump.cc | 2 +- strings/json_lib.c | 4 ++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/cmake/mariadb_connector_c.cmake b/cmake/mariadb_connector_c.cmake index 368a6cc8da5..0f08c3464c4 100644 --- a/cmake/mariadb_connector_c.cmake +++ b/cmake/mariadb_connector_c.cmake @@ -37,3 +37,8 @@ SET(CLIENT_PLUGIN_PVIO_SOCKET STATIC) MESSAGE("== Configuring MariaDB Connector/C") ADD_SUBDIRECTORY(libmariadb) + +#remove after merging libmariadb > v3.0.9 +IF(TARGET caching_sha2_password AND CMAKE_C_FLAGS_DEBUG MATCHES "-Werror") + SET_PROPERTY(TARGET caching_sha2_password APPEND_STRING PROPERTY COMPILE_FLAGS -Wno-unused-function) +ENDIF() diff --git a/sql/spatial.cc b/sql/spatial.cc index 2aca528dd15..5f045f8cbfa 100644 --- a/sql/spatial.cc +++ b/sql/spatial.cc @@ -238,17 +238,17 @@ int Geometry::as_wkt(String *wkt, const char **end) static const uchar type_keyname[]= "type"; -static const int type_keyname_len= 4; +static const uint type_keyname_len= 4; static const uchar coord_keyname[]= "coordinates"; -static const int coord_keyname_len= 11; +static const uint coord_keyname_len= 11; static const uchar geometries_keyname[]= "geometries"; -static const int geometries_keyname_len= 10; +static const uint geometries_keyname_len= 10; static const uchar features_keyname[]= "features"; -static const int features_keyname_len= 8; +static const uint features_keyname_len= 8; static const uchar geometry_keyname[]= "geometry"; -static const int geometry_keyname_len= 8; +static const uint geometry_keyname_len= 8; -static const int max_keyname_len= 11; /*'coordinates' keyname is the longest.*/ +static const uint max_keyname_len= 11; /*'coordinates' keyname is the longest.*/ static const uchar feature_type[]= "feature"; static const int feature_type_len= 7; diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index ed2131bdcee..fa6754fa065 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -392,7 +392,7 @@ buf_dump( if (SHUTTING_DOWN() && !(j % 1024)) { service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, "Dumping buffer pool " - ULINTPF "/" ULINTPF ", " + ULINTPF "/%lu, " "page " ULINTPF "/" ULINTPF, i + 1, srv_buf_pool_instances, j + 1, n_pages); diff --git a/strings/json_lib.c b/strings/json_lib.c index 394517a4ab6..3ce9b0c503f 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -1409,7 +1409,7 @@ int json_find_paths_next(json_engine_t *je, json_find_paths_t *state) if (!json_key_matches(je, &key_name)) continue; } - if (cur_step - state->paths[p_c].last_step == state->cur_depth) + if (cur_step == state->paths[p_c].last_step + state->cur_depth) path_found= TRUE; else { @@ -1442,7 +1442,7 @@ int json_find_paths_next(json_engine_t *je, json_find_paths_t *state) cur_step->n_item == state->array_counters[state->cur_depth]) { /* Array item matches. */ - if (cur_step - state->paths[p_c].last_step == state->cur_depth) + if (cur_step == state->paths[p_c].last_step + state->cur_depth) path_found= TRUE; else {