From a9a4089175a8fd5a9b973f99f87cd0fe181f8f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 14 Dec 2017 13:47:38 +0200 Subject: [PATCH 01/51] Plug a small memory leak in mariabackup --backup --- extra/mariabackup/xtrabackup.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 90a0c829773..f145a94c923 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -3626,7 +3626,6 @@ fail: /* Reset the system variables in the recovery module. */ recv_sys_var_init(); trx_pool_init(); - row_mysql_init(); ut_crc32_init(); crc_init(); From ef9e78c9d41a5ac644f08068e3dabad948b0e30a Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 13 Dec 2017 11:52:53 +0100 Subject: [PATCH 02/51] MDEV-14524 TokuDB is unable to be built on Linux tokudb needs either F_NOCACHE or O_DIRECT, not both --- storage/tokudb/CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/storage/tokudb/CMakeLists.txt b/storage/tokudb/CMakeLists.txt index 4e0291545ec..f01c4aed449 100644 --- a/storage/tokudb/CMakeLists.txt +++ b/storage/tokudb/CMakeLists.txt @@ -4,13 +4,17 @@ IF(CMAKE_VERSION VERSION_LESS "2.8.9") MESSAGE(STATUS "CMake 2.8.9 or higher is required by TokuDB") ELSEIF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") -# tokudb requires F_NOCACHE, O_DIRECT, and designated initializers +# tokudb requires F_NOCACHE or O_DIRECT, and designated initializers CHECK_CXX_SOURCE_COMPILES( " #include struct a {int b; int c; }; struct a d = { .b=1, .c=2 }; -int main() { return F_NOCACHE + O_DIRECT; } +#if defined(O_DIRECT) || defined(F_NOCACHE) +int main() { return 0; } +#else +#error +#endif " TOKUDB_OK) ENDIF() From e4efbfd904d458cf0372b20bc27ba974b3b65a98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 14 Dec 2017 11:43:54 +0200 Subject: [PATCH 03/51] Remove dead code lock_remove_recovered_trx_record_locks() Contrary to what the comment said, trx_resurrect_table_locks() does associate table locks with every recovered transaction that modified any records, ever since this bug fix in MySQL 5.6.12: Bug#16593427 ROLLBACK OF RECOVERED TRANSACTION CORRUPTS NON-ONLINE ADD INDEX --- storage/innobase/include/lock0lock.h | 5 -- storage/innobase/lock/lock0lock.cc | 82 ---------------------------- 2 files changed, 87 deletions(-) diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index 60b07f2fe72..06600960c27 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -964,11 +964,6 @@ struct lock_sys_t{ in the waiting_threads array, protected by lock_sys->wait_mutex */ - ibool rollback_complete; - /*!< TRUE if rollback of all - recovered transactions is - complete. Protected by - lock_sys->mutex */ ulint n_lock_max_wait_time; /*!< Max wait time */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 97bacebaf24..e46d55ae9e1 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -5418,77 +5418,6 @@ lock_remove_all_on_table_for_trx( } } -/*******************************************************************//** -Remove any explicit record locks held by recovering transactions on -the table. -@return number of recovered transactions examined */ -static -ulint -lock_remove_recovered_trx_record_locks( -/*===================================*/ - dict_table_t* table) /*!< in: check if there are any locks - held on records in this table or on the - table itself */ -{ - ut_a(table != NULL); - ut_ad(lock_mutex_own()); - - ulint n_recovered_trx = 0; - - mutex_enter(&trx_sys->mutex); - - for (trx_t* trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); - trx != NULL; - trx = UT_LIST_GET_NEXT(trx_list, trx)) { - - assert_trx_in_rw_list(trx); - - if (!trx->is_recovered) { - continue; - } - - /* Because we are holding the lock_sys->mutex, - implicit locks cannot be converted to explicit ones - while we are scanning the explicit locks. */ - - lock_t* next_lock; - - for (lock_t* lock = UT_LIST_GET_FIRST(trx->lock.trx_locks); - lock != NULL; - lock = next_lock) { - - ut_a(lock->trx == trx); - - /* Recovered transactions can't wait on a lock. */ - - ut_a(!lock_get_wait(lock)); - - next_lock = UT_LIST_GET_NEXT(trx_locks, lock); - - switch (lock_get_type_low(lock)) { - default: - ut_error; - case LOCK_TABLE: - if (lock->un_member.tab_lock.table == table) { - lock_trx_table_locks_remove(lock); - lock_table_remove_low(lock); - } - break; - case LOCK_REC: - if (lock->index->table == table) { - lock_rec_discard(lock); - } - } - } - - ++n_recovered_trx; - } - - mutex_exit(&trx_sys->mutex); - - return(n_recovered_trx); -} - /*********************************************************************//** Removes locks on a table to be dropped or truncated. If remove_also_table_sx_locks is TRUE then table-level S and X locks are @@ -5550,17 +5479,6 @@ lock_remove_all_on_table( } } - /* Note: Recovered transactions don't have table level IX or IS locks - but can have implicit record locks that have been converted to explicit - record locks. Such record locks cannot be freed by traversing the - transaction lock list in dict_table_t (as above). */ - - if (!lock_sys->rollback_complete - && lock_remove_recovered_trx_record_locks(table) == 0) { - - lock_sys->rollback_complete = TRUE; - } - lock_mutex_exit(); } From 34f2f4fa431893db555093edf0577b35c5cb4b0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 15 Dec 2017 13:50:30 +0200 Subject: [PATCH 04/51] MDEV-14660 Assertion failure in lock_move_rec_list_start() after instant ADD COLUMN lock_move_rec_list_start(): Relax a too strict assertion. This function can be invoked on the leftmost leaf page, after all. So, the first record of each page can be a 'default row' record, but the 'default row' record must never be locked. --- .../suite/innodb/r/instant_alter.result | 56 +++++++++++++++++-- mysql-test/suite/innodb/t/instant_alter.test | 22 +++++++- storage/innobase/lock/lock0lock.cc | 7 ++- 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/innodb/r/instant_alter.result b/mysql-test/suite/innodb/r/instant_alter.result index c04deddef1e..581e88b1a2e 100644 --- a/mysql-test/suite/innodb/r/instant_alter.result +++ b/mysql-test/suite/innodb/r/instant_alter.result @@ -353,6 +353,22 @@ id c2 c4 c5 c6 c8 phrase b 3 3 1970-01-01 03:00:42 1970-01-01 03:00:42 NULL 1970-01-01 The quick brown fox jumps over the lazy dog 4 0 1970-01-01 03:00:42 1970-01-01 03:00:42 NULL 1970-01-01 The quick brown fox jumps over the lazy dog 😱 5 9 1970-01-01 03:00:42 1970-01-01 03:00:42 NULL 1970-01-01 ÀÀH binary line of business +CREATE TABLE t4 +(id INT, foo INT DEFAULT 0, c1 VARCHAR(4000), +p GEOMETRY NOT NULL DEFAULT ST_GeomFromText('LINESTRING(0 0,0 1,1 1)'), +PRIMARY KEY(id,foo)) +ENGINE=InnoDB ROW_FORMAT=REDUNDANT; +INSERT INTO t4 (id,c1) VALUES (1, REPEAT('a', 4000)), (2, REPEAT('a', 4000)); +ALTER TABLE t4 ADD COLUMN d1 INT; +BEGIN; +UPDATE t4 SET c1 = repeat('1', 4000), foo=1 WHERE id=1; +INSERT INTO t4 (id,c1) VALUES (1, REPEAT('a', 4000)); +UPDATE t4 SET c1 = repeat('2', 4000), foo=1 WHERE id=2; +ROLLBACK; +BEGIN; +UPDATE t4 SET d1 = 1,foo=2 WHERE id=1; +INSERT INTO t4 (id,foo,c1) VALUES (1, 1, REPEAT('1', 4000)); +COMMIT; CREATE TABLE big (id INT PRIMARY KEY, c1 VARCHAR(4000), c2 VARCHAR(4000), c3 VARCHAR(1000), p POINT NOT NULL DEFAULT ST_GeomFromText('POINT(0 0)'), SPATIAL INDEX(p)) @@ -413,7 +429,7 @@ clust_index_size 3 connection default; InnoDB 0 transactions not purged -DROP TABLE t1,t2,t3,big; +DROP TABLE t1,t2,t3,t4,big; CREATE TABLE t1 (id INT PRIMARY KEY, c2 INT UNIQUE, c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'), @@ -713,6 +729,22 @@ id c2 c4 c5 c6 c8 phrase b 3 3 1970-01-01 03:00:42 1970-01-01 03:00:42 NULL 1970-01-01 The quick brown fox jumps over the lazy dog 4 0 1970-01-01 03:00:42 1970-01-01 03:00:42 NULL 1970-01-01 The quick brown fox jumps over the lazy dog 😱 5 9 1970-01-01 03:00:42 1970-01-01 03:00:42 NULL 1970-01-01 ÀÀH binary line of business +CREATE TABLE t4 +(id INT, foo INT DEFAULT 0, c1 VARCHAR(4000), +p GEOMETRY NOT NULL DEFAULT ST_GeomFromText('LINESTRING(0 0,0 1,1 1)'), +PRIMARY KEY(id,foo)) +ENGINE=InnoDB ROW_FORMAT=COMPACT; +INSERT INTO t4 (id,c1) VALUES (1, REPEAT('a', 4000)), (2, REPEAT('a', 4000)); +ALTER TABLE t4 ADD COLUMN d1 INT; +BEGIN; +UPDATE t4 SET c1 = repeat('1', 4000), foo=1 WHERE id=1; +INSERT INTO t4 (id,c1) VALUES (1, REPEAT('a', 4000)); +UPDATE t4 SET c1 = repeat('2', 4000), foo=1 WHERE id=2; +ROLLBACK; +BEGIN; +UPDATE t4 SET d1 = 1,foo=2 WHERE id=1; +INSERT INTO t4 (id,foo,c1) VALUES (1, 1, REPEAT('1', 4000)); +COMMIT; CREATE TABLE big (id INT PRIMARY KEY, c1 VARCHAR(4000), c2 VARCHAR(4000), c3 VARCHAR(1000), p POINT NOT NULL DEFAULT ST_GeomFromText('POINT(0 0)'), SPATIAL INDEX(p)) @@ -773,7 +805,7 @@ clust_index_size 3 connection default; InnoDB 0 transactions not purged -DROP TABLE t1,t2,t3,big; +DROP TABLE t1,t2,t3,t4,big; CREATE TABLE t1 (id INT PRIMARY KEY, c2 INT UNIQUE, c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'), @@ -1073,6 +1105,22 @@ id c2 c4 c5 c6 c8 phrase b 3 3 1970-01-01 03:00:42 1970-01-01 03:00:42 NULL 1970-01-01 The quick brown fox jumps over the lazy dog 4 0 1970-01-01 03:00:42 1970-01-01 03:00:42 NULL 1970-01-01 The quick brown fox jumps over the lazy dog 😱 5 9 1970-01-01 03:00:42 1970-01-01 03:00:42 NULL 1970-01-01 ÀÀH binary line of business +CREATE TABLE t4 +(id INT, foo INT DEFAULT 0, c1 VARCHAR(4000), +p GEOMETRY NOT NULL DEFAULT ST_GeomFromText('LINESTRING(0 0,0 1,1 1)'), +PRIMARY KEY(id,foo)) +ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +INSERT INTO t4 (id,c1) VALUES (1, REPEAT('a', 4000)), (2, REPEAT('a', 4000)); +ALTER TABLE t4 ADD COLUMN d1 INT; +BEGIN; +UPDATE t4 SET c1 = repeat('1', 4000), foo=1 WHERE id=1; +INSERT INTO t4 (id,c1) VALUES (1, REPEAT('a', 4000)); +UPDATE t4 SET c1 = repeat('2', 4000), foo=1 WHERE id=2; +ROLLBACK; +BEGIN; +UPDATE t4 SET d1 = 1,foo=2 WHERE id=1; +INSERT INTO t4 (id,foo,c1) VALUES (1, 1, REPEAT('1', 4000)); +COMMIT; CREATE TABLE big (id INT PRIMARY KEY, c1 VARCHAR(4000), c2 VARCHAR(4000), c3 VARCHAR(1000), p POINT NOT NULL DEFAULT ST_GeomFromText('POINT(0 0)'), SPATIAL INDEX(p)) @@ -1133,11 +1181,11 @@ clust_index_size 3 connection default; InnoDB 0 transactions not purged -DROP TABLE t1,t2,t3,big; +DROP TABLE t1,t2,t3,t4,big; disconnect analyze; SELECT variable_value-@old_instant instants FROM information_schema.global_status WHERE variable_name = 'innodb_instant_alter_column'; instants -30 +33 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency; diff --git a/mysql-test/suite/innodb/t/instant_alter.test b/mysql-test/suite/innodb/t/instant_alter.test index d73dc02c9bb..8f2261cadf6 100644 --- a/mysql-test/suite/innodb/t/instant_alter.test +++ b/mysql-test/suite/innodb/t/instant_alter.test @@ -236,6 +236,26 @@ ALTER TABLE t3 DROP c3, DROP c7; --disable_info SELECT * FROM t3; +eval CREATE TABLE t4 +(id INT, foo INT DEFAULT 0, c1 VARCHAR(4000), + p GEOMETRY NOT NULL DEFAULT ST_GeomFromText('LINESTRING(0 0,0 1,1 1)'), + PRIMARY KEY(id,foo)) +$engine; + +INSERT INTO t4 (id,c1) VALUES (1, REPEAT('a', 4000)), (2, REPEAT('a', 4000)); +ALTER TABLE t4 ADD COLUMN d1 INT; + +BEGIN; +UPDATE t4 SET c1 = repeat('1', 4000), foo=1 WHERE id=1; +INSERT INTO t4 (id,c1) VALUES (1, REPEAT('a', 4000)); +UPDATE t4 SET c1 = repeat('2', 4000), foo=1 WHERE id=2; +ROLLBACK; + +BEGIN; +UPDATE t4 SET d1 = 1,foo=2 WHERE id=1; +INSERT INTO t4 (id,foo,c1) VALUES (1, 1, REPEAT('1', 4000)); +COMMIT; + eval CREATE TABLE big (id INT PRIMARY KEY, c1 VARCHAR(4000), c2 VARCHAR(4000), c3 VARCHAR(1000), p POINT NOT NULL DEFAULT ST_GeomFromText('POINT(0 0)'), SPATIAL INDEX(p)) @@ -279,7 +299,7 @@ WHERE name = 'test/big'; connection default; --source include/wait_all_purged.inc -DROP TABLE t1,t2,t3,big; +DROP TABLE t1,t2,t3,t4,big; dec $format; } diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index e46d55ae9e1..2f66e178daf 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -3833,8 +3833,9 @@ lock_move_rec_list_start( reset the lock bits on the old */ while (rec1 != rec) { - ut_ad(!page_rec_is_default_row(rec1)); - ut_ad(!page_rec_is_default_row(rec2)); + ut_ad(page_rec_is_default_row(rec1) + == page_rec_is_default_row(rec2)); + ut_d(const rec_t* const prev = rec1); ulint rec1_heap_no; ulint rec2_heap_no; @@ -3858,6 +3859,8 @@ lock_move_rec_list_start( if (rec1_heap_no < lock->un_member.rec_lock.n_bits && lock_rec_reset_nth_bit(lock, rec1_heap_no)) { + ut_ad(!page_rec_is_default_row(prev)); + if (type_mode & LOCK_WAIT) { lock_reset_lock_and_trx_wait(lock); } From 64b11e61b5055cbdcdb97911d426b4f13f14fbd1 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 15 Dec 2017 17:59:33 +0300 Subject: [PATCH 05/51] MDEV-14293: MyRocks lacks basic functionality The error "Unsupported collation on string indexed column %s Use binary collation (latin1_bin, binary, utf8_bin)." is misleading. Change it: - It is now a warning - It is printed only for collations that do not support index-only access (reversible collations that use unpack_info are ok) - The new warning text is: Indexed column %s.%s uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key. --- sql/share/errmsg-utf8.txt | 3 +++ storage/rocksdb/ha_rocksdb.cc | 22 +++++++------------ .../rocksdb/r/add_index_inplace.result | 9 +++++++- .../rocksdb/t/add_index_inplace.test | 9 +++++++- storage/rocksdb/rdb_datadic.cc | 2 +- storage/rocksdb/rdb_datadic.h | 2 ++ 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 528dc7107e6..50ccc6eadd1 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -7753,3 +7753,6 @@ ER_GEOJSON_EMPTY_COORDINATES ER_MYROCKS_CANT_NOPAD_COLLATION eng "MyRocks doesn't currently support collations with \"No pad\" attribute." +ER_MYROCKS_COLLATION_IS_LIMITED + eng "Indexed column %s.%s uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key." + diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index a9d37c1def2..6afba00b3f8 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -5957,9 +5957,11 @@ rdb_is_index_collation_supported(const my_core::Field *const field) { const my_core::enum_field_types type = field->real_type(); /* Handle [VAR](CHAR|BINARY) or TEXT|BLOB */ if (type == MYSQL_TYPE_VARCHAR || type == MYSQL_TYPE_STRING || - type == MYSQL_TYPE_BLOB) { - return RDB_INDEX_COLLATIONS.find(field->charset()->number) != - RDB_INDEX_COLLATIONS.end(); + type == MYSQL_TYPE_BLOB) { + + return (RDB_INDEX_COLLATIONS.find(field->charset()->number) != + RDB_INDEX_COLLATIONS.end()) || + rdb_is_collation_supported(field->charset()); } return true; } @@ -6118,18 +6120,10 @@ int ha_rocksdb::create_cfs( !rdb_is_index_collation_supported( table_arg->key_info[i].key_part[part].field) && !rdb_collation_exceptions->matches(tablename_sys)) { - std::string collation_err; - for (const auto &coll : RDB_INDEX_COLLATIONS) { - if (collation_err != "") { - collation_err += ", "; - } - collation_err += get_charset_name(coll); - } - my_error(ER_UNSUPPORTED_COLLATION, MYF(0), + + my_error(ER_MYROCKS_COLLATION_IS_LIMITED, MYF(ME_JUST_WARNING), tbl_def_arg->full_tablename().c_str(), - table_arg->key_info[i].key_part[part].field->field_name, - collation_err.c_str()); - DBUG_RETURN(HA_EXIT_FAILURE); + table_arg->key_info[i].key_part[part].field->field_name); } } } diff --git a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result index 01fa9f1d35b..ca770c2358a 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result @@ -279,8 +279,15 @@ DROP TABLE t1; set @tmp_rocksdb_strict_collation_check= @@rocksdb_strict_collation_check; set global rocksdb_strict_collation_check=1; CREATE TABLE t1 (a INT, b TEXT); +# MariaDB no longer gives ER_UNSUPPORTED_COLLATION ALTER TABLE t1 ADD KEY kb(b(10)); -ERROR HY000: Unsupported collation on string indexed column test.t1.b Use binary collation (latin1_bin, binary, utf8_bin). +ALTER TABLE t1 ADD PRIMARY KEY(a); +DROP TABLE t1; +CREATE TABLE t1 (a INT, b TEXT collate utf8_general_ci); +# MariaDB no longer gives ER_UNSUPPORTED_COLLATION +ALTER TABLE t1 ADD KEY kb(b(10)); +Warnings: +Warning 4078 Indexed column test.t1.b uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key. ALTER TABLE t1 ADD PRIMARY KEY(a); DROP TABLE t1; set global rocksdb_strict_collation_check= @tmp_rocksdb_strict_collation_check; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test index c1a91c2a5a2..876ef2c9965 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test @@ -173,10 +173,17 @@ set @tmp_rocksdb_strict_collation_check= @@rocksdb_strict_collation_check; set global rocksdb_strict_collation_check=1; CREATE TABLE t1 (a INT, b TEXT); ---error ER_UNSUPPORTED_COLLATION +--echo # MariaDB no longer gives ER_UNSUPPORTED_COLLATION ALTER TABLE t1 ADD KEY kb(b(10)); ALTER TABLE t1 ADD PRIMARY KEY(a); DROP TABLE t1; + +CREATE TABLE t1 (a INT, b TEXT collate utf8_general_ci); +--echo # MariaDB no longer gives ER_UNSUPPORTED_COLLATION +ALTER TABLE t1 ADD KEY kb(b(10)); +ALTER TABLE t1 ADD PRIMARY KEY(a); +DROP TABLE t1; + set global rocksdb_strict_collation_check= @tmp_rocksdb_strict_collation_check; # make sure race condition between connection close and alter on another diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index 3efd3ae6433..df743ff0c48 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -2921,7 +2921,7 @@ std::array rdb_collation_data; mysql_mutex_t rdb_collation_data_mutex; -static bool rdb_is_collation_supported(const my_core::CHARSET_INFO *const cs) { +bool rdb_is_collation_supported(const my_core::CHARSET_INFO *const cs) { return cs->strxfrm_multiply==1 && cs->mbmaxlen == 1 && !(cs->state & (MY_CS_BINSORT | MY_CS_NOPAD)); } diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index b1ecef045d1..5796132de39 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -1358,4 +1358,6 @@ struct Rdb_index_info { uint64 m_ttl_duration = 0; }; +bool rdb_is_collation_supported(const my_core::CHARSET_INFO *const cs); + } // namespace myrocks From 4ef86e36e8a6546434aec41d4dde21919f034820 Mon Sep 17 00:00:00 2001 From: Vesa Pentti Date: Thu, 14 Dec 2017 12:39:12 +0000 Subject: [PATCH 06/51] A fix of mtr bug uncovered by MDEV-12501: passing of parameters in rebootstrap --- mysql-test/mysql-test-run.pl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index aaee6c42058..90ac4c85248 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -329,6 +329,8 @@ my %mysqld_logs; my $opt_debug_sync_timeout= 300; # Default timeout for WAIT_FOR actions. my $warn_seconds = 60; +my $rebootstrap_re= '--innodb[-_](?:page[-_]size|checksum[-_]algorithm|undo[-_]tablespaces|log[-_]group[-_]home[-_]dir|data[-_]home[-_]dir)|data[-_]file[-_]path'; + sub testcase_timeout ($) { my ($tinfo)= @_; if (exists $tinfo->{'case-timeout'}) { @@ -2792,10 +2794,12 @@ sub mysql_server_start($) { { # Some InnoDB options are incompatible with the default bootstrap. # If they are used, re-bootstrap - if ( $extra_opts and - "@$extra_opts" =~ /--innodb[-_](?:page[-_]size|checksum[-_]algorithm|undo[-_]tablespaces|log[-_]group[-_]home[-_]dir|data[-_]home[-_]dir)|data[-_]file[-_]path/ ) + my @rebootstrap_opts; + @rebootstrap_opts = grep {/$rebootstrap_re/o} @$extra_opts if $extra_opts; + if (@rebootstrap_opts) { - mysql_install_db($mysqld, undef, $extra_opts); + mtr_verbose("Re-bootstrap with @rebootstrap_opts"); + mysql_install_db($mysqld, undef, \@rebootstrap_opts); } else { # Copy datadir from installed system db From 7380376370dd9ec93fbcd684c80ab4e5bda70f24 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 16 Dec 2017 16:44:33 +0300 Subject: [PATCH 07/51] MDEV-14293: MyRocks lacks basic functionality Adjust the fix according to the review input: don't introduce a special error caode, use ER_INTERNAL_ERROR catch-all. --- sql/share/errmsg-utf8.txt | 4 ---- storage/rocksdb/ha_rocksdb.cc | 12 +++++++++--- .../mysql-test/rocksdb/r/add_index_inplace.result | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 50ccc6eadd1..74ac7e9f557 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -7752,7 +7752,3 @@ ER_GEOJSON_EMPTY_COORDINATES ER_MYROCKS_CANT_NOPAD_COLLATION eng "MyRocks doesn't currently support collations with \"No pad\" attribute." - -ER_MYROCKS_COLLATION_IS_LIMITED - eng "Indexed column %s.%s uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key." - diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 6afba00b3f8..766c8036fba 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -6121,9 +6121,15 @@ int ha_rocksdb::create_cfs( table_arg->key_info[i].key_part[part].field) && !rdb_collation_exceptions->matches(tablename_sys)) { - my_error(ER_MYROCKS_COLLATION_IS_LIMITED, MYF(ME_JUST_WARNING), - tbl_def_arg->full_tablename().c_str(), - table_arg->key_info[i].key_part[part].field->field_name); + char buf[1024]; + my_snprintf(buf, sizeof(buf), + "Indexed column %s.%s uses a collation that does not " + "allow index-only access in secondary key and has " + "reduced disk space efficiency in primary key.", + tbl_def_arg->full_tablename().c_str(), + table_arg->key_info[i].key_part[part].field->field_name); + + my_error(ER_INTERNAL_ERROR, MYF(ME_JUST_WARNING), buf); } } } diff --git a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result index ca770c2358a..6325dc97cf5 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result @@ -287,7 +287,7 @@ CREATE TABLE t1 (a INT, b TEXT collate utf8_general_ci); # MariaDB no longer gives ER_UNSUPPORTED_COLLATION ALTER TABLE t1 ADD KEY kb(b(10)); Warnings: -Warning 4078 Indexed column test.t1.b uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key. +Warning 1815 Internal error: Indexed column test.t1.b uses a collation that does not allow index-only access in secondary key and has reduced disk space efficiency in primary key. ALTER TABLE t1 ADD PRIMARY KEY(a); DROP TABLE t1; set global rocksdb_strict_collation_check= @tmp_rocksdb_strict_collation_check; From 4bd63bd55158566db8c8b46b29fee590e9596a5e Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 16 Dec 2017 17:45:55 +0300 Subject: [PATCH 08/51] MDEV-14679: RocksdB plugin fails to load with "Loading of unknown plugin ROCKSDB_CFSTATS Set maturity level of all "Sub-plugins" to be the same as the main maturity level. --- storage/rocksdb/ha_rocksdb.cc | 3 ++- storage/rocksdb/ha_rocksdb.h | 3 +++ .../rocksdb/r/mariadb_port_fixes.result | 17 ++++++++++++++ .../rocksdb/t/mariadb_port_fixes.test | 6 +++++ storage/rocksdb/rdb_i_s.cc | 22 +++++++++---------- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 766c8036fba..98a9785713a 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -12499,6 +12499,7 @@ void print_keydup_error(TABLE *table, KEY *key, myf errflag, its name generation. */ + struct st_mysql_storage_engine rocksdb_storage_engine = { MYSQL_HANDLERTON_INTERFACE_VERSION}; @@ -12515,7 +12516,7 @@ maria_declare_plugin(rocksdb_se){ myrocks::rocksdb_status_vars, /* status variables */ myrocks::rocksdb_system_variables, /* system variables */ "1.0", /* string version */ - MariaDB_PLUGIN_MATURITY_ALPHA /* maturity */ + myrocks::MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }, myrocks::rdb_i_s_cfstats, myrocks::rdb_i_s_dbstats, myrocks::rdb_i_s_perf_context, myrocks::rdb_i_s_perf_context_global, diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index d064988d0f3..82819bbf7b2 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -1411,4 +1411,7 @@ private: Rdb_inplace_alter_ctx(const Rdb_inplace_alter_ctx &); Rdb_inplace_alter_ctx &operator=(const Rdb_inplace_alter_ctx &); }; + +const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_ALPHA; + } // namespace myrocks diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index f590fd22dff..9952314cd2c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@ -64,3 +64,20 @@ set global rocksdb_strict_collation_check=off; create table t1 (pk varchar(10) collate latin1_nopad_bin, primary key(pk)) engine=rocksdb; ERROR HY000: MyRocks doesn't currently support collations with "No pad" attribute. set global rocksdb_strict_collation_check=@tmp_rscc; +# +# MDEV-14679: RocksdB plugin fails to load with "Loading of unknown plugin ROCKSDB_CFSTATS +# +select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; +plugin_name plugin_maturity +ROCKSDB Alpha +ROCKSDB_CFSTATS Alpha +ROCKSDB_DBSTATS Alpha +ROCKSDB_PERF_CONTEXT Alpha +ROCKSDB_PERF_CONTEXT_GLOBAL Alpha +ROCKSDB_CF_OPTIONS Alpha +ROCKSDB_COMPACTION_STATS Alpha +ROCKSDB_GLOBAL_INFO Alpha +ROCKSDB_DDL Alpha +ROCKSDB_INDEX_FILE_MAP Alpha +ROCKSDB_LOCKS Alpha +ROCKSDB_TRX Alpha diff --git a/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test b/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test index 70a4f5b05cb..980f2e302b2 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/mariadb_port_fixes.test @@ -67,3 +67,9 @@ set global rocksdb_strict_collation_check=off; create table t1 (pk varchar(10) collate latin1_nopad_bin, primary key(pk)) engine=rocksdb; set global rocksdb_strict_collation_check=@tmp_rscc; + +--echo # +--echo # MDEV-14679: RocksdB plugin fails to load with "Loading of unknown plugin ROCKSDB_CFSTATS +--echo # +select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; + diff --git a/storage/rocksdb/rdb_i_s.cc b/storage/rocksdb/rdb_i_s.cc index 346a2994ec1..424a9e6c1f4 100644 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@ -1484,7 +1484,7 @@ struct st_maria_plugin rdb_i_s_cfstats = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_dbstats = { @@ -1500,7 +1500,7 @@ struct st_maria_plugin rdb_i_s_dbstats = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_perf_context = { @@ -1516,7 +1516,7 @@ struct st_maria_plugin rdb_i_s_perf_context = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_perf_context_global = { @@ -1532,7 +1532,7 @@ struct st_maria_plugin rdb_i_s_perf_context_global = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_cfoptions = { @@ -1548,7 +1548,7 @@ struct st_maria_plugin rdb_i_s_cfoptions = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_global_info = { @@ -1564,7 +1564,7 @@ struct st_maria_plugin rdb_i_s_global_info = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_compact_stats = { @@ -1580,7 +1580,7 @@ struct st_maria_plugin rdb_i_s_compact_stats = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_ddl = { @@ -1596,7 +1596,7 @@ struct st_maria_plugin rdb_i_s_ddl = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_index_file_map = { @@ -1612,7 +1612,7 @@ struct st_maria_plugin rdb_i_s_index_file_map = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_lock_info = { @@ -1628,7 +1628,7 @@ struct st_maria_plugin rdb_i_s_lock_info = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; struct st_maria_plugin rdb_i_s_trx_info = { @@ -1644,6 +1644,6 @@ struct st_maria_plugin rdb_i_s_trx_info = { nullptr, /* status variables */ nullptr, /* system variables */ nullptr, /* config options */ - 0, /* flags */ + MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL }; } // namespace myrocks From d91d1c8dbc9ae1b3459719cf6b6f21007d9be99a Mon Sep 17 00:00:00 2001 From: Vesa Pentti Date: Thu, 14 Dec 2017 09:19:53 +0000 Subject: [PATCH 09/51] Test cleanup related to MDEV-12501 * Removing unnecessary --plugin-maturity=unknown definitions from tests --- mysql-test/include/have_plugin_auth.opt | 2 +- .../encryption/include/have_example_key_management_plugin.opt | 1 - mysql-test/suite/galera/galera_2nodes.cnf | 1 - mysql-test/suite/galera/suite.opt | 1 - mysql-test/suite/galera_3nodes/galera_3nodes.cnf | 1 - mysql-test/suite/innodb/suite.opt | 1 - mysql-test/suite/rpl/suite.opt | 1 - storage/rocksdb/mysql-test/rocksdb/suite.opt | 1 - storage/rocksdb/mysql-test/rocksdb_hotbackup/suite.opt | 1 - storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt | 2 +- storage/rocksdb/mysql-test/rocksdb_stress/suite.opt | 1 - storage/rocksdb/mysql-test/rocksdb_sys_vars/suite.opt | 1 - storage/rocksdb/mysql-test/storage_engine/suite.opt | 1 - 13 files changed, 2 insertions(+), 13 deletions(-) delete mode 100644 mysql-test/suite/galera/suite.opt delete mode 100644 mysql-test/suite/innodb/suite.opt delete mode 100644 mysql-test/suite/rpl/suite.opt delete mode 100644 storage/rocksdb/mysql-test/rocksdb_hotbackup/suite.opt delete mode 100644 storage/rocksdb/mysql-test/rocksdb_stress/suite.opt diff --git a/mysql-test/include/have_plugin_auth.opt b/mysql-test/include/have_plugin_auth.opt index 12a9334fa22..0204e148656 100644 --- a/mysql-test/include/have_plugin_auth.opt +++ b/mysql-test/include/have_plugin_auth.opt @@ -1 +1 @@ ---plugin-load-add=$AUTH_TEST_PLUGIN_SO --plugin-maturity=unknown +--plugin-load-add=$AUTH_TEST_PLUGIN_SO diff --git a/mysql-test/suite/encryption/include/have_example_key_management_plugin.opt b/mysql-test/suite/encryption/include/have_example_key_management_plugin.opt index 66aaef98c54..ce7f1ddef7e 100644 --- a/mysql-test/suite/encryption/include/have_example_key_management_plugin.opt +++ b/mysql-test/suite/encryption/include/have_example_key_management_plugin.opt @@ -1,3 +1,2 @@ --plugin-load-add=$EXAMPLE_KEY_MANAGEMENT_SO --loose-example-key-management ---plugin-maturity=unknown diff --git a/mysql-test/suite/galera/galera_2nodes.cnf b/mysql-test/suite/galera/galera_2nodes.cnf index 8de704dbec8..b24f3603894 100644 --- a/mysql-test/suite/galera/galera_2nodes.cnf +++ b/mysql-test/suite/galera/galera_2nodes.cnf @@ -4,7 +4,6 @@ [mysqld] wsrep-on=1 binlog-format=row -plugin-maturity=unknown innodb-autoinc-lock-mode=2 default-storage-engine=innodb wsrep-provider=@ENV.WSREP_PROVIDER diff --git a/mysql-test/suite/galera/suite.opt b/mysql-test/suite/galera/suite.opt deleted file mode 100644 index 8374626febe..00000000000 --- a/mysql-test/suite/galera/suite.opt +++ /dev/null @@ -1 +0,0 @@ ---plugin-maturity=unknown diff --git a/mysql-test/suite/galera_3nodes/galera_3nodes.cnf b/mysql-test/suite/galera_3nodes/galera_3nodes.cnf index 48b3c002c04..91aa53ad7b1 100644 --- a/mysql-test/suite/galera_3nodes/galera_3nodes.cnf +++ b/mysql-test/suite/galera_3nodes/galera_3nodes.cnf @@ -3,7 +3,6 @@ [mysqld] binlog-format=row -plugin-maturity=unknown innodb-autoinc-lock-mode=2 default-storage-engine=innodb diff --git a/mysql-test/suite/innodb/suite.opt b/mysql-test/suite/innodb/suite.opt deleted file mode 100644 index 8374626febe..00000000000 --- a/mysql-test/suite/innodb/suite.opt +++ /dev/null @@ -1 +0,0 @@ ---plugin-maturity=unknown diff --git a/mysql-test/suite/rpl/suite.opt b/mysql-test/suite/rpl/suite.opt deleted file mode 100644 index 8374626febe..00000000000 --- a/mysql-test/suite/rpl/suite.opt +++ /dev/null @@ -1 +0,0 @@ ---plugin-maturity=unknown diff --git a/storage/rocksdb/mysql-test/rocksdb/suite.opt b/storage/rocksdb/mysql-test/rocksdb/suite.opt index e12c09953a5..8ca2405f1ef 100644 --- a/storage/rocksdb/mysql-test/rocksdb/suite.opt +++ b/storage/rocksdb/mysql-test/rocksdb/suite.opt @@ -1,2 +1 @@ --plugin-load=$HA_ROCKSDB_SO --default-storage-engine=rocksdb ---plugin-maturity=unknown diff --git a/storage/rocksdb/mysql-test/rocksdb_hotbackup/suite.opt b/storage/rocksdb/mysql-test/rocksdb_hotbackup/suite.opt deleted file mode 100644 index 8374626febe..00000000000 --- a/storage/rocksdb/mysql-test/rocksdb_hotbackup/suite.opt +++ /dev/null @@ -1 +0,0 @@ ---plugin-maturity=unknown diff --git a/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt index 47efa90a739..761a2108cec 100644 --- a/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt +++ b/storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt @@ -1 +1 @@ ---ignore-db-dirs=.rocksdb --plugin-load=$HA_ROCKSDB_SO --default-storage-engine=rocksdb --plugin-maturity=unknown +--ignore-db-dirs=.rocksdb --plugin-load=$HA_ROCKSDB_SO --default-storage-engine=rocksdb diff --git a/storage/rocksdb/mysql-test/rocksdb_stress/suite.opt b/storage/rocksdb/mysql-test/rocksdb_stress/suite.opt deleted file mode 100644 index 8374626febe..00000000000 --- a/storage/rocksdb/mysql-test/rocksdb_stress/suite.opt +++ /dev/null @@ -1 +0,0 @@ ---plugin-maturity=unknown diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/suite.opt b/storage/rocksdb/mysql-test/rocksdb_sys_vars/suite.opt index 0effe1dadf7..f6640ca596c 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/suite.opt +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/suite.opt @@ -1,2 +1 @@ --ignore-db-dirs=#rocksdb --plugin-load=$HA_ROCKSDB_SO ---plugin-maturity=unknown diff --git a/storage/rocksdb/mysql-test/storage_engine/suite.opt b/storage/rocksdb/mysql-test/storage_engine/suite.opt index 6ec1095bd9d..e6122c7ed3e 100644 --- a/storage/rocksdb/mysql-test/storage_engine/suite.opt +++ b/storage/rocksdb/mysql-test/storage_engine/suite.opt @@ -1,2 +1 @@ --ignore-db-dirs=#rocksdb --plugin-load=$HA_ROCKSDB_SO --binlog_format=ROW --collation-server=latin1_bin --loose-rocksdb_flush_log_at_trx_commit=0 ---plugin-maturity=unknown From 06f0b23a78ab15b6f3d4465e77ac1bdc747524d4 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 17 Dec 2017 17:53:53 +0200 Subject: [PATCH 10/51] Fixed memory leak in my_rocks rocksdb_sys_vars.rocksdb_update_cf_options_basic.test failed with reports about not freed memory. --- storage/rocksdb/ha_rocksdb.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 98a9785713a..8857b2e8cba 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -12337,6 +12337,7 @@ void rocksdb_set_update_cf_options(THD *const /* unused */, // Basic sanity checking and parsing the options into a map. If this fails // then there's no point to proceed. if (!Rdb_cf_options::parse_cf_options(val, &option_map)) { + my_free(*reinterpret_cast(var_ptr)); *reinterpret_cast(var_ptr) = nullptr; // NO_LINT_DEBUG @@ -12405,6 +12406,7 @@ void rocksdb_set_update_cf_options(THD *const /* unused */, // the CF options. This will results in consistent behavior and avoids // dealing with cases when only a subset of CF-s was successfully updated. if (val) { + my_free(*reinterpret_cast(var_ptr)); *reinterpret_cast(var_ptr) = my_strdup(val, MYF(0)); } else { *reinterpret_cast(var_ptr) = nullptr; From 0acac4fe5fd6d8fab9f13859219f46842153803b Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 18 Dec 2017 01:55:40 +0400 Subject: [PATCH 11/51] MDEV-14593 human-readable XA RECOVER. The 'data' field in the XA RECOVER resultset changed to be charset_bin. It seems to me right and also --binary-as-hex starts working. The XA RECOVER FORMAT='SQL' option implemented. It returns the XID string that fits to be an argument for the XA ... statements. --- mysql-test/r/binary_to_hex.result | 4 + mysql-test/r/xa.result | 53 +++++++++++ mysql-test/t/binary_to_hex.test | 20 +++++ mysql-test/t/xa.test | 48 +++++++++- sql/handler.cc | 140 ++++++++++++++++++++++++++++-- sql/handler.h | 9 ++ sql/share/errmsg-utf8.txt | 4 +- sql/sql_yacc.yy | 21 ++++- 8 files changed, 286 insertions(+), 13 deletions(-) diff --git a/mysql-test/r/binary_to_hex.result b/mysql-test/r/binary_to_hex.result index 345df1de538..51ee5fa1c62 100644 --- a/mysql-test/r/binary_to_hex.result +++ b/mysql-test/r/binary_to_hex.result @@ -115,3 +115,7 @@ c9: 0x000000000101000000000000000000F03F000000000000F03F #Print the table contents in html format
c1c2c3c4c5c6c7c8c9
0x74696E79626C6F622D74657874207265616461626C650x626C6F622D74657874207265616461626C650x6D656469756D626C6F622D74657874207265616461626C650x6C6F6E67626C6F622D74657874207265616461626C650x74657874207265616461626C650x010x630x7661726961626C650x000000000101000000000000000000F03F000000000000F03F
idcol1col2
10xAB1234000000000000000x123ABC
20xDE1234000000000000000x123DEF
DROP TABLE t1, t2; +create table t1 (a int); +formatID gtrid_length bqual_length data +1 3 2 0x7472316271 +DROP TABLE t1; diff --git a/mysql-test/r/xa.result b/mysql-test/r/xa.result index fdcf25f3a12..6c1ff867f89 100644 --- a/mysql-test/r/xa.result +++ b/mysql-test/r/xa.result @@ -60,6 +60,59 @@ a 20 disconnect con1; connection default; +xa start 'tr1'; +insert t1 values (40); +xa end 'tr1'; +xa prepare 'tr1'; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +1 3 0 'tr1' +xa rollback 'tr1'; +xa start 'tr1', 'bq'; +insert t1 values (40); +xa end 'tr1', 'bq'; +xa prepare 'tr1', 'bq'; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +1 3 2 'tr1','bq' +xa rollback 'tr1', 'bq'; +xa start 'tr1', 'bq', 3; +insert t1 values (40); +xa end 'tr1', 'bq', 3; +xa prepare 'tr1', 'bq', 3; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +3 3 2 'tr1','bq',3 +xa rollback 'tr1', 'bq', 3; +xa start 'tr1#$'; +insert t1 values (40); +xa end 'tr1#$'; +xa prepare 'tr1#$'; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +1 5 0 X'7472312324' +xa rollback 'tr1#$'; +xa start 'tr1#$', 'bq'; +insert t1 values (40); +xa end 'tr1#$', 'bq'; +xa prepare 'tr1#$', 'bq'; +xa recover format='SQL'; +formatID gtrid_length bqual_length data +1 5 2 X'7472312324',X'6271' +xa rollback 'tr1#$', 'bq'; +xa start 'tr1#$', 'bq', 3; +insert t1 values (40); +xa end 'tr1#$', 'bq', 3; +xa prepare 'tr1#$', 'bq', 3; +xa recover format='RAW'; +formatID gtrid_length bqual_length data +3 5 2 tr1#$bq +xa recover format='PLAIN'; +ERROR HY000: Unknown XA RECOVER format name: 'PLAIN' +xa recover format='SQL'; +formatID gtrid_length bqual_length data +3 5 2 X'7472312324',X'6271',3 +xa rollback 'tr1#$', 'bq', 3; drop table t1; drop table if exists t1; create table t1(a int, b int, c varchar(20), primary key(a)) engine = innodb; diff --git a/mysql-test/t/binary_to_hex.test b/mysql-test/t/binary_to_hex.test index 8312a246d0c..be4fb301e40 100644 --- a/mysql-test/t/binary_to_hex.test +++ b/mysql-test/t/binary_to_hex.test @@ -72,5 +72,25 @@ SELECT * FROM t2; #Cleanup DROP TABLE t1, t2; +# MDEV-14593 human-readable XA RECOVER + +create table t1 (a int); + +--write_file $MYSQLTEST_VARDIR/tmp/mdev-14593.sql +DELIMITER / +XA START 'tr1', 'bq'/ +INSERT INTO t1 VALUES (0)/ +XA END 'tr1', 'bq'/ +XA PREPARE 'tr1', 'bq'/ +XA RECOVER/ +XA ROLLBACK 'tr1', 'bq'/ +EOF +--exec $MYSQL test --binary-as-hex < $MYSQLTEST_VARDIR/tmp/mdev-14593.sql 2>&1 +remove_file $MYSQLTEST_VARDIR/tmp/mdev-14593.sql; + + +#Cleanup +DROP TABLE t1; + # Wait till all disconnects are completed --source include/wait_until_count_sessions.inc diff --git a/mysql-test/t/xa.test b/mysql-test/t/xa.test index c1f36129d75..7a6208f4b38 100644 --- a/mysql-test/t/xa.test +++ b/mysql-test/t/xa.test @@ -83,6 +83,53 @@ select * from t1; disconnect con1; connection default; + +# MDEV-14593 human-readable XA RECOVER +xa start 'tr1'; +insert t1 values (40); +xa end 'tr1'; +xa prepare 'tr1'; +xa recover format='SQL'; +xa rollback 'tr1'; + +xa start 'tr1', 'bq'; +insert t1 values (40); +xa end 'tr1', 'bq'; +xa prepare 'tr1', 'bq'; +xa recover format='SQL'; +xa rollback 'tr1', 'bq'; + +xa start 'tr1', 'bq', 3; +insert t1 values (40); +xa end 'tr1', 'bq', 3; +xa prepare 'tr1', 'bq', 3; +xa recover format='SQL'; +xa rollback 'tr1', 'bq', 3; + +xa start 'tr1#$'; +insert t1 values (40); +xa end 'tr1#$'; +xa prepare 'tr1#$'; +xa recover format='SQL'; +xa rollback 'tr1#$'; + +xa start 'tr1#$', 'bq'; +insert t1 values (40); +xa end 'tr1#$', 'bq'; +xa prepare 'tr1#$', 'bq'; +xa recover format='SQL'; +xa rollback 'tr1#$', 'bq'; + +xa start 'tr1#$', 'bq', 3; +insert t1 values (40); +xa end 'tr1#$', 'bq', 3; +xa prepare 'tr1#$', 'bq', 3; +xa recover format='RAW'; +--error ER_UNKNOWN_EXPLAIN_FORMAT +xa recover format='PLAIN'; +xa recover format='SQL'; +xa rollback 'tr1#$', 'bq', 3; + drop table t1; # @@ -379,7 +426,6 @@ connection default; DROP TABLE t1, t2; disconnect con2; - # Wait till all disconnects are completed --source include/wait_until_count_sessions.inc diff --git a/sql/handler.cc b/sql/handler.cc index 81c6b60d256..93db0b09abf 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1981,6 +1981,97 @@ int ha_recover(HASH *commit_list) DBUG_RETURN(0); } +/** + return the XID as it appears in the SQL function's arguments. + So this string can be passed to XA START, XA PREPARE etc... + + @note + the 'buf' has to have space for at least SQL_XIDSIZE bytes. +*/ + + +/* + 'a'..'z' 'A'..'Z', '0'..'9' + and '-' '_' ' ' symbols don't have to be + converted. +*/ + +static const char xid_needs_conv[128]= +{ + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 0,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1, + 0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1, + 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0, + 1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1 +}; + +uint get_sql_xid(XID *xid, char *buf) +{ + int tot_len= xid->gtrid_length + xid->bqual_length; + int i; + const char *orig_buf= buf; + + for (i=0; idata)[i]; + if (c >= 128 || xid_needs_conv[c]) + break; + } + + if (i >= tot_len) + { + /* No need to convert characters to hexadecimals. */ + *buf++= '\''; + memcpy(buf, xid->data, xid->gtrid_length); + buf+= xid->gtrid_length; + *buf++= '\''; + if (xid->bqual_length > 0 || xid->formatID != 1) + { + *buf++= ','; + *buf++= '\''; + memcpy(buf, xid->data+xid->gtrid_length, xid->bqual_length); + buf+= xid->bqual_length; + *buf++= '\''; + } + } + else + { + *buf++= 'X'; + *buf++= '\''; + for (i= 0; i < xid->gtrid_length; i++) + { + *buf++=_dig_vec_lower[((uchar*) xid->data)[i] >> 4]; + *buf++=_dig_vec_lower[((uchar*) xid->data)[i] & 0x0f]; + } + *buf++= '\''; + if (xid->bqual_length > 0 || xid->formatID != 1) + { + *buf++= ','; + *buf++= 'X'; + *buf++= '\''; + for (; i < tot_len; i++) + { + *buf++=_dig_vec_lower[((uchar*) xid->data)[i] >> 4]; + *buf++=_dig_vec_lower[((uchar*) xid->data)[i] & 0x0f]; + } + *buf++= '\''; + } + } + + if (xid->formatID != 1) + { + *buf++= ','; + buf+= my_longlong10_to_str_8bit(&my_charset_bin, buf, + MY_INT64_NUM_DECIMAL_DIGITS, -10, xid->formatID); + } + + return buf - orig_buf; +} + + /** return the list of XID's to a client, the same way SHOW commands do. @@ -1990,7 +2081,8 @@ int ha_recover(HASH *commit_list) It can be easily fixed later, if necessary. */ -static my_bool xa_recover_callback(XID_STATE *xs, Protocol *protocol) +static my_bool xa_recover_callback(XID_STATE *xs, Protocol *protocol, + char *data, uint data_len, CHARSET_INFO *data_cs) { if (xs->xa_state == XA_PREPARED) { @@ -1998,8 +2090,7 @@ static my_bool xa_recover_callback(XID_STATE *xs, Protocol *protocol) protocol->store_longlong((longlong) xs->xid.formatID, FALSE); protocol->store_longlong((longlong) xs->xid.gtrid_length, FALSE); protocol->store_longlong((longlong) xs->xid.bqual_length, FALSE); - protocol->store(xs->xid.data, xs->xid.gtrid_length + xs->xid.bqual_length, - &my_charset_bin); + protocol->store(data, data_len, data_cs); if (protocol->write()) return TRUE; } @@ -2007,11 +2098,28 @@ static my_bool xa_recover_callback(XID_STATE *xs, Protocol *protocol) } +static my_bool xa_recover_callback_short(XID_STATE *xs, Protocol *protocol) +{ + return xa_recover_callback(xs, protocol, xs->xid.data, + xs->xid.gtrid_length + xs->xid.bqual_length, &my_charset_bin); +} + + +static my_bool xa_recover_callback_verbose(XID_STATE *xs, Protocol *protocol) +{ + char buf[SQL_XIDSIZE]; + uint len= get_sql_xid(&xs->xid, buf); + return xa_recover_callback(xs, protocol, buf, len, + &my_charset_utf8_general_ci); +} + + bool mysql_xa_recover(THD *thd) { List field_list; Protocol *protocol= thd->protocol; MEM_ROOT *mem_root= thd->mem_root; + my_hash_walk_action action; DBUG_ENTER("mysql_xa_recover"); field_list.push_back(new (mem_root) @@ -2023,16 +2131,32 @@ bool mysql_xa_recover(THD *thd) field_list.push_back(new (mem_root) Item_int(thd, "bqual_length", 0, MY_INT32_NUM_DECIMAL_DIGITS), mem_root); - field_list.push_back(new (mem_root) - Item_empty_string(thd, "data", - XIDDATASIZE), mem_root); + { + uint len; + CHARSET_INFO *cs; + + if (thd->lex->verbose) + { + len= SQL_XIDSIZE; + cs= &my_charset_utf8_general_ci; + action= (my_hash_walk_action) xa_recover_callback_verbose; + } + else + { + len= XIDDATASIZE; + cs= &my_charset_bin; + action= (my_hash_walk_action) xa_recover_callback_short; + } + + field_list.push_back(new (mem_root) + Item_empty_string(thd, "data", len, cs), mem_root); + } if (protocol->send_result_set_metadata(&field_list, Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF)) DBUG_RETURN(1); - if (xid_cache_iterate(thd, (my_hash_walk_action) xa_recover_callback, - protocol)) + if (xid_cache_iterate(thd, action, protocol)) DBUG_RETURN(1); my_eof(thd); DBUG_RETURN(0); diff --git a/sql/handler.h b/sql/handler.h index 9ada2ccbe7c..9941462da8d 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -652,6 +652,15 @@ struct xid_t { }; typedef struct xid_t XID; +/* + The size of XID string representation in the form + 'gtrid', 'bqual', formatID + see xid_t::get_sql_string() for details. +*/ +#define SQL_XIDSIZE (XIDDATASIZE * 2 + 8 + MY_INT64_NUM_DECIMAL_DIGITS) +/* The 'buf' has to have space for at least SQL_XIDSIZE bytes. */ +uint get_sql_xid(XID *xid, char *buf); + /* for recover() handlerton call */ #define MIN_XID_LIST_SIZE 128 #define MAX_XID_LIST_SIZE (1024*128) diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index c55fe6686a4..5d457bc57a4 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -6850,8 +6850,8 @@ ER_CANT_SET_GTID_NEXT_WHEN_OWNING_GTID eng "GTID_NEXT cannot be changed by a client that owns a GTID. The client owns %s. Ownership is released on COMMIT or ROLLBACK" ER_UNKNOWN_EXPLAIN_FORMAT - eng "Unknown EXPLAIN format name: '%s'" - rus "НСизвСстноС имя Ρ„ΠΎΡ€ΠΌΠ°Ρ‚Π° ΠΊΠΎΠΌΠ°Π½Π΄Ρ‹ EXPLAIN: '%s'" + eng "Unknown %s format name: '%s'" + rus "НСизвСстноС имя Ρ„ΠΎΡ€ΠΌΠ°Ρ‚Π° ΠΊΠΎΠΌΠ°Π½Π΄Ρ‹ %s: '%s'" ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION 25006 eng "Cannot execute statement in a READ ONLY transaction" diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index e4661e8f9d1..05f0998683e 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -1682,7 +1682,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize); opt_default_time_precision case_stmt_body opt_bin_mod opt_if_exists_table_element opt_if_not_exists_table_element - opt_recursive + opt_recursive opt_format_xid %type create_or_replace @@ -16810,9 +16810,26 @@ xa: { Lex->sql_command = SQLCOM_XA_ROLLBACK; } - | XA_SYM RECOVER_SYM + | XA_SYM RECOVER_SYM opt_format_xid { Lex->sql_command = SQLCOM_XA_RECOVER; + Lex->verbose= $3; + } + ; + +opt_format_xid: + /* empty */ { $$= false; } + | FORMAT_SYM '=' ident_or_text + { + if (!my_strcasecmp(system_charset_info, $3.str, "SQL")) + $$= true; + else if (!my_strcasecmp(system_charset_info, $3.str, "RAW")) + $$= false; + else + { + my_yyabort_error((ER_UNKNOWN_EXPLAIN_FORMAT, MYF(0), "XA RECOVER", $3.str)); + $$= false; + } } ; From 7fd78055743ba54a3db45f4947674882bf3641e6 Mon Sep 17 00:00:00 2001 From: Vesa Pentti Date: Mon, 11 Dec 2017 19:20:37 +0000 Subject: [PATCH 12/51] MDEV-14315 -- Reflect use of tcmalloc in a system variable and error log * The version of tcmalloc is written to the system variable 'version_malloc_library' if tcmalloc is used, similarly to jemalloc * Extracted method guess_malloc_library() --- include/my_sys.h | 2 ++ mysys/CMakeLists.txt | 3 +- mysys/guess_malloc_library.c | 63 ++++++++++++++++++++++++++++++++++++ sql/sys_vars.cc | 22 +------------ 4 files changed, 68 insertions(+), 22 deletions(-) create mode 100644 mysys/guess_malloc_library.c diff --git a/include/my_sys.h b/include/my_sys.h index fb3e15b64a4..adb7f4826f6 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -154,6 +154,8 @@ typedef struct my_aio_result { /* Extra length needed for filename if one calls my_create_backup_name */ #define MY_BACKUP_NAME_EXTRA_LENGTH 17 +char *guess_malloc_library(); + /* If we have our own safemalloc (for debugging) */ #if defined(SAFEMALLOC) void sf_report_leaked_memory(my_thread_id id); diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index b9f92124cc3..8e4882328f5 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -36,7 +36,8 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c my_default.c string.c thr_alarm.c thr_lock.c thr_mutex.c thr_rwlock.c thr_timer.c tree.c typelib.c base64.c my_memmem.c - my_getpagesize.c + my_getpagesize.c + guess_malloc_library.c lf_alloc-pin.c lf_dynarray.c lf_hash.c safemalloc.c my_new.cc my_atomic.c my_getncpus.c my_safehash.c my_chmod.c my_rnd.c diff --git a/mysys/guess_malloc_library.c b/mysys/guess_malloc_library.c new file mode 100644 index 00000000000..2e640757e11 --- /dev/null +++ b/mysys/guess_malloc_library.c @@ -0,0 +1,63 @@ +/* Copyright (c) 2002, 2015, Oracle and/or its affiliates. + Copyright (c) 2012, 2017, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/* guess_malloc_library() deduces, to the best of its ability, + the currently used malloc library and its version */ + +#include +#include + +char *guess_malloc_library() +{ +#ifndef HAVE_DLOPEN + return (char*) MALLOC_LIBRARY; +#else + static char buf[128]; + + if (strcmp(MALLOC_LIBRARY, "system") != 0) + { + return (char*) MALLOC_LIBRARY; + } + + /* tcmalloc */ + typedef const char* (*tc_version_type)(int*, int*, const char**); + tc_version_type tc_version_func = + (tc_version_type) dlsym(RTLD_DEFAULT, "tc_version"); + if (tc_version_func) + { + int major, minor; + const char* ver_str = tc_version_func(&major, &minor, NULL); + strxnmov(buf, sizeof(buf)-1, "tcmalloc ", ver_str, NULL); + return buf; + } + + /* jemalloc */ + typedef int (*mallctl_type)(const char*, void*, size_t*, void*, size_t); + mallctl_type mallctl_func = + (mallctl_type) dlsym(RTLD_DEFAULT, "mallctl"); + if (mallctl_func) + { + char *ver; + size_t len = sizeof(ver); + mallctl_func("version", &ver, &len, NULL, 0); + strxnmov(buf, sizeof(buf)-1, "jemalloc ", ver, NULL); + return buf; + } + + return (char*) MALLOC_LIBRARY; +#endif +} + diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 78242e0a088..a517ab0965e 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -35,6 +35,7 @@ #include "sql_priv.h" #include "sql_class.h" // set_var.h: THD #include "sys_vars.ic" +#include "my_sys.h" #include "events.h" #include @@ -3511,27 +3512,6 @@ static Sys_var_charptr Sys_version_source_revision( CMD_LINE_HELP_ONLY, IN_SYSTEM_CHARSET, DEFAULT(SOURCE_REVISION)); -static char *guess_malloc_library() -{ - if (strcmp(MALLOC_LIBRARY, "system") == 0) - { -#ifdef HAVE_DLOPEN - typedef int (*mallctl_type)(const char*, void*, size_t*, void*, size_t); - mallctl_type mallctl_func; - mallctl_func= (mallctl_type)dlsym(RTLD_DEFAULT, "mallctl"); - if (mallctl_func) - { - static char buf[128]; - char *ver; - size_t len = sizeof(ver); - mallctl_func("version", &ver, &len, NULL, 0); - strxnmov(buf, sizeof(buf)-1, "jemalloc ", ver, NULL); - return buf; - } -#endif - } - return const_cast(MALLOC_LIBRARY); -} static char *malloc_library; static Sys_var_charptr Sys_malloc_library( "version_malloc_library", "Version of the used malloc library", From 77030649fb1f492b6dd9351a7d4b36e1aeb29f4d Mon Sep 17 00:00:00 2001 From: Takuro Ashie Date: Wed, 13 Dec 2017 15:30:08 +0900 Subject: [PATCH 13/51] Fix typos in some comments --- mysys/lf_alloc-pin.c | 2 +- sql/sql_lex.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysys/lf_alloc-pin.c b/mysys/lf_alloc-pin.c index 8a96fccf16a..4e4917962ab 100644 --- a/mysys/lf_alloc-pin.c +++ b/mysys/lf_alloc-pin.c @@ -161,7 +161,7 @@ LF_PINS *lf_pinbox_get_pins(LF_PINBOX *pinbox) pinstack_top_ver is 32 bits; 16 low bits are the index in the array, to the first element of the list. 16 high bits are a version (every time the 16 low bits are updated, the 16 high bits are - incremented). Versioniong prevents the ABA problem. + incremented). Versioning prevents the ABA problem. */ top_ver= pinbox->pinstack_top_ver; do diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 069b75628e3..92e6dd6c4a2 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -742,7 +742,7 @@ public: /* thread handler */ THD *thd; /* - SELECT_LEX for hidden SELECT in onion which process global + SELECT_LEX for hidden SELECT in union which process global ORDER BY and LIMIT */ st_select_lex *fake_select_lex; From c1e5fef05d87038fff8b55ba763d082a5bec5d78 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 18 Dec 2017 11:25:38 +0400 Subject: [PATCH 14/51] MDEV-14008 Assertion failing: `!is_set() || (m_status == DA_OK_BULK && is_bulk_op()) --- mysql-test/suite/innodb/r/innodb-autoinc.result | 12 ++++++++++++ mysql-test/suite/innodb/t/innodb-autoinc.test | 12 ++++++++++++ sql/field.cc | 4 ++-- sql/field.h | 8 +++++++- storage/innobase/handler/ha_innodb.cc | 4 ++-- storage/xtradb/handler/ha_innodb.cc | 4 ++-- 6 files changed, 37 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-autoinc.result b/mysql-test/suite/innodb/r/innodb-autoinc.result index 879f9dfa238..9aa819de22b 100644 --- a/mysql-test/suite/innodb/r/innodb-autoinc.result +++ b/mysql-test/suite/innodb/r/innodb-autoinc.result @@ -1348,3 +1348,15 @@ t CREATE TABLE `t` ( KEY `i` (`i`) ) ENGINE=InnoDB AUTO_INCREMENT=401 DEFAULT CHARSET=latin1 DROP TABLE t; +# +# MDEV-14008 Assertion failing: `!is_set() || (m_status == DA_OK_BULK && is_bulk_op()) +# +SET sql_mode=STRICT_ALL_TABLES; +CREATE TABLE t1 ( +c1 DOUBLE NOT NULL PRIMARY KEY AUTO_INCREMENT +) ENGINE=InnoDB AUTO_INCREMENT=10000000000000000000; +INSERT INTO t1 VALUES (); +SELECT * FROM t1; +c1 +1e19 +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-autoinc.test b/mysql-test/suite/innodb/t/innodb-autoinc.test index 362be2e055b..ebb6a5d24ff 100644 --- a/mysql-test/suite/innodb/t/innodb-autoinc.test +++ b/mysql-test/suite/innodb/t/innodb-autoinc.test @@ -680,3 +680,15 @@ INSERT INTO t VALUES (NULL); SELECT * FROM t; SHOW CREATE TABLE t; DROP TABLE t; + +--echo # +--echo # MDEV-14008 Assertion failing: `!is_set() || (m_status == DA_OK_BULK && is_bulk_op()) +--echo # + +SET sql_mode=STRICT_ALL_TABLES; +CREATE TABLE t1 ( + c1 DOUBLE NOT NULL PRIMARY KEY AUTO_INCREMENT +) ENGINE=InnoDB AUTO_INCREMENT=10000000000000000000; +INSERT INTO t1 VALUES (); +SELECT * FROM t1; +DROP TABLE t1; diff --git a/sql/field.cc b/sql/field.cc index 7074cc2cbc4..a3c20ec18f2 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -4390,7 +4390,7 @@ double Field_double::val_real(void) return j; } -longlong Field_double::val_int(void) +longlong Field_double::val_int_from_real(bool want_unsigned_result) { ASSERT_COLUMN_MARKED_FOR_READ; double j; @@ -4398,7 +4398,7 @@ longlong Field_double::val_int(void) bool error; float8get(j,ptr); - res= double_to_longlong(j, 0, &error); + res= double_to_longlong(j, want_unsigned_result, &error); if (error) { ErrConvDouble err(j); diff --git a/sql/field.h b/sql/field.h index e7bd5532ae6..c99327bd068 100644 --- a/sql/field.h +++ b/sql/field.h @@ -420,6 +420,10 @@ public: enum_check_fields check_level); virtual double val_real(void)=0; virtual longlong val_int(void)=0; + virtual ulonglong val_uint(void) + { + return (ulonglong) val_int(); + } virtual my_decimal *val_decimal(my_decimal *); inline String *val_str(String *str) { return val_str(str, str); } /* @@ -1554,6 +1558,7 @@ private: class Field_double :public Field_real { + longlong val_int_from_real(bool want_unsigned_result); public: Field_double(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg, uchar null_bit_arg, @@ -1580,7 +1585,8 @@ public: int store(longlong nr, bool unsigned_val); int reset(void) { bzero(ptr,sizeof(double)); return 0; } double val_real(void); - longlong val_int(void); + longlong val_int(void) { return val_int_from_real(false); } + ulonglong val_uint(void) { return (ulonglong) val_int_from_real(true); } String *val_str(String*,String *); bool send_binary(Protocol *protocol); int cmp(const uchar *,const uchar *); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4aab1b8d713..59a8aedd266 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -7274,7 +7274,7 @@ no_commit: table->next_number_field); /* Get the value that MySQL attempted to store in the table.*/ - auto_inc = table->next_number_field->val_int(); + auto_inc = table->next_number_field->val_uint(); switch (error) { case DB_DUPLICATE_KEY: @@ -7735,7 +7735,7 @@ ha_innobase::update_row( ulonglong auto_inc; ulonglong col_max_value; - auto_inc = table->next_number_field->val_int(); + auto_inc = table->next_number_field->val_uint(); /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index f111a576d8f..88b59db9a46 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -7988,7 +7988,7 @@ no_commit: table->next_number_field); /* Get the value that MySQL attempted to store in the table.*/ - auto_inc = table->next_number_field->val_int(); + auto_inc = table->next_number_field->val_uint(); switch (error) { case DB_DUPLICATE_KEY: @@ -8468,7 +8468,7 @@ ha_innobase::update_row( ulonglong auto_inc; ulonglong col_max_value; - auto_inc = table->next_number_field->val_int(); + auto_inc = table->next_number_field->val_uint(); /* We need the upper limit of the col type to check for whether we update the table autoinc counter or not. */ From 8ed78cf7f93f6129c11ee979500971c11e15f6f9 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 18 Dec 2017 12:24:02 +0400 Subject: [PATCH 15/51] xa.test fixed to be thread-stable. --- mysql-test/t/xa.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/t/xa.test b/mysql-test/t/xa.test index 7a6208f4b38..b26f3f3c73f 100644 --- a/mysql-test/t/xa.test +++ b/mysql-test/t/xa.test @@ -80,8 +80,9 @@ xa rollback 'testa','testb'; xa start 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'; select * from t1; - disconnect con1; +--source include/wait_until_count_sessions.inc + connection default; # MDEV-14593 human-readable XA RECOVER From 6ee9cba74502e2da628ac72b21ce42570ff361b9 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 18 Dec 2017 15:21:50 +0400 Subject: [PATCH 16/51] MDEV-10486 MariaDB 10.x does not update rows_examined in performance_schema tables. Save the rows_examined counter before it gets emptied. --- mysql-test/suite/perfschema/r/misc.result | 16 ++++++++++++++++ mysql-test/suite/perfschema/t/misc.test | 15 +++++++++++++++ sql/sql_parse.cc | 5 +++-- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/perfschema/r/misc.result b/mysql-test/suite/perfschema/r/misc.result index 2adf2cba851..7a097a27576 100644 --- a/mysql-test/suite/perfschema/r/misc.result +++ b/mysql-test/suite/perfschema/r/misc.result @@ -118,3 +118,19 @@ B select count(*) from events_statements_history where sql_text like "%..."; count(*) 2 +use test; +create table t1 (id int); +insert into t1 values (1), (2), (3); +truncate performance_schema.events_statements_history; +select * from t1; +id +1 +2 +3 +insert into t1 select RAND()*10000 from t1; +select sql_text, rows_examined from performance_schema.events_statements_history; +sql_text rows_examined +truncate performance_schema.events_statements_history 0 +select * from t1 3 +insert into t1 select RAND()*10000 from t1 6 +drop table t1; diff --git a/mysql-test/suite/perfschema/t/misc.test b/mysql-test/suite/perfschema/t/misc.test index bf3e8afffdc..c9f7dc6bfc0 100644 --- a/mysql-test/suite/perfschema/t/misc.test +++ b/mysql-test/suite/perfschema/t/misc.test @@ -207,3 +207,18 @@ select 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa select _utf8mb4 'васÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑвасÑ' as B; select count(*) from events_statements_history where sql_text like "%..."; + + +# +# MDEV-10486 MariaDB 10.x does not update rows_examined in performance_schema tables +# Verify that the rows_examined counter is set properly. + +use test; +create table t1 (id int); +insert into t1 values (1), (2), (3); +truncate performance_schema.events_statements_history; +select * from t1; +insert into t1 select RAND()*10000 from t1; +select sql_text, rows_examined from performance_schema.events_statements_history; +drop table t1; + diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 5cea264e4a8..3121ee99834 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1957,11 +1957,12 @@ bool dispatch_command(enum enum_server_command command, THD *thd, THD_STAGE_INFO(thd, stage_cleaning_up); thd->reset_query(); - thd->set_examined_row_count(0); // For processlist - thd->set_command(COM_SLEEP); /* Performance Schema Interface instrumentation, end */ MYSQL_END_STATEMENT(thd->m_statement_psi, thd->get_stmt_da()); + thd->set_examined_row_count(0); // For processlist + thd->set_command(COM_SLEEP); + thd->m_statement_psi= NULL; thd->m_digest= NULL; From 2e53b96a0aa9dcb18d8bbe12e5bc7e0aba208540 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 25 Oct 2017 11:07:44 +0300 Subject: [PATCH 17/51] Moved semisync from a plugin to normal server Part of MDEV-13073 AliSQL Optimize performance of semisync Did the following renames to match other similar variables key_ss_mutex_LOCK_binlog_ > key_LOCK_bing key_ss_cond_COND_binlog_send_ -> key_COND_binlog_send COND_binlog_send_ -> COND_binlog_send LOCK_binlog_ -> LOCK_binlog debian/mariadb-server-10.2.install does not install semisync libs. --- debian/mariadb-server-10.3.install | 2 - mysql-test/extra/rpl_tests/rpl_semi_sync.inc | 1 - mysql-test/include/have_semisync.inc | 4 - mysql-test/include/have_semisync.opt | 4 - mysql-test/include/install_semisync.inc | 39 -- mysql-test/include/uninstall_semisync.inc | 29 - mysql-test/r/mysqld--help.result | 28 + .../perfschema/r/dml_setup_instruments.result | 4 +- .../r/rpl_semi_sync_uninstall_plugin.result | 73 --- mysql-test/suite/rpl/t/rpl_mdev359.test | 1 - .../suite/rpl/t/rpl_semi_sync_event.test | 1 - .../rpl/t/rpl_semi_sync_uninstall_plugin.test | 132 ----- .../suite/rpl/t/rpl_semi_sync_wait_point.test | 1 - .../suite/rpl/t/semisync_future-7591.test | 1 - .../suite/rpl/t/semisync_memleak_4066.test | 1 - .../r/sysvars_server_notembedded.result | 98 ++++ .../t/rpl_semi_sync_master_enabled_basic.test | 1 - .../t/rpl_semi_sync_master_timeout_basic.test | 1 - ...pl_semi_sync_master_trace_level_basic.test | 1 - ..._semi_sync_master_wait_no_slave_basic.test | 1 - ...rpl_semi_sync_master_wait_point_basic.test | 1 - .../t/rpl_semi_sync_slave_enabled_basic.test | 1 - ...rpl_semi_sync_slave_trace_level_basic.test | 1 - mysql-test/t/mysqld--help.test | 2 +- plugin/semisync/CMakeLists.txt | 28 - plugin/semisync/semisync_master_plugin.cc | 496 ------------------ plugin/semisync/semisync_slave_plugin.cc | 234 --------- sql/CMakeLists.txt | 3 +- sql/mysqld.cc | 66 ++- sql/replication.h | 14 +- sql/rpl_handler.cc | 8 +- {plugin/semisync => sql}/semisync.cc | 0 {plugin/semisync => sql}/semisync.h | 16 +- {plugin/semisync => sql}/semisync_master.cc | 354 ++++++++++--- {plugin/semisync => sql}/semisync_master.h | 52 +- {plugin/semisync => sql}/semisync_slave.cc | 156 +++++- {plugin/semisync => sql}/semisync_slave.h | 10 +- sql/sys_vars.cc | 182 +++++++ 38 files changed, 859 insertions(+), 1188 deletions(-) delete mode 100644 mysql-test/include/have_semisync.inc delete mode 100644 mysql-test/include/have_semisync.opt delete mode 100644 mysql-test/include/install_semisync.inc delete mode 100644 mysql-test/include/uninstall_semisync.inc delete mode 100644 mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result delete mode 100644 mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test delete mode 100644 plugin/semisync/CMakeLists.txt delete mode 100644 plugin/semisync/semisync_master_plugin.cc delete mode 100644 plugin/semisync/semisync_slave_plugin.cc rename {plugin/semisync => sql}/semisync.cc (100%) rename {plugin/semisync => sql}/semisync.h (87%) rename {plugin/semisync => sql}/semisync_master.cc (78%) rename {plugin/semisync => sql}/semisync_master.h (93%) rename {plugin/semisync => sql}/semisync_slave.cc (50%) rename {plugin/semisync => sql}/semisync_slave.h (92%) diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index 23f8046baaf..604179c7dca 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -52,8 +52,6 @@ usr/lib/mysql/plugin/locales.so usr/lib/mysql/plugin/metadata_lock_info.so usr/lib/mysql/plugin/query_cache_info.so usr/lib/mysql/plugin/query_response_time.so -usr/lib/mysql/plugin/semisync_master.so -usr/lib/mysql/plugin/semisync_slave.so usr/lib/mysql/plugin/server_audit.so usr/lib/mysql/plugin/simple_password_check.so usr/lib/mysql/plugin/sql_errlog.so diff --git a/mysql-test/extra/rpl_tests/rpl_semi_sync.inc b/mysql-test/extra/rpl_tests/rpl_semi_sync.inc index 12053c54f4e..ed56e405e27 100644 --- a/mysql-test/extra/rpl_tests/rpl_semi_sync.inc +++ b/mysql-test/extra/rpl_tests/rpl_semi_sync.inc @@ -4,7 +4,6 @@ # Please check all dependent tests after modifying it # -source include/have_semisync.inc; source include/not_embedded.inc; source include/have_innodb.inc; source include/master-slave.inc; diff --git a/mysql-test/include/have_semisync.inc b/mysql-test/include/have_semisync.inc deleted file mode 100644 index 243fad83717..00000000000 --- a/mysql-test/include/have_semisync.inc +++ /dev/null @@ -1,4 +0,0 @@ -if (`select count(*) < 2 from information_schema.plugins where plugin_name like 'rpl_semi_sync_%'`) -{ - --skip Test requires semisync plugins -} diff --git a/mysql-test/include/have_semisync.opt b/mysql-test/include/have_semisync.opt deleted file mode 100644 index 19e29c7e4de..00000000000 --- a/mysql-test/include/have_semisync.opt +++ /dev/null @@ -1,4 +0,0 @@ ---plugin-load-add=$SEMISYNC_MASTER_SO ---plugin-load-add=$SEMISYNC_SLAVE_SO ---loose-rpl-semi-sync-master ---loose-rpl-semi-sync-slave diff --git a/mysql-test/include/install_semisync.inc b/mysql-test/include/install_semisync.inc deleted file mode 100644 index 9cc6df2072a..00000000000 --- a/mysql-test/include/install_semisync.inc +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. -# ---let $include_filename= install_semisync.inc ---source include/begin_include_file.inc - ---source include/not_embedded.inc ---source include/have_semisync_plugin.inc - ---connection master - ---disable_query_log ---let $value = query_get_value(show variables like 'rpl_semi_sync_master_enabled', Value, 1) -if ($value == No such row) -{ - SET sql_log_bin = 0; - install plugin rpl_semi_sync_master soname 'semisync_master'; - SET GLOBAL rpl_semi_sync_master_enabled = 1; - SET sql_log_bin = 1; -} ---enable_query_log - ---connection slave ---source include/stop_slave_io.inc - ---disable_query_log ---let $value= query_get_value(show variables like 'rpl_semi_sync_slave_enabled', Value, 1) -if ($value == No such row) -{ - SET sql_log_bin = 0; - install plugin rpl_semi_sync_slave soname 'semisync_slave'; - SET GLOBAL rpl_semi_sync_slave_enabled = 1; - SET sql_log_bin = 1; -} -START SLAVE IO_THREAD; ---source include/wait_for_slave_io_to_start.inc ---enable_query_log - ---source include/end_include_file.inc diff --git a/mysql-test/include/uninstall_semisync.inc b/mysql-test/include/uninstall_semisync.inc deleted file mode 100644 index 0a4c55fa4f2..00000000000 --- a/mysql-test/include/uninstall_semisync.inc +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. -# ---let $include_filename= uninstall_semisync.inc ---source include/begin_include_file.inc - ---disable_query_log ---connection slave ---source include/stop_slave_io.inc - -# Uninstall rpl_semi_sync_slave first ---disable_warnings -UNINSTALL PLUGIN rpl_semi_sync_slave; - ---connection master -# After BUG#17638477 fix, uninstallation of rpl_semi_sync_master -# is not allowed when there are semi sync slaves. Hence kill -# all dump threads before uninstalling it. -SET GLOBAL rpl_semi_sync_master_enabled = OFF; ---source include/stop_dump_threads.inc -UNINSTALL PLUGIN rpl_semi_sync_master; ---enable_warnings - ---connection slave -START SLAVE IO_THREAD; ---source include/wait_for_slave_io_to_start.inc ---enable_query_log - ---source include/end_include_file.inc diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index f76957fdcba..032c64b8040 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -998,6 +998,27 @@ The following options may be given as the first argument: --rowid-merge-buff-size=# The size of the buffers used [NOT] IN evaluation via partial matching + --rpl-semi-sync-master-enabled + Enable semi-synchronous replication master (disabled by + default). + --rpl-semi-sync-master-timeout=# + The timeout value (in ms) for semi-synchronous + replication in the master + --rpl-semi-sync-master-trace-level=# + The tracing level for semi-sync replication. + --rpl-semi-sync-master-wait-no-slave + Wait until timeout when no semi-synchronous replication + slave available (enabled by default). + (Defaults to on; use --skip-rpl-semi-sync-master-wait-no-slave to disable.) + --rpl-semi-sync-master-wait-point=name + Should transaction wait for semi-sync ack after having + synced binlog, or after having committed in storage + engine.. One of: AFTER_SYNC, AFTER_COMMIT + --rpl-semi-sync-slave-enabled + Enable semi-synchronous replication slave (disabled by + default). + --rpl-semi-sync-slave-trace-level=# + The tracing level for semi-sync replication. --safe-mode Skip some optimize stages (for testing). Deprecated. --safe-user-create Don't allow new user creation by the user who has no write privileges to the mysql.user table. @@ -1569,6 +1590,13 @@ report-password (No default value) report-port 0 report-user (No default value) rowid-merge-buff-size 8388608 +rpl-semi-sync-master-enabled FALSE +rpl-semi-sync-master-timeout 10000 +rpl-semi-sync-master-trace-level 32 +rpl-semi-sync-master-wait-no-slave TRUE +rpl-semi-sync-master-wait-point AFTER_COMMIT +rpl-semi-sync-slave-enabled FALSE +rpl-semi-sync-slave-trace-level 32 safe-user-create FALSE secure-auth TRUE secure-file-priv (No default value) diff --git a/mysql-test/suite/perfschema/r/dml_setup_instruments.result b/mysql-test/suite/perfschema/r/dml_setup_instruments.result index 83510e5827e..940839b105f 100644 --- a/mysql-test/suite/perfschema/r/dml_setup_instruments.result +++ b/mysql-test/suite/perfschema/r/dml_setup_instruments.result @@ -13,7 +13,7 @@ wait/synch/mutex/sql/HA_DATA_PARTITION::LOCK_auto_inc YES YES wait/synch/mutex/sql/LOCK_active_mi YES YES wait/synch/mutex/sql/LOCK_after_binlog_sync YES YES wait/synch/mutex/sql/LOCK_audit_mask YES YES -wait/synch/mutex/sql/LOCK_binlog_state YES YES +wait/synch/mutex/sql/LOCK_binlog YES YES select * from performance_schema.setup_instruments where name like 'Wait/Synch/Rwlock/sql/%' and name not in ('wait/synch/rwlock/sql/CRYPTO_dynlock_value::lock') @@ -36,6 +36,7 @@ where name like 'Wait/Synch/Cond/sql/%' 'wait/synch/cond/sql/DEBUG_SYNC::cond') order by name limit 10; NAME ENABLED TIMED +wait/synch/cond/sql/COND_binlog_send YES YES wait/synch/cond/sql/COND_flush_thread_cache YES YES wait/synch/cond/sql/COND_group_commit_orderer YES YES wait/synch/cond/sql/COND_gtid_ignore_duplicates YES YES @@ -45,7 +46,6 @@ wait/synch/cond/sql/COND_prepare_ordered YES YES wait/synch/cond/sql/COND_queue_state YES YES wait/synch/cond/sql/COND_rpl_thread YES YES wait/synch/cond/sql/COND_rpl_thread_pool YES YES -wait/synch/cond/sql/COND_rpl_thread_queue YES YES select * from performance_schema.setup_instruments where name='Wait'; select * from performance_schema.setup_instruments diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result b/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result deleted file mode 100644 index 68ad4877927..00000000000 --- a/mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result +++ /dev/null @@ -1,73 +0,0 @@ -include/master-slave.inc -[connection master] -call mtr.add_suppression("Read semi-sync reply network error"); -call mtr.add_suppression("Timeout waiting for reply of binlog"); -INSTALL PLUGIN rpl_semi_sync_master SONAME 'semisync_master'; -connection slave; -INSTALL PLUGIN rpl_semi_sync_slave SONAME 'semisync_slave'; -UNINSTALL PLUGIN rpl_semi_sync_slave; -connection master; -UNINSTALL PLUGIN rpl_semi_sync_master; -CREATE TABLE t1(i int); -INSERT INTO t1 values (1); -DROP TABLE t1; -connection slave; -include/install_semisync.inc -connection master; -connection slave; -connection slave; -show global status like "Slave%_running"; -Variable_name Value -Slave_running ON -Slaves_running 1 -UNINSTALL PLUGIN rpl_semi_sync_slave; -Warnings: -Warning 1620 Plugin is busy and will be uninstalled on shutdown -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; -plugin_name plugin_status -rpl_semi_sync_slave DELETED -connection master; -show global status like "Slave%_connect%"; -Variable_name Value -Slave_connections 2 -Slaves_connected 1 -UNINSTALL PLUGIN rpl_semi_sync_master; -Warnings: -Warning 1620 Plugin is busy and will be uninstalled on shutdown -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; -plugin_name plugin_status -rpl_semi_sync_master DELETED -CREATE TABLE t1(i int); -INSERT INTO t1 values (2); -DROP TABLE t1; -connection slave; -show status like "Rpl_semi_sync_slave_status"; -Variable_name Value -Rpl_semi_sync_slave_status ON -connection master; -show status like "Rpl_semi_sync_master_status"; -Variable_name Value -Rpl_semi_sync_master_status ON -show status like "Rpl_semi_sync_master_clients"; -Variable_name Value -Rpl_semi_sync_master_clients 1 -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; -plugin_name plugin_status -rpl_semi_sync_master DELETED -connection slave; -include/stop_slave.inc -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; -plugin_name plugin_status -connection master; -create table t2 (a int); -drop table t2; -connection slave; -include/start_slave.inc -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; -plugin_name plugin_status -connection master; -CREATE TABLE t1(i int); -INSERT INTO t1 values (3); -DROP TABLE t1; -connection slave; -include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_mdev359.test b/mysql-test/suite/rpl/t/rpl_mdev359.test index 3026c6d363e..5b02ecd72c0 100644 --- a/mysql-test/suite/rpl/t/rpl_mdev359.test +++ b/mysql-test/suite/rpl/t/rpl_mdev359.test @@ -1,4 +1,3 @@ ---source include/have_semisync.inc --source include/not_embedded.inc --source include/have_debug_sync.inc --source include/have_binlog_format_mixed_or_statement.inc diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_event.test b/mysql-test/suite/rpl/t/rpl_semi_sync_event.test index b8f3c8130be..d7685413a07 100644 --- a/mysql-test/suite/rpl/t/rpl_semi_sync_event.test +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_event.test @@ -1,4 +1,3 @@ -source include/have_semisync.inc; source include/not_embedded.inc; source include/have_innodb.inc; source include/master-slave.inc; diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test b/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test deleted file mode 100644 index 360706922ea..00000000000 --- a/mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test +++ /dev/null @@ -1,132 +0,0 @@ -############################################################################### -# Bug#17638477 UNINSTALL AND INSTALL SEMI-SYNC PLUGIN CAUSES SLAVES TO BREAK -# Problem: Uninstallation of Semi sync plugin should be blocked when it is -# in use. -# Test case: Uninstallation of semi sync should be allowed -# On Master: -# 1) When there is no dump thread -# 2) When there are no semi sync slaves (i.e., async replication). -# On Slave: -# 1) When there is no I/O thread -# 2) When there are no semi sync enabled I/O thread (i.e.,async replication). -############################################################################### - ---source include/have_semisync_plugin.inc ---source include/not_embedded.inc ---source include/have_binlog_format_statement.inc ---source include/master-slave.inc - -call mtr.add_suppression("Read semi-sync reply network error"); -call mtr.add_suppression("Timeout waiting for reply of binlog"); - -############################################################################### -# Case 1: Uninstallation of semi sync plugins should be allowed when it is -# not in use i.e., when asynchronous replication is active. -############################################################################### -# Step 1.1: Install semi sync master plugin on master -INSTALL PLUGIN rpl_semi_sync_master SONAME 'semisync_master'; - -# Step 1.2: Install semi sync slave plugin on slave ---connection slave -INSTALL PLUGIN rpl_semi_sync_slave SONAME 'semisync_slave'; - -# Step 1.3: Uninstallation of semisync plugin on master and slave should be -# allowed at this state as there is no semi sync replication enabled between -# master and slave. -UNINSTALL PLUGIN rpl_semi_sync_slave; ---connection master -UNINSTALL PLUGIN rpl_semi_sync_master; - -# Step 1.4: Check that replication is working fine at the end of the test case. -CREATE TABLE t1(i int); -INSERT INTO t1 values (1); -DROP TABLE t1; ---sync_slave_with_master - -############################################################################### -# Case 2: Uninstallation of semi sync plugins should be disallowed -# when it is in use i.e., when semi sync replication is active -############################################################################### -# Step 2.1: Install and enable semi sync replication between master and slave ---source include/install_semisync.inc - -# Step 2.2: Check that rpl_semi_sync_slave uninstallation on Slave is not -# possible at this state ---connection slave -show global status like "Slave%_running"; - -UNINSTALL PLUGIN rpl_semi_sync_slave; -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; - -# Step 2.3: Check that rpl_semi_sync_master uninstallation on Master is not -# possible at this state ---connection master - -# The following is to catch errors if the next uninstall plugin would succeed -show global status like "Slave%_connect%"; - -UNINSTALL PLUGIN rpl_semi_sync_master; -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; - -# Step 2.4: Check that replication is working fine at the end of the test case. -CREATE TABLE t1(i int); -INSERT INTO t1 values (2); -DROP TABLE t1; ---sync_slave_with_master - -# Step 2.5: Make sure rpl_semi_sync_master_status on Master and -# rpl_semi_sync_slave_staus on Slave are ON -show status like "Rpl_semi_sync_slave_status"; - -############################################################################### -# Case 3: Uninstallation of semi sync plugin should be disallowed when there -# are semi sync slaves even though rpl_semi_sync_master_enabled= OFF;. -############################################################################### -# Step 3.1: Disable semi sync on master ---connection master -show status like "Rpl_semi_sync_master_status"; - -# Step 3.2: Check that still Rpl_semi_sync_master_clients is 1 -show status like "Rpl_semi_sync_master_clients"; - -# Step 3.3: Since Rpl_semi_sync_master_clients is 1, uninstallation of -# rpl_semi_sync_master should be disallowed. -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; - -############################################################################### -# Case 4: Uninstallation of semi sync plugin should be allowed when it is not -# in use. Same as Case 1 but this case is to check the case after enabling and -# disabling semi sync replication. -############################################################################### - -# Step 4.1: Stop IO thread on slave. ---connection slave ---source include/stop_slave.inc - -# Step 4.2: Disable semi sync on slave. -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; - ---connection master -# Send something to the slave so that the master would notice that nobody's listening. -create table t2 (a int); drop table t2; -# and wait for plugin to be unloaded automatically -let $wait_condition=select count(*) = 0 from information_schema.plugins where plugin_name like 'rpl_%'; ---source include/wait_condition.inc - ---connection slave - -# Step 4.3: Start IO thread on slave. ---source include/start_slave.inc - -# Step 4.4: Uninstall semi sync plugin, it should be successful now. -select plugin_name,plugin_status from information_schema.plugins where plugin_name like 'rpl_%'; - -# Step 4.7: Check that replication is working fine at the end of the test case ---connection master -CREATE TABLE t1(i int); -INSERT INTO t1 values (3); -DROP TABLE t1; ---sync_slave_with_master - -# Cleanup -source include/rpl_end.inc; diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_wait_point.test b/mysql-test/suite/rpl/t/rpl_semi_sync_wait_point.test index 6e4dc456a27..dcff4030fdb 100644 --- a/mysql-test/suite/rpl/t/rpl_semi_sync_wait_point.test +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_wait_point.test @@ -1,4 +1,3 @@ -source include/have_semisync.inc; source include/not_embedded.inc; source include/have_innodb.inc; diff --git a/mysql-test/suite/rpl/t/semisync_future-7591.test b/mysql-test/suite/rpl/t/semisync_future-7591.test index daf3d2f8571..866041d2579 100644 --- a/mysql-test/suite/rpl/t/semisync_future-7591.test +++ b/mysql-test/suite/rpl/t/semisync_future-7591.test @@ -1,4 +1,3 @@ ---source include/have_semisync.inc --source include/master-slave.inc call mtr.add_suppression("Timeout waiting for reply of binlog*"); diff --git a/mysql-test/suite/rpl/t/semisync_memleak_4066.test b/mysql-test/suite/rpl/t/semisync_memleak_4066.test index f888f764b43..e88e2335696 100644 --- a/mysql-test/suite/rpl/t/semisync_memleak_4066.test +++ b/mysql-test/suite/rpl/t/semisync_memleak_4066.test @@ -1,7 +1,6 @@ # # MDEV-4066 semisync_master + temporary tables causes memory leaks # -source include/have_semisync.inc; source include/have_binlog_format_row.inc; source include/master-slave.inc; diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 8591f4b0c28..376123fa88d 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -4001,6 +4001,104 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME RPL_SEMI_SYNC_MASTER_ENABLED +SESSION_VALUE NULL +GLOBAL_VALUE OFF +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Enable semi-synchronous replication master (disabled by default). +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME RPL_SEMI_SYNC_MASTER_TIMEOUT +SESSION_VALUE NULL +GLOBAL_VALUE 10000 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 10000 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT The timeout value (in ms) for semi-synchronous replication in the master +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_BLOCK_SIZE 1 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME RPL_SEMI_SYNC_MASTER_TRACE_LEVEL +SESSION_VALUE NULL +GLOBAL_VALUE 32 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 32 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT The tracing level for semi-sync replication. +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_BLOCK_SIZE 1 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME RPL_SEMI_SYNC_MASTER_WAIT_NO_SLAVE +SESSION_VALUE NULL +GLOBAL_VALUE ON +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE ON +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Wait until timeout when no semi-synchronous replication slave available (enabled by default). +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME RPL_SEMI_SYNC_MASTER_WAIT_POINT +SESSION_VALUE NULL +GLOBAL_VALUE AFTER_COMMIT +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE AFTER_COMMIT +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE ENUM +VARIABLE_COMMENT Should transaction wait for semi-sync ack after having synced binlog, or after having committed in storage engine. +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST AFTER_SYNC,AFTER_COMMIT +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME RPL_SEMI_SYNC_SLAVE_ENABLED +SESSION_VALUE NULL +GLOBAL_VALUE OFF +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Enable semi-synchronous replication slave (disabled by default). +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME RPL_SEMI_SYNC_SLAVE_TRACE_LEVEL +SESSION_VALUE NULL +GLOBAL_VALUE 32 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 32 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT The tracing level for semi-sync replication. +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_BLOCK_SIZE 1 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME SECURE_AUTH SESSION_VALUE NULL GLOBAL_VALUE ON diff --git a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_enabled_basic.test b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_enabled_basic.test index 2ff03a53c42..da22d0535f4 100644 --- a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_enabled_basic.test +++ b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_enabled_basic.test @@ -6,7 +6,6 @@ # # source include/not_embedded.inc; -source include/have_semisync.inc; select @@global.rpl_semi_sync_master_enabled; SET @start_global_value = @@global.rpl_semi_sync_master_enabled; diff --git a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_timeout_basic.test b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_timeout_basic.test index 74d3c41150b..d312fa1c367 100644 --- a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_timeout_basic.test +++ b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_timeout_basic.test @@ -5,7 +5,6 @@ # 2010-01-21 OBN - Added # source include/not_embedded.inc; -source include/have_semisync.inc; select @@global.rpl_semi_sync_master_timeout; SET @start_global_value = @@global.rpl_semi_sync_master_timeout; diff --git a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_trace_level_basic.test b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_trace_level_basic.test index c41b53fe5e6..3419254bafd 100644 --- a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_trace_level_basic.test +++ b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_trace_level_basic.test @@ -5,7 +5,6 @@ # 2010-01-21 OBN - Added # source include/not_embedded.inc; -source include/have_semisync.inc; select @@global.rpl_semi_sync_master_trace_level; SET @start_global_value = @@global.rpl_semi_sync_master_trace_level; diff --git a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_wait_no_slave_basic.test b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_wait_no_slave_basic.test index d4a46a08140..60ca4425e5b 100644 --- a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_wait_no_slave_basic.test +++ b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_wait_no_slave_basic.test @@ -6,7 +6,6 @@ # # source include/not_embedded.inc; -source include/have_semisync.inc; select @@global.rpl_semi_sync_master_wait_no_slave; SET @start_global_value = @@global.rpl_semi_sync_master_wait_no_slave; diff --git a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_wait_point_basic.test b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_wait_point_basic.test index 8125cf8d653..2f8cd8ec160 100644 --- a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_wait_point_basic.test +++ b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_wait_point_basic.test @@ -1,5 +1,4 @@ source include/not_embedded.inc; -source include/have_semisync.inc; select @@global.rpl_semi_sync_master_wait_point; SET @start_global_value = @@global.rpl_semi_sync_master_wait_point; diff --git a/mysql-test/suite/sys_vars/t/rpl_semi_sync_slave_enabled_basic.test b/mysql-test/suite/sys_vars/t/rpl_semi_sync_slave_enabled_basic.test index c7ce371970d..71be941cbdc 100644 --- a/mysql-test/suite/sys_vars/t/rpl_semi_sync_slave_enabled_basic.test +++ b/mysql-test/suite/sys_vars/t/rpl_semi_sync_slave_enabled_basic.test @@ -6,7 +6,6 @@ # # source include/not_embedded.inc; -source include/have_semisync.inc; select @@global.rpl_semi_sync_slave_enabled; SET @start_global_value = @@global.rpl_semi_sync_slave_enabled; diff --git a/mysql-test/suite/sys_vars/t/rpl_semi_sync_slave_trace_level_basic.test b/mysql-test/suite/sys_vars/t/rpl_semi_sync_slave_trace_level_basic.test index d7e001b7322..74b6ac0a6ae 100644 --- a/mysql-test/suite/sys_vars/t/rpl_semi_sync_slave_trace_level_basic.test +++ b/mysql-test/suite/sys_vars/t/rpl_semi_sync_slave_trace_level_basic.test @@ -5,7 +5,6 @@ # 2010-01-21 OBN - Added # source include/not_embedded.inc; -source include/have_semisync.inc; select @@global.rpl_semi_sync_slave_trace_level; SET @start_global_value = @@global.rpl_semi_sync_slave_trace_level; diff --git a/mysql-test/t/mysqld--help.test b/mysql-test/t/mysqld--help.test index f859d7dd9d6..520384f3ab3 100644 --- a/mysql-test/t/mysqld--help.test +++ b/mysql-test/t/mysqld--help.test @@ -30,7 +30,7 @@ perl; feedback debug temp-pool ssl des-key-file xtradb sequence thread-concurrency super-large-pages mutex-deadlock-detector connect null-audit aria oqgraph sphinx thread-handling - test-sql-discovery rpl-semi-sync query-cache-info + test-sql-discovery query-cache-info query-response-time metadata-lock-info locales unix-socket wsrep file-key-management cracklib-password-check user-variables/; diff --git a/plugin/semisync/CMakeLists.txt b/plugin/semisync/CMakeLists.txt deleted file mode 100644 index 88998fb3093..00000000000 --- a/plugin/semisync/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -SET(SEMISYNC_MASTER_SOURCES - semisync.cc semisync_master.cc semisync_master_plugin.cc - semisync.h semisync_master.h) - -MYSQL_ADD_PLUGIN(semisync_master ${SEMISYNC_MASTER_SOURCES} - RECOMPILE_FOR_EMBEDDED) - -SET(SEMISYNC_SLAVE_SOURCES semisync.cc semisync_slave.cc - semisync_slave_plugin.cc semisync.h semisync_slave.h ) - -MYSQL_ADD_PLUGIN(semisync_slave ${SEMISYNC_SLAVE_SOURCES} - RECOMPILE_FOR_EMBEDDED) - diff --git a/plugin/semisync/semisync_master_plugin.cc b/plugin/semisync/semisync_master_plugin.cc deleted file mode 100644 index b46cf5d79cb..00000000000 --- a/plugin/semisync/semisync_master_plugin.cc +++ /dev/null @@ -1,496 +0,0 @@ -/* Copyright (C) 2007 Google Inc. - Copyright (c) 2008 MySQL AB, 2008-2009 Sun Microsystems, Inc. - Use is subject to license terms. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - - -#include -#include "semisync_master.h" -#include "sql_class.h" // THD - -static ReplSemiSyncMaster repl_semisync; - -C_MODE_START - -int repl_semi_report_binlog_update(Binlog_storage_param *param, - const char *log_file, - my_off_t log_pos, uint32 flags) -{ - int error= 0; - - if (repl_semisync.getMasterEnabled()) - { - /* - Let us store the binlog file name and the position, so that - we know how long to wait for the binlog to the replicated to - the slave in synchronous replication. - */ - error= repl_semisync.writeTranxInBinlog(log_file, - log_pos); - } - - return error; -} - -int repl_semi_request_commit(Trans_param *param) -{ - return 0; -} - -int repl_semi_report_binlog_sync(Binlog_storage_param *param, - const char *log_file, - my_off_t log_pos, uint32 flags) -{ - int error= 0; - if (rpl_semi_sync_master_wait_point == - SEMI_SYNC_MASTER_WAIT_POINT_AFTER_BINLOG_SYNC) - { - error = repl_semisync.commitTrx(log_file, log_pos); - } - - return error; -} - -int repl_semi_report_commit(Trans_param *param) -{ - if (rpl_semi_sync_master_wait_point != - SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT) - { - return 0; - } - - bool is_real_trans= param->flags & TRANS_IS_REAL_TRANS; - - if (is_real_trans && param->log_pos) - { - const char *binlog_name= param->log_file; - return repl_semisync.commitTrx(binlog_name, param->log_pos); - } - return 0; -} - -int repl_semi_report_rollback(Trans_param *param) -{ - return repl_semi_report_commit(param); -} - -int repl_semi_binlog_dump_start(Binlog_transmit_param *param, - const char *log_file, - my_off_t log_pos) -{ - bool semi_sync_slave= repl_semisync.is_semi_sync_slave(); - - if (semi_sync_slave) - { - /* One more semi-sync slave */ - repl_semisync.add_slave(); - - /* - Let's assume this semi-sync slave has already received all - binlog events before the filename and position it requests. - */ - repl_semisync.reportReplyBinlog(param->server_id, log_file, log_pos); - } - sql_print_information("Start %s binlog_dump to slave (server_id: %d), pos(%s, %lu)", - semi_sync_slave ? "semi-sync" : "asynchronous", - param->server_id, log_file, (unsigned long)log_pos); - - return 0; -} - -int repl_semi_binlog_dump_end(Binlog_transmit_param *param) -{ - bool semi_sync_slave= repl_semisync.is_semi_sync_slave(); - - sql_print_information("Stop %s binlog_dump to slave (server_id: %d)", - semi_sync_slave ? "semi-sync" : "asynchronous", - param->server_id); - if (semi_sync_slave) - { - /* One less semi-sync slave */ - repl_semisync.remove_slave(); - } - return 0; -} - -int repl_semi_reserve_header(Binlog_transmit_param *param, - unsigned char *header, - unsigned long size, unsigned long *len) -{ - *len += repl_semisync.reserveSyncHeader(header, size); - return 0; -} - -int repl_semi_before_send_event(Binlog_transmit_param *param, - unsigned char *packet, unsigned long len, - const char *log_file, my_off_t log_pos) -{ - return repl_semisync.updateSyncHeader(packet, - log_file, - log_pos, - param->server_id); -} - -int repl_semi_after_send_event(Binlog_transmit_param *param, - const char *event_buf, unsigned long len) -{ - if (repl_semisync.is_semi_sync_slave()) - { - THD *thd= current_thd; - /* - Possible errors in reading slave reply are ignored deliberately - because we do not want dump thread to quit on this. Error - messages are already reported. - */ - (void) repl_semisync.readSlaveReply(&thd->net, - param->server_id, event_buf); - thd->clear_error(); - } - return 0; -} - -int repl_semi_reset_master(Binlog_transmit_param *param) -{ - if (repl_semisync.resetMaster()) - return 1; - return 0; -} - -C_MODE_END - -/* - semisync system variables - */ -static void fix_rpl_semi_sync_master_timeout(MYSQL_THD thd, - SYS_VAR *var, - void *ptr, - const void *val); - -static void fix_rpl_semi_sync_master_trace_level(MYSQL_THD thd, - SYS_VAR *var, - void *ptr, - const void *val); - -static void fix_rpl_semi_sync_master_enabled(MYSQL_THD thd, - SYS_VAR *var, - void *ptr, - const void *val); - -static MYSQL_SYSVAR_BOOL(enabled, rpl_semi_sync_master_enabled, - PLUGIN_VAR_OPCMDARG, - "Enable semi-synchronous replication master (disabled by default). ", - NULL, // check - &fix_rpl_semi_sync_master_enabled, // update - 0); - -/* NOTE: must match order of rpl_semi_sync_master_wait_point_t */ -static const char *rpl_semi_sync_master_wait_point_names[] = -{ - "AFTER_SYNC", - "AFTER_COMMIT", - NullS -}; - -static TYPELIB rpl_semi_sync_master_wait_point_typelib = -{ - array_elements(rpl_semi_sync_master_wait_point_names) - 1, - "", - rpl_semi_sync_master_wait_point_names, - NULL -}; - -static MYSQL_SYSVAR_ENUM( - wait_point, - rpl_semi_sync_master_wait_point, - PLUGIN_VAR_RQCMDARG, - "Should transaction wait for semi-sync ack after having synced binlog, " - "or after having committed in storeage engine.", - NULL, // check - NULL, // update - SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT, - &rpl_semi_sync_master_wait_point_typelib); - -static MYSQL_SYSVAR_ULONG(timeout, rpl_semi_sync_master_timeout, - PLUGIN_VAR_OPCMDARG, - "The timeout value (in ms) for semi-synchronous replication in the master", - NULL, // check - fix_rpl_semi_sync_master_timeout, // update - 10000, 0, ~0UL, 1); - -static MYSQL_SYSVAR_BOOL(wait_no_slave, rpl_semi_sync_master_wait_no_slave, - PLUGIN_VAR_OPCMDARG, - "Wait until timeout when no semi-synchronous replication slave available (enabled by default). ", - NULL, // check - NULL, // update - 1); - -static MYSQL_SYSVAR_ULONG(trace_level, rpl_semi_sync_master_trace_level, - PLUGIN_VAR_OPCMDARG, - "The tracing level for semi-sync replication.", - NULL, // check - &fix_rpl_semi_sync_master_trace_level, // update - 32, 0, ~0UL, 1); - -static SYS_VAR* semi_sync_master_system_vars[]= { - MYSQL_SYSVAR(enabled), - MYSQL_SYSVAR(wait_point), - MYSQL_SYSVAR(timeout), - MYSQL_SYSVAR(wait_no_slave), - MYSQL_SYSVAR(trace_level), - NULL, -}; - - -static void fix_rpl_semi_sync_master_timeout(MYSQL_THD thd, - SYS_VAR *var, - void *ptr, - const void *val) -{ - *(unsigned long *)ptr= *(unsigned long *)val; - repl_semisync.setWaitTimeout(rpl_semi_sync_master_timeout); - return; -} - -static void fix_rpl_semi_sync_master_trace_level(MYSQL_THD thd, - SYS_VAR *var, - void *ptr, - const void *val) -{ - *(unsigned long *)ptr= *(unsigned long *)val; - repl_semisync.setTraceLevel(rpl_semi_sync_master_trace_level); - return; -} - -static void fix_rpl_semi_sync_master_enabled(MYSQL_THD thd, - SYS_VAR *var, - void *ptr, - const void *val) -{ - *(char *)ptr= *(char *)val; - if (rpl_semi_sync_master_enabled) - { - if (repl_semisync.enableMaster() != 0) - rpl_semi_sync_master_enabled = false; - } - else - { - if (repl_semisync.disableMaster() != 0) - rpl_semi_sync_master_enabled = true; - } - - return; -} - -Trans_observer trans_observer = { - sizeof(Trans_observer), // len - - repl_semi_report_commit, // after_commit - repl_semi_report_rollback, // after_rollback -}; - -Binlog_storage_observer storage_observer = { - sizeof(Binlog_storage_observer), // len - - repl_semi_report_binlog_update, // report_update - repl_semi_report_binlog_sync, // after_sync -}; - -Binlog_transmit_observer transmit_observer = { - sizeof(Binlog_transmit_observer), // len - - repl_semi_binlog_dump_start, // start - repl_semi_binlog_dump_end, // stop - repl_semi_reserve_header, // reserve_header - repl_semi_before_send_event, // before_send_event - repl_semi_after_send_event, // after_send_event - repl_semi_reset_master, // reset -}; - - -#define SHOW_FNAME(name) \ - rpl_semi_sync_master_show_##name - -#define DEF_SHOW_FUNC(name, show_type) \ - static int SHOW_FNAME(name)(MYSQL_THD thd, SHOW_VAR *var, char *buff) \ - { \ - repl_semisync.setExportStats(); \ - var->type= show_type; \ - var->value= (char *)&rpl_semi_sync_master_##name; \ - return 0; \ - } - -DEF_SHOW_FUNC(status, SHOW_BOOL) -DEF_SHOW_FUNC(clients, SHOW_LONG) -DEF_SHOW_FUNC(wait_sessions, SHOW_LONG) -DEF_SHOW_FUNC(trx_wait_time, SHOW_LONGLONG) -DEF_SHOW_FUNC(trx_wait_num, SHOW_LONGLONG) -DEF_SHOW_FUNC(net_wait_time, SHOW_LONGLONG) -DEF_SHOW_FUNC(net_wait_num, SHOW_LONGLONG) -DEF_SHOW_FUNC(avg_net_wait_time, SHOW_LONG) -DEF_SHOW_FUNC(avg_trx_wait_time, SHOW_LONG) - - -/* plugin status variables */ -static SHOW_VAR semi_sync_master_status_vars[]= { - {"Rpl_semi_sync_master_status", - (char*) &SHOW_FNAME(status), - SHOW_SIMPLE_FUNC}, - {"Rpl_semi_sync_master_clients", - (char*) &SHOW_FNAME(clients), - SHOW_SIMPLE_FUNC}, - {"Rpl_semi_sync_master_yes_tx", - (char*) &rpl_semi_sync_master_yes_transactions, - SHOW_LONG}, - {"Rpl_semi_sync_master_no_tx", - (char*) &rpl_semi_sync_master_no_transactions, - SHOW_LONG}, - {"Rpl_semi_sync_master_wait_sessions", - (char*) &SHOW_FNAME(wait_sessions), - SHOW_SIMPLE_FUNC}, - {"Rpl_semi_sync_master_no_times", - (char*) &rpl_semi_sync_master_off_times, - SHOW_LONG}, - {"Rpl_semi_sync_master_timefunc_failures", - (char*) &rpl_semi_sync_master_timefunc_fails, - SHOW_LONG}, - {"Rpl_semi_sync_master_wait_pos_backtraverse", - (char*) &rpl_semi_sync_master_wait_pos_backtraverse, - SHOW_LONG}, - {"Rpl_semi_sync_master_tx_wait_time", - (char*) &SHOW_FNAME(trx_wait_time), - SHOW_SIMPLE_FUNC}, - {"Rpl_semi_sync_master_tx_waits", - (char*) &SHOW_FNAME(trx_wait_num), - SHOW_SIMPLE_FUNC}, - {"Rpl_semi_sync_master_tx_avg_wait_time", - (char*) &SHOW_FNAME(avg_trx_wait_time), - SHOW_SIMPLE_FUNC}, - {"Rpl_semi_sync_master_net_wait_time", - (char*) &SHOW_FNAME(net_wait_time), - SHOW_SIMPLE_FUNC}, - {"Rpl_semi_sync_master_net_waits", - (char*) &SHOW_FNAME(net_wait_num), - SHOW_SIMPLE_FUNC}, - {"Rpl_semi_sync_master_net_avg_wait_time", - (char*) &SHOW_FNAME(avg_net_wait_time), - SHOW_SIMPLE_FUNC}, - {NULL, NULL, SHOW_LONG}, -}; - -#ifdef HAVE_PSI_INTERFACE -PSI_mutex_key key_ss_mutex_LOCK_binlog_; - -static PSI_mutex_info all_semisync_mutexes[]= -{ - { &key_ss_mutex_LOCK_binlog_, "LOCK_binlog_", 0} -}; - -PSI_cond_key key_ss_cond_COND_binlog_send_; - -static PSI_cond_info all_semisync_conds[]= -{ - { &key_ss_cond_COND_binlog_send_, "COND_binlog_send_", 0} -}; -#endif /* HAVE_PSI_INTERFACE */ - -PSI_stage_info stage_waiting_for_semi_sync_ack_from_slave= -{ 0, "Waiting for semi-sync ACK from slave", 0}; - -#ifdef HAVE_PSI_INTERFACE -PSI_stage_info *all_semisync_stages[]= -{ - & stage_waiting_for_semi_sync_ack_from_slave -}; - -static void init_semisync_psi_keys(void) -{ - const char* category= "semisync"; - int count; - - count= array_elements(all_semisync_mutexes); - mysql_mutex_register(category, all_semisync_mutexes, count); - - count= array_elements(all_semisync_conds); - mysql_cond_register(category, all_semisync_conds, count); - - count= array_elements(all_semisync_stages); - mysql_stage_register(category, all_semisync_stages, count); -} -#endif /* HAVE_PSI_INTERFACE */ - -static int semi_sync_master_plugin_init(void *p) -{ -#ifdef HAVE_PSI_INTERFACE - init_semisync_psi_keys(); -#endif - - if (repl_semisync.initObject()) - return 1; - if (register_trans_observer(&trans_observer, p)) - return 1; - if (register_binlog_storage_observer(&storage_observer, p)) - return 1; - if (register_binlog_transmit_observer(&transmit_observer, p)) - return 1; - return 0; -} - -static int semi_sync_master_plugin_deinit(void *p) -{ - if (unregister_trans_observer(&trans_observer, p)) - { - sql_print_error("unregister_trans_observer failed"); - return 1; - } - if (unregister_binlog_storage_observer(&storage_observer, p)) - { - sql_print_error("unregister_binlog_storage_observer failed"); - return 1; - } - if (unregister_binlog_transmit_observer(&transmit_observer, p)) - { - sql_print_error("unregister_binlog_transmit_observer failed"); - return 1; - } - repl_semisync.cleanup(); - sql_print_information("unregister_replicator OK"); - return 0; -} - -struct Mysql_replication semi_sync_master_plugin= { - MYSQL_REPLICATION_INTERFACE_VERSION -}; - -/* - Plugin library descriptor -*/ -maria_declare_plugin(semisync_master) -{ - MYSQL_REPLICATION_PLUGIN, - &semi_sync_master_plugin, - "rpl_semi_sync_master", - "He Zhenxing", - "Semi-synchronous replication master", - PLUGIN_LICENSE_GPL, - semi_sync_master_plugin_init, /* Plugin Init */ - semi_sync_master_plugin_deinit, /* Plugin Deinit */ - 0x0100 /* 1.0 */, - semi_sync_master_status_vars, /* status variables */ - semi_sync_master_system_vars, /* system variables */ - "1.0", - MariaDB_PLUGIN_MATURITY_STABLE -} -maria_declare_plugin_end; - diff --git a/plugin/semisync/semisync_slave_plugin.cc b/plugin/semisync/semisync_slave_plugin.cc deleted file mode 100644 index df9e8e10429..00000000000 --- a/plugin/semisync/semisync_slave_plugin.cc +++ /dev/null @@ -1,234 +0,0 @@ -/* Copyright (C) 2007 Google Inc. - Copyright (C) 2008 MySQL AB - Use is subject to license terms - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - - -#include -#include "semisync_slave.h" -#include - -static ReplSemiSyncSlave repl_semisync; - -/* - indicate whether or not the slave should send a reply to the master. - - This is set to true in repl_semi_slave_read_event if the current - event read is the last event of a transaction. And the value is - checked in repl_semi_slave_queue_event. -*/ -bool semi_sync_need_reply= false; - -C_MODE_START - -int repl_semi_reset_slave(Binlog_relay_IO_param *param) -{ - // TODO: reset semi-sync slave status here - return 0; -} - -int repl_semi_slave_request_dump(Binlog_relay_IO_param *param, - uint32 flags) -{ - MYSQL *mysql= param->mysql; - MYSQL_RES *res= 0; - MYSQL_ROW row; - const char *query; - - if (!repl_semisync.getSlaveEnabled()) - return 0; - - /* Check if master server has semi-sync plugin installed */ - query= "SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled'"; - if (mysql_real_query(mysql, query, strlen(query)) || - !(res= mysql_store_result(mysql))) - { - sql_print_error("Execution failed on master: %s", query); - return 1; - } - - row= mysql_fetch_row(res); - if (!row) - { - /* Master does not support semi-sync */ - sql_print_warning("Master server does not support semi-sync, " - "fallback to asynchronous replication"); - rpl_semi_sync_slave_status= 0; - mysql_free_result(res); - return 0; - } - mysql_free_result(res); - - /* - Tell master dump thread that we want to do semi-sync - replication - */ - query= "SET @rpl_semi_sync_slave= 1"; - if (mysql_real_query(mysql, query, strlen(query))) - { - sql_print_error("Set 'rpl_semi_sync_slave=1' on master failed"); - return 1; - } - mysql_free_result(mysql_store_result(mysql)); - rpl_semi_sync_slave_status= 1; - return 0; -} - -int repl_semi_slave_read_event(Binlog_relay_IO_param *param, - const char *packet, unsigned long len, - const char **event_buf, unsigned long *event_len) -{ - if (rpl_semi_sync_slave_status) - return repl_semisync.slaveReadSyncHeader(packet, len, - &semi_sync_need_reply, - event_buf, event_len); - *event_buf= packet; - *event_len= len; - return 0; -} - -int repl_semi_slave_queue_event(Binlog_relay_IO_param *param, - const char *event_buf, - unsigned long event_len, - uint32 flags) -{ - if (rpl_semi_sync_slave_status && semi_sync_need_reply) - { - /* - We deliberately ignore the error in slaveReply, such error - should not cause the slave IO thread to stop, and the error - messages are already reported. - */ - (void) repl_semisync.slaveReply(param->mysql, - param->master_log_name, - param->master_log_pos); - } - return 0; -} - -int repl_semi_slave_io_start(Binlog_relay_IO_param *param) -{ - return repl_semisync.slaveStart(param); -} - -int repl_semi_slave_io_end(Binlog_relay_IO_param *param) -{ - return repl_semisync.slaveStop(param); -} - -C_MODE_END - -static void fix_rpl_semi_sync_slave_enabled(MYSQL_THD thd, - SYS_VAR *var, - void *ptr, - const void *val) -{ - *(char *)ptr= *(char *)val; - repl_semisync.setSlaveEnabled(rpl_semi_sync_slave_enabled != 0); - return; -} - -static void fix_rpl_semi_sync_trace_level(MYSQL_THD thd, - SYS_VAR *var, - void *ptr, - const void *val) -{ - *(unsigned long *)ptr= *(unsigned long *)val; - repl_semisync.setTraceLevel(rpl_semi_sync_slave_trace_level); - return; -} - -/* plugin system variables */ -static MYSQL_SYSVAR_BOOL(enabled, rpl_semi_sync_slave_enabled, - PLUGIN_VAR_OPCMDARG, - "Enable semi-synchronous replication slave (disabled by default). ", - NULL, // check - &fix_rpl_semi_sync_slave_enabled, // update - 0); - -static MYSQL_SYSVAR_ULONG(trace_level, rpl_semi_sync_slave_trace_level, - PLUGIN_VAR_OPCMDARG, - "The tracing level for semi-sync replication.", - NULL, // check - &fix_rpl_semi_sync_trace_level, // update - 32, 0, ~0UL, 1); - -static SYS_VAR* semi_sync_slave_system_vars[]= { - MYSQL_SYSVAR(enabled), - MYSQL_SYSVAR(trace_level), - NULL, -}; - - -/* plugin status variables */ -static SHOW_VAR semi_sync_slave_status_vars[]= { - {"Rpl_semi_sync_slave_status", - (char*) &rpl_semi_sync_slave_status, SHOW_BOOL}, - {NULL, NULL, SHOW_BOOL}, -}; - -Binlog_relay_IO_observer relay_io_observer = { - sizeof(Binlog_relay_IO_observer), // len - - repl_semi_slave_io_start, // start - repl_semi_slave_io_end, // stop - repl_semi_slave_request_dump, // request_transmit - repl_semi_slave_read_event, // after_read_event - repl_semi_slave_queue_event, // after_queue_event - repl_semi_reset_slave, // reset -}; - -static int semi_sync_slave_plugin_init(void *p) -{ - if (repl_semisync.initObject()) - return 1; - if (register_binlog_relay_io_observer(&relay_io_observer, p)) - return 1; - return 0; -} - -static int semi_sync_slave_plugin_deinit(void *p) -{ - if (unregister_binlog_relay_io_observer(&relay_io_observer, p)) - return 1; - return 0; -} - - -struct Mysql_replication semi_sync_slave_plugin= { - MYSQL_REPLICATION_INTERFACE_VERSION -}; - -/* - Plugin library descriptor -*/ -maria_declare_plugin(semisync_slave) -{ - MYSQL_REPLICATION_PLUGIN, - &semi_sync_slave_plugin, - "rpl_semi_sync_slave", - "He Zhenxing", - "Semi-synchronous replication slave", - PLUGIN_LICENSE_GPL, - semi_sync_slave_plugin_init, /* Plugin Init */ - semi_sync_slave_plugin_deinit, /* Plugin Deinit */ - 0x0100 /* 1.0 */, - semi_sync_slave_status_vars, /* status variables */ - semi_sync_slave_system_vars, /* system variables */ - "1.0", - MariaDB_PLUGIN_MATURITY_STABLE -} -maria_declare_plugin_end; - diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 0f67032bcbe..6c63f3feca3 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -36,7 +36,7 @@ ELSE() ENDIF() INCLUDE_DIRECTORIES( -${CMAKE_SOURCE_DIR}/include +${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql ${PCRE_INCLUDES} ${ZLIB_INCLUDE_DIR} @@ -138,6 +138,7 @@ SET (SQL_SOURCE my_apc.cc mf_iocache_encr.cc item_jsonfunc.cc my_json_writer.cc rpl_gtid.cc rpl_parallel.cc + semisync.cc semisync_master.cc semisync_slave.cc sql_type.cc item_windowfunc.cc sql_window.cc sql_cte.cc diff --git a/sql/mysqld.cc b/sql/mysqld.cc index f3cb39959a7..fc783ae5559 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -97,8 +97,9 @@ #include "set_var.h" #include "rpl_injector.h" - #include "rpl_handler.h" +#include "semisync_master.h" +#include "semisync_slave.h" #ifdef HAVE_SYS_PRCTL_H #include @@ -934,6 +935,7 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, PSI_mutex_key key_RELAYLOG_LOCK_index; PSI_mutex_key key_LOCK_slave_state, key_LOCK_binlog_state, key_LOCK_rpl_thread, key_LOCK_rpl_thread_pool, key_LOCK_parallel_entry; +PSI_mutex_key key_LOCK_binlog; PSI_mutex_key key_LOCK_stats, key_LOCK_global_user_client_stats, key_LOCK_global_table_stats, @@ -1021,7 +1023,8 @@ static PSI_mutex_info all_server_mutexes[]= { &key_LOCK_binlog_state, "LOCK_binlog_state", 0}, { &key_LOCK_rpl_thread, "LOCK_rpl_thread", 0}, { &key_LOCK_rpl_thread_pool, "LOCK_rpl_thread_pool", 0}, - { &key_LOCK_parallel_entry, "LOCK_parallel_entry", 0} + { &key_LOCK_parallel_entry, "LOCK_parallel_entry", 0}, + { &key_LOCK_binlog, "LOCK_binlog", 0} }; PSI_rwlock_key key_rwlock_LOCK_grant, key_rwlock_LOCK_logger, @@ -1062,7 +1065,7 @@ PSI_cond_key key_BINLOG_COND_xid_list, key_BINLOG_update_cond, key_rpl_group_info_sleep_cond, key_TABLE_SHARE_cond, key_user_level_lock_cond, key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache, - key_COND_start_thread, + key_COND_start_thread, key_COND_binlog_send, key_BINLOG_COND_queue_busy; PSI_cond_key key_RELAYLOG_update_cond, key_COND_wakeup_ready, key_COND_wait_commit; @@ -1124,7 +1127,8 @@ static PSI_cond_info all_server_conds[]= { &key_COND_slave_background, "COND_slave_background", 0}, { &key_COND_start_thread, "COND_start_thread", PSI_FLAG_GLOBAL}, { &key_COND_wait_gtid, "COND_wait_gtid", 0}, - { &key_COND_gtid_ignore_duplicates, "COND_gtid_ignore_duplicates", 0} + { &key_COND_gtid_ignore_duplicates, "COND_gtid_ignore_duplicates", 0}, + { &key_COND_binlog_send, "COND_binlog_send", 0} }; PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert, @@ -2217,6 +2221,10 @@ void clean_up(bool print_message) ha_end(); if (tc_log) tc_log->close(); +#ifdef HAVE_REPLICATION + semi_sync_master_deinit(); + semi_sync_slave_deinit(); +#endif delegates_destroy(); xid_cache_free(); tdc_deinit(); @@ -5170,6 +5178,9 @@ static int init_server_components() "this server. However this will be ignored as the " "--log-bin option is not defined."); } + + semi_sync_master_init(); + semi_sync_slave_init(); #endif if (opt_bin_log) @@ -8230,6 +8241,27 @@ static int show_ssl_get_cipher_list(THD *thd, SHOW_VAR *var, char *buff, return 0; } +#define SHOW_FNAME(name) \ + rpl_semi_sync_master_show_##name + +#define DEF_SHOW_FUNC(name, show_type) \ + static int SHOW_FNAME(name)(MYSQL_THD thd, SHOW_VAR *var, char *buff) \ + { \ + repl_semisync_master.setExportStats(); \ + var->type= show_type; \ + var->value= (char *)&rpl_semi_sync_master_##name; \ + return 0; \ + } + +DEF_SHOW_FUNC(status, SHOW_BOOL) +DEF_SHOW_FUNC(clients, SHOW_LONG) +DEF_SHOW_FUNC(wait_sessions, SHOW_LONG) +DEF_SHOW_FUNC(trx_wait_time, SHOW_LONGLONG) +DEF_SHOW_FUNC(trx_wait_num, SHOW_LONGLONG) +DEF_SHOW_FUNC(net_wait_time, SHOW_LONGLONG) +DEF_SHOW_FUNC(net_wait_num, SHOW_LONGLONG) +DEF_SHOW_FUNC(avg_net_wait_time, SHOW_LONG) +DEF_SHOW_FUNC(avg_trx_wait_time, SHOW_LONG) #ifdef HAVE_YASSL @@ -8548,6 +8580,30 @@ SHOW_VAR status_vars[]= { {"Rows_sent", (char*) offsetof(STATUS_VAR, rows_sent), SHOW_LONGLONG_STATUS}, {"Rows_read", (char*) offsetof(STATUS_VAR, rows_read), SHOW_LONGLONG_STATUS}, {"Rows_tmp_read", (char*) offsetof(STATUS_VAR, rows_tmp_read), SHOW_LONGLONG_STATUS}, +#ifdef HAVE_REPLICATION + {"Rpl_semi_sync_master_status", (char*) &SHOW_FNAME(status), SHOW_FUNC}, + {"Rpl_semi_sync_master_clients", (char*) &SHOW_FNAME(clients), SHOW_FUNC}, + {"Rpl_semi_sync_master_yes_tx", (char*) &rpl_semi_sync_master_yes_transactions, SHOW_LONG}, + {"Rpl_semi_sync_master_no_tx", (char*) &rpl_semi_sync_master_no_transactions, SHOW_LONG}, + {"Rpl_semi_sync_master_wait_sessions", (char*) &SHOW_FNAME(wait_sessions), SHOW_FUNC}, + {"Rpl_semi_sync_master_no_times", (char*) &rpl_semi_sync_master_off_times, SHOW_LONG}, + {"Rpl_semi_sync_master_timefunc_failures", (char*) &rpl_semi_sync_master_timefunc_fails, SHOW_LONG}, + {"Rpl_semi_sync_master_wait_pos_backtraverse", (char*) &rpl_semi_sync_master_wait_pos_backtraverse, SHOW_LONG}, + {"Rpl_semi_sync_master_tx_wait_time", (char*) &SHOW_FNAME(trx_wait_time), SHOW_FUNC}, + {"Rpl_semi_sync_master_tx_waits", (char*) &SHOW_FNAME(trx_wait_num), SHOW_FUNC}, + {"Rpl_semi_sync_master_tx_avg_wait_time", (char*) &SHOW_FNAME(avg_trx_wait_time), SHOW_FUNC}, + {"Rpl_semi_sync_master_net_wait_time", (char*) &SHOW_FNAME(net_wait_time), SHOW_FUNC}, + {"Rpl_semi_sync_master_net_waits", (char*) &SHOW_FNAME(net_wait_num), SHOW_FUNC}, + {"Rpl_semi_sync_master_net_avg_wait_time", (char*) &SHOW_FNAME(avg_net_wait_time), SHOW_FUNC}, +#ifdef HAVE_ACC_RECEIVER + {"Rpl_semi_sync_master_request_ack", (char*) &rpl_semi_sync_master_request_ack, SHOW_LONGLONG}, + {"Rpl_semi_sync_master_get_ack", (char*)&rpl_semi_sync_master_get_ack, SHOW_LONGLONG}, +#endif + {"Rpl_semi_sync_slave_status", (char*) &rpl_semi_sync_slave_status, SHOW_BOOL}, +#ifdef HAVE_ACC_RECEIVER + {"Rpl_semi_sync_slave_send_ack", (char*) &rpl_semi_sync_slave_send_ack, SHOW_LONGLONG}, +#endif +#endif /* HAVE_REPLICATION */ #ifdef HAVE_QUERY_CACHE {"Qcache_free_blocks", (char*) &query_cache.free_memory_blocks, SHOW_LONG_NOFLUSH}, {"Qcache_free_memory", (char*) &query_cache.free_memory, SHOW_LONG_NOFLUSH}, @@ -10285,6 +10341,8 @@ PSI_stage_info stage_waiting_for_insert= { 0, "Waiting for INSERT", 0}; PSI_stage_info stage_waiting_for_master_to_send_event= { 0, "Waiting for master to send event", 0}; PSI_stage_info stage_waiting_for_master_update= { 0, "Waiting for master update", 0}; PSI_stage_info stage_waiting_for_relay_log_space= { 0, "Waiting for the slave SQL thread to free enough relay log space", 0}; +PSI_stage_info stage_waiting_for_semi_sync_ack_from_slave= +{ 0, "Waiting for semi-sync ACK from slave", 0}; PSI_stage_info stage_waiting_for_slave_mutex_on_exit= { 0, "Waiting for slave mutex on exit", 0}; PSI_stage_info stage_waiting_for_slave_thread_to_start= { 0, "Waiting for slave thread to start", 0}; PSI_stage_info stage_waiting_for_table_flush= { 0, "Waiting for table flush", 0}; diff --git a/sql/replication.h b/sql/replication.h index 4731c2246ef..d8672310110 100644 --- a/sql/replication.h +++ b/sql/replication.h @@ -18,16 +18,14 @@ /*************************************************************************** NOTE: plugin locking. - This API was created specifically for the semisync plugin and its locking - logic is also matches semisync plugin usage pattern. In particular, a plugin - is locked on Binlog_transmit_observer::transmit_start and is unlocked after - Binlog_transmit_observer::transmit_stop. All other master observable events - happen between these two and don't lock the plugin at all. This works well - for the semisync_master plugin. + + The plugin is locked on Binlog_transmit_observer::transmit_start and is + unlocked after Binlog_transmit_observer::transmit_stop. All other + master observable events happen between these two and don't lock the + plugin at all. Also a plugin is locked on Binlog_relay_IO_observer::thread_start - and unlocked after Binlog_relay_IO_observer::thread_stop. This works well for - the semisync_slave plugin. + and unlocked after Binlog_relay_IO_observer::thread_stop. ***************************************************************************/ #include diff --git a/sql/rpl_handler.cc b/sql/rpl_handler.cc index e3ff2a17a6a..27e411ca6de 100644 --- a/sql/rpl_handler.cc +++ b/sql/rpl_handler.cc @@ -149,13 +149,17 @@ void delegates_destroy() { if (transaction_delegate) transaction_delegate->~Trans_delegate(); + transaction_delegate= 0; if (binlog_storage_delegate) binlog_storage_delegate->~Binlog_storage_delegate(); + binlog_storage_delegate= 0; #ifdef HAVE_REPLICATION if (binlog_transmit_delegate) binlog_transmit_delegate->~Binlog_transmit_delegate(); + binlog_transmit_delegate= 0; if (binlog_relay_io_delegate) binlog_relay_io_delegate->~Binlog_relay_IO_delegate(); + binlog_relay_io_delegate= 0; #endif /* HAVE_REPLICATION */ } @@ -171,13 +175,11 @@ void delegates_destroy() Observer_info *info= iter++; \ for (; info; info= iter++) \ { \ - if (do_lock) plugin_lock(thd, plugin_int_to_ref(info->plugin_int)); \ if (((Observer *)info->observer)->f \ && ((Observer *)info->observer)->f args) \ { \ r= 1; \ - sql_print_error("Run function '" #f "' in plugin '%s' failed", \ - info->plugin_int->name.str); \ + sql_print_error("Run function '" #f "' failed"); \ break; \ } \ } \ diff --git a/plugin/semisync/semisync.cc b/sql/semisync.cc similarity index 100% rename from plugin/semisync/semisync.cc rename to sql/semisync.cc diff --git a/plugin/semisync/semisync.h b/sql/semisync.h similarity index 87% rename from plugin/semisync/semisync.h rename to sql/semisync.h index 28577296817..3142f920f1e 100644 --- a/plugin/semisync/semisync.h +++ b/sql/semisync.h @@ -1,6 +1,5 @@ /* Copyright (C) 2007 Google Inc. Copyright (C) 2008 MySQL AB - Use is subject to license terms This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,18 +18,9 @@ #ifndef SEMISYNC_H #define SEMISYNC_H -#define MYSQL_SERVER -#define HAVE_REPLICATION -#include -#include -#include -#include "unireg.h" -#include -#include "log.h" /* sql_print_information */ - -typedef struct st_mysql_show_var SHOW_VAR; -typedef struct st_mysql_sys_var SYS_VAR; - +#include "mysqld.h" +#include "log_event.h" +#include "replication.h" /** This class is used to trace function calls and other process diff --git a/plugin/semisync/semisync_master.cc b/sql/semisync_master.cc similarity index 78% rename from plugin/semisync/semisync_master.cc rename to sql/semisync_master.cc index 975b2e13253..21c52addce9 100644 --- a/plugin/semisync/semisync_master.cc +++ b/sql/semisync_master.cc @@ -24,34 +24,35 @@ #define TIME_BILLION 1000000000 /* This indicates whether semi-synchronous replication is enabled. */ -char rpl_semi_sync_master_enabled; -unsigned long rpl_semi_sync_master_wait_point = +my_bool rpl_semi_sync_master_enabled; +my_bool rpl_semi_sync_master_wait_no_slave = 1; +my_bool rpl_semi_sync_master_status = 0; +ulong rpl_semi_sync_master_wait_point = SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT; -unsigned long rpl_semi_sync_master_timeout; -unsigned long rpl_semi_sync_master_trace_level; -char rpl_semi_sync_master_status = 0; -unsigned long rpl_semi_sync_master_yes_transactions = 0; -unsigned long rpl_semi_sync_master_no_transactions = 0; -unsigned long rpl_semi_sync_master_off_times = 0; -unsigned long rpl_semi_sync_master_timefunc_fails = 0; -unsigned long rpl_semi_sync_master_wait_timeouts = 0; -unsigned long rpl_semi_sync_master_wait_sessions = 0; -unsigned long rpl_semi_sync_master_wait_pos_backtraverse = 0; -unsigned long rpl_semi_sync_master_avg_trx_wait_time = 0; -unsigned long long rpl_semi_sync_master_trx_wait_num = 0; -unsigned long rpl_semi_sync_master_avg_net_wait_time = 0; -unsigned long long rpl_semi_sync_master_net_wait_num = 0; -unsigned long rpl_semi_sync_master_clients = 0; -unsigned long long rpl_semi_sync_master_net_wait_time = 0; -unsigned long long rpl_semi_sync_master_trx_wait_time = 0; -char rpl_semi_sync_master_wait_no_slave = 1; +ulong rpl_semi_sync_master_timeout; +ulong rpl_semi_sync_master_trace_level; +ulong rpl_semi_sync_master_yes_transactions = 0; +ulong rpl_semi_sync_master_no_transactions = 0; +ulong rpl_semi_sync_master_off_times = 0; +ulong rpl_semi_sync_master_timefunc_fails = 0; +ulong rpl_semi_sync_master_wait_timeouts = 0; +ulong rpl_semi_sync_master_wait_sessions = 0; +ulong rpl_semi_sync_master_wait_pos_backtraverse = 0; +ulong rpl_semi_sync_master_avg_trx_wait_time = 0; +ulonglong rpl_semi_sync_master_trx_wait_num = 0; +ulong rpl_semi_sync_master_avg_net_wait_time = 0; +ulonglong rpl_semi_sync_master_net_wait_num = 0; +ulong rpl_semi_sync_master_clients = 0; +ulonglong rpl_semi_sync_master_net_wait_time = 0; +ulonglong rpl_semi_sync_master_trx_wait_time = 0; +ReplSemiSyncMaster repl_semisync_master; static int getWaitTime(const struct timespec& start_ts); -static unsigned long long timespec_to_usec(const struct timespec *ts) +static ulonglong timespec_to_usec(const struct timespec *ts) { - return (unsigned long long) ts->tv_sec * TIME_MILLION + ts->tv_nsec / TIME_THOUSAND; + return (ulonglong) ts->tv_sec * TIME_MILLION + ts->tv_nsec / TIME_THOUSAND; } /******************************************************************************* @@ -61,7 +62,7 @@ static unsigned long long timespec_to_usec(const struct timespec *ts) ******************************************************************************/ ActiveTranx::ActiveTranx(mysql_mutex_t *lock, - unsigned long trace_level) + ulong trace_level) : Trace(trace_level), allocator_(max_connections), num_entries_(max_connections << 1), /* Transaction hash table size * is set to double the size @@ -141,7 +142,7 @@ int ActiveTranx::insert_tranx_node(const char *log_file_name, if (!ins_node) { sql_print_error("%s: transaction node allocation failed for: (%s, %lu)", - kWho, log_file_name, (unsigned long)log_file_pos); + kWho, log_file_name, (ulong)log_file_pos); result = -1; goto l_end; } @@ -174,8 +175,8 @@ int ActiveTranx::insert_tranx_node(const char *log_file_name, */ sql_print_error("%s: binlog write out-of-order, tail (%s, %lu), " "new node (%s, %lu)", kWho, - trx_rear_->log_name_, (unsigned long)trx_rear_->log_pos_, - ins_node->log_name_, (unsigned long)ins_node->log_pos_); + trx_rear_->log_name_, (ulong)trx_rear_->log_pos_, + ins_node->log_name_, (ulong)ins_node->log_pos_); result = -1; goto l_end; } @@ -187,7 +188,7 @@ int ActiveTranx::insert_tranx_node(const char *log_file_name, if (trace_level_ & kTraceDetail) sql_print_information("%s: insert (%s, %lu) in entry(%u)", kWho, - ins_node->log_name_, (unsigned long)ins_node->log_pos_, + ins_node->log_name_, (ulong)ins_node->log_pos_, hash_val); l_end: @@ -213,7 +214,7 @@ bool ActiveTranx::is_tranx_end_pos(const char *log_file_name, if (trace_level_ & kTraceDetail) sql_print_information("%s: probe (%s, %lu) in entry(%u)", kWho, - log_file_name, (unsigned long)log_file_pos, hash_val); + log_file_name, (ulong)log_file_pos, hash_val); function_exit(kWho, (entry != NULL)); return (entry != NULL); @@ -296,7 +297,7 @@ int ActiveTranx::clear_active_tranx_nodes(const char *log_file_name, if (trace_level_ & kTraceDetail) sql_print_information("%s: cleared %d nodes back until pos (%s, %lu)", kWho, n_frees, - trx_front_->log_name_, (unsigned long)trx_front_->log_pos_); + trx_front_->log_name_, (ulong)trx_front_->log_pos_); } return function_exit(kWho, 0); @@ -358,10 +359,10 @@ int ReplSemiSyncMaster::initObject() setTraceLevel(rpl_semi_sync_master_trace_level); /* Mutex initialization can only be done after MY_INIT(). */ - mysql_mutex_init(key_ss_mutex_LOCK_binlog_, - &LOCK_binlog_, MY_MUTEX_INIT_FAST); - mysql_cond_init(key_ss_cond_COND_binlog_send_, - &COND_binlog_send_, NULL); + mysql_mutex_init(key_LOCK_binlog, + &LOCK_binlog, MY_MUTEX_INIT_FAST); + mysql_cond_init(key_COND_binlog_send, + &COND_binlog_send, NULL); if (rpl_semi_sync_master_enabled) result = enableMaster(); @@ -380,7 +381,7 @@ int ReplSemiSyncMaster::enableMaster() if (!getMasterEnabled()) { - active_tranxs_ = new ActiveTranx(&LOCK_binlog_, trace_level_); + active_tranxs_ = new ActiveTranx(&LOCK_binlog, trace_level_); if (active_tranxs_ != NULL) { commit_file_name_inited_ = false; @@ -389,6 +390,7 @@ int ReplSemiSyncMaster::enableMaster() set_master_enabled(true); state_ = true; + run_hooks_enabled= 1; sql_print_information("Semi-sync replication enabled on the master."); } else @@ -436,8 +438,8 @@ void ReplSemiSyncMaster::cleanup() { if (init_done_) { - mysql_mutex_destroy(&LOCK_binlog_); - mysql_cond_destroy(&COND_binlog_send_); + mysql_mutex_destroy(&LOCK_binlog); + mysql_cond_destroy(&COND_binlog_send); init_done_= 0; } @@ -446,17 +448,17 @@ void ReplSemiSyncMaster::cleanup() void ReplSemiSyncMaster::lock() { - mysql_mutex_lock(&LOCK_binlog_); + mysql_mutex_lock(&LOCK_binlog); } void ReplSemiSyncMaster::unlock() { - mysql_mutex_unlock(&LOCK_binlog_); + mysql_mutex_unlock(&LOCK_binlog); } void ReplSemiSyncMaster::cond_broadcast() { - mysql_cond_broadcast(&COND_binlog_send_); + mysql_cond_broadcast(&COND_binlog_send); } int ReplSemiSyncMaster::cond_timewait(struct timespec *wait_time) @@ -465,8 +467,8 @@ int ReplSemiSyncMaster::cond_timewait(struct timespec *wait_time) int wait_res; function_enter(kWho); - wait_res= mysql_cond_timedwait(&COND_binlog_send_, - &LOCK_binlog_, wait_time); + wait_res= mysql_cond_timedwait(&COND_binlog_send, + &LOCK_binlog, wait_time); return function_exit(kWho, wait_res); } @@ -566,7 +568,7 @@ int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, if (trace_level_ & kTraceDetail) sql_print_information("%s: Got reply at (%s, %lu)", kWho, - log_file_name, (unsigned long)log_file_pos); + log_file_name, (ulong)log_file_pos); } if (rpl_semi_sync_master_wait_sessions > 0) @@ -621,7 +623,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, lock(); /* This must be called after acquired the lock */ - THD_ENTER_COND(NULL, &COND_binlog_send_, &LOCK_binlog_, + THD_ENTER_COND(NULL, &COND_binlog_send, &LOCK_binlog, & stage_waiting_for_semi_sync_ack_from_slave, & old_stage); @@ -632,7 +634,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, if (trace_level_ & kTraceDetail) { sql_print_information("%s: wait pos (%s, %lu), repl(%d)\n", kWho, - trx_wait_binlog_name, (unsigned long)trx_wait_binlog_pos, + trx_wait_binlog_name, (ulong)trx_wait_binlog_pos, (int)is_on()); } @@ -649,7 +651,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, */ if (trace_level_ & kTraceDetail) sql_print_information("%s: Binlog reply is ahead (%s, %lu),", - kWho, reply_file_name_, (unsigned long)reply_file_pos_); + kWho, reply_file_name_, (ulong)reply_file_pos_); break; } } @@ -670,7 +672,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, rpl_semi_sync_master_wait_pos_backtraverse++; if (trace_level_ & kTraceDetail) sql_print_information("%s: move back wait position (%s, %lu),", - kWho, wait_file_name_, (unsigned long)wait_file_pos_); + kWho, wait_file_name_, (ulong)wait_file_pos_); } } else @@ -681,16 +683,16 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, if (trace_level_ & kTraceDetail) sql_print_information("%s: init wait position (%s, %lu),", - kWho, wait_file_name_, (unsigned long)wait_file_pos_); + kWho, wait_file_name_, (ulong)wait_file_pos_); } /* Calcuate the waiting period. */ - long diff_secs = (long) (wait_timeout_ / TIME_THOUSAND); + long diff_secs = (long) (wait_timeout_ / TIME_THOUSAND); long diff_nsecs = (long) ((wait_timeout_ % TIME_THOUSAND) * TIME_MILLION); long nsecs = start_ts.tv_nsec + diff_nsecs; abstime.tv_sec = start_ts.tv_sec + diff_secs + nsecs/TIME_BILLION; abstime.tv_nsec = nsecs % TIME_BILLION; - + /* In semi-synchronous replication, we wait until the binlog-dump * thread has received the reply on the relevant binlog segment from the * replication slave. @@ -700,31 +702,31 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, * these waiting threads. */ rpl_semi_sync_master_wait_sessions++; - + if (trace_level_ & kTraceDetail) sql_print_information("%s: wait %lu ms for binlog sent (%s, %lu)", kWho, wait_timeout_, - wait_file_name_, (unsigned long)wait_file_pos_); - + wait_file_name_, (ulong)wait_file_pos_); + wait_result = cond_timewait(&abstime); rpl_semi_sync_master_wait_sessions--; - + if (wait_result != 0) { /* This is a real wait timeout. */ sql_print_warning("Timeout waiting for reply of binlog (file: %s, pos: %lu), " "semi-sync up to file %s, position %lu.", - trx_wait_binlog_name, (unsigned long)trx_wait_binlog_pos, - reply_file_name_, (unsigned long)reply_file_pos_); + trx_wait_binlog_name, (ulong)trx_wait_binlog_pos, + reply_file_name_, (ulong)reply_file_pos_); rpl_semi_sync_master_wait_timeouts++; - + /* switch semi-sync off */ switch_off(); } else { int wait_time; - + wait_time = getWaitTime(start_ts); if (wait_time < 0) { @@ -732,7 +734,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, { sql_print_error("Replication semi-sync getWaitTime fail at " "wait position (%s, %lu)", - trx_wait_binlog_name, (unsigned long)trx_wait_binlog_pos); + trx_wait_binlog_name, (ulong)trx_wait_binlog_pos); } rpl_semi_sync_master_timefunc_fails++; } @@ -753,7 +755,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, assert(thd_killed(current_thd) || !active_tranxs_ || !active_tranxs_->is_tranx_end_pos(trx_wait_binlog_name, trx_wait_binlog_pos)); - + l_end: /* Update the status counter. */ if (is_on()) @@ -770,7 +772,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, } /* Indicate that semi-sync replication is OFF now. - * + * * What should we do when it is disabled? The problem is that we want * the semi-sync replication enabled again when the slave catches up * later. But, it is not that easy to detect that the slave has caught @@ -842,14 +844,14 @@ int ReplSemiSyncMaster::try_switch_on(int server_id, sql_print_information("Semi-sync replication switched ON with slave (server_id: %d) " "at (%s, %lu)", server_id, log_file_name, - (unsigned long)log_file_pos); + (ulong)log_file_pos); } return function_exit(kWho, 0); } int ReplSemiSyncMaster::reserveSyncHeader(unsigned char *header, - unsigned long size) + ulong size) { const char *kWho = "ReplSemiSyncMaster::reserveSyncHeader"; function_enter(kWho); @@ -870,7 +872,7 @@ int ReplSemiSyncMaster::reserveSyncHeader(unsigned char *header, disableMaster(); return 0; } - + /* Set the magic number and the sync status. By default, no sync * is required. */ @@ -930,13 +932,13 @@ int ReplSemiSyncMaster::updateSyncHeader(unsigned char *packet, { cmp = 1; } - + /* If we are already waiting for some transaction replies which * are later in binlog, do not wait for this one event. */ if (cmp >= 0) { - /* + /* * We only wait if the event is a transaction's ending event. */ assert(active_tranxs_ != NULL); @@ -961,7 +963,7 @@ int ReplSemiSyncMaster::updateSyncHeader(unsigned char *packet, if (trace_level_ & kTraceDetail) sql_print_information("%s: server(%d), (%s, %lu) sync(%d), repl(%d)", kWho, server_id, log_file_name, - (unsigned long)log_file_pos, sync, (int)is_on()); + (ulong)log_file_pos, sync, (int)is_on()); l_end: unlock(); @@ -1181,27 +1183,27 @@ void ReplSemiSyncMaster::setExportStats() rpl_semi_sync_master_status = state_; rpl_semi_sync_master_avg_trx_wait_time= ((rpl_semi_sync_master_trx_wait_num) ? - (unsigned long)((double)rpl_semi_sync_master_trx_wait_time / + (ulong)((double)rpl_semi_sync_master_trx_wait_time / ((double)rpl_semi_sync_master_trx_wait_num)) : 0); rpl_semi_sync_master_avg_net_wait_time= ((rpl_semi_sync_master_net_wait_num) ? - (unsigned long)((double)rpl_semi_sync_master_net_wait_time / + (ulong)((double)rpl_semi_sync_master_net_wait_time / ((double)rpl_semi_sync_master_net_wait_num)) : 0); unlock(); } /* Get the waiting time given the wait's staring time. - * + * * Return: * >= 0: the waiting time in microsecons(us) * < 0: error in get time or time back traverse */ static int getWaitTime(const struct timespec& start_ts) { - unsigned long long start_usecs, end_usecs; + ulonglong start_usecs, end_usecs; struct timespec end_ts; - + /* Starting time in microseconds(us). */ start_usecs = timespec_to_usec(&start_ts); @@ -1216,3 +1218,213 @@ static int getWaitTime(const struct timespec& start_ts) return (int)(end_usecs - start_usecs); } + +/*************************************************************************** + Semisync master interface setup and deinit +***************************************************************************/ + +C_MODE_START + +int repl_semi_report_binlog_update(Binlog_storage_param *param, + const char *log_file, + my_off_t log_pos, uint32 flags) +{ + int error= 0; + + if (repl_semisync_master.getMasterEnabled()) + { + /* + Let us store the binlog file name and the position, so that + we know how long to wait for the binlog to the replicated to + the slave in synchronous replication. + */ + error= repl_semisync_master.writeTranxInBinlog(log_file, + log_pos); + } + + return error; +} + +int repl_semi_request_commit(Trans_param *param) +{ + return 0; +} + +int repl_semi_report_binlog_sync(Binlog_storage_param *param, + const char *log_file, + my_off_t log_pos, uint32 flags) +{ + int error= 0; + if (rpl_semi_sync_master_wait_point == + SEMI_SYNC_MASTER_WAIT_POINT_AFTER_BINLOG_SYNC) + { + error= repl_semisync_master.commitTrx(log_file, log_pos); + } + + return error; +} + +int repl_semi_report_commit(Trans_param *param) +{ + if (rpl_semi_sync_master_wait_point != + SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT) + { + return 0; + } + + bool is_real_trans= param->flags & TRANS_IS_REAL_TRANS; + + if (is_real_trans && param->log_pos) + { + const char *binlog_name= param->log_file; + return repl_semisync_master.commitTrx(binlog_name, param->log_pos); + } + return 0; +} + +int repl_semi_report_rollback(Trans_param *param) +{ + return repl_semi_report_commit(param); +} + +int repl_semi_binlog_dump_start(Binlog_transmit_param *param, + const char *log_file, + my_off_t log_pos) +{ + bool semi_sync_slave= repl_semisync_master.is_semi_sync_slave(); + + if (semi_sync_slave) + { + /* One more semi-sync slave */ + repl_semisync_master.add_slave(); + + /* + Let's assume this semi-sync slave has already received all + binlog events before the filename and position it requests. + */ + repl_semisync_master.reportReplyBinlog(param->server_id, log_file, log_pos); + } + sql_print_information("Start %s binlog_dump to slave (server_id: %d), pos(%s, %lu)", + semi_sync_slave ? "semi-sync" : "asynchronous", + param->server_id, log_file, (ulong)log_pos); + + return 0; +} + +int repl_semi_binlog_dump_end(Binlog_transmit_param *param) +{ + bool semi_sync_slave= repl_semisync_master.is_semi_sync_slave(); + + sql_print_information("Stop %s binlog_dump to slave (server_id: %d)", + semi_sync_slave ? "semi-sync" : "asynchronous", + param->server_id); + if (semi_sync_slave) + { + /* One less semi-sync slave */ + repl_semisync_master.remove_slave(); + } + return 0; +} + +int repl_semi_reserve_header(Binlog_transmit_param *param, + unsigned char *header, + ulong size, ulong *len) +{ + *len += repl_semisync_master.reserveSyncHeader(header, size); + return 0; +} + +int repl_semi_before_send_event(Binlog_transmit_param *param, + unsigned char *packet, ulong len, + const char *log_file, my_off_t log_pos) +{ + return repl_semisync_master.updateSyncHeader(packet, + log_file, + log_pos, + param->server_id); +} + +int repl_semi_after_send_event(Binlog_transmit_param *param, + const char *event_buf, ulong len) +{ + if (repl_semisync_master.is_semi_sync_slave()) + { + THD *thd= current_thd; + /* + Possible errors in reading slave reply are ignored deliberately + because we do not want dump thread to quit on this. Error + messages are already reported. + */ + (void) repl_semisync_master.readSlaveReply(&thd->net, + param->server_id, event_buf); + thd->clear_error(); + } + return 0; +} + +int repl_semi_reset_master(Binlog_transmit_param *param) +{ + if (repl_semisync_master.resetMaster()) + return 1; + return 0; +} + +C_MODE_END + +Trans_observer trans_observer= +{ + sizeof(Trans_observer), // len + + repl_semi_report_commit, // after_commit + repl_semi_report_rollback, // after_rollback +}; + +Binlog_storage_observer storage_observer= +{ + sizeof(Binlog_storage_observer), // len + + repl_semi_report_binlog_update, // report_update + repl_semi_report_binlog_sync, // after_sync +}; + +Binlog_transmit_observer transmit_observer= +{ + sizeof(Binlog_transmit_observer), // len + + repl_semi_binlog_dump_start, // start + repl_semi_binlog_dump_end, // stop + repl_semi_reserve_header, // reserve_header + repl_semi_before_send_event, // before_send_event + repl_semi_after_send_event, // after_send_event + repl_semi_reset_master, // reset +}; + +static bool semi_sync_master_inited= 0; + +int semi_sync_master_init() +{ + void *p= 0; + if (repl_semisync_master.initObject()) + return 1; + if (register_trans_observer(&trans_observer, p)) + return 1; + if (register_binlog_storage_observer(&storage_observer, p)) + return 1; + if (register_binlog_transmit_observer(&transmit_observer, p)) + return 1; + semi_sync_master_inited= 1; + return 0; +} + +void semi_sync_master_deinit() +{ + void *p= 0; + if (!semi_sync_master_inited) + return; + + unregister_trans_observer(&trans_observer, p); + unregister_binlog_storage_observer(&storage_observer, p); + unregister_binlog_transmit_observer(&transmit_observer, p); + repl_semisync_master.cleanup(); + semi_sync_master_inited= 0; +} diff --git a/plugin/semisync/semisync_master.h b/sql/semisync_master.h similarity index 93% rename from plugin/semisync/semisync_master.h rename to sql/semisync_master.h index c2862476ec8..ff1e3dd48b4 100644 --- a/plugin/semisync/semisync_master.h +++ b/sql/semisync_master.h @@ -22,8 +22,8 @@ #include "semisync.h" #ifdef HAVE_PSI_INTERFACE -extern PSI_mutex_key key_ss_mutex_LOCK_binlog_; -extern PSI_cond_key key_ss_cond_COND_binlog_send_; +extern PSI_mutex_key key_LOCK_binlog; +extern PSI_cond_key key_COND_binlog_send; #endif extern PSI_stage_info stage_waiting_for_semi_sync_ack_from_slave; @@ -379,14 +379,14 @@ class ReplSemiSyncMaster /* This cond variable is signaled when enough binlog has been sent to slave, * so that a waiting trx can return the 'ok' to the client for a commit. */ - mysql_cond_t COND_binlog_send_; + mysql_cond_t COND_binlog_send; /* Mutex that protects the following state variables and the active * transaction list. * Under no cirumstances we can acquire mysql_bin_log.LOCK_log if we are * already holding LOCK_binlog_ because it can cause deadlocks. */ - mysql_mutex_t LOCK_binlog_; + mysql_mutex_t LOCK_binlog; /* This is set to true when reply_file_name_ contains meaningful data. */ bool reply_file_name_inited_; @@ -600,26 +600,26 @@ enum rpl_semi_sync_master_wait_point_t { }; /* System and status variables for the master component */ -extern char rpl_semi_sync_master_enabled; -extern char rpl_semi_sync_master_status; -extern unsigned long rpl_semi_sync_master_wait_point; -extern unsigned long rpl_semi_sync_master_clients; -extern unsigned long rpl_semi_sync_master_timeout; -extern unsigned long rpl_semi_sync_master_trace_level; -extern unsigned long rpl_semi_sync_master_yes_transactions; -extern unsigned long rpl_semi_sync_master_no_transactions; -extern unsigned long rpl_semi_sync_master_off_times; -extern unsigned long rpl_semi_sync_master_wait_timeouts; -extern unsigned long rpl_semi_sync_master_timefunc_fails; -extern unsigned long rpl_semi_sync_master_num_timeouts; -extern unsigned long rpl_semi_sync_master_wait_sessions; -extern unsigned long rpl_semi_sync_master_wait_pos_backtraverse; -extern unsigned long rpl_semi_sync_master_avg_trx_wait_time; -extern unsigned long rpl_semi_sync_master_avg_net_wait_time; -extern unsigned long long rpl_semi_sync_master_net_wait_num; -extern unsigned long long rpl_semi_sync_master_trx_wait_num; -extern unsigned long long rpl_semi_sync_master_net_wait_time; -extern unsigned long long rpl_semi_sync_master_trx_wait_time; +extern my_bool rpl_semi_sync_master_enabled; +extern my_bool rpl_semi_sync_master_status; +extern ulong rpl_semi_sync_master_wait_point; +extern ulong rpl_semi_sync_master_clients; +extern ulong rpl_semi_sync_master_timeout; +extern ulong rpl_semi_sync_master_trace_level; +extern ulong rpl_semi_sync_master_yes_transactions; +extern ulong rpl_semi_sync_master_no_transactions; +extern ulong rpl_semi_sync_master_off_times; +extern ulong rpl_semi_sync_master_wait_timeouts; +extern ulong rpl_semi_sync_master_timefunc_fails; +extern ulong rpl_semi_sync_master_num_timeouts; +extern ulong rpl_semi_sync_master_wait_sessions; +extern ulong rpl_semi_sync_master_wait_pos_backtraverse; +extern ulong rpl_semi_sync_master_avg_trx_wait_time; +extern ulong rpl_semi_sync_master_avg_net_wait_time; +extern ulonglong rpl_semi_sync_master_net_wait_num; +extern ulonglong rpl_semi_sync_master_trx_wait_num; +extern ulonglong rpl_semi_sync_master_net_wait_time; +extern ulonglong rpl_semi_sync_master_trx_wait_time; /* This indicates whether we should keep waiting if no semi-sync slave @@ -628,5 +628,9 @@ extern unsigned long long rpl_semi_sync_master_trx_wait_time; 1 (default) : keep waiting until timeout even no available semi-sync slave. */ extern char rpl_semi_sync_master_wait_no_slave; +extern ReplSemiSyncMaster repl_semisync_master; + +int semi_sync_master_init(); +void semi_sync_master_deinit(); #endif /* SEMISYNC_MASTER_H */ diff --git a/plugin/semisync/semisync_slave.cc b/sql/semisync_slave.cc similarity index 50% rename from plugin/semisync/semisync_slave.cc rename to sql/semisync_slave.cc index ff8a40aafac..63bf9dca0e8 100644 --- a/plugin/semisync/semisync_slave.cc +++ b/sql/semisync_slave.cc @@ -18,9 +18,20 @@ #include #include "semisync_slave.h" -char rpl_semi_sync_slave_enabled; -char rpl_semi_sync_slave_status= 0; -unsigned long rpl_semi_sync_slave_trace_level; +my_bool rpl_semi_sync_slave_enabled; +my_bool rpl_semi_sync_slave_status= 0; +ulong rpl_semi_sync_slave_trace_level; +ReplSemiSyncSlave repl_semisync_slave; + +/* + indicate whether or not the slave should send a reply to the master. + + This is set to true in repl_semi_slave_read_event if the current + event read is the last event of a transaction. And the value is + checked in repl_semi_slave_queue_event. +*/ +bool semi_sync_need_reply= false; + int ReplSemiSyncSlave::initObject() { @@ -73,7 +84,7 @@ int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, int ReplSemiSyncSlave::slaveStart(Binlog_relay_IO_param *param) { bool semi_sync= getSlaveEnabled(); - + sql_print_information("Slave I/O thread: Start %s replication to\ master '%s@%s:%d' in log '%s' at position %lu", semi_sync ? "semi-sync" : "asynchronous", @@ -138,3 +149,140 @@ int ReplSemiSyncSlave::slaveReply(MYSQL *mysql, return function_exit(kWho, reply_res); } + +/*************************************************************************** + Semisync slave interface setup and deinit +***************************************************************************/ + +C_MODE_START + +int repl_semi_reset_slave(Binlog_relay_IO_param *param) +{ + // TODO: reset semi-sync slave status here + return 0; +} + +int repl_semi_slave_request_dump(Binlog_relay_IO_param *param, + uint32 flags) +{ + MYSQL *mysql= param->mysql; + MYSQL_RES *res= 0; + MYSQL_ROW row; + const char *query; + + if (!repl_semisync_slave.getSlaveEnabled()) + return 0; + + /* Check if master server has semi-sync plugin installed */ + query= "SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled'"; + if (mysql_real_query(mysql, query, strlen(query)) || + !(res= mysql_store_result(mysql))) + { + sql_print_error("Execution failed on master: %s", query); + return 1; + } + + row= mysql_fetch_row(res); + if (!row) + { + /* Master does not support semi-sync */ + sql_print_warning("Master server does not support semi-sync, " + "fallback to asynchronous replication"); + rpl_semi_sync_slave_status= 0; + mysql_free_result(res); + return 0; + } + mysql_free_result(res); + + /* + Tell master dump thread that we want to do semi-sync + replication + */ + query= "SET @rpl_semi_sync_slave= 1"; + if (mysql_real_query(mysql, query, strlen(query))) + { + sql_print_error("Set 'rpl_semi_sync_slave=1' on master failed"); + return 1; + } + mysql_free_result(mysql_store_result(mysql)); + rpl_semi_sync_slave_status= 1; + return 0; +} + +int repl_semi_slave_read_event(Binlog_relay_IO_param *param, + const char *packet, unsigned long len, + const char **event_buf, unsigned long *event_len) +{ + if (rpl_semi_sync_slave_status) + return repl_semisync_slave.slaveReadSyncHeader(packet, len, + &semi_sync_need_reply, + event_buf, event_len); + *event_buf= packet; + *event_len= len; + return 0; +} + +int repl_semi_slave_queue_event(Binlog_relay_IO_param *param, + const char *event_buf, + unsigned long event_len, + uint32 flags) +{ + if (rpl_semi_sync_slave_status && semi_sync_need_reply) + { + /* + We deliberately ignore the error in slaveReply, such error + should not cause the slave IO thread to stop, and the error + messages are already reported. + */ + (void) repl_semisync_slave.slaveReply(param->mysql, + param->master_log_name, + param->master_log_pos); + } + return 0; +} + +int repl_semi_slave_io_start(Binlog_relay_IO_param *param) +{ + return repl_semisync_slave.slaveStart(param); +} + +int repl_semi_slave_io_end(Binlog_relay_IO_param *param) +{ + return repl_semisync_slave.slaveStop(param); +} + +C_MODE_END + +Binlog_relay_IO_observer relay_io_observer= +{ + sizeof(Binlog_relay_IO_observer), // len + + repl_semi_slave_io_start, // start + repl_semi_slave_io_end, // stop + repl_semi_slave_request_dump, // request_transmit + repl_semi_slave_read_event, // after_read_event + repl_semi_slave_queue_event, // after_queue_event + repl_semi_reset_slave, // reset +}; + +static bool semi_sync_slave_inited= 0; + +int semi_sync_slave_init() +{ + void *p= 0; + if (repl_semisync_slave.initObject()) + return 1; + if (register_binlog_relay_io_observer(&relay_io_observer, p)) + return 1; + semi_sync_slave_inited= 1; + return 0; +} + +void semi_sync_slave_deinit() +{ + void *p= 0; + if (!semi_sync_slave_inited) + return; + unregister_binlog_relay_io_observer(&relay_io_observer, p); + semi_sync_slave_inited= 0; +} diff --git a/plugin/semisync/semisync_slave.h b/sql/semisync_slave.h similarity index 92% rename from plugin/semisync/semisync_slave.h rename to sql/semisync_slave.h index 1bf8cf31972..9cca8bbbdb4 100644 --- a/plugin/semisync/semisync_slave.h +++ b/sql/semisync_slave.h @@ -90,8 +90,12 @@ private: /* System and status variables for the slave component */ -extern char rpl_semi_sync_slave_enabled; -extern unsigned long rpl_semi_sync_slave_trace_level; -extern char rpl_semi_sync_slave_status; +extern my_bool rpl_semi_sync_slave_enabled; +extern my_bool rpl_semi_sync_slave_status; +extern ulong rpl_semi_sync_slave_trace_level; +extern ReplSemiSyncSlave repl_semisync_slave; + +int semi_sync_slave_init(); +void semi_sync_slave_deinit(); #endif /* SEMISYNC_SLAVE_H */ diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 82f8359fefc..e897b6c21ce 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -61,6 +61,8 @@ #include "sql_repl.h" #include "opt_range.h" #include "rpl_parallel.h" +#include "semisync_master.h" +#include "semisync_slave.h" #include /* @@ -3039,9 +3041,189 @@ static Sys_var_replicate_events_marked_for_skip Replicate_events_marked_for_skip "the slave).", GLOBAL_VAR(opt_replicate_events_marked_for_skip), CMD_LINE(REQUIRED_ARG), replicate_events_marked_for_skip_names, DEFAULT(RPL_SKIP_REPLICATE)); + +/* new options for semisync */ + +static bool fix_rpl_semi_sync_master_enabled(sys_var *self, THD *thd, + enum_var_type type) +{ + if (rpl_semi_sync_master_enabled) + { + if (repl_semisync_master.enableMaster() != 0) + rpl_semi_sync_master_enabled= false; +#ifdef HAVE_ACC_RECEIVER + else if (ack_receiver.start()) + { + repl_semisync_master.disableMaster(); + rpl_semi_sync_master_enabled= false; + } +#endif + } + else + { + if (repl_semisync_master.disableMaster() != 0) + rpl_semi_sync_master_enabled= true; +#ifdef HAVE_ACC_RECEIVER + if (!rpl_semi_sync_master_enabled) + ack_receiver.stop(); +#endif + } + return false; +} + +static bool fix_rpl_semi_sync_master_timeout(sys_var *self, THD *thd, + enum_var_type type) +{ + repl_semisync_master.setWaitTimeout(rpl_semi_sync_master_timeout); + return false; +} + +static bool fix_rpl_semi_sync_master_trace_level(sys_var *self, THD *thd, + enum_var_type type) +{ + repl_semisync_master.setTraceLevel(rpl_semi_sync_master_trace_level); +#ifdef HAVE_ACC_RECEIVER + ack_receiver.setTraceLevel(rpl_semi_sync_master_trace_level); +#endif + return false; +} + +static bool fix_rpl_semi_sync_master_wait_point(sys_var *self, THD *thd, + enum_var_type type) +{ +#ifdef HAVE_ACC_RECEIVER + repl_semisync_master.setWaitPoint(rpl_semi_sync_master_wait_point); +#endif + return false; +} + +static bool fix_rpl_semi_sync_master_wait_no_slave(sys_var *self, THD *thd, + enum_var_type type) +{ +#ifdef HAVE_ACC_RECEIVER + repl_semisync_master.checkAndSwitch(); +#endif + return false; +} + +static Sys_var_mybool Sys_semisync_master_enabled( + "rpl_semi_sync_master_enabled", + "Enable semi-synchronous replication master (disabled by default).", + GLOBAL_VAR(rpl_semi_sync_master_enabled), + CMD_LINE(OPT_ARG), DEFAULT(FALSE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(fix_rpl_semi_sync_master_enabled)); + +static Sys_var_ulong Sys_semisync_master_timeout( + "rpl_semi_sync_master_timeout", + "The timeout value (in ms) for semi-synchronous replication in the " + "master", + GLOBAL_VAR(rpl_semi_sync_master_timeout), + CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0,~0L),DEFAULT(10000),BLOCK_SIZE(1), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(fix_rpl_semi_sync_master_timeout)); + +static Sys_var_mybool Sys_semisync_master_wait_no_slave( + "rpl_semi_sync_master_wait_no_slave", + "Wait until timeout when no semi-synchronous replication slave " + "available (enabled by default).", + GLOBAL_VAR(rpl_semi_sync_master_wait_no_slave), + CMD_LINE(OPT_ARG), DEFAULT(TRUE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(fix_rpl_semi_sync_master_wait_no_slave)); + +static Sys_var_ulong Sys_semisync_master_trace_level( + "rpl_semi_sync_master_trace_level", + "The tracing level for semi-sync replication.", + GLOBAL_VAR(rpl_semi_sync_master_trace_level), + CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0,~0L),DEFAULT(32),BLOCK_SIZE(1), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(fix_rpl_semi_sync_master_trace_level)); + +static const char *repl_semisync_wait_point[]= +{"AFTER_SYNC", "AFTER_COMMIT", NullS}; + +static Sys_var_enum Sys_semisync_master_wait_point( + "rpl_semi_sync_master_wait_point", + "Should transaction wait for semi-sync ack after having synced binlog, " + "or after having committed in storage engine.", + GLOBAL_VAR(rpl_semi_sync_master_wait_point), CMD_LINE(REQUIRED_ARG), + repl_semisync_wait_point, DEFAULT(1), + NO_MUTEX_GUARD, NOT_IN_BINLOG,ON_CHECK(0), + ON_UPDATE(fix_rpl_semi_sync_master_wait_point)); + +static bool fix_rpl_semi_sync_slave_enabled(sys_var *self, THD *thd, + enum_var_type type) +{ + repl_semisync_slave.setSlaveEnabled(rpl_semi_sync_slave_enabled != 0); + return false; +} + +static bool fix_rpl_semi_sync_slave_trace_level(sys_var *self, THD *thd, + enum_var_type type) +{ + repl_semisync_slave.setTraceLevel(rpl_semi_sync_slave_trace_level); + return false; +} + +#ifdef HAVE_ACC_RECEIVER +static bool fix_rpl_semi_sync_slave_delay_master(sys_var *self, THD *thd, + enum_var_type type) +{ + repl_semisync_slave.setDelayMaster(rpl_semi_sync_slave_delay_master); + return false; +} + +static bool fix_rpl_semi_sync_slave_kill_conn_timeout(sys_var *self, THD *thd, + enum_var_type type) +{ + repl_semisync_slave.setKillConnTimeout(rpl_semi_sync_slave_kill_conn_timeout); + return false; +} #endif +static Sys_var_mybool Sys_semisync_slave_enabled( + "rpl_semi_sync_slave_enabled", + "Enable semi-synchronous replication slave (disabled by default).", + GLOBAL_VAR(rpl_semi_sync_slave_enabled), + CMD_LINE(OPT_ARG), DEFAULT(FALSE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(fix_rpl_semi_sync_slave_enabled)); + +static Sys_var_ulong Sys_semisync_slave_trace_level( + "rpl_semi_sync_slave_trace_level", + "The tracing level for semi-sync replication.", + GLOBAL_VAR(rpl_semi_sync_slave_trace_level), + CMD_LINE(REQUIRED_ARG), + VALID_RANGE(0,~0L),DEFAULT(32),BLOCK_SIZE(1), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(fix_rpl_semi_sync_slave_trace_level)); + +#ifdef HAVE_ACC_RECEIVER +static Sys_var_mybool Sys_semisync_slave_delay_master( + "rpl_semi_sync_slave_delay_master", + "Only write master info file when ack is needed.", + GLOBAL_VAR(rpl_semi_sync_slave_delay_master), + CMD_LINE(OPT_ARG), DEFAULT(FALSE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(fix_rpl_semi_sync_slave_delay_master)); + +static Sys_var_uint Sys_semisync_slave_kill_conn_timeout( + "rpl_semi_sync_slave_kill_conn_timeout", + "Timeout for the mysql connection used to kill the slave io_thread's " + "connection on master. This timeout comes into play when stop slave " + "is executed.", + GLOBAL_VAR(rpl_semi_sync_slave_kill_conn_timeout), + CMD_LINE(OPT_ARG), + VALID_RANGE(0, UINT_MAX), DEFAULT(5), BLOCK_SIZE(1), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), + ON_UPDATE(fix_rpl_semi_sync_slave_kill_conn_timeout)); +#endif +#endif /* HAVE_REPLICATION */ + static Sys_var_ulong Sys_slow_launch_time( "slow_launch_time", "If creating the thread takes longer than this value (in seconds), " From ea37c129f968593e48170e489788cc481fe9a2bd Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 26 Oct 2017 12:46:45 +0300 Subject: [PATCH 18/51] Removed not used lock argument from read_log_event --- client/mysqlbinlog.cc | 6 ++-- sql/log.cc | 4 +-- sql/log_event.cc | 8 +---- sql/log_event.h | 1 - sql/rpl_parallel.cc | 2 +- sql/rpl_rli.cc | 2 +- sql/slave.cc | 2 +- sql/sql_repl.cc | 71 +++++++++++++++++++++++++++++++++++++++++-- 8 files changed, 78 insertions(+), 18 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index cca3fba5325..d7ffadb1dc6 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -2736,7 +2736,7 @@ static Exit_status check_header(IO_CACHE* file, Format_description_log_event *new_description_event; my_b_seek(file, tmp_pos); /* seek back to event's start */ if (!(new_description_event= (Format_description_log_event*) - Log_event::read_log_event(file, 0, glob_description_event, + Log_event::read_log_event(file, glob_description_event, opt_verify_binlog_checksum))) /* EOF can't be hit here normally, so it's a real error */ { @@ -2770,7 +2770,7 @@ static Exit_status check_header(IO_CACHE* file, { Log_event *ev; my_b_seek(file, tmp_pos); /* seek back to event's start */ - if (!(ev= Log_event::read_log_event(file, 0, glob_description_event, + if (!(ev= Log_event::read_log_event(file, glob_description_event, opt_verify_binlog_checksum))) { /* EOF can't be hit here normally, so it's a real error */ @@ -2884,7 +2884,7 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, char llbuff[21]; my_off_t old_off = my_b_tell(file); - Log_event* ev = Log_event::read_log_event(file, 0, glob_description_event, + Log_event* ev = Log_event::read_log_event(file, glob_description_event, opt_verify_binlog_checksum); if (!ev) { diff --git a/sql/log.cc b/sql/log.cc index e5ff85e4544..2d29f7d626a 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -9918,7 +9918,7 @@ int TC_LOG_BINLOG::recover(LOG_INFO *linfo, const char *last_log_name, for (;;) { while ((ev= Log_event::read_log_event(first_round ? first_log : &log, - 0, fdle, opt_master_verify_checksum)) + fdle, opt_master_verify_checksum)) && ev->is_valid()) { enum Log_event_type typ= ev->get_type_code(); @@ -10159,7 +10159,7 @@ MYSQL_BIN_LOG::do_binlog_recovery(const char *opt_name, bool do_xa_recovery) return 1; } - if ((ev= Log_event::read_log_event(&log, 0, &fdle, + if ((ev= Log_event::read_log_event(&log, &fdle, opt_master_verify_checksum)) && ev->get_type_code() == FORMAT_DESCRIPTION_EVENT) { diff --git a/sql/log_event.cc b/sql/log_event.cc index f1ceaec6456..7fe00b2bf43 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -1854,7 +1854,7 @@ int Log_event::read_log_event(IO_CACHE* file, String* packet, DBUG_RETURN(0); } -Log_event* Log_event::read_log_event(IO_CACHE* file, mysql_mutex_t* log_lock, +Log_event* Log_event::read_log_event(IO_CACHE* file, const Format_description_log_event *fdle, my_bool crc_check) { @@ -1864,9 +1864,6 @@ Log_event* Log_event::read_log_event(IO_CACHE* file, mysql_mutex_t* log_lock, const char *error= 0; Log_event *res= 0; - if (log_lock) - mysql_mutex_lock(log_lock); - switch (read_log_event(file, &event, fdle, BINLOG_CHECKSUM_ALG_OFF)) { case 0: @@ -1903,8 +1900,6 @@ Log_event* Log_event::read_log_event(IO_CACHE* file, mysql_mutex_t* log_lock, res->register_temp_buf(event.release(), true); err: - if (log_lock) - mysql_mutex_unlock(log_lock); if (error) { DBUG_ASSERT(!res); @@ -9735,7 +9730,6 @@ int Execute_load_log_event::do_apply_event(rpl_group_info *rgi) } if (!(lev= (Load_log_event*) Log_event::read_log_event(&file, - (mysql_mutex_t*)0, rli->relay_log.description_event_for_exec, opt_slave_sql_verify_checksum)) || lev->get_type_code() != NEW_LOAD_EVENT) diff --git a/sql/log_event.h b/sql/log_event.h index c8f3241cb3d..9fb14f0eae1 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -1300,7 +1300,6 @@ public: constructor and pass description_event as an argument. */ static Log_event* read_log_event(IO_CACHE* file, - mysql_mutex_t* log_lock, const Format_description_log_event *description_event, my_bool crc_check); diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 4a6e813d73b..2c5ef76681c 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -823,7 +823,7 @@ do_retry: for (;;) { old_offset= cur_offset; - ev= Log_event::read_log_event(&rlog, 0, description_event, + ev= Log_event::read_log_event(&rlog, description_event, opt_slave_sql_verify_checksum); cur_offset= my_b_tell(&rlog); diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index efb256fbe11..28835118fdd 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -538,7 +538,7 @@ read_relay_log_description_event(IO_CACHE *cur_log, ulonglong start_pos, if (my_b_tell(cur_log) >= start_pos) break; - if (!(ev= Log_event::read_log_event(cur_log, 0, fdev, + if (!(ev= Log_event::read_log_event(cur_log, fdev, opt_slave_sql_verify_checksum))) { DBUG_PRINT("info",("could not read event, cur_log->error=%d", diff --git a/sql/slave.cc b/sql/slave.cc index a57312998f1..e54e613ae16 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -7308,7 +7308,7 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) MYSQL_BIN_LOG::open() will write the buffered description event. */ old_pos= rli->event_relay_log_pos; - if ((ev= Log_event::read_log_event(cur_log,0, + if ((ev= Log_event::read_log_event(cur_log, rli->relay_log.description_event_for_exec, opt_slave_sql_verify_checksum))) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 08e9dcf3fe6..d85d984606d 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -875,6 +875,73 @@ get_binlog_list(MEM_ROOT *memroot) DBUG_RETURN(current_list); } +/* + Find the Gtid_list_log_event at the start of a binlog. + + NULL for ok, non-NULL error message for error. + + If ok, then the event is returned in *out_gtid_list. This can be NULL if we + get back to binlogs written by old server version without GTID support. If + so, it means we have reached the point to start from, as no GTID events can + exist in earlier binlogs. +*/ + +static const char * +get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list) +{ + Format_description_log_event init_fdle(BINLOG_VERSION); + Format_description_log_event *fdle; + Log_event *ev; + const char *errormsg = NULL; + + *out_gtid_list= NULL; + + if (!(ev= Log_event::read_log_event(cache, &init_fdle, + opt_master_verify_checksum)) || + ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) + { + if (ev) + delete ev; + return "Could not read format description log event while looking for " + "GTID position in binlog"; + } + + fdle= static_cast(ev); + + for (;;) + { + Log_event_type typ; + + ev= Log_event::read_log_event(cache, fdle, opt_master_verify_checksum); + if (!ev) + { + errormsg= "Could not read GTID list event while looking for GTID " + "position in binlog"; + break; + } + typ= ev->get_type_code(); + if (typ == GTID_LIST_EVENT) + break; /* Done, found it */ + if (typ == START_ENCRYPTION_EVENT) + { + if (fdle->start_decryption((Start_encryption_log_event*) ev)) + errormsg= "Could not set up decryption for binlog."; + } + delete ev; + if (typ == ROTATE_EVENT || typ == STOP_EVENT || + typ == FORMAT_DESCRIPTION_EVENT || typ == START_ENCRYPTION_EVENT) + continue; /* Continue looking */ + + /* We did not find any Gtid_list_log_event, must be old binlog. */ + ev= NULL; + break; + } + + delete fdle; + *out_gtid_list= static_cast(ev); + return errormsg; +} + /* Check if every GTID requested by the slave is contained in this (or a later) @@ -3930,7 +3997,7 @@ bool mysql_show_binlog_events(THD* thd) my_off_t scan_pos = BIN_LOG_HEADER_SIZE; while (scan_pos < pos) { - ev= Log_event::read_log_event(&log, (mysql_mutex_t*)0, description_event, + ev= Log_event::read_log_event(&log, description_event, opt_master_verify_checksum); scan_pos = my_b_tell(&log); if (ev == NULL || !ev->is_valid()) @@ -3964,7 +4031,7 @@ bool mysql_show_binlog_events(THD* thd) my_b_seek(&log, pos); for (event_count = 0; - (ev = Log_event::read_log_event(&log, (mysql_mutex_t*) 0, + (ev = Log_event::read_log_event(&log, description_event, opt_master_verify_checksum)); ) { From 13770edbcb181f06315dab659b592edf6355efc6 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 26 Oct 2017 17:20:20 +0300 Subject: [PATCH 19/51] Changed from using LOCK_log to LOCK_binlog_end_pos for binary log Part of MDEV-13073 AliSQL Optimize performance of semisync The idea it to use a dedicated lock detecting if there is new data in the master's binary log instead of the overused LOCK_log. Changes: - Use dedicated COND variables for the relay and binary log signaling. This was needed as we where the old 'update_cond' variable was used with different mutex's, which could cause deadlocks. - Relay log uses now COND_relay_log_updated and LOCK_log - Binary log uses now COND_bin_log_updated and LOCK_binlog_end_pos - Renamed signal_cnt to relay_signal_cnt (as we now have two signals) - Added some missing error handling in MYSQL_BIN_LOG::new_file_impl() - Reformatted some comments with old style - Renamed m_key_LOCK_binlog_end_pos to key_LOCK_binlog_end_pos - Changed 'signal_update()' to update_binlog_end_pos() which works for both relay and binary log --- mysql-test/suite/perfschema/r/relaylog.result | 8 ++ sql/log.cc | 111 +++++++++++------- sql/log.h | 85 ++++++++++---- sql/mysqld.cc | 18 +-- sql/mysqld.h | 5 +- sql/rpl_parallel.cc | 4 +- sql/rpl_rli.cc | 3 +- sql/slave.cc | 5 +- sql/sql_repl.cc | 2 +- 9 files changed, 158 insertions(+), 83 deletions(-) diff --git a/mysql-test/suite/perfschema/r/relaylog.result b/mysql-test/suite/perfschema/r/relaylog.result index 8dcecf0dc9c..b8655a781eb 100644 --- a/mysql-test/suite/perfschema/r/relaylog.result +++ b/mysql-test/suite/perfschema/r/relaylog.result @@ -62,7 +62,9 @@ where event_name like "%MYSQL_BIN_LOG%" and event_name not like "%MYSQL_BIN_LOG::COND_xid_list" order by event_name; EVENT_NAME COUNT_STAR +wait/synch/cond/sql/MYSQL_BIN_LOG::COND_bin_log_updated MANY wait/synch/cond/sql/MYSQL_BIN_LOG::COND_queue_busy NONE +wait/synch/cond/sql/MYSQL_BIN_LOG::COND_relay_log_updated NONE wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_binlog_background_thread MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_binlog_end_pos MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_index MANY @@ -81,7 +83,9 @@ where event_name like "%MYSQL_RELAY_LOG%" and event_name not like "%MYSQL_RELAY_LOG::update_cond" order by event_name; EVENT_NAME COUNT_STAR SUM_TIMER_WAIT MIN_TIMER_WAIT AVG_TIMER_WAIT MAX_TIMER_WAIT +wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_bin_log_updated 0 0 0 0 0 wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_queue_busy 0 0 0 0 0 +wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_relay_log_updated 0 0 0 0 0 wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_index 0 0 0 0 0 connection slave; "============ Performance schema on slave ============" @@ -142,7 +146,9 @@ where event_name like "%MYSQL_BIN_LOG%" and event_name not like "%MYSQL_BIN_LOG::COND_xid_list" order by event_name; EVENT_NAME COUNT_STAR +wait/synch/cond/sql/MYSQL_BIN_LOG::COND_bin_log_updated NONE wait/synch/cond/sql/MYSQL_BIN_LOG::COND_queue_busy NONE +wait/synch/cond/sql/MYSQL_BIN_LOG::COND_relay_log_updated NONE wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_binlog_background_thread MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_binlog_end_pos MANY wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_index MANY @@ -184,6 +190,8 @@ where event_name like "%MYSQL_RELAY_LOG%" and event_name not like "%MYSQL_RELAY_LOG::update_cond" order by event_name; EVENT_NAME COUNT_STAR +wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_bin_log_updated NONE wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_queue_busy NONE +wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_relay_log_updated MANY wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_index MANY include/stop_slave.inc diff --git a/sql/log.cc b/sql/log.cc index 2d29f7d626a..4f69e341ad7 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -3211,7 +3211,7 @@ MYSQL_BIN_LOG::MYSQL_BIN_LOG(uint *sync_period) group_commit_trigger_lock_wait(0), sync_period_ptr(sync_period), sync_counter(0), state_file_deleted(false), binlog_state_recover_done(false), - is_relay_log(0), signal_cnt(0), + is_relay_log(0), relay_signal_cnt(0), checksum_alg_reset(BINLOG_CHECKSUM_ALG_UNDEF), relay_log_checksum_alg(BINLOG_CHECKSUM_ALG_UNDEF), description_event_for_exec(0), description_event_for_queue(0), @@ -3281,7 +3281,8 @@ void MYSQL_BIN_LOG::cleanup() mysql_mutex_destroy(&LOCK_xid_list); mysql_mutex_destroy(&LOCK_binlog_background_thread); mysql_mutex_destroy(&LOCK_binlog_end_pos); - mysql_cond_destroy(&update_cond); + mysql_cond_destroy(&COND_relay_log_updated); + mysql_cond_destroy(&COND_bin_log_updated); mysql_cond_destroy(&COND_queue_busy); mysql_cond_destroy(&COND_xid_list); mysql_cond_destroy(&COND_binlog_background_thread); @@ -3316,7 +3317,8 @@ void MYSQL_BIN_LOG::init_pthread_objects() mysql_mutex_setflags(&LOCK_index, MYF_NO_DEADLOCK_DETECTION); mysql_mutex_init(key_BINLOG_LOCK_xid_list, &LOCK_xid_list, MY_MUTEX_INIT_FAST); - mysql_cond_init(m_key_update_cond, &update_cond, 0); + mysql_cond_init(m_key_relay_log_update, &COND_relay_log_updated, 0); + mysql_cond_init(m_key_bin_log_update, &COND_bin_log_updated, 0); mysql_cond_init(m_key_COND_queue_busy, &COND_queue_busy, 0); mysql_cond_init(key_BINLOG_COND_xid_list, &COND_xid_list, 0); @@ -3327,7 +3329,7 @@ void MYSQL_BIN_LOG::init_pthread_objects() mysql_cond_init(key_BINLOG_COND_binlog_background_thread_end, &COND_binlog_background_thread_end, 0); - mysql_mutex_init(m_key_LOCK_binlog_end_pos, &LOCK_binlog_end_pos, + mysql_mutex_init(key_LOCK_binlog_end_pos, &LOCK_binlog_end_pos, MY_MUTEX_INIT_SLOW); } @@ -3802,6 +3804,11 @@ bool MYSQL_BIN_LOG::open(const char *log_name, close_purge_index_file(); #endif + /* Notify the io thread that binlog is rotated to a new file */ + if (is_relay_log) + signal_relay_log_update(); + else + update_binlog_end_pos(); DBUG_RETURN(0); err: @@ -5112,7 +5119,12 @@ int MYSQL_BIN_LOG::new_file_impl(bool need_lock) new file name in the current binary log file. */ if ((error= generate_new_name(new_name, name, 0))) + { +#ifdef ENABLE_AND_FIX_HANG + close_on_error= TRUE; +#endif goto end; + } new_name_ptr=new_name; if (log_type == LOG_BIN) @@ -5143,13 +5155,20 @@ int MYSQL_BIN_LOG::new_file_impl(bool need_lock) } bytes_written += r.data_written; } - /* - Update needs to be signalled even if there is no rotate event - log rotation should give the waiting thread a signal to - discover EOF and move on to the next log. - */ - signal_update(); } + + /* + Update needs to be signalled even if there is no rotate event + log rotation should give the waiting thread a signal to + discover EOF and move on to the next log. + */ + if ((error= flush_io_cache(&log_file))) + { + close_on_error= TRUE; + goto end; + } + update_binlog_end_pos(); + old_name=name; name=0; // Don't free name close_flag= LOG_CLOSE_TO_BE_OPENED | LOG_CLOSE_INDEX; @@ -5231,7 +5250,6 @@ end: close(LOG_CLOSE_INDEX); sql_print_error(fatal_log_error, new_name_ptr, errno); } - mysql_mutex_unlock(&LOCK_index); if (need_lock) mysql_mutex_unlock(&LOCK_log); @@ -5280,7 +5298,7 @@ bool MYSQL_BIN_LOG::append_no_lock(Log_event* ev) if (my_b_append_tell(&log_file) > max_size) error= new_file_without_locking(); err: - signal_update(); // Safe as we don't call close + update_binlog_end_pos(); DBUG_RETURN(error); } @@ -5341,7 +5359,7 @@ bool MYSQL_BIN_LOG::write_event_buffer(uchar* buf, uint len) err: my_safe_afree(ebuf, len); if (!error) - signal_update(); + update_binlog_end_pos(); DBUG_RETURN(error); } @@ -6341,6 +6359,7 @@ err: { my_off_t offset= my_b_tell(file); bool check_purge= false; + DBUG_ASSERT(!is_relay_log); if (!error) { @@ -6366,14 +6385,13 @@ err: } else { - /* update binlog_end_pos so it can be read by dump thread - * - * note: must be _after_ the RUN_HOOK(after_flush) or else - * semi-sync-plugin might not have put the transaction into - * it's list before dump-thread tries to send it - */ + /* + update binlog_end_pos so it can be read by dump thread + note: must be _after_ the RUN_HOOK(after_flush) or else + semi-sync might not have put the transaction into + it's list before dump-thread tries to send it + */ update_binlog_end_pos(offset); - if ((error= rotate(false, &check_purge))) check_purge= false; } @@ -7099,7 +7117,7 @@ bool MYSQL_BIN_LOG::write_incident(THD *thd) if (!(error= write_incident_already_locked(thd)) && !(error= flush_and_sync(0))) { - signal_update(); + update_binlog_end_pos(); if ((error= rotate(false, &check_purge))) check_purge= false; } @@ -7140,7 +7158,7 @@ MYSQL_BIN_LOG::write_binlog_checkpoint_event_already_locked(const char *name_arg */ if (!write_event(&ev) && !flush_and_sync(0)) { - signal_update(); + update_binlog_end_pos(); } else { @@ -7839,12 +7857,12 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) first= false; } - /* update binlog_end_pos so it can be read by dump thread - * - * note: must be _after_ the RUN_HOOK(after_flush) or else - * semi-sync-plugin might not have put the transaction into - * it's list before dump-thread tries to send it - */ + /* + update binlog_end_pos so it can be read by dump thread + Note: must be _after_ the RUN_HOOK(after_flush) or else + semi-sync might not have put the transaction into + it's list before dump-thread tries to send it + */ update_binlog_end_pos(commit_offset); if (any_error) @@ -8228,10 +8246,10 @@ void MYSQL_BIN_LOG::wait_for_update_relay_log(THD* thd) DBUG_ENTER("wait_for_update_relay_log"); mysql_mutex_assert_owner(&LOCK_log); - thd->ENTER_COND(&update_cond, &LOCK_log, + thd->ENTER_COND(&COND_relay_log_updated, &LOCK_log, &stage_slave_has_read_all_relay_log, &old_stage); - mysql_cond_wait(&update_cond, &LOCK_log); + mysql_cond_wait(&COND_relay_log_updated, &LOCK_log); thd->EXIT_COND(&old_stage); DBUG_VOID_RETURN; } @@ -8252,6 +8270,24 @@ void MYSQL_BIN_LOG::wait_for_update_relay_log(THD* thd) LOCK_log is released by the caller. */ +int MYSQL_BIN_LOG::wait_for_update_bin_log(THD* thd, + const struct timespec *timeout) +{ + int ret= 0; + DBUG_ENTER("wait_for_update_bin_log"); + + thd_wait_begin(thd, THD_WAIT_BINLOG); + mysql_mutex_assert_owner(&LOCK_binlog_end_pos); + if (!timeout) + mysql_cond_wait(&COND_bin_log_updated, &LOCK_binlog_end_pos); + else + ret= mysql_cond_timedwait(&COND_bin_log_updated, &LOCK_binlog_end_pos, + const_cast(timeout)); + thd_wait_end(thd); + DBUG_RETURN(ret); +} + + int MYSQL_BIN_LOG::wait_for_update_binlog_end_pos(THD* thd, struct timespec *timeout) { @@ -8261,9 +8297,9 @@ int MYSQL_BIN_LOG::wait_for_update_binlog_end_pos(THD* thd, thd_wait_begin(thd, THD_WAIT_BINLOG); mysql_mutex_assert_owner(get_binlog_end_pos_lock()); if (!timeout) - mysql_cond_wait(&update_cond, get_binlog_end_pos_lock()); + mysql_cond_wait(&COND_bin_log_updated, get_binlog_end_pos_lock()); else - ret= mysql_cond_timedwait(&update_cond, get_binlog_end_pos_lock(), + ret= mysql_cond_timedwait(&COND_bin_log_updated, get_binlog_end_pos_lock(), timeout); thd_wait_end(thd); DBUG_RETURN(ret); @@ -8308,7 +8344,8 @@ void MYSQL_BIN_LOG::close(uint exiting) relay_log_checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF); write_event(&s); bytes_written+= s.data_written; - signal_update(); + flush_io_cache(&log_file); + update_binlog_end_pos(); /* When we shut down server, write out the binlog state to a separate @@ -8527,14 +8564,6 @@ bool flush_error_log() return result; } -void MYSQL_BIN_LOG::signal_update() -{ - DBUG_ENTER("MYSQL_BIN_LOG::signal_update"); - signal_cnt++; - mysql_cond_broadcast(&update_cond); - DBUG_VOID_RETURN; -} - #ifdef _WIN32 static void print_buffer_to_nt_eventlog(enum loglevel level, char *buff, size_t length, size_t buffLen) diff --git a/sql/log.h b/sql/log.h index dffb6a80d54..0a82f8813f0 100644 --- a/sql/log.h +++ b/sql/log.h @@ -425,8 +425,10 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG #ifdef HAVE_PSI_INTERFACE /** The instrumentation key to use for @ LOCK_index. */ PSI_mutex_key m_key_LOCK_index; - /** The instrumentation key to use for @ update_cond. */ - PSI_cond_key m_key_update_cond; + /** The instrumentation key to use for @ COND_relay_log_updated */ + PSI_cond_key m_key_relay_log_update; + /** The instrumentation key to use for @ COND_bin_log_updated */ + PSI_cond_key m_key_bin_log_update; /** The instrumentation key to use for opening the log file. */ PSI_file_key m_key_file_log; /** The instrumentation key to use for opening the log index file. */ @@ -488,7 +490,7 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG mysql_mutex_t LOCK_binlog_end_pos; mysql_mutex_t LOCK_xid_list; mysql_cond_t COND_xid_list; - mysql_cond_t update_cond; + mysql_cond_t COND_relay_log_updated, COND_bin_log_updated; ulonglong bytes_written; IO_CACHE index_file; char index_file_name[FN_REFLEN]; @@ -598,7 +600,7 @@ public: /* This is relay log */ bool is_relay_log; - ulong signal_cnt; // update of the counter is checked by heartbeat + ulong relay_signal_cnt; // update of the counter is checked by heartbeat enum enum_binlog_checksum_alg checksum_alg_reset; // to contain a new value when binlog is rotated /* Holds the last seen in Relay-Log FD's checksum alg value. @@ -661,13 +663,15 @@ public: #ifdef HAVE_PSI_INTERFACE void set_psi_keys(PSI_mutex_key key_LOCK_index, - PSI_cond_key key_update_cond, + PSI_cond_key key_relay_log_update, + PSI_cond_key key_bin_log_update, PSI_file_key key_file_log, PSI_file_key key_file_log_index, PSI_file_key key_COND_queue_busy) { m_key_LOCK_index= key_LOCK_index; - m_key_update_cond= key_update_cond; + m_key_relay_log_update= key_relay_log_update; + m_key_bin_log_update= key_bin_log_update; m_key_file_log= key_file_log; m_key_file_log_index= key_file_log_index; m_key_COND_queue_busy= key_COND_queue_busy; @@ -707,7 +711,53 @@ public: DBUG_VOID_RETURN; } void set_max_size(ulong max_size_arg); - void signal_update(); + + /* Handle signaling that relay has been updated */ + void signal_relay_log_update() + { + mysql_mutex_assert_owner(&LOCK_log); + DBUG_ASSERT(is_relay_log); + DBUG_ENTER("MYSQL_BIN_LOG::signal_relay_log_update"); + relay_signal_cnt++; + mysql_cond_broadcast(&COND_relay_log_updated); + DBUG_VOID_RETURN; + } + void signal_bin_log_update() + { + mysql_mutex_assert_owner(&LOCK_binlog_end_pos); + DBUG_ASSERT(!is_relay_log); + DBUG_ENTER("MYSQL_BIN_LOG::signal_bin_log_update"); + mysql_cond_broadcast(&COND_bin_log_updated); + DBUG_VOID_RETURN; + } + void update_binlog_end_pos() + { + if (is_relay_log) + signal_relay_log_update(); + else + { + lock_binlog_end_pos(); + binlog_end_pos= my_b_safe_tell(&log_file); + signal_bin_log_update(); + unlock_binlog_end_pos(); + } + } + void update_binlog_end_pos(my_off_t pos) + { + mysql_mutex_assert_owner(&LOCK_log); + mysql_mutex_assert_not_owner(&LOCK_binlog_end_pos); + lock_binlog_end_pos(); + /* + Note: it would make more sense to assert(pos > binlog_end_pos) + but there are two places triggered by mtr that has pos == binlog_end_pos + i didn't investigate but accepted as it should do no harm + */ + DBUG_ASSERT(pos >= binlog_end_pos); + binlog_end_pos= pos; + signal_bin_log_update(); + unlock_binlog_end_pos(); + } + void wait_for_sufficient_commits(); void binlog_trigger_immediate_group_commit(); void wait_for_update_relay_log(THD* thd); @@ -807,7 +857,7 @@ public: inline char* get_log_fname() { return log_file_name; } inline char* get_name() { return name; } inline mysql_mutex_t* get_log_lock() { return &LOCK_log; } - inline mysql_cond_t* get_log_cond() { return &update_cond; } + inline mysql_cond_t* get_bin_log_cond() { return &COND_bin_log_updated; } inline IO_CACHE* get_log_file() { return &log_file; } inline void lock_index() { mysql_mutex_lock(&LOCK_index);} @@ -831,23 +881,6 @@ public: bool check_strict_gtid_sequence(uint32 domain_id, uint32 server_id, uint64 seq_no); - - void update_binlog_end_pos(my_off_t pos) - { - mysql_mutex_assert_owner(&LOCK_log); - mysql_mutex_assert_not_owner(&LOCK_binlog_end_pos); - lock_binlog_end_pos(); - /** - * note: it would make more sense to assert(pos > binlog_end_pos) - * but there are two places triggered by mtr that has pos == binlog_end_pos - * i didn't investigate but accepted as it should do no harm - */ - DBUG_ASSERT(pos >= binlog_end_pos); - binlog_end_pos= pos; - signal_update(); - unlock_binlog_end_pos(); - } - /** * used when opening new file, and binlog_end_pos moves backwards */ @@ -858,7 +891,7 @@ public: lock_binlog_end_pos(); binlog_end_pos= pos; strcpy(binlog_end_pos_file, file_name); - signal_update(); + signal_bin_log_update(); unlock_binlog_end_pos(); } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index fc783ae5559..5f675a76ecf 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -911,7 +911,7 @@ PSI_mutex_key key_LOCK_des_key_file; PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_BINLOG_LOCK_binlog_background_thread, - m_key_LOCK_binlog_end_pos, + key_LOCK_binlog_end_pos, key_delayed_insert_mutex, key_hash_filo_lock, key_LOCK_active_mi, key_LOCK_connection_count, key_LOCK_crypt, key_LOCK_delayed_create, key_LOCK_delayed_insert, key_LOCK_delayed_status, key_LOCK_error_log, @@ -965,7 +965,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_BINLOG_LOCK_index, "MYSQL_BIN_LOG::LOCK_index", 0}, { &key_BINLOG_LOCK_xid_list, "MYSQL_BIN_LOG::LOCK_xid_list", 0}, { &key_BINLOG_LOCK_binlog_background_thread, "MYSQL_BIN_LOG::LOCK_binlog_background_thread", 0}, - { &m_key_LOCK_binlog_end_pos, "MYSQL_BIN_LOG::LOCK_binlog_end_pos", 0 }, + { &key_LOCK_binlog_end_pos, "MYSQL_BIN_LOG::LOCK_binlog_end_pos", 0 }, { &key_RELAYLOG_LOCK_index, "MYSQL_RELAY_LOG::LOCK_index", 0}, { &key_delayed_insert_mutex, "Delayed_insert::mutex", 0}, { &key_hash_filo_lock, "hash_filo::lock", 0}, @@ -1051,7 +1051,8 @@ static PSI_rwlock_info all_server_rwlocks[]= PSI_cond_key key_PAGE_cond, key_COND_active, key_COND_pool; #endif /* HAVE_MMAP */ -PSI_cond_key key_BINLOG_COND_xid_list, key_BINLOG_update_cond, +PSI_cond_key key_BINLOG_COND_xid_list, + key_BINLOG_COND_bin_log_updated, key_BINLOG_COND_relay_log_updated, key_BINLOG_COND_binlog_background_thread, key_BINLOG_COND_binlog_background_thread_end, key_COND_cache_status_changed, key_COND_manager, @@ -1067,7 +1068,8 @@ PSI_cond_key key_BINLOG_COND_xid_list, key_BINLOG_update_cond, key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache, key_COND_start_thread, key_COND_binlog_send, key_BINLOG_COND_queue_busy; -PSI_cond_key key_RELAYLOG_update_cond, key_COND_wakeup_ready, +PSI_cond_key key_RELAYLOG_COND_relay_log_updated, + key_RELAYLOG_COND_bin_log_updated, key_COND_wakeup_ready, key_COND_wait_commit; PSI_cond_key key_RELAYLOG_COND_queue_busy; PSI_cond_key key_TC_LOG_MMAP_COND_queue_busy; @@ -1088,12 +1090,13 @@ static PSI_cond_info all_server_conds[]= { &key_COND_pool, "TC_LOG_MMAP::COND_pool", 0}, { &key_TC_LOG_MMAP_COND_queue_busy, "TC_LOG_MMAP::COND_queue_busy", 0}, #endif /* HAVE_MMAP */ + { &key_BINLOG_COND_bin_log_updated, "MYSQL_BIN_LOG::COND_bin_log_updated", 0}, { &key_BINLOG_COND_relay_log_updated, "MYSQL_BIN_LOG::COND_relay_log_updated", 0}, { &key_BINLOG_COND_xid_list, "MYSQL_BIN_LOG::COND_xid_list", 0}, - { &key_BINLOG_update_cond, "MYSQL_BIN_LOG::update_cond", 0}, { &key_BINLOG_COND_binlog_background_thread, "MYSQL_BIN_LOG::COND_binlog_background_thread", 0}, { &key_BINLOG_COND_binlog_background_thread_end, "MYSQL_BIN_LOG::COND_binlog_background_thread_end", 0}, { &key_BINLOG_COND_queue_busy, "MYSQL_BIN_LOG::COND_queue_busy", 0}, - { &key_RELAYLOG_update_cond, "MYSQL_RELAY_LOG::update_cond", 0}, + { &key_RELAYLOG_COND_relay_log_updated, "MYSQL_RELAY_LOG::COND_relay_log_updated", 0}, + { &key_RELAYLOG_COND_bin_log_updated, "MYSQL_RELAY_LOG::COND_bin_log_updated", 0}, { &key_RELAYLOG_COND_queue_busy, "MYSQL_RELAY_LOG::COND_queue_busy", 0}, { &key_COND_wakeup_ready, "THD::COND_wakeup_ready", 0}, { &key_COND_wait_commit, "wait_for_commit::COND_wait_commit", 0}, @@ -4244,7 +4247,8 @@ static int init_common_variables() constructor (called before main()). */ mysql_bin_log.set_psi_keys(key_BINLOG_LOCK_index, - key_BINLOG_update_cond, + key_BINLOG_COND_relay_log_updated, + key_BINLOG_COND_bin_log_updated, key_file_binlog, key_file_binlog_index, key_BINLOG_COND_queue_busy); diff --git a/sql/mysqld.h b/sql/mysqld.h index 1ca193310cc..f72132820fe 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -289,7 +289,7 @@ extern PSI_mutex_key key_LOCK_des_key_file; extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_BINLOG_LOCK_binlog_background_thread, - m_key_LOCK_binlog_end_pos, + key_LOCK_binlog_end_pos, key_delayed_insert_mutex, key_hash_filo_lock, key_LOCK_active_mi, key_LOCK_connection_count, key_LOCK_crypt, key_LOCK_delayed_create, key_LOCK_delayed_insert, key_LOCK_delayed_status, key_LOCK_error_log, @@ -340,7 +340,8 @@ extern PSI_cond_key key_BINLOG_COND_xid_list, key_BINLOG_update_cond, key_TABLE_SHARE_cond, key_user_level_lock_cond, key_COND_start_thread, key_COND_thread_count, key_COND_thread_cache, key_COND_flush_thread_cache; -extern PSI_cond_key key_RELAYLOG_update_cond, key_COND_wakeup_ready, +extern PSI_cond_key key_RELAYLOG_COND_relay_log_updated, + key_RELAYLOG_COND_bin_log_updated, key_COND_wakeup_ready, key_COND_wait_commit; extern PSI_cond_key key_RELAYLOG_COND_queue_busy; extern PSI_cond_key key_TC_LOG_MMAP_COND_queue_busy; diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 2c5ef76681c..9ebd19a90e2 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -255,10 +255,8 @@ signal_error_to_sql_driver_thread(THD *thd, rpl_group_info *rgi, int err) rgi->rli->abort_slave= true; rgi->rli->stop_for_until= false; mysql_mutex_lock(rgi->rli->relay_log.get_log_lock()); + rgi->rli->relay_log.signal_relay_log_update(); mysql_mutex_unlock(rgi->rli->relay_log.get_log_lock()); - rgi->rli->relay_log.lock_binlog_end_pos(); - rgi->rli->relay_log.signal_update(); - rgi->rli->relay_log.unlock_binlog_end_pos(); } diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 28835118fdd..6e8b6edb44c 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -70,7 +70,8 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) relay_log_state.init(); #ifdef HAVE_PSI_INTERFACE relay_log.set_psi_keys(key_RELAYLOG_LOCK_index, - key_RELAYLOG_update_cond, + key_RELAYLOG_COND_relay_log_updated, + key_RELAYLOG_COND_bin_log_updated, key_file_relaylog, key_file_relaylog_index, key_RELAYLOG_COND_queue_busy); diff --git a/sql/slave.cc b/sql/slave.cc index e54e613ae16..a8334525345 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -6253,7 +6253,7 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) mysql_mutex_unlock(log_lock); goto err; } - rli->relay_log.signal_update(); + rli->relay_log.signal_relay_log_update(); mysql_mutex_unlock(log_lock); mi->gtid_reconnect_event_skip_count= 0; @@ -6798,7 +6798,8 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) if (got_gtid_event) rli->ign_gtids.update(&event_gtid); } - rli->relay_log.signal_update(); // the slave SQL thread needs to re-check + // the slave SQL thread needs to re-check + rli->relay_log.signal_relay_log_update(); DBUG_PRINT("info", ("master_log_pos: %lu, event originating from %u server, ignored", (ulong) mi->master_log_pos, uint4korr(buf + SERVER_ID_OFFSET))); } diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index d85d984606d..fcbf0ce1bd0 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2437,7 +2437,7 @@ static int wait_new_events(binlog_send_info *info, /* in */ PSI_stage_info old_stage; mysql_bin_log.lock_binlog_end_pos(); - info->thd->ENTER_COND(mysql_bin_log.get_log_cond(), + info->thd->ENTER_COND(mysql_bin_log.get_bin_log_cond(), mysql_bin_log.get_binlog_end_pos_lock(), &stage_master_has_sent_all_binlog_to_slave, &old_stage); From abceaa75428f9b2d64ce64629d010af9aa6eae1f Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 26 Oct 2017 19:14:37 +0300 Subject: [PATCH 20/51] Optimize RUN_HOOK() call RUN_HOOK() is only called if semisync is enabled As the server can't disable the hooks if something is in progress, I added a new variable, run_hooks_enabled, that is set the first time semi sync is used. This means that RUN_HOOK will have no overhead, unless semi sync master or slave has been enabled once. Some of the changes was just to get rid of warnings for embedded server --- sql/handler.cc | 2 +- sql/log.cc | 14 ++++++-------- sql/mysqld.cc | 3 ++- sql/mysqld.h | 1 + sql/rpl_handler.h | 12 +++++++----- sql/semisync_slave.h | 1 + 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index 4e059e0e56c..47eb58e17f3 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1484,7 +1484,7 @@ done: mysql_mutex_assert_not_owner(mysql_bin_log.get_log_lock()); mysql_mutex_assert_not_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); - RUN_HOOK(transaction, after_commit, (thd, FALSE)); + (void) RUN_HOOK(transaction, after_commit, (thd, FALSE)); goto end; /* Come here if error and we need to rollback. */ diff --git a/sql/log.cc b/sql/log.cc index 4f69e341ad7..9d4a622d400 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -6374,11 +6374,9 @@ err: mysql_mutex_assert_owner(&LOCK_log); mysql_mutex_assert_not_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); - bool first= true; - bool last= true; if ((error= RUN_HOOK(binlog_storage, after_flush, (thd, log_file_name, file->pos_in_file, - synced, first, last)))) + synced, true, true)))) { sql_print_error("Failed to run 'after_flush' hooks"); error= 1; @@ -6408,11 +6406,9 @@ err: mysql_mutex_assert_not_owner(&LOCK_log); mysql_mutex_assert_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); - bool first= true; - bool last= true; if (RUN_HOOK(binlog_storage, after_sync, (thd, log_file_name, file->pos_in_file, - first, last))) + true, true))) { error=1; /* error is already printed inside hook */ @@ -7838,7 +7834,8 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) mysql_mutex_assert_owner(&LOCK_log); mysql_mutex_assert_not_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); - bool first= true, last; + bool first __attribute__((unused))= true; + bool last __attribute__((unused)); for (current= queue; current != NULL; current= current->next) { last= current->next == NULL; @@ -7924,7 +7921,8 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) mysql_mutex_assert_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); - bool first= true, last; + bool first __attribute__((unused))= true; + bool last __attribute__((unused)); for (current= queue; current != NULL; current= current->next) { last= current->next == NULL; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5f675a76ecf..71e0aeee473 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -390,7 +390,7 @@ static longlong start_memory_used; /* Global variables */ bool opt_bin_log, opt_bin_log_used=0, opt_ignore_builtin_innodb= 0; -bool opt_bin_log_compress; +bool opt_bin_log_compress, run_hooks_enabled; uint opt_bin_log_compress_min_len; my_bool opt_log, debug_assert_if_crashed_table= 0, opt_help= 0; my_bool debug_assert_on_not_freed_memory= 0; @@ -8950,6 +8950,7 @@ static int mysql_init_variables(void) transactions_multi_engine= 0; rpl_transactions_multi_engine= 0; transactions_gtid_foreign_engine= 0; + run_hooks_enabled= 0; log_bin_basename= NULL; log_bin_index= NULL; diff --git a/sql/mysqld.h b/sql/mysqld.h index f72132820fe..1da95fd13f5 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -109,6 +109,7 @@ extern CHARSET_INFO *character_set_filesystem; extern MY_BITMAP temp_pool; extern bool opt_large_files; extern bool opt_update_log, opt_bin_log, opt_error_log, opt_bin_log_compress; +extern bool run_hooks_enabled; extern uint opt_bin_log_compress_min_len; extern my_bool opt_log, opt_bootstrap; extern my_bool opt_backup_history_log; diff --git a/sql/rpl_handler.h b/sql/rpl_handler.h index afcfd9d55b1..62bd7d2606c 100644 --- a/sql/rpl_handler.h +++ b/sql/rpl_handler.h @@ -206,11 +206,13 @@ extern Binlog_relay_IO_delegate *binlog_relay_io_delegate; #endif /* HAVE_REPLICATION */ /* - if there is no observers in the delegate, we can return 0 - immediately. + if semisync replication is not enabled, we can return immediately. */ -#define RUN_HOOK(group, hook, args) \ - (group ##_delegate->is_empty() ? \ - 0 : group ##_delegate->hook args) +#ifdef HAVE_REPLICATION +#define RUN_HOOK(group, hook, args) \ + (unlikely(run_hooks_enabled) ? group ##_delegate->hook args : 0) +#else +#define RUN_HOOK(group, hook, args) 0 +#endif /* HAVE_REPLICATION */ #endif /* RPL_HANDLER_H */ diff --git a/sql/semisync_slave.h b/sql/semisync_slave.h index 9cca8bbbdb4..6bc10b0d479 100644 --- a/sql/semisync_slave.h +++ b/sql/semisync_slave.h @@ -44,6 +44,7 @@ public: return slave_enabled_; } void setSlaveEnabled(bool enabled) { + run_hooks_enabled|= enabled; slave_enabled_ = enabled; } From e972125f11d8f37bc263b113e85ed064257a92ee Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Wed, 22 Nov 2017 17:10:34 +0200 Subject: [PATCH 21/51] MDEV-13073 This part merges the Ali semisync related changes and specifically the ack receiving functionality. Semisync is turned to be static instead of plugin so its functions are invoked at the same points as RUN_HOOKS. The RUN_HOOKS and the observer interface remain to be removed by later patch. Todo: React on killed status by repl_semisync_master.wait_after_sync(). Currently Repl_semi_sync_master::commit_trx does not check the killed status. There were few bugfixes found that are present in mysql and its unclear whether/how they are covered. Those include: Bug#15985893: GTID SKIPPED EVENTS ON MASTER CAUSE SEMI SYNC TIME-OUTS Bug#17932935 CALLING IS_SEMI_SYNC_SLAVE() IN EACH FUNCTION CALL HAS BAD PERFORMANCE Bug#20574628: SEMI-SYNC REPLICATION PERFORMANCE DEGRADES WITH A HIGH NUMBER OF THREADS --- mysql-test/extra/rpl_tests/rpl_semi_sync.inc | 1 + .../binlog_encryption/rpl_semi_sync.result | 6 +- .../perfschema/r/dml_setup_instruments.result | 4 +- mysql-test/suite/perfschema/r/relaylog.result | 2 + mysql-test/suite/rpl/r/rpl_semi_sync.result | 6 +- .../suite/rpl/r/rpl_semi_sync_event.result | 1 + .../r/rpl_semi_sync_event_after_sync.result | 1 + .../suite/rpl/t/rpl_semi_sync_event.test | 2 + .../rpl_semi_sync_master_enabled_basic.result | 6 + .../t/rpl_semi_sync_master_enabled_basic.test | 29 + sql/CMakeLists.txt | 1 + sql/handler.cc | 8 + sql/log.cc | 80 ++- sql/log.h | 11 +- sql/mysqld.cc | 33 +- sql/mysqld.h | 1 + sql/rpl_handler.h | 4 + sql/rpl_mi.h | 5 + sql/rpl_rli.cc | 3 +- sql/semisync.h | 16 + sql/semisync_master.cc | 584 ++++++++---------- sql/semisync_master.h | 84 ++- sql/semisync_master_ack_receiver.cc | 308 +++++++++ sql/semisync_master_ack_receiver.h | 119 ++++ sql/semisync_slave.cc | 286 ++++----- sql/semisync_slave.h | 50 +- sql/slave.cc | 34 +- sql/sql_class.cc | 16 +- sql/sql_class.h | 22 +- sql/sql_repl.cc | 126 ++-- sql/sys_vars.cc | 15 - sql/transaction.cc | 28 +- 32 files changed, 1197 insertions(+), 695 deletions(-) create mode 100644 sql/semisync_master_ack_receiver.cc create mode 100644 sql/semisync_master_ack_receiver.h diff --git a/mysql-test/extra/rpl_tests/rpl_semi_sync.inc b/mysql-test/extra/rpl_tests/rpl_semi_sync.inc index ed56e405e27..393b49372e1 100644 --- a/mysql-test/extra/rpl_tests/rpl_semi_sync.inc +++ b/mysql-test/extra/rpl_tests/rpl_semi_sync.inc @@ -16,6 +16,7 @@ connection master; call mtr.add_suppression("Timeout waiting for reply of binlog"); call mtr.add_suppression("Read semi-sync reply"); call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +call mtr.add_suppression("mysqld: Got an error reading communication packets"); connection slave; call mtr.add_suppression("Master server does not support semi-sync"); call mtr.add_suppression("Semi-sync slave .* reply"); diff --git a/mysql-test/suite/binlog_encryption/rpl_semi_sync.result b/mysql-test/suite/binlog_encryption/rpl_semi_sync.result index 6d574681d73..106efb555d3 100644 --- a/mysql-test/suite/binlog_encryption/rpl_semi_sync.result +++ b/mysql-test/suite/binlog_encryption/rpl_semi_sync.result @@ -4,6 +4,7 @@ connection master; call mtr.add_suppression("Timeout waiting for reply of binlog"); call mtr.add_suppression("Read semi-sync reply"); call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +call mtr.add_suppression("mysqld: Got an error reading communication packets"); connection slave; call mtr.add_suppression("Master server does not support semi-sync"); call mtr.add_suppression("Semi-sync slave .* reply"); @@ -176,7 +177,7 @@ Variable_name Value Rpl_semi_sync_master_yes_tx 14 show status like 'Rpl_semi_sync_master_clients'; Variable_name Value -Rpl_semi_sync_master_clients 1 +Rpl_semi_sync_master_clients 0 [ semi-sync replication of these transactions will fail ] insert into t1 values (500); [ master status should be OFF ] @@ -321,7 +322,6 @@ connection slave; include/stop_slave.inc reset slave; connection master; -kill query _tid; connection slave; include/start_slave.inc connection master; @@ -353,7 +353,6 @@ include/stop_slave.inc reset slave; connection master; reset master; -kill query _tid; set sql_log_bin=0; grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; flush privileges; @@ -404,7 +403,6 @@ SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; Variable_name Value Rpl_semi_sync_slave_status OFF connection master; -kill query _tid; [ Semi-sync status on master should be ON ] show status like 'Rpl_semi_sync_master_clients'; Variable_name Value diff --git a/mysql-test/suite/perfschema/r/dml_setup_instruments.result b/mysql-test/suite/perfschema/r/dml_setup_instruments.result index 940839b105f..3f4d6107717 100644 --- a/mysql-test/suite/perfschema/r/dml_setup_instruments.result +++ b/mysql-test/suite/perfschema/r/dml_setup_instruments.result @@ -4,6 +4,7 @@ where name like 'Wait/Synch/Mutex/sql/%' and name not in ('wait/synch/mutex/sql/DEBUG_SYNC::mutex') order by name limit 10; NAME ENABLED TIMED +wait/synch/mutex/sql/Ack_receiver::m_mutex YES YES wait/synch/mutex/sql/Cversion_lock YES YES wait/synch/mutex/sql/Delayed_insert::mutex YES YES wait/synch/mutex/sql/Event_scheduler::LOCK_scheduler_state YES YES @@ -13,7 +14,6 @@ wait/synch/mutex/sql/HA_DATA_PARTITION::LOCK_auto_inc YES YES wait/synch/mutex/sql/LOCK_active_mi YES YES wait/synch/mutex/sql/LOCK_after_binlog_sync YES YES wait/synch/mutex/sql/LOCK_audit_mask YES YES -wait/synch/mutex/sql/LOCK_binlog YES YES select * from performance_schema.setup_instruments where name like 'Wait/Synch/Rwlock/sql/%' and name not in ('wait/synch/rwlock/sql/CRYPTO_dynlock_value::lock') @@ -36,6 +36,7 @@ where name like 'Wait/Synch/Cond/sql/%' 'wait/synch/cond/sql/DEBUG_SYNC::cond') order by name limit 10; NAME ENABLED TIMED +wait/synch/cond/sql/Ack_receiver::m_cond YES YES wait/synch/cond/sql/COND_binlog_send YES YES wait/synch/cond/sql/COND_flush_thread_cache YES YES wait/synch/cond/sql/COND_group_commit_orderer YES YES @@ -45,7 +46,6 @@ wait/synch/cond/sql/COND_parallel_entry YES YES wait/synch/cond/sql/COND_prepare_ordered YES YES wait/synch/cond/sql/COND_queue_state YES YES wait/synch/cond/sql/COND_rpl_thread YES YES -wait/synch/cond/sql/COND_rpl_thread_pool YES YES select * from performance_schema.setup_instruments where name='Wait'; select * from performance_schema.setup_instruments diff --git a/mysql-test/suite/perfschema/r/relaylog.result b/mysql-test/suite/perfschema/r/relaylog.result index b8655a781eb..3fcf7367b53 100644 --- a/mysql-test/suite/perfschema/r/relaylog.result +++ b/mysql-test/suite/perfschema/r/relaylog.result @@ -86,6 +86,7 @@ EVENT_NAME COUNT_STAR SUM_TIMER_WAIT MIN_TIMER_WAIT AVG_TIMER_WAIT MAX_TIMER_WAI wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_bin_log_updated 0 0 0 0 0 wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_queue_busy 0 0 0 0 0 wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_relay_log_updated 0 0 0 0 0 +wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_binlog_end_pos 0 0 0 0 0 wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_index 0 0 0 0 0 connection slave; "============ Performance schema on slave ============" @@ -193,5 +194,6 @@ EVENT_NAME COUNT_STAR wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_bin_log_updated NONE wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_queue_busy NONE wait/synch/cond/sql/MYSQL_RELAY_LOG::COND_relay_log_updated MANY +wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_binlog_end_pos NONE wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_index MANY include/stop_slave.inc diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync.result b/mysql-test/suite/rpl/r/rpl_semi_sync.result index 6d574681d73..106efb555d3 100644 --- a/mysql-test/suite/rpl/r/rpl_semi_sync.result +++ b/mysql-test/suite/rpl/r/rpl_semi_sync.result @@ -4,6 +4,7 @@ connection master; call mtr.add_suppression("Timeout waiting for reply of binlog"); call mtr.add_suppression("Read semi-sync reply"); call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +call mtr.add_suppression("mysqld: Got an error reading communication packets"); connection slave; call mtr.add_suppression("Master server does not support semi-sync"); call mtr.add_suppression("Semi-sync slave .* reply"); @@ -176,7 +177,7 @@ Variable_name Value Rpl_semi_sync_master_yes_tx 14 show status like 'Rpl_semi_sync_master_clients'; Variable_name Value -Rpl_semi_sync_master_clients 1 +Rpl_semi_sync_master_clients 0 [ semi-sync replication of these transactions will fail ] insert into t1 values (500); [ master status should be OFF ] @@ -321,7 +322,6 @@ connection slave; include/stop_slave.inc reset slave; connection master; -kill query _tid; connection slave; include/start_slave.inc connection master; @@ -353,7 +353,6 @@ include/stop_slave.inc reset slave; connection master; reset master; -kill query _tid; set sql_log_bin=0; grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; flush privileges; @@ -404,7 +403,6 @@ SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; Variable_name Value Rpl_semi_sync_slave_status OFF connection master; -kill query _tid; [ Semi-sync status on master should be ON ] show status like 'Rpl_semi_sync_master_clients'; Variable_name Value diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_event.result b/mysql-test/suite/rpl/r/rpl_semi_sync_event.result index c347ff410ac..917e7c2b02b 100644 --- a/mysql-test/suite/rpl/r/rpl_semi_sync_event.result +++ b/mysql-test/suite/rpl/r/rpl_semi_sync_event.result @@ -5,6 +5,7 @@ call mtr.add_suppression("Timeout waiting for reply of binlog"); call mtr.add_suppression("Semi-sync master .* waiting for slave reply"); call mtr.add_suppression("Read semi-sync reply"); call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +call mtr.add_suppression("mysqld: Got an error reading communication packets"); connection slave; call mtr.add_suppression("Master server does not support semi-sync"); call mtr.add_suppression("Semi-sync slave .* reply"); diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_event_after_sync.result b/mysql-test/suite/rpl/r/rpl_semi_sync_event_after_sync.result index c237eb8df47..24daf0d72b5 100644 --- a/mysql-test/suite/rpl/r/rpl_semi_sync_event_after_sync.result +++ b/mysql-test/suite/rpl/r/rpl_semi_sync_event_after_sync.result @@ -6,6 +6,7 @@ call mtr.add_suppression("Timeout waiting for reply of binlog"); call mtr.add_suppression("Semi-sync master .* waiting for slave reply"); call mtr.add_suppression("Read semi-sync reply"); call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +call mtr.add_suppression("mysqld: Got an error reading communication packets"); connection slave; call mtr.add_suppression("Master server does not support semi-sync"); call mtr.add_suppression("Semi-sync slave .* reply"); diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_event.test b/mysql-test/suite/rpl/t/rpl_semi_sync_event.test index d7685413a07..4d96fd694ec 100644 --- a/mysql-test/suite/rpl/t/rpl_semi_sync_event.test +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_event.test @@ -10,6 +10,8 @@ call mtr.add_suppression("Timeout waiting for reply of binlog"); call mtr.add_suppression("Semi-sync master .* waiting for slave reply"); call mtr.add_suppression("Read semi-sync reply"); call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +call mtr.add_suppression("mysqld: Got an error reading communication packets"); + connection slave; call mtr.add_suppression("Master server does not support semi-sync"); call mtr.add_suppression("Semi-sync slave .* reply"); diff --git a/mysql-test/suite/sys_vars/r/rpl_semi_sync_master_enabled_basic.result b/mysql-test/suite/sys_vars/r/rpl_semi_sync_master_enabled_basic.result index 7454f0b0089..0c90462df59 100644 --- a/mysql-test/suite/sys_vars/r/rpl_semi_sync_master_enabled_basic.result +++ b/mysql-test/suite/sys_vars/r/rpl_semi_sync_master_enabled_basic.result @@ -65,6 +65,12 @@ set global rpl_semi_sync_master_enabled=1e1; ERROR 42000: Incorrect argument type to variable 'rpl_semi_sync_master_enabled' set global rpl_semi_sync_master_enabled="some text"; ERROR 42000: Variable 'rpl_semi_sync_master_enabled' can't be set to the value of 'some text' +connect con1,localhost,root,,; +connect con2,localhost,root,,; +disconnect con1; +disconnect con2; +connection default; +SET @@global.rpl_semi_sync_master_enabled = 1; SET @@global.rpl_semi_sync_master_enabled = @start_global_value; select @@global.rpl_semi_sync_master_enabled; @@global.rpl_semi_sync_master_enabled diff --git a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_enabled_basic.test b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_enabled_basic.test index da22d0535f4..68653d3a9a7 100644 --- a/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_enabled_basic.test +++ b/mysql-test/suite/sys_vars/t/rpl_semi_sync_master_enabled_basic.test @@ -51,6 +51,35 @@ set global rpl_semi_sync_master_enabled=1e1; --error ER_WRONG_VALUE_FOR_VAR set global rpl_semi_sync_master_enabled="some text"; +# +# Test conflicting concurrent setting +# +--let $val_saved= `SELECT @@global.rpl_semi_sync_master_enabled` +connect (con1,localhost,root,,); +connect (con2,localhost,root,,); +--let $iter=100 +--disable_query_log +while ($iter) +{ + --connection con1 + --send_eval SET @@global.rpl_semi_sync_master_enabled = $iter % 2 + + --connection con2 + --send_eval SET @@global.rpl_semi_sync_master_enabled = ($iter + 1) % 2 + + --connection con1 + reap; + --connection con2 + reap; + + --dec $iter +} +--enable_query_log +disconnect con1; +disconnect con2; + +--connection default +--eval SET @@global.rpl_semi_sync_master_enabled = $val_saved # # Cleanup diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 6c63f3feca3..fadee80491b 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -139,6 +139,7 @@ SET (SQL_SOURCE my_json_writer.cc rpl_gtid.cc rpl_parallel.cc semisync.cc semisync_master.cc semisync_slave.cc + semisync_master_ack_receiver.cc sql_type.cc item_windowfunc.cc sql_window.cc sql_cte.cc diff --git a/sql/handler.cc b/sql/handler.cc index 47eb58e17f3..7ea02278abc 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -50,6 +50,7 @@ #ifdef WITH_ARIA_STORAGE_ENGINE #include "../storage/maria/ha_maria.h" #endif +#include "semisync_master.h" #include "wsrep_mysqld.h" #include "wsrep.h" @@ -1485,6 +1486,10 @@ done: mysql_mutex_assert_not_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); (void) RUN_HOOK(transaction, after_commit, (thd, FALSE)); +#ifdef REPLICATION + repl_semisync_master.waitAfterCommit(thd, all); + DEBUG_SYNC(thd, "after_group_after_commit"); +#endif goto end; /* Come here if error and we need to rollback. */ @@ -1730,6 +1735,9 @@ int ha_rollback_trans(THD *thd, bool all) ER_WARNING_NOT_COMPLETE_ROLLBACK, ER_THD(thd, ER_WARNING_NOT_COMPLETE_ROLLBACK)); (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); +#ifdef REPLICATION + repl_semisync_master.waitAfterRollback(thd, all); +#endif DBUG_RETURN(error); } diff --git a/sql/log.cc b/sql/log.cc index 9d4a622d400..a866d72d785 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -53,6 +53,7 @@ #include "debug_sync.h" #include "sql_show.h" #include "my_pthread.h" +#include "semisync_master.h" #include "wsrep_mysqld.h" #include "sp_rcontext.h" #include "sp_head.h" @@ -3329,7 +3330,7 @@ void MYSQL_BIN_LOG::init_pthread_objects() mysql_cond_init(key_BINLOG_COND_binlog_background_thread_end, &COND_binlog_background_thread_end, 0); - mysql_mutex_init(key_LOCK_binlog_end_pos, &LOCK_binlog_end_pos, + mysql_mutex_init(m_key_LOCK_binlog_end_pos, &LOCK_binlog_end_pos, MY_MUTEX_INIT_SLOW); } @@ -5250,6 +5251,7 @@ end: close(LOG_CLOSE_INDEX); sql_print_error(fatal_log_error, new_name_ptr, errno); } + mysql_mutex_unlock(&LOCK_index); if (need_lock) mysql_mutex_unlock(&LOCK_log); @@ -6376,7 +6378,12 @@ err: mysql_mutex_assert_not_owner(&LOCK_commit_ordered); if ((error= RUN_HOOK(binlog_storage, after_flush, (thd, log_file_name, file->pos_in_file, - synced, true, true)))) + synced, true, true))) +#ifdef REPLICATION + || repl_semisync_master.reportBinlogUpdate(thd, log_file_name, + file->pos_in_file) +#endif + ) { sql_print_error("Failed to run 'after_flush' hooks"); error= 1; @@ -6408,7 +6415,12 @@ err: mysql_mutex_assert_not_owner(&LOCK_commit_ordered); if (RUN_HOOK(binlog_storage, after_sync, (thd, log_file_name, file->pos_in_file, - true, true))) + true, true)) +#ifdef REPLICATION + || repl_semisync_master.waitAfterSync(log_file_name, + file->pos_in_file) +#endif + ) { error=1; /* error is already printed inside hook */ @@ -7589,7 +7601,11 @@ MYSQL_BIN_LOG::write_transaction_to_binlog_events(group_commit_entry *entry) else if (is_leader) trx_group_commit_leader(entry); else if (!entry->queued_by_other) + { + DEBUG_SYNC(entry->thd, "after_semisync_queue"); + entry->thd->wait_for_wakeup_ready(); + } else { /* @@ -7840,11 +7856,20 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) { last= current->next == NULL; if (!current->error && - RUN_HOOK(binlog_storage, after_flush, - (current->thd, - current->cache_mngr->last_commit_pos_file, - current->cache_mngr->last_commit_pos_offset, synced, - first, last))) + (RUN_HOOK(binlog_storage, after_flush, + (current->thd, + current->cache_mngr->last_commit_pos_file, + current->cache_mngr->last_commit_pos_offset, synced, + first, last)) +#ifdef REPLICATION + || (DBUG_EVALUATE_IF("failed_report_binlog_update", 1, 0) || + repl_semisync_master. + reportBinlogUpdate(current->thd, + current->cache_mngr->last_commit_pos_file, + current->cache_mngr-> + last_commit_pos_offset)) +#endif + )) { current->error= ER_ERROR_ON_WRITE; current->commit_errno= -1; @@ -7927,12 +7952,22 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) { last= current->next == NULL; if (!current->error && - RUN_HOOK(binlog_storage, after_sync, + (RUN_HOOK(binlog_storage, after_sync, (current->thd, current->cache_mngr->last_commit_pos_file, current->cache_mngr->last_commit_pos_offset, - first, last))) + first, last)) +#ifdef REPLICATION + || (DBUG_EVALUATE_IF("simulate_after_sync_hook_error", 1, 0) || + repl_semisync_master.waitAfterSync(current->cache_mngr-> + last_commit_pos_file, + current->cache_mngr-> + last_commit_pos_offset)) +#endif + )) { - /* error is already printed inside hook */ + const char *hook_name= rpl_semi_sync_master_enabled ? + "'waitAfterSync'" : "binlog_storage 'after_sync'"; + sql_print_error("Failed to call '%s'", hook_name); } first= false; } @@ -8268,24 +8303,6 @@ void MYSQL_BIN_LOG::wait_for_update_relay_log(THD* thd) LOCK_log is released by the caller. */ -int MYSQL_BIN_LOG::wait_for_update_bin_log(THD* thd, - const struct timespec *timeout) -{ - int ret= 0; - DBUG_ENTER("wait_for_update_bin_log"); - - thd_wait_begin(thd, THD_WAIT_BINLOG); - mysql_mutex_assert_owner(&LOCK_binlog_end_pos); - if (!timeout) - mysql_cond_wait(&COND_bin_log_updated, &LOCK_binlog_end_pos); - else - ret= mysql_cond_timedwait(&COND_bin_log_updated, &LOCK_binlog_end_pos, - const_cast(timeout)); - thd_wait_end(thd); - DBUG_RETURN(ret); -} - - int MYSQL_BIN_LOG::wait_for_update_binlog_end_pos(THD* thd, struct timespec *timeout) { @@ -10427,7 +10444,7 @@ get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list) *out_gtid_list= NULL; - if (!(ev= Log_event::read_log_event(cache, 0, &init_fdle, + if (!(ev= Log_event::read_log_event(cache, &init_fdle, opt_master_verify_checksum)) || ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) { @@ -10443,7 +10460,7 @@ get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list) { Log_event_type typ; - ev= Log_event::read_log_event(cache, 0, fdle, opt_master_verify_checksum); + ev= Log_event::read_log_event(cache, fdle, opt_master_verify_checksum); if (!ev) { errormsg= "Could not read GTID list event while looking for GTID " @@ -10473,6 +10490,7 @@ get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list) return errormsg; } + struct st_mysql_storage_engine binlog_storage_engine= { MYSQL_HANDLERTON_INTERFACE_VERSION }; diff --git a/sql/log.h b/sql/log.h index 0a82f8813f0..02ace7c7921 100644 --- a/sql/log.h +++ b/sql/log.h @@ -349,6 +349,11 @@ public: /* for documentation of mutexes held in various places in code */ }; +/* Tell the io thread if we can delay the master info sync. */ +#define SEMI_SYNC_SLAVE_DELAY_SYNC 1 +/* Tell the io thread if the current event needs a ack. */ +#define SEMI_SYNC_NEED_ACK 2 + class MYSQL_QUERY_LOG: public MYSQL_LOG { public: @@ -435,6 +440,8 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG PSI_file_key m_key_file_log_index; PSI_file_key m_key_COND_queue_busy; + /** The instrumentation key to use for LOCK_binlog_end_pos. */ + PSI_mutex_key m_key_LOCK_binlog_end_pos; #endif struct group_commit_entry @@ -667,7 +674,8 @@ public: PSI_cond_key key_bin_log_update, PSI_file_key key_file_log, PSI_file_key key_file_log_index, - PSI_file_key key_COND_queue_busy) + PSI_file_key key_COND_queue_busy, + PSI_mutex_key key_LOCK_binlog_end_pos) { m_key_LOCK_index= key_LOCK_index; m_key_relay_log_update= key_relay_log_update; @@ -675,6 +683,7 @@ public: m_key_file_log= key_file_log; m_key_file_log_index= key_file_log_index; m_key_COND_queue_busy= key_COND_queue_busy; + m_key_LOCK_binlog_end_pos= key_LOCK_binlog_end_pos; } #endif diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 71e0aeee473..86759b8ed8b 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -933,6 +933,7 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_LOCK_thread_count, key_LOCK_thread_cache, key_PARTITION_LOCK_auto_inc; PSI_mutex_key key_RELAYLOG_LOCK_index; +PSI_mutex_key key_LOCK_relaylog_end_pos; PSI_mutex_key key_LOCK_slave_state, key_LOCK_binlog_state, key_LOCK_rpl_thread, key_LOCK_rpl_thread_pool, key_LOCK_parallel_entry; PSI_mutex_key key_LOCK_binlog; @@ -947,6 +948,8 @@ PSI_mutex_key key_LOCK_after_binlog_sync; PSI_mutex_key key_LOCK_prepare_ordered, key_LOCK_commit_ordered, key_LOCK_slave_background; PSI_mutex_key key_TABLE_SHARE_LOCK_share; +PSI_mutex_key key_ss_mutex_LOCK_binlog_; +PSI_mutex_key key_ss_mutex_Ack_receiver_mutex; static PSI_mutex_info all_server_mutexes[]= { @@ -967,6 +970,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_BINLOG_LOCK_binlog_background_thread, "MYSQL_BIN_LOG::LOCK_binlog_background_thread", 0}, { &key_LOCK_binlog_end_pos, "MYSQL_BIN_LOG::LOCK_binlog_end_pos", 0 }, { &key_RELAYLOG_LOCK_index, "MYSQL_RELAY_LOG::LOCK_index", 0}, + { &key_LOCK_relaylog_end_pos, "MYSQL_RELAY_LOG::LOCK_binlog_end_pos", 0}, { &key_delayed_insert_mutex, "Delayed_insert::mutex", 0}, { &key_hash_filo_lock, "hash_filo::lock", 0}, { &key_LOCK_active_mi, "LOCK_active_mi", PSI_FLAG_GLOBAL}, @@ -1024,6 +1028,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_LOCK_rpl_thread, "LOCK_rpl_thread", 0}, { &key_LOCK_rpl_thread_pool, "LOCK_rpl_thread_pool", 0}, { &key_LOCK_parallel_entry, "LOCK_parallel_entry", 0}, + { &key_ss_mutex_Ack_receiver_mutex, "Ack_receiver::m_mutex", 0}, { &key_LOCK_binlog, "LOCK_binlog", 0} }; @@ -1078,6 +1083,7 @@ PSI_cond_key key_COND_rpl_thread_queue, key_COND_rpl_thread, key_COND_parallel_entry, key_COND_group_commit_orderer, key_COND_prepare_ordered, key_COND_slave_background; PSI_cond_key key_COND_wait_gtid, key_COND_gtid_ignore_duplicates; +PSI_cond_key key_ss_cond_Ack_receiver_cond; static PSI_cond_info all_server_conds[]= { @@ -1131,6 +1137,7 @@ static PSI_cond_info all_server_conds[]= { &key_COND_start_thread, "COND_start_thread", PSI_FLAG_GLOBAL}, { &key_COND_wait_gtid, "COND_wait_gtid", 0}, { &key_COND_gtid_ignore_duplicates, "COND_gtid_ignore_duplicates", 0}, + { &key_ss_cond_Ack_receiver_cond, "Ack_receiver::m_cond", 0}, { &key_COND_binlog_send, "COND_binlog_send", 0} }; @@ -1138,6 +1145,7 @@ PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert, key_thread_handle_manager, key_thread_main, key_thread_one_connection, key_thread_signal_hand, key_thread_slave_background, key_rpl_parallel_thread; +PSI_thread_key key_ss_thread_Ack_receiver_thread; static PSI_thread_info all_server_threads[]= { @@ -1164,6 +1172,7 @@ static PSI_thread_info all_server_threads[]= { &key_thread_one_connection, "one_connection", 0}, { &key_thread_signal_hand, "signal_handler", PSI_FLAG_GLOBAL}, { &key_thread_slave_background, "slave_background", PSI_FLAG_GLOBAL}, + { &key_ss_thread_Ack_receiver_thread, "Ack_receiver", PSI_FLAG_GLOBAL}, { &key_rpl_parallel_thread, "rpl_parallel_thread", 0} }; @@ -1743,6 +1752,7 @@ static void close_connections(void) Events::deinit(); slave_prepare_for_shutdown(); mysql_bin_log.stop_background_thread(); + ack_receiver.stop(); /* Give threads time to die. @@ -2226,7 +2236,6 @@ void clean_up(bool print_message) tc_log->close(); #ifdef HAVE_REPLICATION semi_sync_master_deinit(); - semi_sync_slave_deinit(); #endif delegates_destroy(); xid_cache_free(); @@ -4251,7 +4260,8 @@ static int init_common_variables() key_BINLOG_COND_bin_log_updated, key_file_binlog, key_file_binlog_index, - key_BINLOG_COND_queue_busy); + key_BINLOG_COND_queue_busy, + key_LOCK_binlog_end_pos); #endif /* @@ -5183,8 +5193,12 @@ static int init_server_components() "--log-bin option is not defined."); } - semi_sync_master_init(); - semi_sync_slave_init(); + if (repl_semisync_master.initObject() || + repl_semisync_slave.initObject()) + { + sql_print_error("Could not initialize semisync."); + unireg_abort(1); + } #endif if (opt_bin_log) @@ -8599,14 +8613,10 @@ SHOW_VAR status_vars[]= { {"Rpl_semi_sync_master_net_wait_time", (char*) &SHOW_FNAME(net_wait_time), SHOW_FUNC}, {"Rpl_semi_sync_master_net_waits", (char*) &SHOW_FNAME(net_wait_num), SHOW_FUNC}, {"Rpl_semi_sync_master_net_avg_wait_time", (char*) &SHOW_FNAME(avg_net_wait_time), SHOW_FUNC}, -#ifdef HAVE_ACC_RECEIVER {"Rpl_semi_sync_master_request_ack", (char*) &rpl_semi_sync_master_request_ack, SHOW_LONGLONG}, {"Rpl_semi_sync_master_get_ack", (char*)&rpl_semi_sync_master_get_ack, SHOW_LONGLONG}, -#endif {"Rpl_semi_sync_slave_status", (char*) &rpl_semi_sync_slave_status, SHOW_BOOL}, -#ifdef HAVE_ACC_RECEIVER {"Rpl_semi_sync_slave_send_ack", (char*) &rpl_semi_sync_slave_send_ack, SHOW_LONGLONG}, -#endif #endif /* HAVE_REPLICATION */ #ifdef HAVE_QUERY_CACHE {"Qcache_free_blocks", (char*) &query_cache.free_memory_blocks, SHOW_LONG_NOFLUSH}, @@ -8950,7 +8960,7 @@ static int mysql_init_variables(void) transactions_multi_engine= 0; rpl_transactions_multi_engine= 0; transactions_gtid_foreign_engine= 0; - run_hooks_enabled= 0; + run_hooks_enabled= 0; // don't run hooks, semisync does not need 'em log_bin_basename= NULL; log_bin_index= NULL; @@ -10348,6 +10358,8 @@ PSI_stage_info stage_waiting_for_master_update= { 0, "Waiting for master update" PSI_stage_info stage_waiting_for_relay_log_space= { 0, "Waiting for the slave SQL thread to free enough relay log space", 0}; PSI_stage_info stage_waiting_for_semi_sync_ack_from_slave= { 0, "Waiting for semi-sync ACK from slave", 0}; +PSI_stage_info stage_waiting_for_semi_sync_slave={ 0, "Waiting for semi-sync slave connection", 0}; +PSI_stage_info stage_reading_semi_sync_ack={ 0, "Reading semi-sync ACK from slave", 0}; PSI_stage_info stage_waiting_for_slave_mutex_on_exit= { 0, "Waiting for slave mutex on exit", 0}; PSI_stage_info stage_waiting_for_slave_thread_to_start= { 0, "Waiting for slave thread to start", 0}; PSI_stage_info stage_waiting_for_table_flush= { 0, "Waiting for table flush", 0}; @@ -10508,6 +10520,9 @@ PSI_stage_info *all_server_stages[]= & stage_gtid_wait_other_connection, & stage_slave_background_process_request, & stage_slave_background_wait_request, + & stage_waiting_for_semi_sync_ack_from_slave, + & stage_waiting_for_semi_sync_slave, + & stage_reading_semi_sync_ack, & stage_waiting_for_deadlock_kill }; diff --git a/sql/mysqld.h b/sql/mysqld.h index 1da95fd13f5..5399ec91b19 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -310,6 +310,7 @@ extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_LOCK_start_thread, key_LOCK_error_messages, key_LOCK_thread_count, key_PARTITION_LOCK_auto_inc; extern PSI_mutex_key key_RELAYLOG_LOCK_index; +extern PSI_mutex_key key_LOCK_relaylog_end_pos; extern PSI_mutex_key key_LOCK_slave_state, key_LOCK_binlog_state, key_LOCK_rpl_thread, key_LOCK_rpl_thread_pool, key_LOCK_parallel_entry; diff --git a/sql/rpl_handler.h b/sql/rpl_handler.h index 62bd7d2606c..f8b11adbb69 100644 --- a/sql/rpl_handler.h +++ b/sql/rpl_handler.h @@ -209,6 +209,10 @@ extern Binlog_relay_IO_delegate *binlog_relay_io_delegate; if semisync replication is not enabled, we can return immediately. */ #ifdef HAVE_REPLICATION +/* + As semisync is unpluggined and its hooks are turned into static + invocations all other hooks are not run for optimization sake. +*/ #define RUN_HOOK(group, hook, args) \ (unlikely(run_hooks_enabled) ? group ##_delegate->hook args : 0) #else diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index 610bc77b683..14d74dc4bb7 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -311,6 +311,11 @@ class Master_info : public Slave_reporting_capability /* The parallel replication mode. */ enum_slave_parallel_mode parallel_mode; + /* + semi_ack is used to identify if the current binlog event needs an + ACK from slave, or if delay_master is enabled. + */ + int semi_ack; }; int init_master_info(Master_info* mi, const char* master_info_fname, diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 6e8b6edb44c..321eef97700 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -74,7 +74,8 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) key_RELAYLOG_COND_bin_log_updated, key_file_relaylog, key_file_relaylog_index, - key_RELAYLOG_COND_queue_busy); + key_RELAYLOG_COND_queue_busy, + key_LOCK_relaylog_end_pos); #endif group_relay_log_name[0]= event_relay_log_name[0]= diff --git a/sql/semisync.h b/sql/semisync.h index 3142f920f1e..cf791b36c1b 100644 --- a/sql/semisync.h +++ b/sql/semisync.h @@ -47,6 +47,20 @@ public: return exit_code; } + inline bool function_exit(const char *func_name, bool exit_code) + { + if (trace_level_ & kTraceFunction) + sql_print_information("<--- %s exit (%s)", func_name, + exit_code ? "True" : "False"); + return exit_code; + } + + inline void function_exit(const char *func_name) + { + if (trace_level_ & kTraceFunction) + sql_print_information("<--- %s exit", func_name); + } + Trace() :trace_level_(0L) {} @@ -79,5 +93,7 @@ public: #define REPLY_MAGIC_NUM_OFFSET 0 #define REPLY_BINLOG_POS_OFFSET (REPLY_MAGIC_NUM_OFFSET + REPLY_MAGIC_NUM_LEN) #define REPLY_BINLOG_NAME_OFFSET (REPLY_BINLOG_POS_OFFSET + REPLY_BINLOG_POS_LEN) +#define REPLY_MESSAGE_MAX_LENGTH \ + (REPLY_MAGIC_NUM_LEN + REPLY_BINLOG_POS_LEN + REPLY_BINLOG_NAME_LEN) #endif /* SEMISYNC_H */ diff --git a/sql/semisync_master.cc b/sql/semisync_master.cc index 21c52addce9..de91e30beec 100644 --- a/sql/semisync_master.cc +++ b/sql/semisync_master.cc @@ -24,7 +24,9 @@ #define TIME_BILLION 1000000000 /* This indicates whether semi-synchronous replication is enabled. */ -my_bool rpl_semi_sync_master_enabled; +my_bool rpl_semi_sync_master_enabled= 0; +unsigned long long rpl_semi_sync_master_request_ack = 0; +unsigned long long rpl_semi_sync_master_get_ack = 0; my_bool rpl_semi_sync_master_wait_no_slave = 1; my_bool rpl_semi_sync_master_status = 0; ulong rpl_semi_sync_master_wait_point = @@ -47,6 +49,15 @@ ulonglong rpl_semi_sync_master_net_wait_time = 0; ulonglong rpl_semi_sync_master_trx_wait_time = 0; ReplSemiSyncMaster repl_semisync_master; +Ack_receiver ack_receiver; + +/* + structure to save transaction log filename and position +*/ +typedef struct Trans_binlog_info { + my_off_t log_pos; + char log_file[FN_REFLEN]; +} Trans_binlog_info; static int getWaitTime(const struct timespec& start_ts); @@ -336,7 +347,8 @@ ReplSemiSyncMaster::ReplSemiSyncMaster() wait_file_pos_(0), master_enabled_(false), wait_timeout_(0L), - state_(0) + state_(0), + wait_point_(0) { strcpy(reply_file_name_, ""); strcpy(wait_file_name_, ""); @@ -345,18 +357,13 @@ ReplSemiSyncMaster::ReplSemiSyncMaster() int ReplSemiSyncMaster::initObject() { int result; - const char *kWho = "ReplSemiSyncMaster::initObject"; - if (init_done_) - { - fprintf(stderr, "%s called twice\n", kWho); - return 1; - } init_done_ = true; /* References to the parameter works after set_options(). */ setWaitTimeout(rpl_semi_sync_master_timeout); setTraceLevel(rpl_semi_sync_master_trace_level); + setWaitPoint(rpl_semi_sync_master_wait_point); /* Mutex initialization can only be done after MY_INIT(). */ mysql_mutex_init(key_LOCK_binlog, @@ -365,9 +372,22 @@ int ReplSemiSyncMaster::initObject() &COND_binlog_send, NULL); if (rpl_semi_sync_master_enabled) + { result = enableMaster(); + if (!result) + result= ack_receiver.start(); /* Start the ACK thread. */ + } else + { result = disableMaster(); + } + + /* + If rpl_semi_sync_master_wait_no_slave is disabled, let's temporarily + switch off semisync to avoid hang if there's none active slave. + */ + if (!rpl_semi_sync_master_wait_no_slave) + switch_off(); return result; } @@ -390,7 +410,6 @@ int ReplSemiSyncMaster::enableMaster() set_master_enabled(true); state_ = true; - run_hooks_enabled= 1; sql_print_information("Semi-sync replication enabled on the master."); } else @@ -498,12 +517,50 @@ void ReplSemiSyncMaster::remove_slave() unlock(); } -bool ReplSemiSyncMaster::is_semi_sync_slave() +int ReplSemiSyncMaster::reportReplyPacket(uint32 server_id, const uchar *packet, + ulong packet_len) { - int null_value; - long long val= 0; - get_user_var_int("rpl_semi_sync_slave", &val, &null_value); - return val; + const char *kWho = "ReplSemiSyncMaster::reportReplyPacket"; + int result= -1; + char log_file_name[FN_REFLEN+1]; + my_off_t log_file_pos; + ulong log_file_len = 0; + + function_enter(kWho); + + if (unlikely(packet[REPLY_MAGIC_NUM_OFFSET] != ReplSemiSyncMaster::kPacketMagicNum)) + { + sql_print_error("Read semi-sync reply magic number error"); + goto l_end; + } + + if (unlikely(packet_len < REPLY_BINLOG_NAME_OFFSET)) + { + sql_print_error("Read semi-sync reply length error: packet is too small"); + goto l_end; + } + + log_file_pos = uint8korr(packet + REPLY_BINLOG_POS_OFFSET); + log_file_len = packet_len - REPLY_BINLOG_NAME_OFFSET; + if (unlikely(log_file_len >= FN_REFLEN)) + { + sql_print_error("Read semi-sync reply binlog file length too large"); + goto l_end; + } + strncpy(log_file_name, (const char*)packet + REPLY_BINLOG_NAME_OFFSET, log_file_len); + log_file_name[log_file_len] = 0; + + DBUG_ASSERT(dirname_length(log_file_name) == 0); + + if (trace_level_ & kTraceDetail) + sql_print_information("%s: Got reply(%s, %lu) from server %u", + kWho, log_file_name, (ulong)log_file_pos, server_id); + + rpl_semi_sync_master_get_ack++; + reportReplyBinlog(server_id, log_file_name, log_file_pos); + +l_end: + return function_exit(kWho, result); } int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, @@ -602,6 +659,121 @@ int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, return function_exit(kWho, 0); } +int ReplSemiSyncMaster::waitAfterSync(const char *log_file, my_off_t log_pos) +{ + if (!getMasterEnabled()) + return 0; + + int ret= 0; + if(log_pos && + waitPoint() == SEMI_SYNC_MASTER_WAIT_POINT_AFTER_BINLOG_SYNC) + ret= commitTrx(log_file + dirname_length(log_file), log_pos); + + return ret; +} + +int ReplSemiSyncMaster::waitAfterCommit(THD* thd, bool all) +{ + if (!getMasterEnabled()) + return 0; + + int ret= 0; + const char *log_file; + my_off_t log_pos; + + bool is_real_trans= + (all || thd->transaction.all.ha_list == 0); + /* + The coordinates are propagated to this point having been computed + in reportBinlogUpdate + */ + Trans_binlog_info *log_info= thd->semisync_info; + log_file= log_info && log_info->log_file[0] ? log_info->log_file : 0; + log_pos= log_info ? log_info->log_pos : 0; + + DBUG_ASSERT(!log_file || dirname_length(log_file) == 0); + + if (is_real_trans && + log_pos && + waitPoint() == SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT) + ret= commitTrx(log_file, log_pos); + + if (is_real_trans && log_info) + { + log_info->log_file[0]= 0; + log_info->log_pos= 0; + } + + return ret; +} + +int ReplSemiSyncMaster::waitAfterRollback(THD *thd, bool all) +{ + return waitAfterCommit(thd, all); +} + +/** + The method runs after flush to binary log is done. +*/ +int ReplSemiSyncMaster::reportBinlogUpdate(THD* thd, const char *log_file, + my_off_t log_pos) +{ + if (getMasterEnabled()) + { + Trans_binlog_info *log_info; + + if (!(log_info= thd->semisync_info)) + { + if(!(log_info= + (Trans_binlog_info*) my_malloc(sizeof(Trans_binlog_info), MYF(0)))) + return 1; + thd->semisync_info= log_info; + } + strcpy(log_info->log_file, log_file + dirname_length(log_file)); + log_info->log_pos = log_pos; + + return writeTranxInBinlog(log_info->log_file, log_pos); + } + + return 0; +} + +void ReplSemiSyncMaster::dump_start(THD* thd, + const char *log_file, + my_off_t log_pos) +{ + if (!thd->semi_sync_slave) + return; + + if (ack_receiver.add_slave(thd)) + { + sql_print_error("Failed to register slave to semi-sync ACK receiver " + "thread. Turning off semisync"); + thd->semi_sync_slave= 0; + return; + } + + add_slave(); + reportReplyBinlog(thd->variables.server_id, log_file + dirname_length(log_file), log_pos); + sql_print_information("Start semi-sync binlog_dump to slave (server_id: %d), pos(%s, %lu", + thd->variables.server_id, log_file, (unsigned long)log_pos); + + return; +} + +void ReplSemiSyncMaster::dump_end(THD* thd) +{ + if (!thd->semi_sync_slave) + return; + + sql_print_information("Stop semi-sync binlog_dump to slave (server_id: %d)", thd->variables.server_id); + + remove_slave(); + ack_receiver.remove_slave(thd); + + return; +} + int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, my_off_t trx_wait_binlog_pos) { @@ -850,42 +1022,23 @@ int ReplSemiSyncMaster::try_switch_on(int server_id, return function_exit(kWho, 0); } -int ReplSemiSyncMaster::reserveSyncHeader(unsigned char *header, - ulong size) +int ReplSemiSyncMaster::reserveSyncHeader(String* packet) { const char *kWho = "ReplSemiSyncMaster::reserveSyncHeader"; function_enter(kWho); - int hlen=0; - if (!is_semi_sync_slave()) - { - hlen= 0; - } - else - { - /* No enough space for the extra header, disable semi-sync master */ - if (sizeof(kSyncHeader) > size) - { - sql_print_warning("No enough space in the packet " - "for semi-sync extra header, " - "semi-sync replication disabled"); - disableMaster(); - return 0; - } - - /* Set the magic number and the sync status. By default, no sync - * is required. - */ - memcpy(header, kSyncHeader, sizeof(kSyncHeader)); - hlen= sizeof(kSyncHeader); - } - return function_exit(kWho, hlen); + /* Set the magic number and the sync status. By default, no sync + * is required. + */ + packet->append(reinterpret_cast(kSyncHeader), + sizeof(kSyncHeader)); + return function_exit(kWho, 0); } -int ReplSemiSyncMaster::updateSyncHeader(unsigned char *packet, +int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, const char *log_file_name, my_off_t log_file_pos, - uint32 server_id) + bool* need_sync) { const char *kWho = "ReplSemiSyncMaster::updateSyncHeader"; int cmp = 0; @@ -894,8 +1047,11 @@ int ReplSemiSyncMaster::updateSyncHeader(unsigned char *packet, /* If the semi-sync master is not enabled, or the slave is not a semi-sync * target, do not request replies from the slave. */ - if (!getMasterEnabled() || !is_semi_sync_slave()) + if (!getMasterEnabled() || !thd->semi_sync_slave) + { + *need_sync = false; return 0; + } function_enter(kWho); @@ -903,12 +1059,15 @@ int ReplSemiSyncMaster::updateSyncHeader(unsigned char *packet, /* This is the real check inside the mutex. */ if (!getMasterEnabled()) - goto l_end; // sync= false at this point in time + { + assert(sync == false); + goto l_end; + } if (is_on()) { /* semi-sync is ON */ - /* sync= false; No sync unless a transaction is involved. */ + sync = false; /* No sync unless a transaction is involved. */ if (reply_file_name_inited_) { @@ -962,8 +1121,9 @@ int ReplSemiSyncMaster::updateSyncHeader(unsigned char *packet, if (trace_level_ & kTraceDetail) sql_print_information("%s: server(%d), (%s, %lu) sync(%d), repl(%d)", - kWho, server_id, log_file_name, + kWho, thd->variables.server_id, log_file_name, (ulong)log_file_pos, sync, (int)is_on()); + *need_sync= sync; l_end: unlock(); @@ -1033,6 +1193,10 @@ int ReplSemiSyncMaster::writeTranxInBinlog(const char* log_file_name, log_file_name, (ulong)log_file_pos); switch_off(); } + else + { + rpl_semi_sync_master_request_ack++; + } } l_end: @@ -1041,19 +1205,12 @@ int ReplSemiSyncMaster::writeTranxInBinlog(const char* log_file_name, return function_exit(kWho, result); } -int ReplSemiSyncMaster::readSlaveReply(NET *net, uint32 server_id, - const char *event_buf) +int ReplSemiSyncMaster::flushNet(THD *thd, + const char *event_buf) { - const char *kWho = "ReplSemiSyncMaster::readSlaveReply"; - const unsigned char *packet; - char log_file_name[FN_REFLEN]; - my_off_t log_file_pos; - ulong log_file_len = 0; - ulong packet_len; + const char *kWho = "ReplSemiSyncMaster::flushNet"; int result = -1; - struct timespec start_ts; - ulong trc_level = trace_level_; - LINT_INIT_STRUCT(start_ts); + NET* net= &thd->net; function_enter(kWho); @@ -1065,9 +1222,6 @@ int ReplSemiSyncMaster::readSlaveReply(NET *net, uint32 server_id, goto l_end; } - if (trc_level & kTraceNetWait) - set_timespec(start_ts, 0); - /* We flush to make sure that the current event is sent to the network, * instead of being buffered in the TCP/IP stack. */ @@ -1079,82 +1233,35 @@ int ReplSemiSyncMaster::readSlaveReply(NET *net, uint32 server_id, } net_clear(net, 0); - if (trc_level & kTraceDetail) - sql_print_information("%s: Wait for replica's reply", kWho); - - /* Wait for the network here. Though binlog dump thread can indefinitely wait - * here, transactions would not wait indefintely. - * Transactions wait on binlog replies detected by binlog dump threads. If - * binlog dump threads wait too long, transactions will timeout and continue. - */ - packet_len = my_net_read(net); - - if (trc_level & kTraceNetWait) - { - int wait_time = getWaitTime(start_ts); - if (wait_time < 0) - { - sql_print_error("Semi-sync master wait for reply " - "fail to get wait time."); - rpl_semi_sync_master_timefunc_fails++; - } - else - { - rpl_semi_sync_master_net_wait_num++; - rpl_semi_sync_master_net_wait_time += wait_time; - } - } - - if (packet_len == packet_error || packet_len < REPLY_BINLOG_NAME_OFFSET) - { - if (packet_len == packet_error) - sql_print_error("Read semi-sync reply network error: %s (errno: %d)", - net->last_error, net->last_errno); - else - sql_print_error("Read semi-sync reply length error: %s (errno: %d)", - net->last_error, net->last_errno); - goto l_end; - } - - packet = net->read_pos; - if (packet[REPLY_MAGIC_NUM_OFFSET] != ReplSemiSyncMaster::kPacketMagicNum) - { - sql_print_error("Read semi-sync reply magic number error"); - goto l_end; - } - - log_file_pos = uint8korr(packet + REPLY_BINLOG_POS_OFFSET); - log_file_len = packet_len - REPLY_BINLOG_NAME_OFFSET; - if (log_file_len >= FN_REFLEN) - { - sql_print_error("Read semi-sync reply binlog file length too large"); - goto l_end; - } - strncpy(log_file_name, (const char*)packet + REPLY_BINLOG_NAME_OFFSET, log_file_len); - log_file_name[log_file_len] = 0; - - if (trc_level & kTraceDetail) - sql_print_information("%s: Got reply (%s, %lu)", - kWho, log_file_name, (ulong)log_file_pos); - - result = reportReplyBinlog(server_id, log_file_name, log_file_pos); + net->pkt_nr++; + result = 0; + rpl_semi_sync_master_net_wait_num++; l_end: + thd->clear_error(); return function_exit(kWho, result); } - -int ReplSemiSyncMaster::resetMaster() +int ReplSemiSyncMaster::afterResetMaster() { - const char *kWho = "ReplSemiSyncMaster::resetMaster"; + const char *kWho = "ReplSemiSyncMaster::afterResetMaster"; int result = 0; function_enter(kWho); + if (rpl_semi_sync_master_enabled) + { + sql_print_information("Enable Semi-sync Master after reset master"); + enableMaster(); + } lock(); - state_ = getMasterEnabled()? 1 : 0; + if (rpl_semi_sync_master_clients == 0 && + !rpl_semi_sync_master_wait_no_slave) + state_ = 0; + else + state_ = getMasterEnabled()? 1 : 0; wait_file_name_inited_ = false; reply_file_name_inited_ = false; @@ -1176,6 +1283,31 @@ int ReplSemiSyncMaster::resetMaster() return function_exit(kWho, result); } +int ReplSemiSyncMaster::beforeResetMaster() +{ + const char *kWho = "ReplSemiSyncMaster::beforeResetMaster"; + int result = 0; + + function_enter(kWho); + + if (rpl_semi_sync_master_enabled) + disableMaster(); + + return function_exit(kWho, result); +} + +void ReplSemiSyncMaster::checkAndSwitch() +{ + lock(); + if (getMasterEnabled() && is_on()) + { + if (!rpl_semi_sync_master_wait_no_slave + && rpl_semi_sync_master_clients == 0) + switch_off(); + } + unlock(); +} + void ReplSemiSyncMaster::setExportStats() { lock(); @@ -1219,212 +1351,8 @@ static int getWaitTime(const struct timespec& start_ts) return (int)(end_usecs - start_usecs); } -/*************************************************************************** - Semisync master interface setup and deinit -***************************************************************************/ - -C_MODE_START - -int repl_semi_report_binlog_update(Binlog_storage_param *param, - const char *log_file, - my_off_t log_pos, uint32 flags) -{ - int error= 0; - - if (repl_semisync_master.getMasterEnabled()) - { - /* - Let us store the binlog file name and the position, so that - we know how long to wait for the binlog to the replicated to - the slave in synchronous replication. - */ - error= repl_semisync_master.writeTranxInBinlog(log_file, - log_pos); - } - - return error; -} - -int repl_semi_request_commit(Trans_param *param) -{ - return 0; -} - -int repl_semi_report_binlog_sync(Binlog_storage_param *param, - const char *log_file, - my_off_t log_pos, uint32 flags) -{ - int error= 0; - if (rpl_semi_sync_master_wait_point == - SEMI_SYNC_MASTER_WAIT_POINT_AFTER_BINLOG_SYNC) - { - error= repl_semisync_master.commitTrx(log_file, log_pos); - } - - return error; -} - -int repl_semi_report_commit(Trans_param *param) -{ - if (rpl_semi_sync_master_wait_point != - SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT) - { - return 0; - } - - bool is_real_trans= param->flags & TRANS_IS_REAL_TRANS; - - if (is_real_trans && param->log_pos) - { - const char *binlog_name= param->log_file; - return repl_semisync_master.commitTrx(binlog_name, param->log_pos); - } - return 0; -} - -int repl_semi_report_rollback(Trans_param *param) -{ - return repl_semi_report_commit(param); -} - -int repl_semi_binlog_dump_start(Binlog_transmit_param *param, - const char *log_file, - my_off_t log_pos) -{ - bool semi_sync_slave= repl_semisync_master.is_semi_sync_slave(); - - if (semi_sync_slave) - { - /* One more semi-sync slave */ - repl_semisync_master.add_slave(); - - /* - Let's assume this semi-sync slave has already received all - binlog events before the filename and position it requests. - */ - repl_semisync_master.reportReplyBinlog(param->server_id, log_file, log_pos); - } - sql_print_information("Start %s binlog_dump to slave (server_id: %d), pos(%s, %lu)", - semi_sync_slave ? "semi-sync" : "asynchronous", - param->server_id, log_file, (ulong)log_pos); - - return 0; -} - -int repl_semi_binlog_dump_end(Binlog_transmit_param *param) -{ - bool semi_sync_slave= repl_semisync_master.is_semi_sync_slave(); - - sql_print_information("Stop %s binlog_dump to slave (server_id: %d)", - semi_sync_slave ? "semi-sync" : "asynchronous", - param->server_id); - if (semi_sync_slave) - { - /* One less semi-sync slave */ - repl_semisync_master.remove_slave(); - } - return 0; -} - -int repl_semi_reserve_header(Binlog_transmit_param *param, - unsigned char *header, - ulong size, ulong *len) -{ - *len += repl_semisync_master.reserveSyncHeader(header, size); - return 0; -} - -int repl_semi_before_send_event(Binlog_transmit_param *param, - unsigned char *packet, ulong len, - const char *log_file, my_off_t log_pos) -{ - return repl_semisync_master.updateSyncHeader(packet, - log_file, - log_pos, - param->server_id); -} - -int repl_semi_after_send_event(Binlog_transmit_param *param, - const char *event_buf, ulong len) -{ - if (repl_semisync_master.is_semi_sync_slave()) - { - THD *thd= current_thd; - /* - Possible errors in reading slave reply are ignored deliberately - because we do not want dump thread to quit on this. Error - messages are already reported. - */ - (void) repl_semisync_master.readSlaveReply(&thd->net, - param->server_id, event_buf); - thd->clear_error(); - } - return 0; -} - -int repl_semi_reset_master(Binlog_transmit_param *param) -{ - if (repl_semisync_master.resetMaster()) - return 1; - return 0; -} - -C_MODE_END - -Trans_observer trans_observer= -{ - sizeof(Trans_observer), // len - - repl_semi_report_commit, // after_commit - repl_semi_report_rollback, // after_rollback -}; - -Binlog_storage_observer storage_observer= -{ - sizeof(Binlog_storage_observer), // len - - repl_semi_report_binlog_update, // report_update - repl_semi_report_binlog_sync, // after_sync -}; - -Binlog_transmit_observer transmit_observer= -{ - sizeof(Binlog_transmit_observer), // len - - repl_semi_binlog_dump_start, // start - repl_semi_binlog_dump_end, // stop - repl_semi_reserve_header, // reserve_header - repl_semi_before_send_event, // before_send_event - repl_semi_after_send_event, // after_send_event - repl_semi_reset_master, // reset -}; - -static bool semi_sync_master_inited= 0; - -int semi_sync_master_init() -{ - void *p= 0; - if (repl_semisync_master.initObject()) - return 1; - if (register_trans_observer(&trans_observer, p)) - return 1; - if (register_binlog_storage_observer(&storage_observer, p)) - return 1; - if (register_binlog_transmit_observer(&transmit_observer, p)) - return 1; - semi_sync_master_inited= 1; - return 0; -} - void semi_sync_master_deinit() { - void *p= 0; - if (!semi_sync_master_inited) - return; - - unregister_trans_observer(&trans_observer, p); - unregister_binlog_storage_observer(&storage_observer, p); - unregister_binlog_transmit_observer(&transmit_observer, p); repl_semisync_master.cleanup(); - semi_sync_master_inited= 0; + ack_receiver.cleanup(); } diff --git a/sql/semisync_master.h b/sql/semisync_master.h index ff1e3dd48b4..66fac17cd45 100644 --- a/sql/semisync_master.h +++ b/sql/semisync_master.h @@ -20,14 +20,13 @@ #define SEMISYNC_MASTER_H #include "semisync.h" +#include "semisync_master_ack_receiver.h" #ifdef HAVE_PSI_INTERFACE extern PSI_mutex_key key_LOCK_binlog; extern PSI_cond_key key_COND_binlog_send; #endif -extern PSI_stage_info stage_waiting_for_semi_sync_ack_from_slave; - struct TranxNode { char log_name_[FN_REFLEN]; my_off_t log_pos_; @@ -432,6 +431,9 @@ class ReplSemiSyncMaster bool state_; /* whether semi-sync is switched */ + /*Waiting for ACK before/after innodb commit*/ + ulong wait_point_; + void lock(); void unlock(); void cond_broadcast(); @@ -473,6 +475,17 @@ class ReplSemiSyncMaster wait_timeout_ = wait_timeout; } + /*set the ACK point, after binlog sync or after transaction commit*/ + void setWaitPoint(unsigned long ack_point) + { + wait_point_ = ack_point; + } + + ulong waitPoint() //no cover line + { + return wait_point_; //no cover line + } + /* Initialize this class after MySQL parameters are initialized. this * function should be called once at bootstrap time. */ @@ -490,8 +503,9 @@ class ReplSemiSyncMaster /* Remove a semi-sync replication slave */ void remove_slave(); - /* Is the slave servered by the thread requested semi-sync */ - bool is_semi_sync_slave(); + /* It parses a reply packet and call reportReplyBinlog to handle it. */ + int reportReplyPacket(uint32 server_id, const uchar *packet, + ulong packet_len); /* In semi-sync replication, reports up to which binlog position we have * received replies from the slave indicating that it already get the events. @@ -527,42 +541,61 @@ class ReplSemiSyncMaster int commitTrx(const char* trx_wait_binlog_name, my_off_t trx_wait_binlog_pos); + /*Wait for ACK after writing/sync binlog to file*/ + int waitAfterSync(const char* log_file, my_off_t log_pos); + + /*Wait for ACK after commting the transaction*/ + int waitAfterCommit(THD* thd, bool all); + + /*Wait after the transaction is rollback*/ + int waitAfterRollback(THD *thd, bool all); + /*Store the current binlog position in active_tranxs_. This position should + * be acked by slave*/ + int reportBinlogUpdate(THD *thd, const char *log_file,my_off_t log_pos); + + void dump_start(THD* thd, + const char *log_file, + my_off_t log_pos); + + void dump_end(THD* thd); + /* Reserve space in the replication event packet header: * . slave semi-sync off: 1 byte - (0) * . slave semi-sync on: 3 byte - (0, 0xef, 0/1} - * + * * Input: - * header - (IN) the header buffer - * size - (IN) size of the header buffer + * packet - (IN) the header buffer * * Return: * size of the bytes reserved for header */ - int reserveSyncHeader(unsigned char *header, unsigned long size); + int reserveSyncHeader(String* packet); /* Update the sync bit in the packet header to indicate to the slave whether * the master will wait for the reply of the event. If semi-sync is switched * off and we detect that the slave is catching up, we switch semi-sync on. * * Input: + * THD - (IN) current dump thread * packet - (IN) the packet containing the replication event * log_file_name - (IN) the event ending position's file name * log_file_pos - (IN) the event ending position's file offset + * need_sync - (IN) identify if flushNet is needed to call. * server_id - (IN) master server id number * * Return: * 0: success; non-zero: error */ - int updateSyncHeader(unsigned char *packet, + int updateSyncHeader(THD* thd, unsigned char *packet, const char *log_file_name, - my_off_t log_file_pos, - uint32 server_id); + my_off_t log_file_pos, + bool* need_sync); /* Called when a transaction finished writing binlog events. * . update the 'largest' transactions' binlog event position * . insert the ending position in the active transaction list if * semi-sync is on - * + * * Input: (the transaction events' ending binlog position) * log_file_name - (IN) transaction ending position's file name * log_file_pos - (IN) transaction ending position's file offset @@ -574,16 +607,8 @@ class ReplSemiSyncMaster /* Read the slave's reply so that we know how much progress the slave makes * on receive replication events. - * - * Input: - * net - (IN) the connection to master - * server_id - (IN) master server id number - * event_buf - (IN) pointer to the event packet - * - * Return: - * 0: success; non-zero: error */ - int readSlaveReply(NET *net, uint32 server_id, const char *event_buf); + int flushNet(THD* thd, const char *event_buf); /* Export internal statistics for semi-sync replication. */ void setExportStats(); @@ -591,7 +616,12 @@ class ReplSemiSyncMaster /* 'reset master' command is issued from the user and semi-sync need to * go off for that. */ - int resetMaster(); + int afterResetMaster(); + + /*called before reset master*/ + int beforeResetMaster(); + + void checkAndSwitch(); }; enum rpl_semi_sync_master_wait_point_t { @@ -599,6 +629,9 @@ enum rpl_semi_sync_master_wait_point_t { SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT, }; +extern ReplSemiSyncMaster repl_semisync_master; +extern Ack_receiver ack_receiver; + /* System and status variables for the master component */ extern my_bool rpl_semi_sync_master_enabled; extern my_bool rpl_semi_sync_master_status; @@ -620,6 +653,8 @@ extern ulonglong rpl_semi_sync_master_net_wait_num; extern ulonglong rpl_semi_sync_master_trx_wait_num; extern ulonglong rpl_semi_sync_master_net_wait_time; extern ulonglong rpl_semi_sync_master_trx_wait_time; +extern unsigned long long rpl_semi_sync_master_request_ack; +extern unsigned long long rpl_semi_sync_master_get_ack; /* This indicates whether we should keep waiting if no semi-sync slave @@ -630,7 +665,10 @@ extern ulonglong rpl_semi_sync_master_trx_wait_time; extern char rpl_semi_sync_master_wait_no_slave; extern ReplSemiSyncMaster repl_semisync_master; -int semi_sync_master_init(); +extern PSI_stage_info stage_waiting_for_semi_sync_ack_from_slave; +extern PSI_stage_info stage_reading_semi_sync_ack; +extern PSI_stage_info stage_waiting_for_semi_sync_slave; + void semi_sync_master_deinit(); #endif /* SEMISYNC_MASTER_H */ diff --git a/sql/semisync_master_ack_receiver.cc b/sql/semisync_master_ack_receiver.cc new file mode 100644 index 00000000000..eee35cc122f --- /dev/null +++ b/sql/semisync_master_ack_receiver.cc @@ -0,0 +1,308 @@ +/* Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include +#include "semisync_master.h" +#include "semisync_master_ack_receiver.h" + +extern PSI_mutex_key key_ss_mutex_Ack_receiver_mutex; +extern PSI_cond_key key_ss_cond_Ack_receiver_cond; +extern PSI_thread_key key_ss_thread_Ack_receiver_thread; +extern ReplSemiSyncMaster repl_semisync; + +/* Callback function of ack receive thread */ +pthread_handler_t ack_receive_handler(void *arg) +{ + Ack_receiver *recv= reinterpret_cast(arg); + + my_thread_init(); + recv->run(); + my_thread_end(); + + return NULL; +} + +Ack_receiver::Ack_receiver() +{ + const char *kWho = "Ack_receiver::Ack_receiver"; + function_enter(kWho); + + m_status= ST_DOWN; + mysql_mutex_init(key_ss_mutex_Ack_receiver_mutex, &m_mutex, + MY_MUTEX_INIT_FAST); + mysql_cond_init(key_ss_cond_Ack_receiver_cond, &m_cond, NULL); + m_pid= 0; + + function_exit(kWho); +} + +void Ack_receiver::cleanup() +{ + const char *kWho = "Ack_receiver::~Ack_receiver"; + function_enter(kWho); + + stop(); + mysql_mutex_destroy(&m_mutex); + mysql_cond_destroy(&m_cond); + + function_exit(kWho); +} + +bool Ack_receiver::start() +{ + const char *kWho = "Ack_receiver::start"; + function_enter(kWho); + + mysql_mutex_lock(&m_mutex); + if(m_status == ST_DOWN) + { + pthread_attr_t attr; + + m_status= ST_UP; + + if (DBUG_EVALUATE_IF("rpl_semisync_simulate_create_thread_failure", 1, 0) || + pthread_attr_init(&attr) != 0 || + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE) != 0 || +#ifndef _WIN32 + pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM) != 0 || +#endif + mysql_thread_create(key_ss_thread_Ack_receiver_thread, &m_pid, + &attr, ack_receive_handler, this)) + { + sql_print_error("Failed to start semi-sync ACK receiver thread, " + " could not create thread(errno:%d)", errno); + + m_status= ST_DOWN; + mysql_mutex_unlock(&m_mutex); + + return function_exit(kWho, true); + } + (void) pthread_attr_destroy(&attr); + } + mysql_mutex_unlock(&m_mutex); + + return function_exit(kWho, false); +} + +void Ack_receiver::stop() +{ + const char *kWho = "Ack_receiver::stop"; + function_enter(kWho); + + mysql_mutex_lock(&m_mutex); + if (m_status == ST_UP) + { + m_status= ST_STOPPING; + mysql_cond_broadcast(&m_cond); + + while (m_status == ST_STOPPING) + mysql_cond_wait(&m_cond, &m_mutex); + + DBUG_ASSERT(m_status == ST_DOWN); + + m_pid= 0; + } + mysql_mutex_unlock(&m_mutex); + + function_exit(kWho); +} + +bool Ack_receiver::add_slave(THD *thd) +{ + Slave *slave; + const char *kWho = "Ack_receiver::add_slave"; + function_enter(kWho); + + if (!(slave= new Slave)) + return function_exit(kWho, true); + + slave->thd= thd; + slave->vio= *thd->net.vio; + slave->vio.mysql_socket.m_psi= NULL; + slave->vio.read_timeout= 1; + + mysql_mutex_lock(&m_mutex); + m_slaves.push_back(slave); + m_slaves_changed= true; + mysql_cond_broadcast(&m_cond); + mysql_mutex_unlock(&m_mutex); + + return function_exit(kWho, false); +} + +void Ack_receiver::remove_slave(THD *thd) +{ + I_List_iterator it(m_slaves); + Slave *slave; + const char *kWho = "Ack_receiver::remove_slave"; + function_enter(kWho); + + mysql_mutex_lock(&m_mutex); + + while ((slave= it++)) + { + if (slave->thd == thd) + { + delete slave; + m_slaves_changed= true; + break; + } + } + mysql_mutex_unlock(&m_mutex); + function_exit(kWho); +} + +inline void Ack_receiver::set_stage_info(const PSI_stage_info &stage) +{ + MYSQL_SET_STAGE(stage.m_key, __FILE__, __LINE__); +} + +inline void Ack_receiver::wait_for_slave_connection() +{ + set_stage_info(stage_waiting_for_semi_sync_slave); + mysql_cond_wait(&m_cond, &m_mutex); +} + +my_socket Ack_receiver::get_slave_sockets(fd_set *fds, uint *count) +{ + my_socket max_fd= INVALID_SOCKET; + Slave *slave; + I_List_iterator it(m_slaves); + + *count= 0; + FD_ZERO(fds); + while ((slave= it++)) + { + (*count)++; + my_socket fd= slave->sock_fd(); + max_fd= (fd > max_fd ? fd : max_fd); + FD_SET(fd, fds); + } + + return max_fd; +} + +/* Auxilary function to initialize a NET object with given net buffer. */ +static void init_net(NET *net, unsigned char *buff, unsigned int buff_len) +{ + memset(net, 0, sizeof(NET)); + net->max_packet= buff_len; + net->buff= buff; + net->buff_end= buff + buff_len; + net->read_pos= net->buff; +} + +void Ack_receiver::run() +{ + // skip LOCK_global_system_variables due to the 3rd arg + THD *thd= new THD(next_thread_id(), false, true); + NET net; + unsigned char net_buff[REPLY_MESSAGE_MAX_LENGTH]; + fd_set read_fds; + my_socket max_fd= INVALID_SOCKET; + Slave *slave; + + my_thread_init(); + + DBUG_ENTER("Ack_receiver::run"); + + sql_print_information("Starting ack receiver thread"); + thd->system_thread= SYSTEM_THREAD_SEMISYNC_MASTER_BACKGROUND; + thd->thread_stack= (char*) &thd; + thd->store_globals(); + thd->security_ctx->skip_grants(); + thread_safe_increment32(&service_thread_count); + thd->set_command(COM_DAEMON); + init_net(&net, net_buff, REPLY_MESSAGE_MAX_LENGTH); + + mysql_mutex_lock(&m_mutex); + m_slaves_changed= true; + mysql_mutex_unlock(&m_mutex); + + while (1) + { + fd_set fds; + int ret; + uint slave_count; + + mysql_mutex_lock(&m_mutex); + if (unlikely(m_status == ST_STOPPING)) + goto end; + + set_stage_info(stage_waiting_for_semi_sync_ack_from_slave); + if (unlikely(m_slaves_changed)) + { + if (unlikely(m_slaves.is_empty())) + { + wait_for_slave_connection(); + mysql_mutex_unlock(&m_mutex); + continue; + } + + max_fd= get_slave_sockets(&read_fds, &slave_count); + m_slaves_changed= false; + DBUG_PRINT("info", ("fd count %u, max_fd %d", slave_count, max_fd)); + } + + struct timeval tv= {1, 0}; + fds= read_fds; + /* select requires max fd + 1 for the first argument */ + ret= select(max_fd+1, &fds, NULL, NULL, &tv); + if (ret <= 0) + { + mysql_mutex_unlock(&m_mutex); + + ret= DBUG_EVALUATE_IF("rpl_semisync_simulate_select_error", -1, ret); + + if (ret == -1) + sql_print_information("Failed to select() on semi-sync dump sockets, " + "error: errno=%d", socket_errno); + /* Sleep 1us, so other threads can catch the m_mutex easily. */ + my_sleep(1); + continue; + } + + set_stage_info(stage_reading_semi_sync_ack); + I_List_iterator it(m_slaves); + + while ((slave= it++)) + { + if (FD_ISSET(slave->sock_fd(), &fds)) + { + ulong len; + + net_clear(&net, 0); + net.vio= &slave->vio; + + len= my_net_read(&net); + if (likely(len != packet_error)) + repl_semisync_master.reportReplyPacket(slave->server_id(), + net.read_pos, len); + else if (net.last_errno == ER_NET_READ_ERROR) + FD_CLR(slave->sock_fd(), &read_fds); + } + } + mysql_mutex_unlock(&m_mutex); + } +end: + sql_print_information("Stopping ack receiver thread"); + m_status= ST_DOWN; + delete thd; + thread_safe_decrement32(&service_thread_count); + signal_thd_deleted(); + mysql_cond_broadcast(&m_cond); + mysql_mutex_unlock(&m_mutex); + DBUG_VOID_RETURN; +} diff --git a/sql/semisync_master_ack_receiver.h b/sql/semisync_master_ack_receiver.h new file mode 100644 index 00000000000..25307131bad --- /dev/null +++ b/sql/semisync_master_ack_receiver.h @@ -0,0 +1,119 @@ +/* Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef SEMISYNC_MASTER_ACK_RECEIVER_DEFINED +#define SEMISYNC_MASTER_ACK_RECEIVER_DEFINED + +#include "my_global.h" +#include "my_pthread.h" +#include "sql_class.h" +#include "semisync.h" +/** + Ack_receiver is responsible to control ack receive thread and maintain + slave information used by ack receive thread. + + There are mainly four operations on ack receive thread: + start: start ack receive thread + stop: stop ack receive thread + add_slave: maintain a new semisync slave's information + remove_slave: remove a semisync slave's information + */ +class Ack_receiver : public ReplSemiSyncBase +{ +public: + Ack_receiver(); + ~Ack_receiver() {} + void cleanup(); + /** + Notify ack receiver to receive acks on the dump session. + + It adds the given dump thread into the slave list and wakes + up ack thread if it is waiting for any slave coming. + + @param[in] thd THD of a dump thread. + + @return it return false if succeeds, otherwise true is returned. + */ + bool add_slave(THD *thd); + + /** + Notify ack receiver not to receive ack on the dump session. + + it removes the given dump thread from slave list. + + @param[in] thd THD of a dump thread. + */ + void remove_slave(THD *thd); + + /** + Start ack receive thread + + @return it return false if succeeds, otherwise true is returned. + */ + bool start(); + + /** + Stop ack receive thread + */ + void stop(); + + /** + The core of ack receive thread. + + It monitors all slaves' sockets and receives acks when they come. + */ + void run(); + + void setTraceLevel(unsigned long trace_level) + { + trace_level_= trace_level; + } +private: + enum status {ST_UP, ST_DOWN, ST_STOPPING}; + uint8 m_status; + /* + Protect m_status, m_slaves_changed and m_slaves. ack thread and other + session may access the variables at the same time. + */ + mysql_mutex_t m_mutex; + mysql_cond_t m_cond; + /* If slave list is updated(add or remove). */ + bool m_slaves_changed; + + class Slave :public ilink + { +public: + THD *thd; + Vio vio; + + my_socket sock_fd() { return vio.mysql_socket.fd; } + uint server_id() { return thd->variables.server_id; } + }; + + I_List m_slaves; + + pthread_t m_pid; + +/* Declare them private, so no one can copy the object. */ + Ack_receiver(const Ack_receiver &ack_receiver); + Ack_receiver& operator=(const Ack_receiver &ack_receiver); + + void set_stage_info(const PSI_stage_info &stage); + void wait_for_slave_connection(); + my_socket get_slave_sockets(fd_set *fds, uint *count); +}; + +extern Ack_receiver ack_receiver; +#endif diff --git a/sql/semisync_slave.cc b/sql/semisync_slave.cc index 63bf9dca0e8..012f807d28e 100644 --- a/sql/semisync_slave.cc +++ b/sql/semisync_slave.cc @@ -18,10 +18,13 @@ #include #include "semisync_slave.h" -my_bool rpl_semi_sync_slave_enabled; +ReplSemiSyncSlave repl_semisync_slave; + +my_bool rpl_semi_sync_slave_enabled= 0; + +char rpl_semi_sync_slave_delay_master; my_bool rpl_semi_sync_slave_status= 0; ulong rpl_semi_sync_slave_trace_level; -ReplSemiSyncSlave repl_semisync_slave; /* indicate whether or not the slave should send a reply to the master. @@ -31,30 +34,27 @@ ReplSemiSyncSlave repl_semisync_slave; checked in repl_semi_slave_queue_event. */ bool semi_sync_need_reply= false; - +unsigned int rpl_semi_sync_slave_kill_conn_timeout; +unsigned long long rpl_semi_sync_slave_send_ack = 0; int ReplSemiSyncSlave::initObject() { int result= 0; - const char *kWho = "ReplSemiSyncSlave::initObject"; - if (init_done_) - { - fprintf(stderr, "%s called twice\n", kWho); - return 1; - } init_done_ = true; /* References to the parameter works after set_options(). */ setSlaveEnabled(rpl_semi_sync_slave_enabled); setTraceLevel(rpl_semi_sync_slave_trace_level); + setDelayMaster(rpl_semi_sync_slave_delay_master); + setKillConnTimeout(rpl_semi_sync_slave_kill_conn_timeout); return result; } int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, unsigned long total_len, - bool *need_reply, + int *semi_flags, const char **payload, unsigned long *payload_len) { @@ -62,128 +62,119 @@ int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, int read_res = 0; function_enter(kWho); - if ((unsigned char)(header[0]) == kPacketMagicNum) + if (rpl_semi_sync_slave_status) { - *need_reply = (header[1] & kPacketFlagSync); - *payload_len = total_len - 2; - *payload = header + 2; + if (DBUG_EVALUATE_IF("semislave_corrupt_log", 0, 1) + && (unsigned char)(header[0]) == kPacketMagicNum) + { + semi_sync_need_reply = (header[1] & kPacketFlagSync); + *payload_len = total_len - 2; + *payload = header + 2; - if (trace_level_ & kTraceDetail) - sql_print_information("%s: reply - %d", kWho, *need_reply); - } - else - { - sql_print_error("Missing magic number for semi-sync packet, packet " - "len: %lu", total_len); - read_res = -1; + if (trace_level_ & kTraceDetail) + sql_print_information("%s: reply - %d", kWho, semi_sync_need_reply); + + if (semi_sync_need_reply) + *semi_flags |= SEMI_SYNC_NEED_ACK; + if (isDelayMaster()) + *semi_flags |= SEMI_SYNC_SLAVE_DELAY_SYNC; + } + else + { + sql_print_error("Missing magic number for semi-sync packet, packet " + "len: %lu", total_len); + read_res = -1; + } + } else { + *payload= header; + *payload_len= total_len; } return function_exit(kWho, read_res); } -int ReplSemiSyncSlave::slaveStart(Binlog_relay_IO_param *param) +int ReplSemiSyncSlave::slaveStart(Master_info *mi) { bool semi_sync= getSlaveEnabled(); sql_print_information("Slave I/O thread: Start %s replication to\ master '%s@%s:%d' in log '%s' at position %lu", semi_sync ? "semi-sync" : "asynchronous", - param->user, param->host, param->port, - param->master_log_name[0] ? param->master_log_name : "FIRST", - (unsigned long)param->master_log_pos); + const_cast(mi->user), mi->host, mi->port, + const_cast(mi->master_log_name), + (unsigned long)(mi->master_log_pos)); if (semi_sync && !rpl_semi_sync_slave_status) rpl_semi_sync_slave_status= 1; + + /*clear the counter*/ + rpl_semi_sync_slave_send_ack= 0; return 0; } -int ReplSemiSyncSlave::slaveStop(Binlog_relay_IO_param *param) +int ReplSemiSyncSlave::slaveStop(Master_info *mi) { if (rpl_semi_sync_slave_status) rpl_semi_sync_slave_status= 0; - if (mysql_reply) - mysql_close(mysql_reply); - mysql_reply= 0; + if (getSlaveEnabled()) + killConnection(mi->mysql); return 0; } -int ReplSemiSyncSlave::slaveReply(MYSQL *mysql, - const char *binlog_filename, - my_off_t binlog_filepos) +int ReplSemiSyncSlave::resetSlave(Master_info *mi) { - const char *kWho = "ReplSemiSyncSlave::slaveReply"; - NET *net= &mysql->net; - uchar reply_buffer[REPLY_MAGIC_NUM_LEN - + REPLY_BINLOG_POS_LEN - + REPLY_BINLOG_NAME_LEN]; - int reply_res, name_len = strlen(binlog_filename); - - function_enter(kWho); - - /* Prepare the buffer of the reply. */ - reply_buffer[REPLY_MAGIC_NUM_OFFSET] = kPacketMagicNum; - int8store(reply_buffer + REPLY_BINLOG_POS_OFFSET, binlog_filepos); - memcpy(reply_buffer + REPLY_BINLOG_NAME_OFFSET, - binlog_filename, - name_len + 1 /* including trailing '\0' */); - - if (trace_level_ & kTraceDetail) - sql_print_information("%s: reply (%s, %lu)", kWho, - binlog_filename, (ulong)binlog_filepos); - - net_clear(net, 0); - /* Send the reply. */ - reply_res = my_net_write(net, reply_buffer, - name_len + REPLY_BINLOG_NAME_OFFSET); - if (!reply_res) - { - reply_res = net_flush(net); - if (reply_res) - sql_print_error("Semi-sync slave net_flush() reply failed"); - } - else - { - sql_print_error("Semi-sync slave send reply failed: %s (%d)", - net->last_error, net->last_errno); - } - - return function_exit(kWho, reply_res); -} - -/*************************************************************************** - Semisync slave interface setup and deinit -***************************************************************************/ - -C_MODE_START - -int repl_semi_reset_slave(Binlog_relay_IO_param *param) -{ - // TODO: reset semi-sync slave status here return 0; } -int repl_semi_slave_request_dump(Binlog_relay_IO_param *param, - uint32 flags) +void ReplSemiSyncSlave::killConnection(MYSQL *mysql) { - MYSQL *mysql= param->mysql; + if (!mysql) + return; + + char kill_buffer[30]; + MYSQL *kill_mysql = NULL; + kill_mysql = mysql_init(kill_mysql); + mysql_options(kill_mysql, MYSQL_OPT_CONNECT_TIMEOUT, &kill_conn_timeout_); + mysql_options(kill_mysql, MYSQL_OPT_READ_TIMEOUT, &kill_conn_timeout_); + mysql_options(kill_mysql, MYSQL_OPT_WRITE_TIMEOUT, &kill_conn_timeout_); + + bool ret= (!mysql_real_connect(kill_mysql, mysql->host, + mysql->user, mysql->passwd,0, mysql->port, mysql->unix_socket, 0)); + if (DBUG_EVALUATE_IF("semisync_slave_failed_kill", 1, 0) || ret) + { + sql_print_information("cannot connect to master to kill slave io_thread's " + "connection"); + if (!ret) + mysql_close(kill_mysql); + return; + } + uint kill_buffer_length = my_snprintf(kill_buffer, 30, "KILL %lu", + mysql->thread_id); + mysql_real_query(kill_mysql, kill_buffer, kill_buffer_length); + mysql_close(kill_mysql); +} + +int ReplSemiSyncSlave::requestTransmit(Master_info *mi) +{ + MYSQL *mysql= mi->mysql; MYSQL_RES *res= 0; MYSQL_ROW row; const char *query; - if (!repl_semisync_slave.getSlaveEnabled()) + if (!getSlaveEnabled()) return 0; - /* Check if master server has semi-sync plugin installed */ query= "SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled'"; if (mysql_real_query(mysql, query, strlen(query)) || !(res= mysql_store_result(mysql))) { - sql_print_error("Execution failed on master: %s", query); + sql_print_error("Execution failed on master: %s, error :%s", query, mysql_error(mysql)); return 1; } row= mysql_fetch_row(res); - if (!row) + if (DBUG_EVALUATE_IF("master_not_support_semisync", 1, 0) + || !row) { /* Master does not support semi-sync */ sql_print_warning("Master server does not support semi-sync, " @@ -195,8 +186,8 @@ int repl_semi_slave_request_dump(Binlog_relay_IO_param *param, mysql_free_result(res); /* - Tell master dump thread that we want to do semi-sync - replication + Tell master dump thread that we want to do semi-sync + replication */ query= "SET @rpl_semi_sync_slave= 1"; if (mysql_real_query(mysql, query, strlen(query))) @@ -206,83 +197,56 @@ int repl_semi_slave_request_dump(Binlog_relay_IO_param *param, } mysql_free_result(mysql_store_result(mysql)); rpl_semi_sync_slave_status= 1; + return 0; } -int repl_semi_slave_read_event(Binlog_relay_IO_param *param, - const char *packet, unsigned long len, - const char **event_buf, unsigned long *event_len) +int ReplSemiSyncSlave::slaveReply(Master_info *mi) { - if (rpl_semi_sync_slave_status) - return repl_semisync_slave.slaveReadSyncHeader(packet, len, - &semi_sync_need_reply, - event_buf, event_len); - *event_buf= packet; - *event_len= len; - return 0; -} + const char *kWho = "ReplSemiSyncSlave::slaveReply"; + MYSQL* mysql= mi->mysql; + const char *binlog_filename= const_cast(mi->master_log_name); + my_off_t binlog_filepos= mi->master_log_pos; + + NET *net= &mysql->net; + uchar reply_buffer[REPLY_MAGIC_NUM_LEN + + REPLY_BINLOG_POS_LEN + + REPLY_BINLOG_NAME_LEN]; + int reply_res = 0; + int name_len = strlen(binlog_filename); + + function_enter(kWho); -int repl_semi_slave_queue_event(Binlog_relay_IO_param *param, - const char *event_buf, - unsigned long event_len, - uint32 flags) -{ if (rpl_semi_sync_slave_status && semi_sync_need_reply) { - /* - We deliberately ignore the error in slaveReply, such error - should not cause the slave IO thread to stop, and the error - messages are already reported. - */ - (void) repl_semisync_slave.slaveReply(param->mysql, - param->master_log_name, - param->master_log_pos); + /* Prepare the buffer of the reply. */ + reply_buffer[REPLY_MAGIC_NUM_OFFSET] = kPacketMagicNum; + int8store(reply_buffer + REPLY_BINLOG_POS_OFFSET, binlog_filepos); + memcpy(reply_buffer + REPLY_BINLOG_NAME_OFFSET, + binlog_filename, + name_len + 1 /* including trailing '\0' */); + + if (trace_level_ & kTraceDetail) + sql_print_information("%s: reply (%s, %lu)", kWho, + binlog_filename, (ulong)binlog_filepos); + + net_clear(net, 0); + /* Send the reply. */ + reply_res = my_net_write(net, reply_buffer, + name_len + REPLY_BINLOG_NAME_OFFSET); + if (!reply_res) + { + reply_res = DBUG_EVALUATE_IF("semislave_failed_net_flush", 1, net_flush(net)); + if (reply_res) + sql_print_error("Semi-sync slave net_flush() reply failed"); + rpl_semi_sync_slave_send_ack++; + } + else + { + sql_print_error("Semi-sync slave send reply failed: %s (%d)", + net->last_error, net->last_errno); + } } - return 0; -} - -int repl_semi_slave_io_start(Binlog_relay_IO_param *param) -{ - return repl_semisync_slave.slaveStart(param); -} - -int repl_semi_slave_io_end(Binlog_relay_IO_param *param) -{ - return repl_semisync_slave.slaveStop(param); -} - -C_MODE_END - -Binlog_relay_IO_observer relay_io_observer= -{ - sizeof(Binlog_relay_IO_observer), // len - - repl_semi_slave_io_start, // start - repl_semi_slave_io_end, // stop - repl_semi_slave_request_dump, // request_transmit - repl_semi_slave_read_event, // after_read_event - repl_semi_slave_queue_event, // after_queue_event - repl_semi_reset_slave, // reset -}; - -static bool semi_sync_slave_inited= 0; - -int semi_sync_slave_init() -{ - void *p= 0; - if (repl_semisync_slave.initObject()) - return 1; - if (register_binlog_relay_io_observer(&relay_io_observer, p)) - return 1; - semi_sync_slave_inited= 1; - return 0; -} - -void semi_sync_slave_deinit() -{ - void *p= 0; - if (!semi_sync_slave_inited) - return; - unregister_binlog_relay_io_observer(&relay_io_observer, p); - semi_sync_slave_inited= 0; + + return function_exit(kWho, reply_res); } diff --git a/sql/semisync_slave.h b/sql/semisync_slave.h index 6bc10b0d479..0df4445ee4a 100644 --- a/sql/semisync_slave.h +++ b/sql/semisync_slave.h @@ -19,6 +19,10 @@ #define SEMISYNC_SLAVE_H #include "semisync.h" +#include "my_global.h" +#include "sql_priv.h" +#include "rpl_mi.h" +#include "mysql.h" /** The extension class for the slave of semi-synchronous replication @@ -44,49 +48,54 @@ public: return slave_enabled_; } void setSlaveEnabled(bool enabled) { - run_hooks_enabled|= enabled; slave_enabled_ = enabled; } + + bool isDelayMaster(){ + return delay_master_; + } + + void setDelayMaster(bool enabled) { + delay_master_ = enabled; + } + + void setKillConnTimeout(unsigned int timeout) { + kill_conn_timeout_ = timeout; + } /* A slave reads the semi-sync packet header and separate the metadata * from the payload data. - * + * * Input: * header - (IN) packet header pointer * total_len - (IN) total packet length: metadata + payload - * need_reply - (IN) whether the master is waiting for the reply + * semi_flags - (IN) store flags: SEMI_SYNC_SLAVE_DELAY_SYNC and SEMI_SYNC_NEED_ACK * payload - (IN) payload: the replication event * payload_len - (IN) payload length * * Return: * 0: success; non-zero: error */ - int slaveReadSyncHeader(const char *header, unsigned long total_len, bool *need_reply, + int slaveReadSyncHeader(const char *header, unsigned long total_len, int *semi_flags, const char **payload, unsigned long *payload_len); /* A slave replies to the master indicating its replication process. It * indicates that the slave has received all events before the specified * binlog position. - * - * Input: - * mysql - (IN) the mysql network connection - * binlog_filename - (IN) the reply point's binlog file name - * binlog_filepos - (IN) the reply point's binlog file offset - * - * Return: - * 0: success; non-zero: error */ - int slaveReply(MYSQL *mysql, const char *binlog_filename, - my_off_t binlog_filepos); - - int slaveStart(Binlog_relay_IO_param *param); - int slaveStop(Binlog_relay_IO_param *param); + int slaveReply(Master_info* mi); + int slaveStart(Master_info *mi); + int slaveStop(Master_info *mi); + int requestTransmit(Master_info*); + void killConnection(MYSQL *mysql); + int resetSlave(Master_info *mi); private: /* True when initObject has been called */ bool init_done_; bool slave_enabled_; /* semi-sycn is enabled on the slave */ - MYSQL *mysql_reply; /* connection to send reply */ + bool delay_master_; + unsigned int kill_conn_timeout_; }; @@ -96,7 +105,8 @@ extern my_bool rpl_semi_sync_slave_status; extern ulong rpl_semi_sync_slave_trace_level; extern ReplSemiSyncSlave repl_semisync_slave; -int semi_sync_slave_init(); -void semi_sync_slave_deinit(); +extern char rpl_semi_sync_slave_delay_master; +extern unsigned int rpl_semi_sync_slave_kill_conn_timeout; +extern unsigned long long rpl_semi_sync_slave_send_ack; #endif /* SEMISYNC_SLAVE_H */ diff --git a/sql/slave.cc b/sql/slave.cc index a8334525345..b4a817d6ecd 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -61,6 +61,7 @@ #include "debug_sync.h" #include "rpl_parallel.h" #include "sql_show.h" +#include "semisync_slave.h" #define FLAGSTR(V,F) ((V)&(F)?#F" ":"") @@ -3590,7 +3591,9 @@ static int request_dump(THD *thd, MYSQL* mysql, Master_info* mi, before_request_transmit, (thd, mi, binlog_flags))) DBUG_RETURN(1); - + if (repl_semisync_slave.requestTransmit(mi)) + DBUG_RETURN(1); + // TODO if big log files: Change next to int8store() int4store(buf, (ulong) mi->master_log_pos); int2store(buf + 4, binlog_flags); @@ -4615,7 +4618,9 @@ pthread_handler_t handle_slave_io(void *arg) } - if (RUN_HOOK(binlog_relay_io, thread_start, (thd, mi))) + if (RUN_HOOK(binlog_relay_io, thread_start, (thd, mi)) || + (DBUG_EVALUATE_IF("failed_slave_start", 1, 0) + || repl_semisync_slave.slaveStart(mi))) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, ER_THD(thd, ER_SLAVE_FATAL_ERROR), @@ -4805,9 +4810,13 @@ Stopping slave I/O thread due to out-of-memory error from master"); retry_count=0; // ok event, reset retry counter THD_STAGE_INFO(thd, stage_queueing_master_event_to_the_relay_log); event_buf= (const char*)mysql->net.read_pos + 1; + mi->semi_ack= 0; if (RUN_HOOK(binlog_relay_io, after_read_event, (thd, mi,(const char*)mysql->net.read_pos + 1, - event_len, &event_buf, &event_len))) + event_len, &event_buf, &event_len)) || + repl_semisync_slave. + slaveReadSyncHeader((const char*)mysql->net.read_pos + 1, event_len, + &(mi->semi_ack), &event_buf, &event_len)) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, ER_THD(thd, ER_SLAVE_FATAL_ERROR), @@ -4868,7 +4877,10 @@ Stopping slave I/O thread due to out-of-memory error from master"); } if (RUN_HOOK(binlog_relay_io, after_queue_event, - (thd, mi, event_buf, event_len, synced))) + (thd, mi, event_buf, event_len, synced)) || + (rpl_semi_sync_slave_status && + (mi->semi_ack & SEMI_SYNC_NEED_ACK) && + repl_semisync_slave.slaveReply(mi))) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, ER_THD(thd, ER_SLAVE_FATAL_ERROR), @@ -4877,7 +4889,16 @@ Stopping slave I/O thread due to out-of-memory error from master"); } if (mi->using_gtid == Master_info::USE_GTID_NO && - flush_master_info(mi, TRUE, TRUE)) + /* + If rpl_semi_sync_slave_delay_master is enabled, we will flush + master info only when ack is needed. This may lead to at least one + group transaction delay but affords better performance improvement. + */ + (!repl_semisync_slave.getSlaveEnabled() || + (!(mi->semi_ack & SEMI_SYNC_SLAVE_DELAY_SYNC) || + (mi->semi_ack & (SEMI_SYNC_NEED_ACK)))) && + (DBUG_EVALUATE_IF("failed_flush_master_info", 1, 0) || + flush_master_info(mi, TRUE, TRUE))) { sql_print_error("Failed to flush master info file"); goto err; @@ -4931,7 +4952,8 @@ err: IO_RPL_LOG_NAME, mi->master_log_pos, tmp.c_ptr_safe()); } - RUN_HOOK(binlog_relay_io, thread_stop, (thd, mi)); + (void) RUN_HOOK(binlog_relay_io, thread_stop, (thd, mi)); + repl_semisync_slave.slaveStop(mi); thd->reset_query(); thd->reset_db(NULL, 0); if (mysql) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 35ed9be74f9..d0aa0818e99 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -706,7 +706,7 @@ extern "C" void thd_kill_timeout(THD* thd) } -THD::THD(my_thread_id id, bool is_wsrep_applier) +THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock) :Statement(&main_lex, &main_mem_root, STMT_CONVENTIONAL_EXECUTION, /* statement id */ 0), rli_fake(0), rgi_fake(0), rgi_slave(NULL), @@ -893,7 +893,7 @@ THD::THD(my_thread_id id, bool is_wsrep_applier) /* Call to init() below requires fully initialized Open_tables_state. */ reset_open_tables_state(this); - init(); + init(skip_global_sys_var_lock); #if defined(ENABLED_PROFILING) profiling.set_thd(this); #endif @@ -1264,10 +1264,11 @@ const Type_handler *THD::type_handler_for_date() const Init common variables that has to be reset on start and on change_user */ -void THD::init(void) +void THD::init(bool skip_lock) { DBUG_ENTER("thd::init"); - mysql_mutex_lock(&LOCK_global_system_variables); + if (!skip_lock) + mysql_mutex_lock(&LOCK_global_system_variables); plugin_thdvar_init(this); /* plugin_thd_var_init() sets variables= global_system_variables, which @@ -1280,8 +1281,8 @@ void THD::init(void) ::strmake(default_master_connection_buff, global_system_variables.default_master_connection.str, variables.default_master_connection.length); - - mysql_mutex_unlock(&LOCK_global_system_variables); + if (!skip_lock) + mysql_mutex_unlock(&LOCK_global_system_variables); user_time.val= start_time= start_time_sec_part= 0; @@ -4193,7 +4194,8 @@ my_bool thd_net_is_killed() void thd_increment_bytes_received(void *thd, ulong length) { - ((THD*) thd)->status_var.bytes_received+= length; + if (thd != NULL) // MDEV-13073 Ack collector having NULL + ((THD*) thd)->status_var.bytes_received+= length; } diff --git a/sql/sql_class.h b/sql/sql_class.h index 4249bc6bb5b..66c490c6acf 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -1569,7 +1569,8 @@ enum enum_thread_type SYSTEM_THREAD_EVENT_WORKER= 16, SYSTEM_THREAD_BINLOG_BACKGROUND= 32, SYSTEM_THREAD_SLAVE_BACKGROUND= 64, - SYSTEM_THREAD_GENERIC= 128 + SYSTEM_THREAD_GENERIC= 128, + SYSTEM_THREAD_SEMISYNC_MASTER_BACKGROUND= 256 }; inline char const * @@ -1585,6 +1586,7 @@ show_system_thread(enum_thread_type thread) RETURN_NAME_AS_STRING(SYSTEM_THREAD_EVENT_SCHEDULER); RETURN_NAME_AS_STRING(SYSTEM_THREAD_EVENT_WORKER); RETURN_NAME_AS_STRING(SYSTEM_THREAD_SLAVE_BACKGROUND); + RETURN_NAME_AS_STRING(SYSTEM_THREAD_SEMISYNC_MASTER_BACKGROUND); default: sprintf(buf, "", thread); return buf; @@ -2261,7 +2263,8 @@ public: /* Needed by MariaDB semi sync replication */ Trans_binlog_info *semisync_info; - + /* If this is a semisync slave connection. */ + bool semi_sync_slave; ulonglong client_capabilities; /* What the client supports */ ulong max_client_packet_length; @@ -3147,11 +3150,20 @@ public: /* Debug Sync facility. See debug_sync.cc. */ struct st_debug_sync_control *debug_sync_control; #endif /* defined(ENABLED_DEBUG_SYNC) */ - THD(my_thread_id id, bool is_wsrep_applier= false); + /** + @param id thread identifier + @param is_wsrep_applier thread type + @param skip_lock instruct whether @c LOCK_global_system_variables + is already locked, to not acquire it then. + */ + THD(my_thread_id id, bool is_wsrep_applier= false, bool skip_lock= false); ~THD(); - - void init(void); + /** + @param skip_lock instruct whether @c LOCK_global_system_variables + is already locked, to not acquire it then. + */ + void init(bool skip_lock= false); /* Initialize memory roots necessary for query processing and (!) pre-allocate memory for it. We can't do that in THD constructor because diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index fcbf0ce1bd0..36fa6120584 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -30,7 +30,8 @@ #include #include "rpl_handler.h" #include "debug_sync.h" -#include "log.h" // get_gtid_list_event +#include "semisync_master.h" +#include "semisync_slave.h" enum enum_gtid_until_state { GTID_UNTIL_NOT_DONE, @@ -160,6 +161,7 @@ struct binlog_send_info { bool clear_initial_log_pos; bool should_stop; + size_t dirlen; binlog_send_info(THD *thd_arg, String *packet_arg, ushort flags_arg, char *lfn) @@ -315,14 +317,30 @@ static int reset_transmit_packet(binlog_send_info *info, ushort flags, if (RUN_HOOK(binlog_transmit, reserve_header, (info->thd, flags, packet))) { + /* RUN_HOOK() must return zero when thd->semi_sync_slave */ + DBUG_ASSERT(!info->thd->semi_sync_slave); + info->error= ER_UNKNOWN_ERROR; *errmsg= "Failed to run hook 'reserve_header'"; ret= 1; } + if (info->thd->semi_sync_slave) + { + repl_semisync_master.reserveSyncHeader(packet); + } + *ev_offset= packet->length(); return ret; } +inline bool is_semi_sync_slave() +{ + int null_value; + long long val= 0; + get_user_var_int("rpl_semi_sync_slave", &val, &null_value); + return val; +} + static int send_file(THD *thd) { NET* net = &thd->net; @@ -875,73 +893,6 @@ get_binlog_list(MEM_ROOT *memroot) DBUG_RETURN(current_list); } -/* - Find the Gtid_list_log_event at the start of a binlog. - - NULL for ok, non-NULL error message for error. - - If ok, then the event is returned in *out_gtid_list. This can be NULL if we - get back to binlogs written by old server version without GTID support. If - so, it means we have reached the point to start from, as no GTID events can - exist in earlier binlogs. -*/ - -static const char * -get_gtid_list_event(IO_CACHE *cache, Gtid_list_log_event **out_gtid_list) -{ - Format_description_log_event init_fdle(BINLOG_VERSION); - Format_description_log_event *fdle; - Log_event *ev; - const char *errormsg = NULL; - - *out_gtid_list= NULL; - - if (!(ev= Log_event::read_log_event(cache, &init_fdle, - opt_master_verify_checksum)) || - ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) - { - if (ev) - delete ev; - return "Could not read format description log event while looking for " - "GTID position in binlog"; - } - - fdle= static_cast(ev); - - for (;;) - { - Log_event_type typ; - - ev= Log_event::read_log_event(cache, fdle, opt_master_verify_checksum); - if (!ev) - { - errormsg= "Could not read GTID list event while looking for GTID " - "position in binlog"; - break; - } - typ= ev->get_type_code(); - if (typ == GTID_LIST_EVENT) - break; /* Done, found it */ - if (typ == START_ENCRYPTION_EVENT) - { - if (fdle->start_decryption((Start_encryption_log_event*) ev)) - errormsg= "Could not set up decryption for binlog."; - } - delete ev; - if (typ == ROTATE_EVENT || typ == STOP_EVENT || - typ == FORMAT_DESCRIPTION_EVENT || typ == START_ENCRYPTION_EVENT) - continue; /* Continue looking */ - - /* We did not find any Gtid_list_log_event, must be old binlog. */ - ev= NULL; - break; - } - - delete fdle; - *out_gtid_list= static_cast(ev); - return errormsg; -} - /* Check if every GTID requested by the slave is contained in this (or a later) @@ -1673,6 +1624,7 @@ send_event_to_slave(binlog_send_info *info, Log_event_type event_type, enum enum_binlog_checksum_alg current_checksum_alg= info->current_checksum_alg; slave_connection_state *gtid_state= &info->gtid_state; slave_connection_state *until_gtid_state= info->until_gtid_state; + bool need_sync= false; if (event_type == GTID_LIST_EVENT && info->using_gtid_state && until_gtid_state) @@ -1984,7 +1936,10 @@ send_event_to_slave(binlog_send_info *info, Log_event_type event_type, pos= my_b_tell(log); if (RUN_HOOK(binlog_transmit, before_send_event, - (info->thd, info->flags, packet, info->log_file_name, pos))) + (info->thd, info->flags, packet, info->log_file_name, pos)) || + repl_semisync_master.updateSyncHeader(info->thd, (uchar *)packet->c_ptr(), + info->log_file_name + info->dirlen, + pos, &need_sync)) { info->error= ER_UNKNOWN_ERROR; return "run 'before_send_event' hook failed"; @@ -2012,6 +1967,8 @@ send_event_to_slave(binlog_send_info *info, Log_event_type event_type, info->error= ER_UNKNOWN_ERROR; return "Failed to run hook 'after_send_event'"; } + if (need_sync) + repl_semisync_master.flushNet(info->thd, packet->c_ptr()); return NULL; /* Success */ } @@ -2748,7 +2705,7 @@ static int send_one_binlog_file(binlog_send_info *info, /** end of file or error */ return (int)end_pos; } - + info->dirlen= dirname_length(info->log_file_name); /** * send events from current position up to end_pos */ @@ -2770,6 +2727,7 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, binlog_send_info infoobj(thd, packet, flags, linfo.log_file_name); binlog_send_info *info= &infoobj; + bool has_transmit_started= false; int old_max_allowed_packet= thd->variables.max_allowed_packet; thd->variables.max_allowed_packet= MAX_MAX_ALLOWED_PACKET; @@ -2792,6 +2750,11 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, info->error= ER_UNKNOWN_ERROR; goto err; } + has_transmit_started= true; + + /* Check if the dump thread is created by a slave with semisync enabled. */ + thd->semi_sync_slave = is_semi_sync_slave(); + repl_semisync_master.dump_start(thd, log_ident, pos); /* heartbeat_period from @master_heartbeat_period user variable @@ -2908,7 +2871,11 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, err: THD_STAGE_INFO(thd, stage_waiting_to_finalize_termination); - RUN_HOOK(binlog_transmit, transmit_stop, (thd, flags)); + (void) RUN_HOOK(binlog_transmit, transmit_stop, (thd, flags)); + if (has_transmit_started) + { + repl_semisync_master.dump_end(thd); + } if (info->thd->killed == KILL_SLAVE_SAME_ID) { @@ -3374,7 +3341,9 @@ int reset_slave(THD *thd, Master_info* mi) else if (global_system_variables.log_warnings > 1) sql_print_information("Deleted Master_info file '%s'.", fname); - RUN_HOOK(binlog_relay_io, after_reset_slave, (thd, mi)); + (void) RUN_HOOK(binlog_relay_io, after_reset_slave, (thd, mi)); + if (rpl_semi_sync_slave_enabled) + repl_semisync_slave.resetSlave(mi); err: mi->unlock_slave_threads(); if (error) @@ -3876,11 +3845,14 @@ int reset_master(THD* thd, rpl_gtid *init_state, uint32 init_state_len, return 1; } - if (mysql_bin_log.reset_logs(thd, 1, init_state, init_state_len, - next_log_number)) - return 1; - RUN_HOOK(binlog_transmit, after_reset_master, (thd, 0 /* flags */)); - return 0; + bool ret= 0; + /* Temporarily disable master semisync before reseting master. */ + repl_semisync_master.beforeResetMaster(); + ret= mysql_bin_log.reset_logs(thd, 1, init_state, init_state_len, + next_log_number); + (void) RUN_HOOK(binlog_transmit, after_reset_master, (thd, 0 /* flags */)); + repl_semisync_master.afterResetMaster(); + return ret; } diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index e897b6c21ce..9eac6017c7c 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3051,22 +3051,18 @@ static bool fix_rpl_semi_sync_master_enabled(sys_var *self, THD *thd, { if (repl_semisync_master.enableMaster() != 0) rpl_semi_sync_master_enabled= false; -#ifdef HAVE_ACC_RECEIVER else if (ack_receiver.start()) { repl_semisync_master.disableMaster(); rpl_semi_sync_master_enabled= false; } -#endif } else { if (repl_semisync_master.disableMaster() != 0) rpl_semi_sync_master_enabled= true; -#ifdef HAVE_ACC_RECEIVER if (!rpl_semi_sync_master_enabled) ack_receiver.stop(); -#endif } return false; } @@ -3082,27 +3078,21 @@ static bool fix_rpl_semi_sync_master_trace_level(sys_var *self, THD *thd, enum_var_type type) { repl_semisync_master.setTraceLevel(rpl_semi_sync_master_trace_level); -#ifdef HAVE_ACC_RECEIVER ack_receiver.setTraceLevel(rpl_semi_sync_master_trace_level); -#endif return false; } static bool fix_rpl_semi_sync_master_wait_point(sys_var *self, THD *thd, enum_var_type type) { -#ifdef HAVE_ACC_RECEIVER repl_semisync_master.setWaitPoint(rpl_semi_sync_master_wait_point); -#endif return false; } static bool fix_rpl_semi_sync_master_wait_no_slave(sys_var *self, THD *thd, enum_var_type type) { -#ifdef HAVE_ACC_RECEIVER repl_semisync_master.checkAndSwitch(); -#endif return false; } @@ -3168,7 +3158,6 @@ static bool fix_rpl_semi_sync_slave_trace_level(sys_var *self, THD *thd, return false; } -#ifdef HAVE_ACC_RECEIVER static bool fix_rpl_semi_sync_slave_delay_master(sys_var *self, THD *thd, enum_var_type type) { @@ -3182,8 +3171,6 @@ static bool fix_rpl_semi_sync_slave_kill_conn_timeout(sys_var *self, THD *thd, repl_semisync_slave.setKillConnTimeout(rpl_semi_sync_slave_kill_conn_timeout); return false; } -#endif - static Sys_var_mybool Sys_semisync_slave_enabled( "rpl_semi_sync_slave_enabled", @@ -3202,7 +3189,6 @@ static Sys_var_ulong Sys_semisync_slave_trace_level( NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), ON_UPDATE(fix_rpl_semi_sync_slave_trace_level)); -#ifdef HAVE_ACC_RECEIVER static Sys_var_mybool Sys_semisync_slave_delay_master( "rpl_semi_sync_slave_delay_master", "Only write master info file when ack is needed.", @@ -3221,7 +3207,6 @@ static Sys_var_uint Sys_semisync_slave_kill_conn_timeout( VALID_RANGE(0, UINT_MAX), DEFAULT(5), BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(0), ON_UPDATE(fix_rpl_semi_sync_slave_kill_conn_timeout)); -#endif #endif /* HAVE_REPLICATION */ static Sys_var_ulong Sys_slow_launch_time( diff --git a/sql/transaction.cc b/sql/transaction.cc index cbd875e3114..f6ccf5a1930 100644 --- a/sql/transaction.cc +++ b/sql/transaction.cc @@ -24,7 +24,7 @@ #include "rpl_handler.h" #include "debug_sync.h" // DEBUG_SYNC #include "sql_acl.h" - +#include "semisync_master.h" #ifndef EMBEDDED_LIBRARY /** @@ -318,9 +318,19 @@ bool trans_commit(THD *thd) transaction, so the hooks for rollback will be called. */ if (res) + { (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); +#ifdef HAVE_REPLICATION + repl_semisync_master.waitAfterRollback(thd, FALSE); +#endif + } else + { (void) RUN_HOOK(transaction, after_commit, (thd, FALSE)); +#ifdef HAVE_REPLICATION + repl_semisync_master.waitAfterCommit(thd, FALSE); +#endif + } thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_KEEP_LOG); thd->transaction.all.reset(); thd->lex->start_transaction_opt= 0; @@ -414,6 +424,9 @@ bool trans_rollback(THD *thd) DBUG_PRINT("info", ("clearing SERVER_STATUS_IN_TRANS")); res= ha_rollback_trans(thd, TRUE); (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); +#ifdef HAVE_REPLICATION + repl_semisync_master.waitAfterRollback(thd, FALSE); +#endif thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_KEEP_LOG); /* Reset the binlog transaction marker */ thd->variables.option_bits&= ~OPTION_GTID_BEGIN; @@ -526,9 +539,19 @@ bool trans_commit_stmt(THD *thd) transaction, so the hooks for rollback will be called. */ if (res) + { (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); +#ifdef HAVE_REPLICATION + repl_semisync_master.waitAfterRollback(thd, FALSE); +#endif + } else + { (void) RUN_HOOK(transaction, after_commit, (thd, FALSE)); +#ifdef HAVE_REPLICATION + repl_semisync_master.waitAfterCommit(thd, FALSE); +#endif + } thd->transaction.stmt.reset(); @@ -568,6 +591,9 @@ bool trans_rollback_stmt(THD *thd) } (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); +#ifdef HAVE_REPLICATION + repl_semisync_master.waitAfterRollback(thd, FALSE); +#endif thd->transaction.stmt.reset(); From 74b35b68744ce552f09124e27ef1785af69d7989 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Wed, 22 Nov 2017 19:34:42 +0200 Subject: [PATCH 22/51] MDEV-13073. This part patch weeds out RUN_HOOK from the server as semisync is defined statically. Consequently the observer interfaces are removed as well. --- libmysqld/CMakeLists.txt | 2 +- .../perfschema/r/dml_setup_instruments.result | 4 +- .../r/sysvars_server_notembedded,32bit.rdiff | 93 ++- sql/CMakeLists.txt | 2 +- sql/handler.cc | 8 +- sql/log.cc | 75 +-- sql/mysqld.cc | 12 +- sql/mysqld.h | 1 - sql/rpl_handler.cc | 555 ------------------ sql/rpl_handler.h | 222 ------- sql/semisync_master.cc | 8 +- sql/semisync_master.h | 2 +- sql/semisync_slave.h | 2 + sql/slave.cc | 26 +- sql/sql_parse.cc | 1 - sql/sql_repl.cc | 59 +- sql/transaction.cc | 7 - 17 files changed, 137 insertions(+), 942 deletions(-) delete mode 100644 sql/rpl_handler.cc delete mode 100644 sql/rpl_handler.h diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index abb19fd9932..19962df0bec 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -90,7 +90,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/scheduler.cc ../sql/sql_audit.cc ../sql/sql_alter.cc ../sql/sql_partition_admin.cc ../sql/event_parse_data.cc - ../sql/sql_signal.cc ../sql/rpl_handler.cc + ../sql/sql_signal.cc ../sql/sys_vars.cc ${CMAKE_BINARY_DIR}/sql/sql_builtin.cc ../sql/mdl.cc ../sql/transaction.cc diff --git a/mysql-test/suite/perfschema/r/dml_setup_instruments.result b/mysql-test/suite/perfschema/r/dml_setup_instruments.result index 3f4d6107717..ff184806e2e 100644 --- a/mysql-test/suite/perfschema/r/dml_setup_instruments.result +++ b/mysql-test/suite/perfschema/r/dml_setup_instruments.result @@ -4,7 +4,7 @@ where name like 'Wait/Synch/Mutex/sql/%' and name not in ('wait/synch/mutex/sql/DEBUG_SYNC::mutex') order by name limit 10; NAME ENABLED TIMED -wait/synch/mutex/sql/Ack_receiver::m_mutex YES YES +wait/synch/mutex/sql/Ack_receiver::mutex YES YES wait/synch/mutex/sql/Cversion_lock YES YES wait/synch/mutex/sql/Delayed_insert::mutex YES YES wait/synch/mutex/sql/Event_scheduler::LOCK_scheduler_state YES YES @@ -36,7 +36,7 @@ where name like 'Wait/Synch/Cond/sql/%' 'wait/synch/cond/sql/DEBUG_SYNC::cond') order by name limit 10; NAME ENABLED TIMED -wait/synch/cond/sql/Ack_receiver::m_cond YES YES +wait/synch/cond/sql/Ack_receiver::cond YES YES wait/synch/cond/sql/COND_binlog_send YES YES wait/synch/cond/sql/COND_flush_thread_cache YES YES wait/synch/cond/sql/COND_group_commit_orderer YES YES diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff index 8b0749810e9..cb9e84b81c8 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff @@ -1,5 +1,5 @@ ---- sysvars_server_notembedded.result 2017-11-17 17:00:22.470630255 +0100 -+++ sysvars_server_notembedded,32bit.reject 2017-11-17 19:12:42.732453556 +0100 +--- sysvars_server_notembedded.result 2017-12-15 20:57:40.174654761 +0200 ++++ sysvars_server_notembedded,32bit.reject 2017-12-15 21:02:20.476044700 +0200 @@ -58,7 +58,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 1 @@ -1116,7 +1116,46 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -4048,7 +4048,7 @@ +@@ -4034,10 +4034,10 @@ + GLOBAL_VALUE_ORIGIN COMPILE-TIME + DEFAULT_VALUE 10000 + VARIABLE_SCOPE GLOBAL +-VARIABLE_TYPE BIGINT UNSIGNED ++VARIABLE_TYPE INT UNSIGNED + VARIABLE_COMMENT The timeout value (in ms) for semi-synchronous replication in the master + NUMERIC_MIN_VALUE 0 +-NUMERIC_MAX_VALUE 18446744073709551615 ++NUMERIC_MAX_VALUE 4294967295 + NUMERIC_BLOCK_SIZE 1 + ENUM_VALUE_LIST NULL + READ_ONLY NO +@@ -4048,10 +4048,10 @@ + GLOBAL_VALUE_ORIGIN COMPILE-TIME + DEFAULT_VALUE 32 + VARIABLE_SCOPE GLOBAL +-VARIABLE_TYPE BIGINT UNSIGNED ++VARIABLE_TYPE INT UNSIGNED + VARIABLE_COMMENT The tracing level for semi-sync replication. + NUMERIC_MIN_VALUE 0 +-NUMERIC_MAX_VALUE 18446744073709551615 ++NUMERIC_MAX_VALUE 4294967295 + NUMERIC_BLOCK_SIZE 1 + ENUM_VALUE_LIST NULL + READ_ONLY NO +@@ -4132,10 +4132,10 @@ + GLOBAL_VALUE_ORIGIN COMPILE-TIME + DEFAULT_VALUE 32 + VARIABLE_SCOPE GLOBAL +-VARIABLE_TYPE BIGINT UNSIGNED ++VARIABLE_TYPE INT UNSIGNED + VARIABLE_COMMENT The tracing level for semi-sync replication. + NUMERIC_MIN_VALUE 0 +-NUMERIC_MAX_VALUE 18446744073709551615 ++NUMERIC_MAX_VALUE 4294967295 + NUMERIC_BLOCK_SIZE 1 + ENUM_VALUE_LIST NULL + READ_ONLY NO +@@ -4174,7 +4174,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 1 VARIABLE_SCOPE SESSION @@ -1125,7 +1164,7 @@ VARIABLE_COMMENT Uniquely identifies the server instance in the community of replication partners NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 4294967295 -@@ -4230,7 +4230,7 @@ +@@ -4356,7 +4356,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -1134,7 +1173,7 @@ VARIABLE_COMMENT Maximum number of parallel threads to use on slave for events in a single replication domain. When using multiple domains, this can be used to limit a single domain from grabbing all threads and thus stalling other domains. The default of 0 means to allow a domain to grab as many threads as it wants, up to the value of slave_parallel_threads. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16383 -@@ -4272,7 +4272,7 @@ +@@ -4398,7 +4398,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1073741824 VARIABLE_SCOPE GLOBAL @@ -1143,7 +1182,7 @@ VARIABLE_COMMENT The maximum packet length to sent successfully from the master to slave. NUMERIC_MIN_VALUE 1024 NUMERIC_MAX_VALUE 1073741824 -@@ -4300,7 +4300,7 @@ +@@ -4426,7 +4426,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 131072 VARIABLE_SCOPE GLOBAL @@ -1152,7 +1191,7 @@ VARIABLE_COMMENT Limit on how much memory SQL threads should use per parallel replication thread when reading ahead in the relay log looking for opportunities for parallel replication. Only used when --slave-parallel-threads > 0. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2147483647 -@@ -4328,7 +4328,7 @@ +@@ -4454,7 +4454,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -1161,7 +1200,7 @@ VARIABLE_COMMENT If non-zero, number of threads to spawn to apply in parallel events on the slave that were group-committed on the master or were logged with GTID in different replication domains. Note that these threads are in addition to the IO and SQL threads, which are always created by a replication slave NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16383 -@@ -4342,7 +4342,7 @@ +@@ -4468,7 +4468,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -1170,7 +1209,7 @@ VARIABLE_COMMENT Alias for slave_parallel_threads NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16383 -@@ -4398,7 +4398,7 @@ +@@ -4524,7 +4524,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 10 VARIABLE_SCOPE GLOBAL @@ -1179,7 +1218,7 @@ VARIABLE_COMMENT Number of times the slave SQL thread will retry a transaction in case it failed with a deadlock, elapsed lock wait timeout or listed in slave_transaction_retry_errors, before giving up and stopping NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -4426,7 +4426,7 @@ +@@ -4552,7 +4552,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -1188,7 +1227,7 @@ VARIABLE_COMMENT Interval of the slave SQL thread will retry a transaction in case it failed with a deadlock or elapsed lock wait timeout or listed in slave_transaction_retry_errors NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 3600 -@@ -4257,7 +4257,7 @@ +@@ -4580,7 +4580,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2 VARIABLE_SCOPE GLOBAL @@ -1197,7 +1236,7 @@ VARIABLE_COMMENT If creating the thread takes longer than this value (in seconds), the Slow_launch_threads counter will be incremented NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 31536000 -@@ -4316,7 +4316,7 @@ +@@ -4639,7 +4639,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Each thread that needs to do a sort allocates a buffer of this size NUMERIC_MIN_VALUE 1024 @@ -1206,7 +1245,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -4621,7 +4621,7 @@ +@@ -4944,7 +4944,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 256 VARIABLE_SCOPE GLOBAL @@ -1215,7 +1254,7 @@ VARIABLE_COMMENT The soft upper limit for number of cached stored routines for one connection. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 524288 -@@ -4719,7 +4719,7 @@ +@@ -5042,7 +5042,7 @@ GLOBAL_VALUE_ORIGIN AUTO DEFAULT_VALUE 400 VARIABLE_SCOPE GLOBAL @@ -1224,7 +1263,7 @@ VARIABLE_COMMENT The number of cached table definitions NUMERIC_MIN_VALUE 400 NUMERIC_MAX_VALUE 524288 -@@ -4733,7 +4733,7 @@ +@@ -5056,7 +5056,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2000 VARIABLE_SCOPE GLOBAL @@ -1233,7 +1272,7 @@ VARIABLE_COMMENT The number of cached open tables NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1048576 -@@ -4761,7 +4761,7 @@ +@@ -5126,7 +5126,7 @@ GLOBAL_VALUE_ORIGIN AUTO DEFAULT_VALUE 256 VARIABLE_SCOPE GLOBAL @@ -1242,7 +1281,7 @@ VARIABLE_COMMENT How many threads we should keep in a cache for reuse. These are freed after 5 minutes of idle time NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16384 -@@ -4775,7 +4775,7 @@ +@@ -5140,7 +5140,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 10 VARIABLE_SCOPE GLOBAL @@ -1251,7 +1290,7 @@ VARIABLE_COMMENT Permits the application to give the threads system a hint for the desired number of threads that should be run at the same time.This variable has no effect, and is deprecated. It will be removed in a future release. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 512 -@@ -4980,15 +4980,15 @@ +@@ -5345,15 +5345,15 @@ READ_ONLY YES COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME TMP_DISK_TABLE_SIZE @@ -1271,7 +1310,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -5002,7 +5002,7 @@ +@@ -5367,7 +5367,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. Same as tmp_table_size. NUMERIC_MIN_VALUE 1024 @@ -1280,7 +1319,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -5016,7 +5016,7 @@ +@@ -5381,7 +5381,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Alias for tmp_memory_table_size. If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. NUMERIC_MIN_VALUE 1024 @@ -1289,7 +1328,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -5027,7 +5027,7 @@ +@@ -5392,7 +5392,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8192 VARIABLE_SCOPE SESSION @@ -1298,7 +1337,7 @@ VARIABLE_COMMENT Allocation block size for transactions to be stored in binary log NUMERIC_MIN_VALUE 1024 NUMERIC_MAX_VALUE 134217728 -@@ -5041,7 +5041,7 @@ +@@ -5406,7 +5406,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 4096 VARIABLE_SCOPE SESSION @@ -1307,7 +1346,7 @@ VARIABLE_COMMENT Persistent buffer for transactions to be stored in binary log NUMERIC_MIN_VALUE 1024 NUMERIC_MAX_VALUE 134217728 -@@ -5139,7 +5139,7 @@ +@@ -5504,7 +5504,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 28800 VARIABLE_SCOPE SESSION @@ -1316,7 +1355,7 @@ VARIABLE_COMMENT The number of seconds the server waits for activity on a connection before closing it NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 31536000 -@@ -5243,7 +5243,7 @@ +@@ -5609,7 +5609,7 @@ COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME OPEN_FILES_LIMIT VARIABLE_SCOPE GLOBAL @@ -1325,7 +1364,7 @@ VARIABLE_COMMENT If this is not 0, then mysqld will use this value to reserve file descriptors to use with setrlimit(). If this value is 0 or autoset then mysqld will reserve max_connections*5 or max_connections + table_cache*2 (whichever is larger) number of file descriptors NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -5256,7 +5256,7 @@ +@@ -5622,7 +5622,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Sets the internal state of the RAND() generator for replication purposes NUMERIC_MIN_VALUE 0 @@ -1334,7 +1373,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -5266,7 +5266,7 @@ +@@ -5632,7 +5632,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Sets the internal state of the RAND() generator for replication purposes NUMERIC_MIN_VALUE 0 @@ -1343,7 +1382,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -5351,7 +5351,7 @@ +@@ -5727,7 +5727,7 @@ VARIABLE_NAME LOG_TC_SIZE GLOBAL_VALUE_ORIGIN AUTO VARIABLE_SCOPE GLOBAL diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index fadee80491b..24e1dd27d02 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -122,7 +122,7 @@ SET (SQL_SOURCE rpl_rli.cc rpl_mi.cc sql_servers.cc sql_audit.cc sql_connect.cc scheduler.cc sql_partition_admin.cc sql_profile.cc event_parse_data.cc sql_alter.cc - sql_signal.cc rpl_handler.cc mdl.cc sql_admin.cc + sql_signal.cc mdl.cc sql_admin.cc transaction.cc sys_vars.cc sql_truncate.cc datadict.cc sql_reload.cc item_inetfunc.cc diff --git a/sql/handler.cc b/sql/handler.cc index 7ea02278abc..a0b1c0ebf07 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -23,7 +23,7 @@ #include "mariadb.h" #include "sql_priv.h" #include "unireg.h" -#include "rpl_handler.h" +#include "rpl_rli.h" #include "sql_cache.h" // query_cache, query_cache_* #include "sql_connect.h" // global_table_stats #include "key.h" // key_copy, key_unpack, key_cmp_if_same, key_cmp @@ -1485,8 +1485,7 @@ done: mysql_mutex_assert_not_owner(mysql_bin_log.get_log_lock()); mysql_mutex_assert_not_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); - (void) RUN_HOOK(transaction, after_commit, (thd, FALSE)); -#ifdef REPLICATION +#ifdef HAVE_REPLICATION repl_semisync_master.waitAfterCommit(thd, all); DEBUG_SYNC(thd, "after_group_after_commit"); #endif @@ -1734,8 +1733,7 @@ int ha_rollback_trans(THD *thd, bool all) push_warning(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARNING_NOT_COMPLETE_ROLLBACK, ER_THD(thd, ER_WARNING_NOT_COMPLETE_ROLLBACK)); - (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); -#ifdef REPLICATION +#ifdef HAVE_REPLICATION repl_semisync_master.waitAfterRollback(thd, all); #endif DBUG_RETURN(error); diff --git a/sql/log.cc b/sql/log.cc index a866d72d785..cd757cac30e 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -49,7 +49,6 @@ #endif #include "sql_plugin.h" -#include "rpl_handler.h" #include "debug_sync.h" #include "sql_show.h" #include "my_pthread.h" @@ -6376,19 +6375,15 @@ err: mysql_mutex_assert_owner(&LOCK_log); mysql_mutex_assert_not_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); - if ((error= RUN_HOOK(binlog_storage, after_flush, - (thd, log_file_name, file->pos_in_file, - synced, true, true))) -#ifdef REPLICATION - || repl_semisync_master.reportBinlogUpdate(thd, log_file_name, - file->pos_in_file) -#endif - ) +#ifdef HAVE_REPLICATION + if (repl_semisync_master.reportBinlogUpdate(thd, log_file_name, + file->pos_in_file)) { sql_print_error("Failed to run 'after_flush' hooks"); error= 1; } else +#endif { /* update binlog_end_pos so it can be read by dump thread @@ -6413,18 +6408,14 @@ err: mysql_mutex_assert_not_owner(&LOCK_log); mysql_mutex_assert_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); - if (RUN_HOOK(binlog_storage, after_sync, - (thd, log_file_name, file->pos_in_file, - true, true)) -#ifdef REPLICATION - || repl_semisync_master.waitAfterSync(log_file_name, - file->pos_in_file) -#endif - ) +#ifdef HAVE_REPLICATION + if (repl_semisync_master.waitAfterSync(log_file_name, + file->pos_in_file)) { error=1; /* error is already printed inside hook */ } +#endif /* Take mutex to protect against a reader seeing partial writes of 64-bit @@ -7850,33 +7841,23 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) mysql_mutex_assert_owner(&LOCK_log); mysql_mutex_assert_not_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); - bool first __attribute__((unused))= true; - bool last __attribute__((unused)); + for (current= queue; current != NULL; current= current->next) { - last= current->next == NULL; +#ifdef HAVE_REPLICATION if (!current->error && - (RUN_HOOK(binlog_storage, after_flush, - (current->thd, - current->cache_mngr->last_commit_pos_file, - current->cache_mngr->last_commit_pos_offset, synced, - first, last)) -#ifdef REPLICATION - || (DBUG_EVALUATE_IF("failed_report_binlog_update", 1, 0) || - repl_semisync_master. - reportBinlogUpdate(current->thd, - current->cache_mngr->last_commit_pos_file, - current->cache_mngr-> - last_commit_pos_offset)) -#endif - )) + repl_semisync_master. + reportBinlogUpdate(current->thd, + current->cache_mngr->last_commit_pos_file, + current->cache_mngr-> + last_commit_pos_offset)) { current->error= ER_ERROR_ON_WRITE; current->commit_errno= -1; current->error_cache= NULL; any_error= true; } - first= false; +#endif } /* @@ -7951,24 +7932,14 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) for (current= queue; current != NULL; current= current->next) { last= current->next == NULL; - if (!current->error && - (RUN_HOOK(binlog_storage, after_sync, - (current->thd, current->cache_mngr->last_commit_pos_file, - current->cache_mngr->last_commit_pos_offset, - first, last)) -#ifdef REPLICATION - || (DBUG_EVALUATE_IF("simulate_after_sync_hook_error", 1, 0) || - repl_semisync_master.waitAfterSync(current->cache_mngr-> - last_commit_pos_file, - current->cache_mngr-> - last_commit_pos_offset)) +#ifdef HAVE_REPLICATION + if (!current->error) + current->error= + repl_semisync_master.waitAfterSync(current->cache_mngr-> + last_commit_pos_file, + current->cache_mngr-> + last_commit_pos_offset); #endif - )) - { - const char *hook_name= rpl_semi_sync_master_enabled ? - "'waitAfterSync'" : "binlog_storage 'after_sync'"; - sql_print_error("Failed to call '%s'", hook_name); - } first= false; } } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 86759b8ed8b..f6a4ea60755 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -97,7 +97,6 @@ #include "set_var.h" #include "rpl_injector.h" -#include "rpl_handler.h" #include "semisync_master.h" #include "semisync_slave.h" @@ -390,7 +389,7 @@ static longlong start_memory_used; /* Global variables */ bool opt_bin_log, opt_bin_log_used=0, opt_ignore_builtin_innodb= 0; -bool opt_bin_log_compress, run_hooks_enabled; +bool opt_bin_log_compress; uint opt_bin_log_compress_min_len; my_bool opt_log, debug_assert_if_crashed_table= 0, opt_help= 0; my_bool debug_assert_on_not_freed_memory= 0; @@ -2237,7 +2236,6 @@ void clean_up(bool print_message) #ifdef HAVE_REPLICATION semi_sync_master_deinit(); #endif - delegates_destroy(); xid_cache_free(); tdc_deinit(); mdl_destroy(); @@ -5154,13 +5152,6 @@ static int init_server_components() xid_cache_init(); - /* - initialize delegates for extension observers, errors have already - been reported in the function - */ - if (delegates_init()) - unireg_abort(1); - /* need to configure logging before initializing storage engines */ if (!opt_bin_log_used && !WSREP_ON) { @@ -8960,7 +8951,6 @@ static int mysql_init_variables(void) transactions_multi_engine= 0; rpl_transactions_multi_engine= 0; transactions_gtid_foreign_engine= 0; - run_hooks_enabled= 0; // don't run hooks, semisync does not need 'em log_bin_basename= NULL; log_bin_index= NULL; diff --git a/sql/mysqld.h b/sql/mysqld.h index 5399ec91b19..9a2cde9b145 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -109,7 +109,6 @@ extern CHARSET_INFO *character_set_filesystem; extern MY_BITMAP temp_pool; extern bool opt_large_files; extern bool opt_update_log, opt_bin_log, opt_error_log, opt_bin_log_compress; -extern bool run_hooks_enabled; extern uint opt_bin_log_compress_min_len; extern my_bool opt_log, opt_bootstrap; extern my_bool opt_backup_history_log; diff --git a/sql/rpl_handler.cc b/sql/rpl_handler.cc deleted file mode 100644 index 27e411ca6de..00000000000 --- a/sql/rpl_handler.cc +++ /dev/null @@ -1,555 +0,0 @@ -/* Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -#include "mariadb.h" -#include "sql_priv.h" -#include "unireg.h" - -#include "rpl_mi.h" -#include "sql_repl.h" -#include "log_event.h" -#include "rpl_filter.h" -#include -#include "rpl_handler.h" - -Trans_delegate *transaction_delegate; -Binlog_storage_delegate *binlog_storage_delegate; -#ifdef HAVE_REPLICATION -Binlog_transmit_delegate *binlog_transmit_delegate; -Binlog_relay_IO_delegate *binlog_relay_io_delegate; -#endif /* HAVE_REPLICATION */ - -/* - structure to save transaction log filename and position -*/ -typedef struct Trans_binlog_info { - my_off_t log_pos; - char log_file[FN_REFLEN]; -} Trans_binlog_info; - -int get_user_var_int(const char *name, - long long int *value, int *null_value) -{ - bool null_val; - user_var_entry *entry= - (user_var_entry*) my_hash_search(¤t_thd->user_vars, - (uchar*) name, strlen(name)); - if (!entry) - return 1; - *value= entry->val_int(&null_val); - if (null_value) - *null_value= null_val; - return 0; -} - -int get_user_var_real(const char *name, - double *value, int *null_value) -{ - bool null_val; - user_var_entry *entry= - (user_var_entry*) my_hash_search(¤t_thd->user_vars, - (uchar*) name, strlen(name)); - if (!entry) - return 1; - *value= entry->val_real(&null_val); - if (null_value) - *null_value= null_val; - return 0; -} - -int get_user_var_str(const char *name, char *value, - size_t len, unsigned int precision, int *null_value) -{ - String str; - bool null_val; - user_var_entry *entry= - (user_var_entry*) my_hash_search(¤t_thd->user_vars, - (uchar*) name, strlen(name)); - if (!entry) - return 1; - entry->val_str(&null_val, &str, precision); - strncpy(value, str.c_ptr(), len); - if (null_value) - *null_value= null_val; - return 0; -} - -int delegates_init() -{ - static my_aligned_storage trans_mem; - static my_aligned_storage storage_mem; -#ifdef HAVE_REPLICATION - static my_aligned_storage transmit_mem; - static my_aligned_storage relay_io_mem; -#endif - - void *place_trans_mem= trans_mem.data; - void *place_storage_mem= storage_mem.data; - - transaction_delegate= new (place_trans_mem) Trans_delegate; - - if (!transaction_delegate->is_inited()) - { - sql_print_error("Initialization of transaction delegates failed. " - "Please report a bug."); - return 1; - } - - binlog_storage_delegate= new (place_storage_mem) Binlog_storage_delegate; - - if (!binlog_storage_delegate->is_inited()) - { - sql_print_error("Initialization binlog storage delegates failed. " - "Please report a bug."); - return 1; - } - -#ifdef HAVE_REPLICATION - void *place_transmit_mem= transmit_mem.data; - void *place_relay_io_mem= relay_io_mem.data; - - binlog_transmit_delegate= new (place_transmit_mem) Binlog_transmit_delegate; - - if (!binlog_transmit_delegate->is_inited()) - { - sql_print_error("Initialization of binlog transmit delegates failed. " - "Please report a bug."); - return 1; - } - - binlog_relay_io_delegate= new (place_relay_io_mem) Binlog_relay_IO_delegate; - - if (!binlog_relay_io_delegate->is_inited()) - { - sql_print_error("Initialization binlog relay IO delegates failed. " - "Please report a bug."); - return 1; - } -#endif - - return 0; -} - -void delegates_destroy() -{ - if (transaction_delegate) - transaction_delegate->~Trans_delegate(); - transaction_delegate= 0; - if (binlog_storage_delegate) - binlog_storage_delegate->~Binlog_storage_delegate(); - binlog_storage_delegate= 0; -#ifdef HAVE_REPLICATION - if (binlog_transmit_delegate) - binlog_transmit_delegate->~Binlog_transmit_delegate(); - binlog_transmit_delegate= 0; - if (binlog_relay_io_delegate) - binlog_relay_io_delegate->~Binlog_relay_IO_delegate(); - binlog_relay_io_delegate= 0; -#endif /* HAVE_REPLICATION */ -} - -/* - This macro is used by almost all the Delegate methods to iterate - over all the observers running given callback function of the - delegate. - */ -#define FOREACH_OBSERVER(r, f, do_lock, args) \ - param.server_id= thd->variables.server_id; \ - read_lock(); \ - Observer_info_iterator iter= observer_info_iter(); \ - Observer_info *info= iter++; \ - for (; info; info= iter++) \ - { \ - if (((Observer *)info->observer)->f \ - && ((Observer *)info->observer)->f args) \ - { \ - r= 1; \ - sql_print_error("Run function '" #f "' failed"); \ - break; \ - } \ - } \ - unlock(); - - -int Trans_delegate::after_commit(THD *thd, bool all) -{ - Trans_param param; - Trans_binlog_info *log_info; - bool is_real_trans= (all || thd->transaction.all.ha_list == 0); - int ret= 0; - - param.flags = is_real_trans ? TRANS_IS_REAL_TRANS : 0; - - log_info= thd->semisync_info; - - param.log_file= log_info && log_info->log_file[0] ? log_info->log_file : 0; - param.log_pos= log_info ? log_info->log_pos : 0; - - FOREACH_OBSERVER(ret, after_commit, false, (¶m)); - - /* - This is the end of a real transaction or autocommit statement, we - can mark the memory unused. - */ - if (is_real_trans && log_info) - { - log_info->log_file[0]= 0; - log_info->log_pos= 0; - } - return ret; -} - -int Trans_delegate::after_rollback(THD *thd, bool all) -{ - Trans_param param; - Trans_binlog_info *log_info; - bool is_real_trans= (all || thd->transaction.all.ha_list == 0); - int ret= 0; - - param.flags = is_real_trans ? TRANS_IS_REAL_TRANS : 0; - - log_info= thd->semisync_info; - - param.log_file= log_info && log_info->log_file[0] ? log_info->log_file : 0; - param.log_pos= log_info ? log_info->log_pos : 0; - - FOREACH_OBSERVER(ret, after_rollback, false, (¶m)); - - /* - This is the end of a real transaction or autocommit statement, we - can mark the memory unused. - */ - if (is_real_trans && log_info) - { - log_info->log_file[0]= 0; - log_info->log_pos= 0; - } - return ret; -} - -int Binlog_storage_delegate::after_flush(THD *thd, - const char *log_file, - my_off_t log_pos, - bool synced, - bool first_in_group, - bool last_in_group) -{ - Binlog_storage_param param; - Trans_binlog_info *log_info; - uint32 flags=0; - int ret= 0; - - if (synced) - flags |= BINLOG_STORAGE_IS_SYNCED; - if (first_in_group) - flags|= BINLOG_GROUP_COMMIT_LEADER; - if (last_in_group) - flags|= BINLOG_GROUP_COMMIT_TRAILER; - - if (!(log_info= thd->semisync_info)) - { - if(!(log_info= - (Trans_binlog_info*) my_malloc(sizeof(Trans_binlog_info), MYF(0)))) - return 1; - thd->semisync_info= log_info; - } - - strmake_buf(log_info->log_file, log_file+dirname_length(log_file)); - log_info->log_pos = log_pos; - - FOREACH_OBSERVER(ret, after_flush, false, - (¶m, log_info->log_file, log_info->log_pos, flags)); - return ret; -} - -int Binlog_storage_delegate::after_sync(THD *thd, - const char *log_file, - my_off_t log_pos, - bool first_in_group, - bool last_in_group) -{ - Binlog_storage_param param; - uint32 flags=0; - - if (first_in_group) - flags|= BINLOG_GROUP_COMMIT_LEADER; - if (last_in_group) - flags|= BINLOG_GROUP_COMMIT_TRAILER; - - int ret= 0; - FOREACH_OBSERVER(ret, after_sync, false, - (¶m, log_file+dirname_length(log_file), log_pos, flags)); - - return ret; -} - -#ifdef HAVE_REPLICATION -int Binlog_transmit_delegate::transmit_start(THD *thd, ushort flags, - const char *log_file, - my_off_t log_pos) -{ - Binlog_transmit_param param; - param.flags= flags; - - int ret= 0; - FOREACH_OBSERVER(ret, transmit_start, true, (¶m, log_file, log_pos)); - return ret; -} - -int Binlog_transmit_delegate::transmit_stop(THD *thd, ushort flags) -{ - Binlog_transmit_param param; - param.flags= flags; - - int ret= 0; - FOREACH_OBSERVER(ret, transmit_stop, false, (¶m)); - return ret; -} - -int Binlog_transmit_delegate::reserve_header(THD *thd, ushort flags, - String *packet) -{ - /* NOTE2ME: Maximum extra header size for each observer, I hope 32 - bytes should be enough for each Observer to reserve their extra - header. If later found this is not enough, we can increase this - /HEZX - */ -#define RESERVE_HEADER_SIZE 32 - unsigned char header[RESERVE_HEADER_SIZE]; - ulong hlen; - Binlog_transmit_param param; - param.flags= flags; - param.server_id= thd->variables.server_id; - - int ret= 0; - read_lock(); - Observer_info_iterator iter= observer_info_iter(); - Observer_info *info= iter++; - for (; info; info= iter++) - { - hlen= 0; - if (((Observer *)info->observer)->reserve_header - && ((Observer *)info->observer)->reserve_header(¶m, - header, - RESERVE_HEADER_SIZE, - &hlen)) - { - ret= 1; - break; - } - if (hlen == 0) - continue; - if (hlen > RESERVE_HEADER_SIZE || packet->append((char *)header, hlen)) - { - ret= 1; - break; - } - } - unlock(); - return ret; -} - -int Binlog_transmit_delegate::before_send_event(THD *thd, ushort flags, - String *packet, - const char *log_file, - my_off_t log_pos) -{ - Binlog_transmit_param param; - param.flags= flags; - - int ret= 0; - FOREACH_OBSERVER(ret, before_send_event, false, - (¶m, (uchar *)packet->c_ptr(), - packet->length(), - log_file+dirname_length(log_file), log_pos)); - return ret; -} - -int Binlog_transmit_delegate::after_send_event(THD *thd, ushort flags, - String *packet) -{ - Binlog_transmit_param param; - param.flags= flags; - - int ret= 0; - FOREACH_OBSERVER(ret, after_send_event, false, - (¶m, packet->c_ptr(), packet->length())); - return ret; -} - -int Binlog_transmit_delegate::after_reset_master(THD *thd, ushort flags) - -{ - Binlog_transmit_param param; - param.flags= flags; - - int ret= 0; - FOREACH_OBSERVER(ret, after_reset_master, false, (¶m)); - return ret; -} - -void Binlog_relay_IO_delegate::init_param(Binlog_relay_IO_param *param, - Master_info *mi) -{ - param->mysql= mi->mysql; - param->user= mi->user; - param->host= mi->host; - param->port= mi->port; - param->master_log_name= mi->master_log_name; - param->master_log_pos= mi->master_log_pos; -} - -int Binlog_relay_IO_delegate::thread_start(THD *thd, Master_info *mi) -{ - Binlog_relay_IO_param param; - init_param(¶m, mi); - - int ret= 0; - FOREACH_OBSERVER(ret, thread_start, true, (¶m)); - return ret; -} - - -int Binlog_relay_IO_delegate::thread_stop(THD *thd, Master_info *mi) -{ - - Binlog_relay_IO_param param; - init_param(¶m, mi); - - int ret= 0; - FOREACH_OBSERVER(ret, thread_stop, false, (¶m)); - return ret; -} - -int Binlog_relay_IO_delegate::before_request_transmit(THD *thd, - Master_info *mi, - ushort flags) -{ - Binlog_relay_IO_param param; - init_param(¶m, mi); - - int ret= 0; - FOREACH_OBSERVER(ret, before_request_transmit, false, (¶m, (uint32)flags)); - return ret; -} - -int Binlog_relay_IO_delegate::after_read_event(THD *thd, Master_info *mi, - const char *packet, ulong len, - const char **event_buf, - ulong *event_len) -{ - Binlog_relay_IO_param param; - init_param(¶m, mi); - - int ret= 0; - FOREACH_OBSERVER(ret, after_read_event, false, - (¶m, packet, len, event_buf, event_len)); - return ret; -} - -int Binlog_relay_IO_delegate::after_queue_event(THD *thd, Master_info *mi, - const char *event_buf, - ulong event_len, - bool synced) -{ - Binlog_relay_IO_param param; - init_param(¶m, mi); - - uint32 flags=0; - if (synced) - flags |= BINLOG_STORAGE_IS_SYNCED; - - int ret= 0; - FOREACH_OBSERVER(ret, after_queue_event, false, - (¶m, event_buf, event_len, flags)); - return ret; -} - -int Binlog_relay_IO_delegate::after_reset_slave(THD *thd, Master_info *mi) - -{ - Binlog_relay_IO_param param; - init_param(¶m, mi); - - int ret= 0; - FOREACH_OBSERVER(ret, after_reset_slave, false, (¶m)); - return ret; -} -#endif /* HAVE_REPLICATION */ - -int register_trans_observer(Trans_observer *observer, void *p) -{ - return transaction_delegate->add_observer(observer, (st_plugin_int *)p); -} - -int unregister_trans_observer(Trans_observer *observer, void *p) -{ - return transaction_delegate->remove_observer(observer, (st_plugin_int *)p); -} - -int register_binlog_storage_observer(Binlog_storage_observer *observer, void *p) -{ - return binlog_storage_delegate->add_observer(observer, (st_plugin_int *)p); -} - -int unregister_binlog_storage_observer(Binlog_storage_observer *observer, void *p) -{ - return binlog_storage_delegate->remove_observer(observer, (st_plugin_int *)p); -} - -#ifdef HAVE_REPLICATION -int register_binlog_transmit_observer(Binlog_transmit_observer *observer, void *p) -{ - return binlog_transmit_delegate->add_observer(observer, (st_plugin_int *)p); -} - -int unregister_binlog_transmit_observer(Binlog_transmit_observer *observer, void *p) -{ - return binlog_transmit_delegate->remove_observer(observer, (st_plugin_int *)p); -} - -int register_binlog_relay_io_observer(Binlog_relay_IO_observer *observer, void *p) -{ - return binlog_relay_io_delegate->add_observer(observer, (st_plugin_int *)p); -} - -int unregister_binlog_relay_io_observer(Binlog_relay_IO_observer *observer, void *p) -{ - return binlog_relay_io_delegate->remove_observer(observer, (st_plugin_int *)p); -} -#else -int register_binlog_transmit_observer(Binlog_transmit_observer *observer, void *p) -{ - return 0; -} - -int unregister_binlog_transmit_observer(Binlog_transmit_observer *observer, void *p) -{ - return 0; -} - -int register_binlog_relay_io_observer(Binlog_relay_IO_observer *observer, void *p) -{ - return 0; -} - -int unregister_binlog_relay_io_observer(Binlog_relay_IO_observer *observer, void *p) -{ - return 0; -} -#endif /* HAVE_REPLICATION */ diff --git a/sql/rpl_handler.h b/sql/rpl_handler.h deleted file mode 100644 index f8b11adbb69..00000000000 --- a/sql/rpl_handler.h +++ /dev/null @@ -1,222 +0,0 @@ -/* Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -#ifndef RPL_HANDLER_H -#define RPL_HANDLER_H - -#include "sql_priv.h" -#include "rpl_mi.h" -#include "rpl_rli.h" -#include "sql_plugin.h" -#include "replication.h" - -class Observer_info { -public: - void *observer; - st_plugin_int *plugin_int; - - Observer_info(void *ob, st_plugin_int *p) - :observer(ob), plugin_int(p) - { } -}; - -class Delegate { -public: - typedef List Observer_info_list; - typedef List_iterator Observer_info_iterator; - - int add_observer(void *observer, st_plugin_int *plugin) - { - int ret= FALSE; - if (!inited) - return TRUE; - write_lock(); - Observer_info_iterator iter(observer_info_list); - Observer_info *info= iter++; - while (info && info->observer != observer) - info= iter++; - if (!info) - { - info= new Observer_info(observer, plugin); - if (!info || observer_info_list.push_back(info, &memroot)) - ret= TRUE; - } - else - ret= TRUE; - unlock(); - return ret; - } - - int remove_observer(void *observer, st_plugin_int *plugin) - { - int ret= FALSE; - if (!inited) - return TRUE; - write_lock(); - Observer_info_iterator iter(observer_info_list); - Observer_info *info= iter++; - while (info && info->observer != observer) - info= iter++; - if (info) - { - iter.remove(); - delete info; - } - else - ret= TRUE; - unlock(); - return ret; - } - - inline Observer_info_iterator observer_info_iter() - { - return Observer_info_iterator(observer_info_list); - } - - inline bool is_empty() - { - return observer_info_list.is_empty(); - } - - inline int read_lock() - { - if (!inited) - return TRUE; - return rw_rdlock(&lock); - } - - inline int write_lock() - { - if (!inited) - return TRUE; - return rw_wrlock(&lock); - } - - inline int unlock() - { - if (!inited) - return TRUE; - return rw_unlock(&lock); - } - - inline bool is_inited() - { - return inited; - } - - Delegate() - { - inited= FALSE; - if (my_rwlock_init(&lock, NULL)) - return; - init_sql_alloc(&memroot, 1024, 0, MYF(0)); - inited= TRUE; - } - ~Delegate() - { - inited= FALSE; - rwlock_destroy(&lock); - free_root(&memroot, MYF(0)); - } - -private: - Observer_info_list observer_info_list; - rw_lock_t lock; - MEM_ROOT memroot; - bool inited; -}; - -class Trans_delegate - :public Delegate { -public: - typedef Trans_observer Observer; - int before_commit(THD *thd, bool all); - int before_rollback(THD *thd, bool all); - int after_commit(THD *thd, bool all); - int after_rollback(THD *thd, bool all); -}; - -class Binlog_storage_delegate - :public Delegate { -public: - typedef Binlog_storage_observer Observer; - int after_flush(THD *thd, const char *log_file, - my_off_t log_pos, bool synced, - bool first_in_group, bool last_in_group); - int after_sync(THD *thd, const char *log_file, my_off_t log_pos, - bool first_in_group, bool last_in_group); -}; - -#ifdef HAVE_REPLICATION -class Binlog_transmit_delegate - :public Delegate { -public: - typedef Binlog_transmit_observer Observer; - int transmit_start(THD *thd, ushort flags, - const char *log_file, my_off_t log_pos); - int transmit_stop(THD *thd, ushort flags); - int reserve_header(THD *thd, ushort flags, String *packet); - int before_send_event(THD *thd, ushort flags, - String *packet, const - char *log_file, my_off_t log_pos ); - int after_send_event(THD *thd, ushort flags, - String *packet); - int after_reset_master(THD *thd, ushort flags); -}; - -class Binlog_relay_IO_delegate - :public Delegate { -public: - typedef Binlog_relay_IO_observer Observer; - int thread_start(THD *thd, Master_info *mi); - int thread_stop(THD *thd, Master_info *mi); - int before_request_transmit(THD *thd, Master_info *mi, ushort flags); - int after_read_event(THD *thd, Master_info *mi, - const char *packet, ulong len, - const char **event_buf, ulong *event_len); - int after_queue_event(THD *thd, Master_info *mi, - const char *event_buf, ulong event_len, - bool synced); - int after_reset_slave(THD *thd, Master_info *mi); -private: - void init_param(Binlog_relay_IO_param *param, Master_info *mi); -}; -#endif /* HAVE_REPLICATION */ - -int delegates_init(); -void delegates_destroy(); - -extern Trans_delegate *transaction_delegate; -extern Binlog_storage_delegate *binlog_storage_delegate; -#ifdef HAVE_REPLICATION -extern Binlog_transmit_delegate *binlog_transmit_delegate; -extern Binlog_relay_IO_delegate *binlog_relay_io_delegate; -#endif /* HAVE_REPLICATION */ - -/* - if semisync replication is not enabled, we can return immediately. -*/ -#ifdef HAVE_REPLICATION -/* - As semisync is unpluggined and its hooks are turned into static - invocations all other hooks are not run for optimization sake. -*/ -#define RUN_HOOK(group, hook, args) \ - (unlikely(run_hooks_enabled) ? group ##_delegate->hook args : 0) -#else -#define RUN_HOOK(group, hook, args) 0 -#endif /* HAVE_REPLICATION */ - -#endif /* RPL_HANDLER_H */ diff --git a/sql/semisync_master.cc b/sql/semisync_master.cc index de91e30beec..8adbce179e1 100644 --- a/sql/semisync_master.cc +++ b/sql/semisync_master.cc @@ -738,19 +738,19 @@ int ReplSemiSyncMaster::reportBinlogUpdate(THD* thd, const char *log_file, return 0; } -void ReplSemiSyncMaster::dump_start(THD* thd, +int ReplSemiSyncMaster::dump_start(THD* thd, const char *log_file, my_off_t log_pos) { if (!thd->semi_sync_slave) - return; + return 0; if (ack_receiver.add_slave(thd)) { sql_print_error("Failed to register slave to semi-sync ACK receiver " "thread. Turning off semisync"); thd->semi_sync_slave= 0; - return; + return 1; } add_slave(); @@ -758,7 +758,7 @@ void ReplSemiSyncMaster::dump_start(THD* thd, sql_print_information("Start semi-sync binlog_dump to slave (server_id: %d), pos(%s, %lu", thd->variables.server_id, log_file, (unsigned long)log_pos); - return; + return 0; } void ReplSemiSyncMaster::dump_end(THD* thd) diff --git a/sql/semisync_master.h b/sql/semisync_master.h index 66fac17cd45..737ad46dd27 100644 --- a/sql/semisync_master.h +++ b/sql/semisync_master.h @@ -553,7 +553,7 @@ class ReplSemiSyncMaster * be acked by slave*/ int reportBinlogUpdate(THD *thd, const char *log_file,my_off_t log_pos); - void dump_start(THD* thd, + int dump_start(THD* thd, const char *log_file, my_off_t log_pos); diff --git a/sql/semisync_slave.h b/sql/semisync_slave.h index 0df4445ee4a..d67e1a05070 100644 --- a/sql/semisync_slave.h +++ b/sql/semisync_slave.h @@ -24,6 +24,8 @@ #include "rpl_mi.h" #include "mysql.h" +class Master_info; + /** The extension class for the slave of semi-synchronous replication */ diff --git a/sql/slave.cc b/sql/slave.cc index b4a817d6ecd..acca99481e8 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -43,7 +43,6 @@ #include #include #include -#include "rpl_handler.h" #include #include #include @@ -3587,10 +3586,6 @@ static int request_dump(THD *thd, MYSQL* mysql, Master_info* mi, if (opt_log_slave_updates && opt_replicate_annotate_row_events) binlog_flags|= BINLOG_SEND_ANNOTATE_ROWS_EVENT; - if (RUN_HOOK(binlog_relay_io, - before_request_transmit, - (thd, mi, binlog_flags))) - DBUG_RETURN(1); if (repl_semisync_slave.requestTransmit(mi)) DBUG_RETURN(1); @@ -4618,9 +4613,8 @@ pthread_handler_t handle_slave_io(void *arg) } - if (RUN_HOOK(binlog_relay_io, thread_start, (thd, mi)) || - (DBUG_EVALUATE_IF("failed_slave_start", 1, 0) - || repl_semisync_slave.slaveStart(mi))) + if (DBUG_EVALUATE_IF("failed_slave_start", 1, 0) + || repl_semisync_slave.slaveStart(mi)) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, ER_THD(thd, ER_SLAVE_FATAL_ERROR), @@ -4811,10 +4805,7 @@ Stopping slave I/O thread due to out-of-memory error from master"); THD_STAGE_INFO(thd, stage_queueing_master_event_to_the_relay_log); event_buf= (const char*)mysql->net.read_pos + 1; mi->semi_ack= 0; - if (RUN_HOOK(binlog_relay_io, after_read_event, - (thd, mi,(const char*)mysql->net.read_pos + 1, - event_len, &event_buf, &event_len)) || - repl_semisync_slave. + if (repl_semisync_slave. slaveReadSyncHeader((const char*)mysql->net.read_pos + 1, event_len, &(mi->semi_ack), &event_buf, &event_len)) { @@ -4865,9 +4856,6 @@ Stopping slave I/O thread due to out-of-memory error from master"); tokenamount -= network_read_len; } - /* XXX: 'synced' should be updated by queue_event to indicate - whether event has been synced to disk */ - bool synced= 0; if (queue_event(mi, event_buf, event_len)) { mi->report(ERROR_LEVEL, ER_SLAVE_RELAY_LOG_WRITE_FAILURE, NULL, @@ -4876,11 +4864,8 @@ Stopping slave I/O thread due to out-of-memory error from master"); goto err; } - if (RUN_HOOK(binlog_relay_io, after_queue_event, - (thd, mi, event_buf, event_len, synced)) || - (rpl_semi_sync_slave_status && - (mi->semi_ack & SEMI_SYNC_NEED_ACK) && - repl_semisync_slave.slaveReply(mi))) + if (rpl_semi_sync_slave_status && (mi->semi_ack & SEMI_SYNC_NEED_ACK) && + repl_semisync_slave.slaveReply(mi)) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, ER_THD(thd, ER_SLAVE_FATAL_ERROR), @@ -4952,7 +4937,6 @@ err: IO_RPL_LOG_NAME, mi->master_log_pos, tmp.c_ptr_safe()); } - (void) RUN_HOOK(binlog_relay_io, thread_stop, (thd, mi)); repl_semisync_slave.slaveStop(mi); thd->reset_query(); thd->reset_db(NULL, 0); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 9ce6c6b8787..60ab84069b0 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -82,7 +82,6 @@ #include #include #include -#include "rpl_handler.h" #include "rpl_mi.h" #include "sql_digest.h" diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 36fa6120584..c9e804135a5 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -28,7 +28,6 @@ #include "log_event.h" #include "rpl_filter.h" #include -#include "rpl_handler.h" #include "debug_sync.h" #include "semisync_master.h" #include "semisync_slave.h" @@ -315,24 +314,35 @@ static int reset_transmit_packet(binlog_send_info *info, ushort flags, packet->length(0); packet->set("\0", 1, &my_charset_bin); - if (RUN_HOOK(binlog_transmit, reserve_header, (info->thd, flags, packet))) - { - /* RUN_HOOK() must return zero when thd->semi_sync_slave */ - DBUG_ASSERT(!info->thd->semi_sync_slave); - - info->error= ER_UNKNOWN_ERROR; - *errmsg= "Failed to run hook 'reserve_header'"; - ret= 1; - } if (info->thd->semi_sync_slave) { - repl_semisync_master.reserveSyncHeader(packet); + if (repl_semisync_master.reserveSyncHeader(packet)) + { + info->error= ER_UNKNOWN_ERROR; + *errmsg= "Failed to run hook 'reserve_header'"; + ret= 1; + } } *ev_offset= packet->length(); return ret; } +int get_user_var_int(const char *name, + long long int *value, int *null_value) +{ + bool null_val; + user_var_entry *entry= + (user_var_entry*) my_hash_search(¤t_thd->user_vars, + (uchar*) name, strlen(name)); + if (!entry) + return 1; + *value= entry->val_int(&null_val); + if (null_value) + *null_value= null_val; + return 0; +} + inline bool is_semi_sync_slave() { int null_value; @@ -1935,9 +1945,7 @@ send_event_to_slave(binlog_send_info *info, Log_event_type event_type, THD_STAGE_INFO(info->thd, stage_sending_binlog_event_to_slave); pos= my_b_tell(log); - if (RUN_HOOK(binlog_transmit, before_send_event, - (info->thd, info->flags, packet, info->log_file_name, pos)) || - repl_semisync_master.updateSyncHeader(info->thd, (uchar *)packet->c_ptr(), + if (repl_semisync_master.updateSyncHeader(info->thd, (uchar *)packet->c_ptr(), info->log_file_name + info->dirlen, pos, &need_sync)) { @@ -1961,14 +1969,11 @@ send_event_to_slave(binlog_send_info *info, Log_event_type event_type, } } - if (RUN_HOOK(binlog_transmit, after_send_event, - (info->thd, info->flags, packet))) + if (need_sync && repl_semisync_master.flushNet(info->thd, packet->c_ptr())) { info->error= ER_UNKNOWN_ERROR; return "Failed to run hook 'after_send_event'"; } - if (need_sync) - repl_semisync_master.flushNet(info->thd, packet->c_ptr()); return NULL; /* Success */ } @@ -2740,21 +2745,16 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, if (init_binlog_sender(info, &linfo, log_ident, &pos)) goto err; - /* - run hook first when all check has been made that slave seems to - be requesting a reasonable position. i.e when transmit actually starts - */ - if (RUN_HOOK(binlog_transmit, transmit_start, (thd, flags, log_ident, pos))) + has_transmit_started= true; + + /* Check if the dump thread is created by a slave with semisync enabled. */ + thd->semi_sync_slave = is_semi_sync_slave(); + if (repl_semisync_master.dump_start(thd, log_ident, pos)) { info->errmsg= "Failed to run hook 'transmit_start'"; info->error= ER_UNKNOWN_ERROR; goto err; } - has_transmit_started= true; - - /* Check if the dump thread is created by a slave with semisync enabled. */ - thd->semi_sync_slave = is_semi_sync_slave(); - repl_semisync_master.dump_start(thd, log_ident, pos); /* heartbeat_period from @master_heartbeat_period user variable @@ -2871,7 +2871,6 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, err: THD_STAGE_INFO(thd, stage_waiting_to_finalize_termination); - (void) RUN_HOOK(binlog_transmit, transmit_stop, (thd, flags)); if (has_transmit_started) { repl_semisync_master.dump_end(thd); @@ -3341,7 +3340,6 @@ int reset_slave(THD *thd, Master_info* mi) else if (global_system_variables.log_warnings > 1) sql_print_information("Deleted Master_info file '%s'.", fname); - (void) RUN_HOOK(binlog_relay_io, after_reset_slave, (thd, mi)); if (rpl_semi_sync_slave_enabled) repl_semisync_slave.resetSlave(mi); err: @@ -3850,7 +3848,6 @@ int reset_master(THD* thd, rpl_gtid *init_state, uint32 init_state_len, repl_semisync_master.beforeResetMaster(); ret= mysql_bin_log.reset_logs(thd, 1, init_state, init_state_len, next_log_number); - (void) RUN_HOOK(binlog_transmit, after_reset_master, (thd, 0 /* flags */)); repl_semisync_master.afterResetMaster(); return ret; } diff --git a/sql/transaction.cc b/sql/transaction.cc index f6ccf5a1930..349f4549d31 100644 --- a/sql/transaction.cc +++ b/sql/transaction.cc @@ -21,7 +21,6 @@ #include "mariadb.h" #include "sql_priv.h" #include "transaction.h" -#include "rpl_handler.h" #include "debug_sync.h" // DEBUG_SYNC #include "sql_acl.h" #include "semisync_master.h" @@ -319,14 +318,12 @@ bool trans_commit(THD *thd) */ if (res) { - (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); #ifdef HAVE_REPLICATION repl_semisync_master.waitAfterRollback(thd, FALSE); #endif } else { - (void) RUN_HOOK(transaction, after_commit, (thd, FALSE)); #ifdef HAVE_REPLICATION repl_semisync_master.waitAfterCommit(thd, FALSE); #endif @@ -423,7 +420,6 @@ bool trans_rollback(THD *thd) ~(SERVER_STATUS_IN_TRANS | SERVER_STATUS_IN_TRANS_READONLY); DBUG_PRINT("info", ("clearing SERVER_STATUS_IN_TRANS")); res= ha_rollback_trans(thd, TRUE); - (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); #ifdef HAVE_REPLICATION repl_semisync_master.waitAfterRollback(thd, FALSE); #endif @@ -540,14 +536,12 @@ bool trans_commit_stmt(THD *thd) */ if (res) { - (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); #ifdef HAVE_REPLICATION repl_semisync_master.waitAfterRollback(thd, FALSE); #endif } else { - (void) RUN_HOOK(transaction, after_commit, (thd, FALSE)); #ifdef HAVE_REPLICATION repl_semisync_master.waitAfterCommit(thd, FALSE); #endif @@ -590,7 +584,6 @@ bool trans_rollback_stmt(THD *thd) trans_reset_one_shot_chistics(thd); } - (void) RUN_HOOK(transaction, after_rollback, (thd, FALSE)); #ifdef HAVE_REPLICATION repl_semisync_master.waitAfterRollback(thd, FALSE); #endif From c0ea3056b64779c798f90d3ba9a4f8d37eb14665 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Wed, 22 Nov 2017 23:38:16 +0200 Subject: [PATCH 23/51] MDEV-13073. This is a part with a new test (refined Ali's one) and affected result files. Specifically to rpl_semi_sync_after_sync*, the changed results reflect a fact that thanks to fixes in the dump thread functionality there's no longer zombie thread to kill neither such thread represent a semisync client (so the counter drops). --- mysql-test/r/mysqld--help.result | 8 + .../r/perf_buildin_semisync_issue40.result | 367 ++++++++++++++++ .../rpl/r/rpl_semi_sync_after_sync.result | 6 +- .../rpl/r/rpl_semi_sync_after_sync_row.result | 6 +- .../perf_buildin_semisync_issue40-master.opt | 1 + .../t/perf_buildin_semisync_issue40-slave.opt | 1 + .../rpl/t/perf_buildin_semisync_issue40.test | 395 ++++++++++++++++++ .../r/sysvars_server_notembedded.result | 28 ++ 8 files changed, 804 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result create mode 100644 mysql-test/suite/rpl/t/perf_buildin_semisync_issue40-master.opt create mode 100644 mysql-test/suite/rpl/t/perf_buildin_semisync_issue40-slave.opt create mode 100644 mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 032c64b8040..e9c7047e492 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -1014,9 +1014,15 @@ The following options may be given as the first argument: Should transaction wait for semi-sync ack after having synced binlog, or after having committed in storage engine.. One of: AFTER_SYNC, AFTER_COMMIT + --rpl-semi-sync-slave-delay-master + Only write master info file when ack is needed. --rpl-semi-sync-slave-enabled Enable semi-synchronous replication slave (disabled by default). + --rpl-semi-sync-slave-kill-conn-timeout[=#] + Timeout for the mysql connection used to kill the slave + io_thread's connection on master. This timeout comes into + play when stop slave is executed. --rpl-semi-sync-slave-trace-level=# The tracing level for semi-sync replication. --safe-mode Skip some optimize stages (for testing). Deprecated. @@ -1595,7 +1601,9 @@ rpl-semi-sync-master-timeout 10000 rpl-semi-sync-master-trace-level 32 rpl-semi-sync-master-wait-no-slave TRUE rpl-semi-sync-master-wait-point AFTER_COMMIT +rpl-semi-sync-slave-delay-master FALSE rpl-semi-sync-slave-enabled FALSE +rpl-semi-sync-slave-kill-conn-timeout 5 rpl-semi-sync-slave-trace-level 32 safe-user-create FALSE secure-auth TRUE diff --git a/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result b/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result new file mode 100644 index 00000000000..17ecea99bbe --- /dev/null +++ b/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result @@ -0,0 +1,367 @@ +include/master-slave.inc +[connection master] +CALL mtr.add_suppression("Failed to start semi-sync ACK receiver thread.*"); +CALL mtr.add_suppression("Failed to register slave to semi-sync ACK receiver thread.*"); +CALL mtr.add_suppression("Failed to stop ack receiver thread on pthread_join.*"); +CALL mtr.add_suppression("Got an error reading communication packets:*"); +CALL mtr.add_suppression("Timeout waiting for reply of binlog*"); +CALL mtr.add_suppression("slaveReadSyncHeader*"); +CALL mtr.add_suppression("Missing magic number for semi-sync*"); +CALL mtr.add_suppression("Got timeout reading communication packets*"); +CALL mtr.add_suppression("Failed to call*"); +CALL mtr.add_suppression("Execution failed on master*"); +CALL mtr.add_suppression("Failed on request_dump()*"); +CALL mtr.add_suppression("Semi-sync master failed on*"); +CALL mtr.add_suppression("Master command COM_BINLOG_DUMP failed*"); +CALL mtr.add_suppression("on master failed*"); +CALL mtr.add_suppression("Master server does not support semi-sync*"); +CALL mtr.add_suppression("Semi-sync slave net_flush*"); +CALL mtr.add_suppression("Failed to flush master info*"); +CALL mtr.add_suppression("Request to stop slave SQL Thread received while apply*"); +connection master; +[ enable semi-sync on master ] +set global rpl_semi_sync_master_enabled = 1; +show variables like 'rpl_semi_sync_master_enabled'; +Variable_name Value +rpl_semi_sync_master_enabled ON +connection slave; +[ enable semi-sync on slave ] +stop slave; +set global rpl_semi_sync_slave_enabled = 1; +start slave; +show status like 'rpl_semi_sync_slave%'; +Variable_name Value +Rpl_semi_sync_slave_send_ack 0 +Rpl_semi_sync_slave_status ON +connection master; +CREATE TABLE t1(a INT) ENGINE=InnoDB; +connection slave; +connection master; +connect con1,localhost,root,,; +connect con2,localhost,root,,; +connect con3,localhost,root,,; +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +show status like "rpl_semi_sync_master_yes_tx"; +Variable_name Value +Rpl_semi_sync_master_yes_tx 1 +######################################### +# Test rpl_semi_sync_master_wait_point # +######################################### +# Test after_sync and after_commit first. +#Test after_sync +connection con1; +SET GLOBAL rpl_semi_sync_master_timeout = 1000000; +SET GLOBAL rpl_semi_sync_master_wait_point= 'AFTER_SYNC'; +SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL after_sync_done WAIT_FOR end"; +INSERT into t1 values (1);; +connection con2; +SET DEBUG_SYNC= "now WAIT_FOR after_sync_done"; +connection slave; +#slave can see record (1) after sync slave with master +select * from t1; +a +1 +connection con2; +#con2 shouldn't see record (1) +select * from t1; +a +SET DEBUG_SYNC= "now SIGNAL end"; +connection con1; +connection con1; +select * from t1; +a +1 +truncate table t1; +connection slave; +connection con1; +SET DEBUG_SYNC= 'reset'; +SET DEBUG_SYNC= "commit_before_get_LOCK_log SIGNAL before_fetch_done WAIT_FOR more_queue"; +INSERT into t1 VALUES (1);; +connection con2; +SET DEBUG_SYNC= "now WAIT_FOR before_fetch_done"; +SET DEBUG_SYNC= "after_semisync_queue SIGNAL more_queue"; +INSERT INTO t1 VALUES (2); +connection con1; +connection con1; +SET DEBUG_SYNC= 'reset'; +SET DEBUG_SYNC= "commit_before_get_LOCK_log SIGNAL before_fetch_done WAIT_FOR disable_semisync"; +INSERT into t1 VALUES (3);; +connection con2; +SET DEBUG_SYNC= "now WAIT_FOR before_fetch_done"; +SET GLOBAL rpl_semi_sync_master_enabled= 0; +SET DEBUG_SYNC= "now SIGNAL disable_semisync"; +connection con1; +SET GLOBAL rpl_semi_sync_master_enabled = 1; +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +#Test after_commit +connection con1; +SET GLOBAL rpl_semi_sync_master_wait_point= 'AFTER_COMMIT'; +SET DEBUG_SYNC= "after_group_after_commit SIGNAL after_commit_done WAIT_FOR end"; +INSERT into t1 values (4);; +connection con2; +SET DEBUG_SYNC= "now WAIT_FOR after_commit_done"; +connection slave; +select * from t1; +a +1 +2 +3 +4 +connection con2; +select * from t1; +a +1 +2 +3 +4 +SET DEBUG_SYNC= "now SIGNAL end"; +connection con1; +connection con1; +select * from t1; +a +1 +2 +3 +4 +truncate table t1; +####################################################### +# Test some other options in order to cover the patch # +####################################################### +connection slave; +# Test rpl_semi_sync_slave_trace_level +SET GLOBAL rpl_semi_sync_slave_trace_level= 1; +SET GLOBAL rpl_semi_sync_slave_trace_level= 16; +SET GLOBAL rpl_semi_sync_slave_trace_level= 64; +SET GLOBAL rpl_semi_sync_slave_trace_level= 128; +SET GLOBAL rpl_semi_sync_slave_trace_level= 32; +connection master; +# Test rpl_semi_sync_master_trace_level +SET GLOBAL rpl_semi_sync_master_trace_level= 1; +SET GLOBAL rpl_semi_sync_master_trace_level= 16; +SET GLOBAL rpl_semi_sync_master_trace_level= 64; +SET GLOBAL rpl_semi_sync_master_trace_level= 128; +SET GLOBAL rpl_semi_sync_master_trace_level= 32; +# Test rpl_semi_sync_master_timeout +SET GLOBAL rpl_semi_sync_master_timeout= 1000; +SET GLOBAL rpl_semi_sync_master_timeout= 10000; +SET GLOBAL rpl_semi_sync_master_timeout = 1000000; +# Test rpl_semi_sync_slave_kill_conn_timeout +SET GLOBAL rpl_semi_sync_slave_kill_conn_timeout= 10; +SET GLOBAL rpl_semi_sync_slave_kill_conn_timeout= 20; +SET GLOBAL rpl_semi_sync_slave_kill_conn_timeout= 60; +SET GLOBAL rpl_semi_sync_slave_kill_conn_timeout= 5; +############################################ +# Test rpl_semi_sync_master_wait_no_slave # +############################################ +SET GLOBAL rpl_semi_sync_master_wait_no_slave = 1; +connection slave; +STOP SLAVE IO_THREAD; +include/wait_for_slave_io_to_stop.inc +connection con1; +SET GLOBAL rpl_semi_sync_master_timeout = 1000; +INSERT INTO t1 values (1);; +connection con1; +# Rpl_semi_sync_master_no_tx should be non-zero +SHOW STATUS LIKE 'Rpl_semi_sync_master_no_tx' +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +connection con1; +INSERT INTO t1 values (2); +connection slave; +connection con1; +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +connection slave; +STOP SLAVE IO_THREAD; +include/wait_for_slave_io_to_stop.inc +connection con1; +SET GLOBAL rpl_semi_sync_master_wait_no_slave= 0; +SET GLOBAL rpl_semi_sync_master_timeout= 1000000000; +INSERT INTO t1 values (3); +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 0 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status OFF +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +connection con1; +SET GLOBAL rpl_semi_sync_master_timeout= 10000000; +SET GLOBAL rpl_semi_sync_master_wait_no_slave= 1; +INSERT INTO t1 values (4); +connection slave; +connection con1; +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +########################################## +# Test rpl_semi_sync_slave_delay_master # +########################################## +connection slave; +SET GLOBAL rpl_semi_sync_slave_delay_master= 1; +START SLAVE IO_THREAD; +Warnings: +Note 1254 Slave is already running +include/wait_for_slave_io_to_start.inc +connection con1; +INSERT INTO t1 values (3); +include/sync_slave_io_with_master.inc +connection con1; +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +show status like 'Rpl_semi_sync_master_status'; +Variable_name Value +Rpl_semi_sync_master_status ON +connection slave; +connection slave; +select * from t1 order by a; +a +1 +2 +3 +3 +4 +connection con1; +select * from t1 order by a; +a +1 +2 +3 +3 +4 +connection slave; +SET GLOBAL rpl_semi_sync_slave_delay_master = 0; +STOP SLAVE IO_THREAD; +include/wait_for_slave_io_to_stop.inc +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +########################################################## +# Test rpl_semi_sync_master_enabled and new ACK thread # +######################################################### +connection con1; +SET GLOBAL rpl_semi_sync_master_enabled = 0; +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +INSERT INTO t1 VALUES (1); +SET GLOBAL rpl_semi_sync_master_enabled = 1; +INSERT INTO t1 VALUES (2); +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +# Test failure of select error . +SET GLOBAL debug = 'd,rpl_semisync_simulate_select_error'; +Warnings: +Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +INSERT INTO t1 VALUES(3); +connection slave; +connection con1; +# Test failure of pthread_create +SET GLOBAL rpl_semi_sync_master_enabled = 0; +SET GLOBAL debug = 'd,rpl_semisync_simulate_create_thread_failure'; +Warnings: +Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL rpl_semi_sync_master_enabled= ON; +SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled'; +Variable_name Value +rpl_semi_sync_master_enabled OFF +# Test failure of pthread_join +SET GLOBAL DEBUG = 'd,rpl_semisync_simulate_thread_join_failure'; +Warnings: +Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL rpl_semi_sync_master_enabled= ON; +SET GLOBAL rpl_semi_sync_master_enabled= OFF; +# +# Failure on registering semisync slave +# +SET GLOBAL debug= 'd,rpl_semisync_simulate_add_slave_failure'; +Warnings: +Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +SET GLOBAL rpl_semi_sync_master_enabled= ON; +connection slave; +STOP SLAVE IO_THREAD; +include/wait_for_slave_io_to_stop.inc +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +connection con1; +# Should be Zero. +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 0 +SET GLOBAL debug=''; +Warnings: +Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +connection con1; +connection slave; +show status like 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 0 +################################################################## +# Test fixing of BUG#70669 # +#SLAVE CAN'T CONTINUE REPLICATION AFTER MASTER'S CRASH RECOVERY # +################################################################# +connection con1; +SET GLOBAL sync_binlog = 1; +CREATE TABLE t2 (c1 INT); +connection slave; +connection con1; +INSERT INTO t2 values (1); +connection slave; +SELECT * FROM t2; +c1 +1 +connection con2; +connection con1; +connection slave; +show tables like 't2'; +Tables_in_test (t2) +t2 +select * from t2; +c1 +1 +connection con1; +INSERT INTO t2 VALUES (2); +connection con2; +INSERT INTO t2 VALUES (3); +connection con1; +connection con2; +connection con1; +SET GLOBAL sync_binlog = 0; +DROP TABLE t2; +connection con2; +connection slave; +show tables like 't2'; +Tables_in_test (t2) +connection con2; +#cleanup +connection master; +SET DEBUG_SYNC= 'reset'; +disconnect con1; +disconnect con2; +disconnect con3; +SET GLOBAL rpl_semi_sync_master_timeout= 10000; +SET GLOBAL rpl_semi_sync_master_enabled = 0; +DROP TABLE t1; +connection slave; +SET GLOBAL rpl_semi_sync_slave_enabled = 0; +stop slave; +start slave; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_after_sync.result b/mysql-test/suite/rpl/r/rpl_semi_sync_after_sync.result index 927113726fa..c61340f3967 100644 --- a/mysql-test/suite/rpl/r/rpl_semi_sync_after_sync.result +++ b/mysql-test/suite/rpl/r/rpl_semi_sync_after_sync.result @@ -5,6 +5,7 @@ connection master; call mtr.add_suppression("Timeout waiting for reply of binlog"); call mtr.add_suppression("Read semi-sync reply"); call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +call mtr.add_suppression("mysqld: Got an error reading communication packets"); connection slave; call mtr.add_suppression("Master server does not support semi-sync"); call mtr.add_suppression("Semi-sync slave .* reply"); @@ -177,7 +178,7 @@ Variable_name Value Rpl_semi_sync_master_yes_tx 16 show status like 'Rpl_semi_sync_master_clients'; Variable_name Value -Rpl_semi_sync_master_clients 1 +Rpl_semi_sync_master_clients 0 [ semi-sync replication of these transactions will fail ] insert into t1 values (500); [ master status should be OFF ] @@ -322,7 +323,6 @@ connection slave; include/stop_slave.inc reset slave; connection master; -kill query _tid; connection slave; include/start_slave.inc connection master; @@ -354,7 +354,6 @@ include/stop_slave.inc reset slave; connection master; reset master; -kill query _tid; set sql_log_bin=0; grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; flush privileges; @@ -405,7 +404,6 @@ SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; Variable_name Value Rpl_semi_sync_slave_status OFF connection master; -kill query _tid; [ Semi-sync status on master should be ON ] show status like 'Rpl_semi_sync_master_clients'; Variable_name Value diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_after_sync_row.result b/mysql-test/suite/rpl/r/rpl_semi_sync_after_sync_row.result index 30280551ce2..6a23f24b66d 100644 --- a/mysql-test/suite/rpl/r/rpl_semi_sync_after_sync_row.result +++ b/mysql-test/suite/rpl/r/rpl_semi_sync_after_sync_row.result @@ -5,6 +5,7 @@ connection master; call mtr.add_suppression("Timeout waiting for reply of binlog"); call mtr.add_suppression("Read semi-sync reply"); call mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT."); +call mtr.add_suppression("mysqld: Got an error reading communication packets"); connection slave; call mtr.add_suppression("Master server does not support semi-sync"); call mtr.add_suppression("Semi-sync slave .* reply"); @@ -177,7 +178,7 @@ Variable_name Value Rpl_semi_sync_master_yes_tx 14 show status like 'Rpl_semi_sync_master_clients'; Variable_name Value -Rpl_semi_sync_master_clients 1 +Rpl_semi_sync_master_clients 0 [ semi-sync replication of these transactions will fail ] insert into t1 values (500); [ master status should be OFF ] @@ -322,7 +323,6 @@ connection slave; include/stop_slave.inc reset slave; connection master; -kill query _tid; connection slave; include/start_slave.inc connection master; @@ -354,7 +354,6 @@ include/stop_slave.inc reset slave; connection master; reset master; -kill query _tid; set sql_log_bin=0; grant replication slave on *.* to rpl@127.0.0.1 identified by 'rpl_password'; flush privileges; @@ -405,7 +404,6 @@ SHOW STATUS LIKE 'Rpl_semi_sync_slave_status'; Variable_name Value Rpl_semi_sync_slave_status OFF connection master; -kill query _tid; [ Semi-sync status on master should be ON ] show status like 'Rpl_semi_sync_master_clients'; Variable_name Value diff --git a/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40-master.opt b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40-master.opt new file mode 100644 index 00000000000..2672d4ff35e --- /dev/null +++ b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40-master.opt @@ -0,0 +1 @@ +--binlog_format=row diff --git a/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40-slave.opt b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40-slave.opt new file mode 100644 index 00000000000..2672d4ff35e --- /dev/null +++ b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40-slave.opt @@ -0,0 +1 @@ +--binlog_format=row diff --git a/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test new file mode 100644 index 00000000000..3edd001af82 --- /dev/null +++ b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test @@ -0,0 +1,395 @@ +--source include/have_innodb.inc +--source include/have_debug_sync.inc +--source include/master-slave.inc + +CALL mtr.add_suppression("Failed to start semi-sync ACK receiver thread.*"); +CALL mtr.add_suppression("Failed to register slave to semi-sync ACK receiver thread.*"); +CALL mtr.add_suppression("Failed to stop ack receiver thread on pthread_join.*"); +CALL mtr.add_suppression("Got an error reading communication packets:*"); +CALL mtr.add_suppression("Timeout waiting for reply of binlog*"); +CALL mtr.add_suppression("slaveReadSyncHeader*"); +CALL mtr.add_suppression("Missing magic number for semi-sync*"); +CALL mtr.add_suppression("Got timeout reading communication packets*"); +CALL mtr.add_suppression("Failed to call*"); +CALL mtr.add_suppression("Execution failed on master*"); +CALL mtr.add_suppression("Failed on request_dump()*"); +CALL mtr.add_suppression("Semi-sync master failed on*"); +CALL mtr.add_suppression("Master command COM_BINLOG_DUMP failed*"); +CALL mtr.add_suppression("on master failed*"); +CALL mtr.add_suppression("Master server does not support semi-sync*"); +CALL mtr.add_suppression("Semi-sync slave net_flush*"); +CALL mtr.add_suppression("Failed to flush master info*"); +CALL mtr.add_suppression("Request to stop slave SQL Thread received while apply*"); + +connection master; +echo [ enable semi-sync on master ]; +set global rpl_semi_sync_master_enabled = 1; +show variables like 'rpl_semi_sync_master_enabled'; + +connection slave; +echo [ enable semi-sync on slave ]; +stop slave; +set global rpl_semi_sync_slave_enabled = 1; +start slave; +show status like 'rpl_semi_sync_slave%'; + +connection master; +CREATE TABLE t1(a INT) ENGINE=InnoDB; +sync_slave_with_master; + +connection master; +connect(con1,localhost,root,,); +connect(con2,localhost,root,,); +connect(con3,localhost,root,,); + +show status like 'Rpl_semi_sync_master_clients'; +show status like "rpl_semi_sync_master_yes_tx"; + +--echo ######################################### +--echo # Test rpl_semi_sync_master_wait_point # +--echo ######################################### +--echo # Test after_sync and after_commit first. + +--echo #Test after_sync +connection con1; +# Let's set a very large timeout value for testing purpose. +SET GLOBAL rpl_semi_sync_master_timeout = 1000000; +SET GLOBAL rpl_semi_sync_master_wait_point= 'AFTER_SYNC'; +SET DEBUG_SYNC= "commit_before_get_LOCK_commit_ordered SIGNAL after_sync_done WAIT_FOR end"; +--send INSERT into t1 values (1); + +connection con2; +SET DEBUG_SYNC= "now WAIT_FOR after_sync_done"; + +sync_slave_with_master; +--echo #slave can see record (1) after sync slave with master +select * from t1; + +connection con2; +--echo #con2 shouldn't see record (1) +select * from t1; +SET DEBUG_SYNC= "now SIGNAL end"; + +connection con1; +reap; + +connection con1; +select * from t1; +truncate table t1; + +sync_slave_with_master; + +# Test more threads in one semisync queue +connection con1; +SET DEBUG_SYNC= 'reset'; +SET DEBUG_SYNC= "commit_before_get_LOCK_log SIGNAL before_fetch_done WAIT_FOR more_queue"; +#SET DEBUG_SYNC= "before_semisync_fetch SIGNAL before_fetch_done WAIT_FOR more_queue"; +--send INSERT into t1 VALUES (1); + +connection con2; +SET DEBUG_SYNC= "now WAIT_FOR before_fetch_done"; +SET DEBUG_SYNC= "after_semisync_queue SIGNAL more_queue"; +INSERT INTO t1 VALUES (2); + +connection con1; +reap; + +# Test more threads in one semisync queue, but disable semisync before +# waiting. +connection con1; +SET DEBUG_SYNC= 'reset'; +SET DEBUG_SYNC= "commit_before_get_LOCK_log SIGNAL before_fetch_done WAIT_FOR disable_semisync"; +#SET DEBUG_SYNC= "before_semisync_fetch SIGNAL before_fetch_done WAIT_FOR more_queue"; +#SET DEBUG_SYNC= "before_semisync_fetch SIGNAL before_fetch_done WAIT_FOR disable_semisync"; +--send INSERT into t1 VALUES (3); + +connection con2; +SET DEBUG_SYNC= "now WAIT_FOR before_fetch_done"; +SET GLOBAL rpl_semi_sync_master_enabled= 0; +SET DEBUG_SYNC= "now SIGNAL disable_semisync"; + +connection con1; +reap; +SET GLOBAL rpl_semi_sync_master_enabled = 1; +show status like 'Rpl_semi_sync_master_clients'; + +--echo #Test after_commit +connection con1; +SET GLOBAL rpl_semi_sync_master_wait_point= 'AFTER_COMMIT'; +SET DEBUG_SYNC= "after_group_after_commit SIGNAL after_commit_done WAIT_FOR end"; +--send INSERT into t1 values (4); + +connection con2; +SET DEBUG_SYNC= "now WAIT_FOR after_commit_done"; + +sync_slave_with_master; +select * from t1; + +connection con2; +select * from t1; +SET DEBUG_SYNC= "now SIGNAL end"; + +connection con1; +reap; + +connection con1; +select * from t1; +truncate table t1; + +--echo ####################################################### +--echo # Test some other options in order to cover the patch # +--echo ####################################################### +connection slave; +--echo # Test rpl_semi_sync_slave_trace_level +SET GLOBAL rpl_semi_sync_slave_trace_level= 1; +SET GLOBAL rpl_semi_sync_slave_trace_level= 16; +SET GLOBAL rpl_semi_sync_slave_trace_level= 64; +SET GLOBAL rpl_semi_sync_slave_trace_level= 128; +SET GLOBAL rpl_semi_sync_slave_trace_level= 32; +connection master; +--echo # Test rpl_semi_sync_master_trace_level +SET GLOBAL rpl_semi_sync_master_trace_level= 1; +SET GLOBAL rpl_semi_sync_master_trace_level= 16; +SET GLOBAL rpl_semi_sync_master_trace_level= 64; +SET GLOBAL rpl_semi_sync_master_trace_level= 128; +SET GLOBAL rpl_semi_sync_master_trace_level= 32; +--echo # Test rpl_semi_sync_master_timeout +SET GLOBAL rpl_semi_sync_master_timeout= 1000; +SET GLOBAL rpl_semi_sync_master_timeout= 10000; +SET GLOBAL rpl_semi_sync_master_timeout = 1000000; + +--echo # Test rpl_semi_sync_slave_kill_conn_timeout +SET GLOBAL rpl_semi_sync_slave_kill_conn_timeout= 10; +SET GLOBAL rpl_semi_sync_slave_kill_conn_timeout= 20; +SET GLOBAL rpl_semi_sync_slave_kill_conn_timeout= 60; +SET GLOBAL rpl_semi_sync_slave_kill_conn_timeout= 5; + +--echo ############################################ +--echo # Test rpl_semi_sync_master_wait_no_slave # +--echo ############################################ +SET GLOBAL rpl_semi_sync_master_wait_no_slave = 1; +connection slave; +STOP SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_stop.inc + +connection con1; +SET GLOBAL rpl_semi_sync_master_timeout = 1000; +--send INSERT INTO t1 values (1); + +connection con1; +reap; +echo # Rpl_semi_sync_master_no_tx should be non-zero +SHOW STATUS LIKE 'Rpl_semi_sync_master_no_tx'; + +# test rpl_semi_sync_master_wait_no_slave = 0 +connection slave; +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +connection con1; +INSERT INTO t1 values (2); +sync_slave_with_master; +connection con1; +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; + +connection slave; +STOP SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_stop.inc + +connection con1; +SET GLOBAL rpl_semi_sync_master_wait_no_slave= 0; +SET GLOBAL rpl_semi_sync_master_timeout= 1000000000; +INSERT INTO t1 values (3); +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; + + +connection slave; +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +connection con1; +SET GLOBAL rpl_semi_sync_master_timeout= 10000000; +SET GLOBAL rpl_semi_sync_master_wait_no_slave= 1; +INSERT INTO t1 values (4); +sync_slave_with_master; + +connection con1; +show status like 'Rpl_semi_sync_master_status'; +show status like 'Rpl_semi_sync_master_clients'; + +--echo ########################################## +--echo # Test rpl_semi_sync_slave_delay_master # +--echo ########################################## + +connection slave; +SET GLOBAL rpl_semi_sync_slave_delay_master= 1; +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +connection con1; +INSERT INTO t1 values (3); +--source include/sync_slave_io_with_master.inc + +connection con1; +show status like 'Rpl_semi_sync_master_clients'; +show status like 'Rpl_semi_sync_master_status'; + +sync_slave_with_master; + +connection slave; +select * from t1 order by a; +connection con1; +select * from t1 order by a; + +connection slave; +SET GLOBAL rpl_semi_sync_slave_delay_master = 0; +STOP SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_stop.inc +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +--echo ########################################################## +--echo # Test rpl_semi_sync_master_enabled and new ACK thread # +--echo ######################################################### +connection con1; +SET GLOBAL rpl_semi_sync_master_enabled = 0; +show status like 'Rpl_semi_sync_master_clients'; +INSERT INTO t1 VALUES (1); +SET GLOBAL rpl_semi_sync_master_enabled = 1; +INSERT INTO t1 VALUES (2); +show status like 'Rpl_semi_sync_master_clients'; + +--echo # Test failure of select error . +SET GLOBAL debug = 'd,rpl_semisync_simulate_select_error'; +# It can still receive ACK from semi-sync slave +INSERT INTO t1 VALUES(3); +sync_slave_with_master; + +connection con1; +--echo # Test failure of pthread_create +SET GLOBAL rpl_semi_sync_master_enabled = 0; +SET GLOBAL debug = 'd,rpl_semisync_simulate_create_thread_failure'; +SET GLOBAL rpl_semi_sync_master_enabled= ON; +--sleep 3 +SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled'; + +--echo # Test failure of pthread_join +SET GLOBAL DEBUG = 'd,rpl_semisync_simulate_thread_join_failure'; +SET GLOBAL rpl_semi_sync_master_enabled= ON; +--sleep 3 +SET GLOBAL rpl_semi_sync_master_enabled= OFF; +--sleep 3 + +--echo # +--echo # Failure on registering semisync slave +--echo # +SET GLOBAL debug= 'd,rpl_semisync_simulate_add_slave_failure'; +SET GLOBAL rpl_semi_sync_master_enabled= ON; + +connection slave; +STOP SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_stop.inc +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +connection con1; +--echo # Should be Zero. +show status like 'Rpl_semi_sync_master_clients'; +SET GLOBAL debug=''; + +connection slave; +--disable_warnings +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc +--enable_warnings + +connection con1; +sync_slave_with_master; + +show status like 'Rpl_semi_sync_master_clients'; + +--echo ################################################################## +--echo # Test fixing of BUG#70669 # +--echo #SLAVE CAN'T CONTINUE REPLICATION AFTER MASTER'S CRASH RECOVERY # +--echo ################################################################# +connection con1; +SET GLOBAL sync_binlog = 1; +CREATE TABLE t2 (c1 INT); +sync_slave_with_master; + +connection con1; +# Block the session before its events are synced to disk +#SET DEBUG_SYNC = 'before_sync_binlog_file SIGNAL before_sync_done WAIT_FOR continue'; +send INSERT INTO t2 values (1); + +--sleep 3 +connection slave; +SELECT * FROM t2; + +connection con2; +#SET DEBUG_SYNC= "now WAIT_FOR before_sync_done"; +#SET DEBUG_SYNC = "now SIGNAL continue"; + +connection con1; +reap; + +sync_slave_with_master; +show tables like 't2'; +select * from t2; + +connection con1; +#SET DEBUG_SYNC= "before_update_pos SIGNAL leader_ready WAIT_FOR follower_ready"; +send INSERT INTO t2 VALUES (2); + +connection con2; +#SET DEBUG_SYNC= "now WAIT_FOR leader_ready"; +#SET DEBUG_SYNC= "after_sync_queue SIGNAL follower_ready"; +send INSERT INTO t2 VALUES (3); + +connection con1; +reap; +connection con2; +reap; + +connection con1; +#SET DEBUG_SYNC = 'before_sync_binlog_file SIGNAL before_sync_done WAIT_FOR continue'; +SET GLOBAL sync_binlog = 0; + +# Todo: fix this simulation and implement the intended sync protocol. +# As a workaround the DROP sender explicitly okays +# which naturally increments the binlog position. +#send DROP TABLE t2; +DROP TABLE t2; + +connection con2; +#SET DEBUG_SYNC= "now WAIT_FOR before_sync_done"; + +sync_slave_with_master; + +# t2 should be dropped +show tables like 't2'; + +connection con2; +#SET DEBUG_SYNC = "now SIGNAL continue"; + +# This block is commented out on purpose. See the todo/workaround above. +#connection con1; +#reap; + + +--echo #cleanup +connection master; +SET DEBUG_SYNC= 'reset'; +disconnect con1; +disconnect con2; +disconnect con3; +SET GLOBAL rpl_semi_sync_master_timeout= 10000; +SET GLOBAL rpl_semi_sync_master_enabled = 0; +DROP TABLE t1; +connection slave; +SET GLOBAL rpl_semi_sync_slave_enabled = 0; +stop slave;start slave; + +--source include/rpl_end.inc diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 376123fa88d..c9aaa2b4dd5 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -4071,6 +4071,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST AFTER_SYNC,AFTER_COMMIT READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME RPL_SEMI_SYNC_SLAVE_DELAY_MASTER +SESSION_VALUE NULL +GLOBAL_VALUE OFF +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Only write master info file when ack is needed. +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME RPL_SEMI_SYNC_SLAVE_ENABLED SESSION_VALUE NULL GLOBAL_VALUE OFF @@ -4085,6 +4099,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL +VARIABLE_NAME RPL_SEMI_SYNC_SLAVE_KILL_CONN_TIMEOUT +SESSION_VALUE NULL +GLOBAL_VALUE 5 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 5 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE INT UNSIGNED +VARIABLE_COMMENT Timeout for the mysql connection used to kill the slave io_thread's connection on master. This timeout comes into play when stop slave is executed. +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 4294967295 +NUMERIC_BLOCK_SIZE 1 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME RPL_SEMI_SYNC_SLAVE_TRACE_LEVEL SESSION_VALUE NULL GLOBAL_VALUE 32 From 6a84e3407df448b28230602ca4e748c85bce52bc Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Mon, 27 Nov 2017 13:00:08 +0200 Subject: [PATCH 24/51] MDEV-13073. This patch replaces semisync's native function_enter,exit and its custom trace faciltiy with standard DBUG_ based equivalents. --- sql/semisync.h | 26 ---- sql/semisync_master.cc | 189 +++++++++++++--------------- sql/semisync_master_ack_receiver.cc | 35 +++--- sql/semisync_slave.cc | 21 ++-- 4 files changed, 113 insertions(+), 158 deletions(-) diff --git a/sql/semisync.h b/sql/semisync.h index cf791b36c1b..b2769591df4 100644 --- a/sql/semisync.h +++ b/sql/semisync.h @@ -35,32 +35,6 @@ public: unsigned long trace_level_; /* the level for tracing */ - inline void function_enter(const char *func_name) - { - if (trace_level_ & kTraceFunction) - sql_print_information("---> %s enter", func_name); - } - inline int function_exit(const char *func_name, int exit_code) - { - if (trace_level_ & kTraceFunction) - sql_print_information("<--- %s exit (%d)", func_name, exit_code); - return exit_code; - } - - inline bool function_exit(const char *func_name, bool exit_code) - { - if (trace_level_ & kTraceFunction) - sql_print_information("<--- %s exit (%s)", func_name, - exit_code ? "True" : "False"); - return exit_code; - } - - inline void function_exit(const char *func_name) - { - if (trace_level_ & kTraceFunction) - sql_print_information("<--- %s exit", func_name); - } - Trace() :trace_level_(0L) {} diff --git a/sql/semisync_master.cc b/sql/semisync_master.cc index 8adbce179e1..299dd3f2251 100644 --- a/sql/semisync_master.cc +++ b/sql/semisync_master.cc @@ -142,18 +142,18 @@ int ActiveTranx::compare(const char *log_file_name1, my_off_t log_file_pos1, int ActiveTranx::insert_tranx_node(const char *log_file_name, my_off_t log_file_pos) { - const char *kWho = "ActiveTranx:insert_tranx_node"; TranxNode *ins_node; int result = 0; unsigned int hash_val; - function_enter(kWho); + DBUG_ENTER("Active_tranx:insert_tranx_node"); ins_node = allocator_.allocate_node(); if (!ins_node) { sql_print_error("%s: transaction node allocation failed for: (%s, %lu)", - kWho, log_file_name, (ulong)log_file_pos); + "Active_tranx:insert_tranx_node", + log_file_name, (ulong)log_file_pos); result = -1; goto l_end; } @@ -185,7 +185,7 @@ int ActiveTranx::insert_tranx_node(const char *log_file_name, * mysql_bin_log.LOCK_log when appending events. */ sql_print_error("%s: binlog write out-of-order, tail (%s, %lu), " - "new node (%s, %lu)", kWho, + "new node (%s, %lu)", "Active_tranx:insert_tranx_node", trx_rear_->log_name_, (ulong)trx_rear_->log_pos_, ins_node->log_name_, (ulong)ins_node->log_pos_); result = -1; @@ -197,20 +197,19 @@ int ActiveTranx::insert_tranx_node(const char *log_file_name, ins_node->hash_next_ = trx_htb_[hash_val]; trx_htb_[hash_val] = ins_node; - if (trace_level_ & kTraceDetail) - sql_print_information("%s: insert (%s, %lu) in entry(%u)", kWho, + DBUG_PRINT("semisync", ("%s: insert (%s, %lu) in entry(%u)", + "Active_tranx:insert_tranx_node", ins_node->log_name_, (ulong)ins_node->log_pos_, - hash_val); - + hash_val)); l_end: - return function_exit(kWho, result); + + DBUG_RETURN(result); } bool ActiveTranx::is_tranx_end_pos(const char *log_file_name, my_off_t log_file_pos) { - const char *kWho = "ActiveTranx::is_tranx_end_pos"; - function_enter(kWho); + DBUG_ENTER("Active_tranx::is_tranx_end_pos"); unsigned int hash_val = get_hash_value(log_file_name, log_file_pos); TranxNode *entry = trx_htb_[hash_val]; @@ -223,21 +222,19 @@ bool ActiveTranx::is_tranx_end_pos(const char *log_file_name, entry = entry->hash_next_; } - if (trace_level_ & kTraceDetail) - sql_print_information("%s: probe (%s, %lu) in entry(%u)", kWho, - log_file_name, (ulong)log_file_pos, hash_val); + DBUG_PRINT("semisync", ("%s: probe (%s, %lu) in entry(%u)", + "Active_tranx::is_tranx_end_pos", + log_file_name, (ulong)log_file_pos, hash_val)); - function_exit(kWho, (entry != NULL)); - return (entry != NULL); + DBUG_RETURN(entry != NULL); } int ActiveTranx::clear_active_tranx_nodes(const char *log_file_name, my_off_t log_file_pos) { - const char *kWho = "ActiveTranx::::clear_active_tranx_nodes"; TranxNode *new_front; - function_enter(kWho); + DBUG_ENTER("Active_tranx::::clear_active_tranx_nodes"); if (log_file_name != NULL) { @@ -271,8 +268,8 @@ int ActiveTranx::clear_active_tranx_nodes(const char *log_file_name, trx_rear_ = NULL; } - if (trace_level_ & kTraceDetail) - sql_print_information("%s: cleared all nodes", kWho); + DBUG_PRINT("semisync", ("%s: cleared all nodes", + "Active_tranx::::clear_active_tranx_nodes")); } else if (new_front != trx_front_) { @@ -305,13 +302,13 @@ int ActiveTranx::clear_active_tranx_nodes(const char *log_file_name, trx_front_ = new_front; allocator_.free_nodes_before(trx_front_); - if (trace_level_ & kTraceDetail) - sql_print_information("%s: cleared %d nodes back until pos (%s, %lu)", - kWho, n_frees, - trx_front_->log_name_, (ulong)trx_front_->log_pos_); + DBUG_PRINT("semisync", ("%s: cleared %d nodes back until pos (%s, %lu)", + "Active_tranx::::clear_active_tranx_nodes", + n_frees, + trx_front_->log_name_, (ulong)trx_front_->log_pos_)); } - return function_exit(kWho, 0); + DBUG_RETURN(0); } @@ -482,13 +479,14 @@ void ReplSemiSyncMaster::cond_broadcast() int ReplSemiSyncMaster::cond_timewait(struct timespec *wait_time) { - const char *kWho = "ReplSemiSyncMaster::cond_timewait()"; int wait_res; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::cond_timewait()"); + wait_res= mysql_cond_timedwait(&COND_binlog_send, &LOCK_binlog, wait_time); - return function_exit(kWho, wait_res); + + DBUG_RETURN(wait_res); } void ReplSemiSyncMaster::add_slave() @@ -520,13 +518,12 @@ void ReplSemiSyncMaster::remove_slave() int ReplSemiSyncMaster::reportReplyPacket(uint32 server_id, const uchar *packet, ulong packet_len) { - const char *kWho = "ReplSemiSyncMaster::reportReplyPacket"; int result= -1; char log_file_name[FN_REFLEN+1]; my_off_t log_file_pos; ulong log_file_len = 0; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::report_reply_packet"); if (unlikely(packet[REPLY_MAGIC_NUM_OFFSET] != ReplSemiSyncMaster::kPacketMagicNum)) { @@ -552,30 +549,30 @@ int ReplSemiSyncMaster::reportReplyPacket(uint32 server_id, const uchar *packet, DBUG_ASSERT(dirname_length(log_file_name) == 0); - if (trace_level_ & kTraceDetail) - sql_print_information("%s: Got reply(%s, %lu) from server %u", - kWho, log_file_name, (ulong)log_file_pos, server_id); + DBUG_PRINT("semisync", ("%s: Got reply(%s, %lu) from server %u", + "Repl_semi_sync_master::report_reply_packet", + log_file_name, (ulong)log_file_pos, server_id)); rpl_semi_sync_master_get_ack++; reportReplyBinlog(server_id, log_file_name, log_file_pos); l_end: - return function_exit(kWho, result); + + DBUG_RETURN(result); } int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, const char *log_file_name, my_off_t log_file_pos) { - const char *kWho = "ReplSemiSyncMaster::reportReplyBinlog"; int cmp; bool can_release_threads = false; bool need_copy_send_pos = true; - if (!(getMasterEnabled())) - return 0; + DBUG_ENTER("Repl_semi_sync_master::report_reply_binlog"); - function_enter(kWho); + if (!(getMasterEnabled())) + DBUG_RETURN(0); lock(); @@ -623,9 +620,9 @@ int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, assert(active_tranxs_ != NULL); active_tranxs_->clear_active_tranx_nodes(log_file_name, log_file_pos); - if (trace_level_ & kTraceDetail) - sql_print_information("%s: Got reply at (%s, %lu)", kWho, - log_file_name, (ulong)log_file_pos); + DBUG_PRINT("semisync", ("%s: Got reply at (%s, %lu)", + "Repl_semi_sync_master::report_reply_binlog", + log_file_name, (ulong)log_file_pos)); } if (rpl_semi_sync_master_wait_sessions > 0) @@ -650,13 +647,13 @@ int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, if (can_release_threads) { - if (trace_level_ & kTraceDetail) - sql_print_information("%s: signal all waiting threads.", kWho); + DBUG_PRINT("semisync", ("%s: signal all waiting threads.", + "Repl_semi_sync_master::report_reply_binlog")); cond_broadcast(); } - return function_exit(kWho, 0); + DBUG_RETURN(0); } int ReplSemiSyncMaster::waitAfterSync(const char *log_file, my_off_t log_pos) @@ -777,9 +774,8 @@ void ReplSemiSyncMaster::dump_end(THD* thd) int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, my_off_t trx_wait_binlog_pos) { - const char *kWho = "ReplSemiSyncMaster::commitTrx"; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::commit_trx"); if (getMasterEnabled() && trx_wait_binlog_name) { @@ -803,12 +799,10 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, if (!getMasterEnabled() || !is_on()) goto l_end; - if (trace_level_ & kTraceDetail) - { - sql_print_information("%s: wait pos (%s, %lu), repl(%d)\n", kWho, + DBUG_PRINT("semisync", ("%s: wait pos (%s, %lu), repl(%d)\n", + "Repl_semi_sync_master::commit_trx", trx_wait_binlog_name, (ulong)trx_wait_binlog_pos, - (int)is_on()); - } + (int)is_on())); while (is_on() && !thd_killed(current_thd)) { @@ -821,9 +815,10 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, /* We have already sent the relevant binlog to the slave: no need to * wait here. */ - if (trace_level_ & kTraceDetail) - sql_print_information("%s: Binlog reply is ahead (%s, %lu),", - kWho, reply_file_name_, (ulong)reply_file_pos_); + DBUG_PRINT("semisync", ("%s: Binlog reply is ahead (%s, %lu),", + "Repl_semi_sync_master::commit_trx", + reply_file_name_, + (ulong)reply_file_pos_)); break; } } @@ -842,9 +837,9 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, wait_file_pos_ = trx_wait_binlog_pos; rpl_semi_sync_master_wait_pos_backtraverse++; - if (trace_level_ & kTraceDetail) - sql_print_information("%s: move back wait position (%s, %lu),", - kWho, wait_file_name_, (ulong)wait_file_pos_); + DBUG_PRINT("semisync", ("%s: move back wait position (%s, %lu),", + "Repl_semi_sync_master::commit_trx", + wait_file_name_, (ulong)wait_file_pos_)); } } else @@ -853,9 +848,9 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, wait_file_pos_ = trx_wait_binlog_pos; wait_file_name_inited_ = true; - if (trace_level_ & kTraceDetail) - sql_print_information("%s: init wait position (%s, %lu),", - kWho, wait_file_name_, (ulong)wait_file_pos_); + DBUG_PRINT("semisync", ("%s: init wait position (%s, %lu),", + "Repl_semi_sync_master::commit_trx", + wait_file_name_, (ulong)wait_file_pos_)); } /* Calcuate the waiting period. */ @@ -875,10 +870,10 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, */ rpl_semi_sync_master_wait_sessions++; - if (trace_level_ & kTraceDetail) - sql_print_information("%s: wait %lu ms for binlog sent (%s, %lu)", - kWho, wait_timeout_, - wait_file_name_, (ulong)wait_file_pos_); + DBUG_PRINT("semisync", ("%s: wait %lu ms for binlog sent (%s, %lu)", + "Repl_semi_sync_master::commit_trx", + wait_timeout_, + wait_file_name_, (ulong)wait_file_pos_)); wait_result = cond_timewait(&abstime); rpl_semi_sync_master_wait_sessions--; @@ -902,12 +897,10 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, wait_time = getWaitTime(start_ts); if (wait_time < 0) { - if (trace_level_ & kTraceGeneral) - { - sql_print_error("Replication semi-sync getWaitTime fail at " - "wait position (%s, %lu)", - trx_wait_binlog_name, (ulong)trx_wait_binlog_pos); - } + DBUG_PRINT("semisync", ("Replication semi-sync getWaitTime fail at " + "wait position (%s, %lu)", + trx_wait_binlog_name, + (ulong)trx_wait_binlog_pos)); rpl_semi_sync_master_timefunc_fails++; } else @@ -940,7 +933,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, THD_EXIT_COND(NULL, & old_stage); } - return function_exit(kWho, 0); + DBUG_RETURN(0); } /* Indicate that semi-sync replication is OFF now. @@ -963,10 +956,10 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, */ int ReplSemiSyncMaster::switch_off() { - const char *kWho = "ReplSemiSyncMaster::switch_off"; int result; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::switch_off"); + state_ = false; /* Clear the active transaction list. */ @@ -979,17 +972,16 @@ int ReplSemiSyncMaster::switch_off() sql_print_information("Semi-sync replication switched OFF."); cond_broadcast(); /* wake up all waiting threads */ - return function_exit(kWho, result); + DBUG_RETURN(result); } int ReplSemiSyncMaster::try_switch_on(int server_id, const char *log_file_name, my_off_t log_file_pos) { - const char *kWho = "ReplSemiSyncMaster::try_switch_on"; bool semi_sync_on = false; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::try_switch_on"); /* If the current sending event's position is larger than or equal to the * 'largest' commit transaction binlog position, the slave is already @@ -1019,20 +1011,19 @@ int ReplSemiSyncMaster::try_switch_on(int server_id, (ulong)log_file_pos); } - return function_exit(kWho, 0); + DBUG_RETURN(0); } int ReplSemiSyncMaster::reserveSyncHeader(String* packet) { - const char *kWho = "ReplSemiSyncMaster::reserveSyncHeader"; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::reserve_sync_header"); /* Set the magic number and the sync status. By default, no sync * is required. */ packet->append(reinterpret_cast(kSyncHeader), sizeof(kSyncHeader)); - return function_exit(kWho, 0); + DBUG_RETURN(0); } int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, @@ -1040,21 +1031,20 @@ int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, my_off_t log_file_pos, bool* need_sync) { - const char *kWho = "ReplSemiSyncMaster::updateSyncHeader"; int cmp = 0; bool sync = false; + DBUG_ENTER("Repl_semi_sync_master::update_sync_header"); + /* If the semi-sync master is not enabled, or the slave is not a semi-sync * target, do not request replies from the slave. */ if (!getMasterEnabled() || !thd->semi_sync_slave) { *need_sync = false; - return 0; + DBUG_RETURN(0); } - function_enter(kWho); - lock(); /* This is the real check inside the mutex. */ @@ -1119,10 +1109,10 @@ int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, } } - if (trace_level_ & kTraceDetail) - sql_print_information("%s: server(%d), (%s, %lu) sync(%d), repl(%d)", - kWho, thd->variables.server_id, log_file_name, - (ulong)log_file_pos, sync, (int)is_on()); + DBUG_PRINT("semisync", ("%s: server(%lu), (%s, %lu) sync(%d), repl(%d)", + "Repl_semi_sync_master::update_sync_header", + thd->variables.server_id, log_file_name, + (ulong)log_file_pos, sync, (int)is_on())); *need_sync= sync; l_end: @@ -1136,16 +1126,15 @@ int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, (packet)[2] = kPacketFlagSync; } - return function_exit(kWho, 0); + DBUG_RETURN(0); } int ReplSemiSyncMaster::writeTranxInBinlog(const char* log_file_name, my_off_t log_file_pos) { - const char *kWho = "ReplSemiSyncMaster::writeTranxInBinlog"; int result = 0; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::write_tranx_in_binlog"); lock(); @@ -1202,17 +1191,16 @@ int ReplSemiSyncMaster::writeTranxInBinlog(const char* log_file_name, l_end: unlock(); - return function_exit(kWho, result); + DBUG_RETURN(result); } int ReplSemiSyncMaster::flushNet(THD *thd, const char *event_buf) { - const char *kWho = "ReplSemiSyncMaster::flushNet"; int result = -1; NET* net= &thd->net; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::flush_net"); assert((unsigned char)event_buf[1] == kPacketMagicNum); if ((unsigned char)event_buf[2] != kPacketFlagSync) @@ -1239,15 +1227,15 @@ int ReplSemiSyncMaster::flushNet(THD *thd, l_end: thd->clear_error(); - return function_exit(kWho, result); + + DBUG_RETURN(result); } int ReplSemiSyncMaster::afterResetMaster() { - const char *kWho = "ReplSemiSyncMaster::afterResetMaster"; int result = 0; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::after_reset_master"); if (rpl_semi_sync_master_enabled) { @@ -1280,20 +1268,19 @@ int ReplSemiSyncMaster::afterResetMaster() unlock(); - return function_exit(kWho, result); + DBUG_RETURN(result); } int ReplSemiSyncMaster::beforeResetMaster() { - const char *kWho = "ReplSemiSyncMaster::beforeResetMaster"; int result = 0; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_master::before_reset_master"); if (rpl_semi_sync_master_enabled) disableMaster(); - return function_exit(kWho, result); + DBUG_RETURN(result); } void ReplSemiSyncMaster::checkAndSwitch() diff --git a/sql/semisync_master_ack_receiver.cc b/sql/semisync_master_ack_receiver.cc index eee35cc122f..2ad48ac1b1c 100644 --- a/sql/semisync_master_ack_receiver.cc +++ b/sql/semisync_master_ack_receiver.cc @@ -36,8 +36,7 @@ pthread_handler_t ack_receive_handler(void *arg) Ack_receiver::Ack_receiver() { - const char *kWho = "Ack_receiver::Ack_receiver"; - function_enter(kWho); + DBUG_ENTER("Ack_receiver::Ack_receiver"); m_status= ST_DOWN; mysql_mutex_init(key_ss_mutex_Ack_receiver_mutex, &m_mutex, @@ -45,25 +44,23 @@ Ack_receiver::Ack_receiver() mysql_cond_init(key_ss_cond_Ack_receiver_cond, &m_cond, NULL); m_pid= 0; - function_exit(kWho); + DBUG_VOID_RETURN; } void Ack_receiver::cleanup() { - const char *kWho = "Ack_receiver::~Ack_receiver"; - function_enter(kWho); + DBUG_ENTER("Ack_receiver::~Ack_receiver"); stop(); mysql_mutex_destroy(&m_mutex); mysql_cond_destroy(&m_cond); - function_exit(kWho); + DBUG_VOID_RETURN; } bool Ack_receiver::start() { - const char *kWho = "Ack_receiver::start"; - function_enter(kWho); + DBUG_ENTER("Ack_receiver::start"); mysql_mutex_lock(&m_mutex); if(m_status == ST_DOWN) @@ -87,19 +84,18 @@ bool Ack_receiver::start() m_status= ST_DOWN; mysql_mutex_unlock(&m_mutex); - return function_exit(kWho, true); + DBUG_RETURN(true); } (void) pthread_attr_destroy(&attr); } mysql_mutex_unlock(&m_mutex); - return function_exit(kWho, false); + DBUG_RETURN(false); } void Ack_receiver::stop() { - const char *kWho = "Ack_receiver::stop"; - function_enter(kWho); + DBUG_ENTER("Ack_receiver::stop"); mysql_mutex_lock(&m_mutex); if (m_status == ST_UP) @@ -116,17 +112,16 @@ void Ack_receiver::stop() } mysql_mutex_unlock(&m_mutex); - function_exit(kWho); + DBUG_VOID_RETURN; } bool Ack_receiver::add_slave(THD *thd) { Slave *slave; - const char *kWho = "Ack_receiver::add_slave"; - function_enter(kWho); + DBUG_ENTER("Ack_receiver::add_slave"); if (!(slave= new Slave)) - return function_exit(kWho, true); + DBUG_RETURN(true); slave->thd= thd; slave->vio= *thd->net.vio; @@ -139,15 +134,14 @@ bool Ack_receiver::add_slave(THD *thd) mysql_cond_broadcast(&m_cond); mysql_mutex_unlock(&m_mutex); - return function_exit(kWho, false); + DBUG_RETURN(false); } void Ack_receiver::remove_slave(THD *thd) { I_List_iterator it(m_slaves); Slave *slave; - const char *kWho = "Ack_receiver::remove_slave"; - function_enter(kWho); + DBUG_ENTER("Ack_receiver::remove_slave"); mysql_mutex_lock(&m_mutex); @@ -161,7 +155,8 @@ void Ack_receiver::remove_slave(THD *thd) } } mysql_mutex_unlock(&m_mutex); - function_exit(kWho); + + DBUG_VOID_RETURN; } inline void Ack_receiver::set_stage_info(const PSI_stage_info &stage) diff --git a/sql/semisync_slave.cc b/sql/semisync_slave.cc index 012f807d28e..7cfc1879dd5 100644 --- a/sql/semisync_slave.cc +++ b/sql/semisync_slave.cc @@ -58,9 +58,8 @@ int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, const char **payload, unsigned long *payload_len) { - const char *kWho = "ReplSemiSyncSlave::slaveReadSyncHeader"; int read_res = 0; - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_slave::slave_read_sync_header"); if (rpl_semi_sync_slave_status) { @@ -71,8 +70,9 @@ int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, *payload_len = total_len - 2; *payload = header + 2; - if (trace_level_ & kTraceDetail) - sql_print_information("%s: reply - %d", kWho, semi_sync_need_reply); + DBUG_PRINT("semisync", ("%s: reply - %d", + "Repl_semi_sync_slave::slave_read_sync_header", + semi_sync_need_reply)); if (semi_sync_need_reply) *semi_flags |= SEMI_SYNC_NEED_ACK; @@ -90,7 +90,7 @@ int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, *payload_len= total_len; } - return function_exit(kWho, read_res); + DBUG_RETURN(read_res); } int ReplSemiSyncSlave::slaveStart(Master_info *mi) @@ -203,7 +203,6 @@ int ReplSemiSyncSlave::requestTransmit(Master_info *mi) int ReplSemiSyncSlave::slaveReply(Master_info *mi) { - const char *kWho = "ReplSemiSyncSlave::slaveReply"; MYSQL* mysql= mi->mysql; const char *binlog_filename= const_cast(mi->master_log_name); my_off_t binlog_filepos= mi->master_log_pos; @@ -215,7 +214,7 @@ int ReplSemiSyncSlave::slaveReply(Master_info *mi) int reply_res = 0; int name_len = strlen(binlog_filename); - function_enter(kWho); + DBUG_ENTER("Repl_semi_sync_slave::slave_reply"); if (rpl_semi_sync_slave_status && semi_sync_need_reply) { @@ -226,9 +225,9 @@ int ReplSemiSyncSlave::slaveReply(Master_info *mi) binlog_filename, name_len + 1 /* including trailing '\0' */); - if (trace_level_ & kTraceDetail) - sql_print_information("%s: reply (%s, %lu)", kWho, - binlog_filename, (ulong)binlog_filepos); + DBUG_PRINT("semisync", ("%s: reply (%s, %lu)", + "Repl_semi_sync_slave::slave_reply", + binlog_filename, (ulong)binlog_filepos)); net_clear(net, 0); /* Send the reply. */ @@ -248,5 +247,5 @@ int ReplSemiSyncSlave::slaveReply(Master_info *mi) } } - return function_exit(kWho, reply_res); + DBUG_RETURN(reply_res); } From f279d3c43aa7536b0f9bf46df8f4a3ef02918be0 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Sat, 25 Nov 2017 18:54:42 +0200 Subject: [PATCH 25/51] MDEV-13073. This part converts the Ali patch`s identifiers to the MariaDB standard. Also some renaming is done as well as white spaces removal. --- .../r/perf_buildin_semisync_issue40.result | 2 +- .../rpl/t/perf_buildin_semisync_issue40.test | 2 +- sql/handler.cc | 4 +- sql/log.cc | 24 +- sql/mysqld.cc | 19 +- sql/semisync.cc | 16 +- sql/semisync.h | 16 +- sql/semisync_master.cc | 277 +++++++++--------- sql/semisync_master.h | 134 ++++----- sql/semisync_master_ack_receiver.cc | 18 +- sql/semisync_master_ack_receiver.h | 4 +- sql/semisync_slave.cc | 50 ++-- sql/semisync_slave.h | 51 ++-- sql/slave.cc | 14 +- sql/sql_repl.cc | 17 +- sql/sys_vars.cc | 25 +- sql/transaction.cc | 12 +- 17 files changed, 347 insertions(+), 338 deletions(-) diff --git a/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result b/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result index 17ecea99bbe..96b07f665d4 100644 --- a/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result +++ b/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result @@ -5,7 +5,7 @@ CALL mtr.add_suppression("Failed to register slave to semi-sync ACK receiver thr CALL mtr.add_suppression("Failed to stop ack receiver thread on pthread_join.*"); CALL mtr.add_suppression("Got an error reading communication packets:*"); CALL mtr.add_suppression("Timeout waiting for reply of binlog*"); -CALL mtr.add_suppression("slaveReadSyncHeader*"); +CALL mtr.add_suppression("slave_read_sync_header*"); CALL mtr.add_suppression("Missing magic number for semi-sync*"); CALL mtr.add_suppression("Got timeout reading communication packets*"); CALL mtr.add_suppression("Failed to call*"); diff --git a/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test index 3edd001af82..113e691e0af 100644 --- a/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test +++ b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test @@ -7,7 +7,7 @@ CALL mtr.add_suppression("Failed to register slave to semi-sync ACK receiver thr CALL mtr.add_suppression("Failed to stop ack receiver thread on pthread_join.*"); CALL mtr.add_suppression("Got an error reading communication packets:*"); CALL mtr.add_suppression("Timeout waiting for reply of binlog*"); -CALL mtr.add_suppression("slaveReadSyncHeader*"); +CALL mtr.add_suppression("slave_read_sync_header*"); CALL mtr.add_suppression("Missing magic number for semi-sync*"); CALL mtr.add_suppression("Got timeout reading communication packets*"); CALL mtr.add_suppression("Failed to call*"); diff --git a/sql/handler.cc b/sql/handler.cc index a0b1c0ebf07..3ea9902e57b 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1486,7 +1486,7 @@ done: mysql_mutex_assert_not_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); #ifdef HAVE_REPLICATION - repl_semisync_master.waitAfterCommit(thd, all); + repl_semisync_master.wait_after_commit(thd, all); DEBUG_SYNC(thd, "after_group_after_commit"); #endif goto end; @@ -1734,7 +1734,7 @@ int ha_rollback_trans(THD *thd, bool all) ER_WARNING_NOT_COMPLETE_ROLLBACK, ER_THD(thd, ER_WARNING_NOT_COMPLETE_ROLLBACK)); #ifdef HAVE_REPLICATION - repl_semisync_master.waitAfterRollback(thd, all); + repl_semisync_master.wait_after_rollback(thd, all); #endif DBUG_RETURN(error); } diff --git a/sql/log.cc b/sql/log.cc index cd757cac30e..bdf0b6fdc59 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -6376,8 +6376,8 @@ err: mysql_mutex_assert_not_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); #ifdef HAVE_REPLICATION - if (repl_semisync_master.reportBinlogUpdate(thd, log_file_name, - file->pos_in_file)) + if (repl_semisync_master.report_binlog_update(thd, log_file_name, + file->pos_in_file)) { sql_print_error("Failed to run 'after_flush' hooks"); error= 1; @@ -6409,8 +6409,8 @@ err: mysql_mutex_assert_owner(&LOCK_after_binlog_sync); mysql_mutex_assert_not_owner(&LOCK_commit_ordered); #ifdef HAVE_REPLICATION - if (repl_semisync_master.waitAfterSync(log_file_name, - file->pos_in_file)) + if (repl_semisync_master.wait_after_sync(log_file_name, + file->pos_in_file)) { error=1; /* error is already printed inside hook */ @@ -7847,10 +7847,10 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) #ifdef HAVE_REPLICATION if (!current->error && repl_semisync_master. - reportBinlogUpdate(current->thd, - current->cache_mngr->last_commit_pos_file, - current->cache_mngr-> - last_commit_pos_offset)) + report_binlog_update(current->thd, + current->cache_mngr->last_commit_pos_file, + current->cache_mngr-> + last_commit_pos_offset)) { current->error= ER_ERROR_ON_WRITE; current->commit_errno= -1; @@ -7935,10 +7935,10 @@ MYSQL_BIN_LOG::trx_group_commit_leader(group_commit_entry *leader) #ifdef HAVE_REPLICATION if (!current->error) current->error= - repl_semisync_master.waitAfterSync(current->cache_mngr-> - last_commit_pos_file, - current->cache_mngr-> - last_commit_pos_offset); + repl_semisync_master.wait_after_sync(current->cache_mngr-> + last_commit_pos_file, + current->cache_mngr-> + last_commit_pos_offset); #endif first= false; } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index f6a4ea60755..3f64e8537ae 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -947,8 +947,7 @@ PSI_mutex_key key_LOCK_after_binlog_sync; PSI_mutex_key key_LOCK_prepare_ordered, key_LOCK_commit_ordered, key_LOCK_slave_background; PSI_mutex_key key_TABLE_SHARE_LOCK_share; -PSI_mutex_key key_ss_mutex_LOCK_binlog_; -PSI_mutex_key key_ss_mutex_Ack_receiver_mutex; +PSI_mutex_key key_LOCK_ack_receiver; static PSI_mutex_info all_server_mutexes[]= { @@ -1027,7 +1026,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_LOCK_rpl_thread, "LOCK_rpl_thread", 0}, { &key_LOCK_rpl_thread_pool, "LOCK_rpl_thread_pool", 0}, { &key_LOCK_parallel_entry, "LOCK_parallel_entry", 0}, - { &key_ss_mutex_Ack_receiver_mutex, "Ack_receiver::m_mutex", 0}, + { &key_LOCK_ack_receiver, "Ack_receiver::mutex", 0}, { &key_LOCK_binlog, "LOCK_binlog", 0} }; @@ -1082,7 +1081,7 @@ PSI_cond_key key_COND_rpl_thread_queue, key_COND_rpl_thread, key_COND_parallel_entry, key_COND_group_commit_orderer, key_COND_prepare_ordered, key_COND_slave_background; PSI_cond_key key_COND_wait_gtid, key_COND_gtid_ignore_duplicates; -PSI_cond_key key_ss_cond_Ack_receiver_cond; +PSI_cond_key key_COND_ack_receiver; static PSI_cond_info all_server_conds[]= { @@ -1136,7 +1135,7 @@ static PSI_cond_info all_server_conds[]= { &key_COND_start_thread, "COND_start_thread", PSI_FLAG_GLOBAL}, { &key_COND_wait_gtid, "COND_wait_gtid", 0}, { &key_COND_gtid_ignore_duplicates, "COND_gtid_ignore_duplicates", 0}, - { &key_ss_cond_Ack_receiver_cond, "Ack_receiver::m_cond", 0}, + { &key_COND_ack_receiver, "Ack_receiver::cond", 0}, { &key_COND_binlog_send, "COND_binlog_send", 0} }; @@ -1144,7 +1143,7 @@ PSI_thread_key key_thread_bootstrap, key_thread_delayed_insert, key_thread_handle_manager, key_thread_main, key_thread_one_connection, key_thread_signal_hand, key_thread_slave_background, key_rpl_parallel_thread; -PSI_thread_key key_ss_thread_Ack_receiver_thread; +PSI_thread_key key_thread_ack_receiver; static PSI_thread_info all_server_threads[]= { @@ -1171,7 +1170,7 @@ static PSI_thread_info all_server_threads[]= { &key_thread_one_connection, "one_connection", 0}, { &key_thread_signal_hand, "signal_handler", PSI_FLAG_GLOBAL}, { &key_thread_slave_background, "slave_background", PSI_FLAG_GLOBAL}, - { &key_ss_thread_Ack_receiver_thread, "Ack_receiver", PSI_FLAG_GLOBAL}, + { &key_thread_ack_receiver, "Ack_receiver", PSI_FLAG_GLOBAL}, { &key_rpl_parallel_thread, "rpl_parallel_thread", 0} }; @@ -5184,8 +5183,8 @@ static int init_server_components() "--log-bin option is not defined."); } - if (repl_semisync_master.initObject() || - repl_semisync_slave.initObject()) + if (repl_semisync_master.init_object() || + repl_semisync_slave.init_object()) { sql_print_error("Could not initialize semisync."); unireg_abort(1); @@ -8256,7 +8255,7 @@ static int show_ssl_get_cipher_list(THD *thd, SHOW_VAR *var, char *buff, #define DEF_SHOW_FUNC(name, show_type) \ static int SHOW_FNAME(name)(MYSQL_THD thd, SHOW_VAR *var, char *buff) \ { \ - repl_semisync_master.setExportStats(); \ + repl_semisync_master.set_export_stats(); \ var->type= show_type; \ var->value= (char *)&rpl_semi_sync_master_##name; \ return 0; \ diff --git a/sql/semisync.cc b/sql/semisync.cc index df37f03ec2f..a8a11f091db 100644 --- a/sql/semisync.cc +++ b/sql/semisync.cc @@ -19,14 +19,14 @@ #include #include "semisync.h" -const unsigned char ReplSemiSyncBase::kPacketMagicNum = 0xef; -const unsigned char ReplSemiSyncBase::kPacketFlagSync = 0x01; +const unsigned char Repl_semi_sync_base::k_packet_magic_num= 0xef; +const unsigned char Repl_semi_sync_base::k_packet_flag_sync= 0x01; -const unsigned long Trace::kTraceGeneral = 0x0001; -const unsigned long Trace::kTraceDetail = 0x0010; -const unsigned long Trace::kTraceNetWait = 0x0020; -const unsigned long Trace::kTraceFunction = 0x0040; +const unsigned long Trace::k_trace_general= 0x0001; +const unsigned long Trace::k_trace_detail= 0x0010; +const unsigned long Trace::k_trace_net_wait= 0x0020; +const unsigned long Trace::k_trace_function= 0x0040; -const unsigned char ReplSemiSyncBase::kSyncHeader[2] = - {ReplSemiSyncBase::kPacketMagicNum, 0}; +const unsigned char Repl_semi_sync_base::k_sync_header[2]= + {Repl_semi_sync_base::k_packet_magic_num, 0}; diff --git a/sql/semisync.h b/sql/semisync.h index b2769591df4..bf43ba5c8d8 100644 --- a/sql/semisync.h +++ b/sql/semisync.h @@ -28,10 +28,10 @@ */ class Trace { public: - static const unsigned long kTraceFunction; - static const unsigned long kTraceGeneral; - static const unsigned long kTraceDetail; - static const unsigned long kTraceNetWait; + static const unsigned long k_trace_function; + static const unsigned long k_trace_general; + static const unsigned long k_trace_detail; + static const unsigned long k_trace_net_wait; unsigned long trace_level_; /* the level for tracing */ @@ -46,14 +46,14 @@ public: /** Base class for semi-sync master and slave classes */ -class ReplSemiSyncBase +class Repl_semi_sync_base :public Trace { public: - static const unsigned char kSyncHeader[2]; /* three byte packet header */ + static const unsigned char k_sync_header[2]; /* three byte packet header */ /* Constants in network packet header. */ - static const unsigned char kPacketMagicNum; - static const unsigned char kPacketFlagSync; + static const unsigned char k_packet_magic_num; + static const unsigned char k_packet_flag_sync; }; /* The layout of a semisync slave reply packet: diff --git a/sql/semisync_master.cc b/sql/semisync_master.cc index 299dd3f2251..5b594d20c71 100644 --- a/sql/semisync_master.cc +++ b/sql/semisync_master.cc @@ -48,7 +48,7 @@ ulong rpl_semi_sync_master_clients = 0; ulonglong rpl_semi_sync_master_net_wait_time = 0; ulonglong rpl_semi_sync_master_trx_wait_time = 0; -ReplSemiSyncMaster repl_semisync_master; +Repl_semi_sync_master repl_semisync_master; Ack_receiver ack_receiver; /* @@ -59,7 +59,7 @@ typedef struct Trans_binlog_info { char log_file[FN_REFLEN]; } Trans_binlog_info; -static int getWaitTime(const struct timespec& start_ts); +static int get_wait_time(const struct timespec& start_ts); static ulonglong timespec_to_usec(const struct timespec *ts) { @@ -68,12 +68,12 @@ static ulonglong timespec_to_usec(const struct timespec *ts) /******************************************************************************* * - * class : manage all active transaction nodes + * class : manage all active transaction nodes * ******************************************************************************/ -ActiveTranx::ActiveTranx(mysql_mutex_t *lock, - ulong trace_level) +Active_tranx::Active_tranx(mysql_mutex_t *lock, + ulong trace_level) : Trace(trace_level), allocator_(max_connections), num_entries_(max_connections << 1), /* Transaction hash table size * is set to double the size @@ -85,22 +85,22 @@ ActiveTranx::ActiveTranx(mysql_mutex_t *lock, trx_rear_ = NULL; /* Create the hash table to find a transaction's ending event. */ - trx_htb_ = new TranxNode *[num_entries_]; + trx_htb_ = new Tranx_node *[num_entries_]; for (int idx = 0; idx < num_entries_; ++idx) trx_htb_[idx] = NULL; sql_print_information("Semi-sync replication initialized for transactions."); } -ActiveTranx::~ActiveTranx() +Active_tranx::~Active_tranx() { delete [] trx_htb_; trx_htb_ = NULL; num_entries_ = 0; } -unsigned int ActiveTranx::calc_hash(const unsigned char *key, - unsigned int length) +unsigned int Active_tranx::calc_hash(const unsigned char *key, + unsigned int length) { unsigned int nr = 1, nr2 = 4; @@ -113,8 +113,8 @@ unsigned int ActiveTranx::calc_hash(const unsigned char *key, return((unsigned int) nr); } -unsigned int ActiveTranx::get_hash_value(const char *log_file_name, - my_off_t log_file_pos) +unsigned int Active_tranx::get_hash_value(const char *log_file_name, + my_off_t log_file_pos) { unsigned int hash1 = calc_hash((const unsigned char *)log_file_name, strlen(log_file_name)); @@ -124,8 +124,8 @@ unsigned int ActiveTranx::get_hash_value(const char *log_file_name, return (hash1 + hash2) % num_entries_; } -int ActiveTranx::compare(const char *log_file_name1, my_off_t log_file_pos1, - const char *log_file_name2, my_off_t log_file_pos2) +int Active_tranx::compare(const char *log_file_name1, my_off_t log_file_pos1, + const char *log_file_name2, my_off_t log_file_pos2) { int cmp = strcmp(log_file_name1, log_file_name2); @@ -139,10 +139,10 @@ int ActiveTranx::compare(const char *log_file_name1, my_off_t log_file_pos1, return 0; } -int ActiveTranx::insert_tranx_node(const char *log_file_name, - my_off_t log_file_pos) +int Active_tranx::insert_tranx_node(const char *log_file_name, + my_off_t log_file_pos) { - TranxNode *ins_node; + Tranx_node *ins_node; int result = 0; unsigned int hash_val; @@ -206,13 +206,13 @@ int ActiveTranx::insert_tranx_node(const char *log_file_name, DBUG_RETURN(result); } -bool ActiveTranx::is_tranx_end_pos(const char *log_file_name, - my_off_t log_file_pos) +bool Active_tranx::is_tranx_end_pos(const char *log_file_name, + my_off_t log_file_pos) { DBUG_ENTER("Active_tranx::is_tranx_end_pos"); unsigned int hash_val = get_hash_value(log_file_name, log_file_pos); - TranxNode *entry = trx_htb_[hash_val]; + Tranx_node *entry = trx_htb_[hash_val]; while (entry != NULL) { @@ -229,10 +229,10 @@ bool ActiveTranx::is_tranx_end_pos(const char *log_file_name, DBUG_RETURN(entry != NULL); } -int ActiveTranx::clear_active_tranx_nodes(const char *log_file_name, - my_off_t log_file_pos) +int Active_tranx::clear_active_tranx_nodes(const char *log_file_name, + my_off_t log_file_pos) { - TranxNode *new_front; + Tranx_node *new_front; DBUG_ENTER("Active_tranx::::clear_active_tranx_nodes"); @@ -258,7 +258,7 @@ int ActiveTranx::clear_active_tranx_nodes(const char *log_file_name, /* No active transaction nodes after the call. */ /* Clear the hash table. */ - memset(trx_htb_, 0, num_entries_ * sizeof(TranxNode *)); + memset(trx_htb_, 0, num_entries_ * sizeof(Tranx_node *)); allocator_.free_all_nodes(); /* Clear the active transaction list. */ @@ -273,7 +273,7 @@ int ActiveTranx::clear_active_tranx_nodes(const char *log_file_name, } else if (new_front != trx_front_) { - TranxNode *curr_node, *next_node; + Tranx_node *curr_node, *next_node; /* Delete all transaction nodes before the confirmation point. */ int n_frees = 0; @@ -285,7 +285,7 @@ int ActiveTranx::clear_active_tranx_nodes(const char *log_file_name, /* Remove the node from the hash table. */ unsigned int hash_val = get_hash_value(curr_node->log_name_, curr_node->log_pos_); - TranxNode **hash_ptr = &(trx_htb_[hash_val]); + Tranx_node **hash_ptr = &(trx_htb_[hash_val]); while ((*hash_ptr) != NULL) { if ((*hash_ptr) == curr_node) @@ -314,28 +314,28 @@ int ActiveTranx::clear_active_tranx_nodes(const char *log_file_name, /******************************************************************************* * - * class: the basic code layer for sync-replication master. - * class: the basic code layer for sync-replication slave. + * class: the basic code layer for syncsync master. + * class: the basic code layer for syncsync slave. * * The most important functions during semi-syn replication listed: * * Master: - * . reportReplyBinlog(): called by the binlog dump thread when it receives + * . report_reply_binlog(): called by the binlog dump thread when it receives * the slave's status information. - * . updateSyncHeader(): based on transaction waiting information, decide + * . update_sync_header(): based on transaction waiting information, decide * whether to request the slave to reply. - * . writeTranxInBinlog(): called by the transaction thread when it finishes + * . write_tranx_in_binlog(): called by the transaction thread when it finishes * writing all transaction events in binlog. - * . commitTrx(): transaction thread wait for the slave reply. + * . commit_trx(): transaction thread wait for the slave reply. * * Slave: - * . slaveReadSyncHeader(): read the semi-sync header from the master, get the - * sync status and get the payload for events. - * . slaveReply(): reply to the master about the replication progress. + * . slave_read_sync_header(): read the semi-sync header from the master, get + * the sync status and get the payload for events. + * . slave_reply(): reply to the master about the replication progress. * ******************************************************************************/ -ReplSemiSyncMaster::ReplSemiSyncMaster() +Repl_semi_sync_master::Repl_semi_sync_master() : active_tranxs_(NULL), init_done_(false), reply_file_name_inited_(false), @@ -351,16 +351,16 @@ ReplSemiSyncMaster::ReplSemiSyncMaster() strcpy(wait_file_name_, ""); } -int ReplSemiSyncMaster::initObject() +int Repl_semi_sync_master::init_object() { int result; init_done_ = true; /* References to the parameter works after set_options(). */ - setWaitTimeout(rpl_semi_sync_master_timeout); - setTraceLevel(rpl_semi_sync_master_trace_level); - setWaitPoint(rpl_semi_sync_master_wait_point); + set_wait_timeout(rpl_semi_sync_master_timeout); + set_trace_level(rpl_semi_sync_master_trace_level); + set_wait_point(rpl_semi_sync_master_wait_point); /* Mutex initialization can only be done after MY_INIT(). */ mysql_mutex_init(key_LOCK_binlog, @@ -370,13 +370,13 @@ int ReplSemiSyncMaster::initObject() if (rpl_semi_sync_master_enabled) { - result = enableMaster(); + result = enable_master(); if (!result) result= ack_receiver.start(); /* Start the ACK thread. */ } else { - result = disableMaster(); + result = disable_master(); } /* @@ -389,16 +389,16 @@ int ReplSemiSyncMaster::initObject() return result; } -int ReplSemiSyncMaster::enableMaster() +int Repl_semi_sync_master::enable_master() { int result = 0; /* Must have the lock when we do enable of disable. */ lock(); - if (!getMasterEnabled()) + if (!get_master_enabled()) { - active_tranxs_ = new ActiveTranx(&LOCK_binlog, trace_level_); + active_tranxs_ = new Active_tranx(&LOCK_binlog, trace_level_); if (active_tranxs_ != NULL) { commit_file_name_inited_ = false; @@ -421,12 +421,12 @@ int ReplSemiSyncMaster::enableMaster() return result; } -int ReplSemiSyncMaster::disableMaster() +int Repl_semi_sync_master::disable_master() { /* Must have the lock when we do enable of disable. */ lock(); - if (getMasterEnabled()) + if (get_master_enabled()) { /* Switch off the semi-sync first so that waiting transaction will be * waken up. @@ -450,7 +450,7 @@ int ReplSemiSyncMaster::disableMaster() return 0; } -void ReplSemiSyncMaster::cleanup() +void Repl_semi_sync_master::cleanup() { if (init_done_) { @@ -462,22 +462,22 @@ void ReplSemiSyncMaster::cleanup() delete active_tranxs_; } -void ReplSemiSyncMaster::lock() +void Repl_semi_sync_master::lock() { mysql_mutex_lock(&LOCK_binlog); } -void ReplSemiSyncMaster::unlock() +void Repl_semi_sync_master::unlock() { mysql_mutex_unlock(&LOCK_binlog); } -void ReplSemiSyncMaster::cond_broadcast() +void Repl_semi_sync_master::cond_broadcast() { mysql_cond_broadcast(&COND_binlog_send); } -int ReplSemiSyncMaster::cond_timewait(struct timespec *wait_time) +int Repl_semi_sync_master::cond_timewait(struct timespec *wait_time) { int wait_res; @@ -489,20 +489,20 @@ int ReplSemiSyncMaster::cond_timewait(struct timespec *wait_time) DBUG_RETURN(wait_res); } -void ReplSemiSyncMaster::add_slave() +void Repl_semi_sync_master::add_slave() { lock(); rpl_semi_sync_master_clients++; unlock(); } -void ReplSemiSyncMaster::remove_slave() +void Repl_semi_sync_master::remove_slave() { lock(); rpl_semi_sync_master_clients--; /* Only switch off if semi-sync is enabled and is on */ - if (getMasterEnabled() && is_on()) + if (get_master_enabled() && is_on()) { /* If user has chosen not to wait if no semi-sync slave available and the last semi-sync slave exits, turn off semi-sync on master @@ -515,8 +515,9 @@ void ReplSemiSyncMaster::remove_slave() unlock(); } -int ReplSemiSyncMaster::reportReplyPacket(uint32 server_id, const uchar *packet, - ulong packet_len) +int Repl_semi_sync_master::report_reply_packet(uint32 server_id, + const uchar *packet, + ulong packet_len) { int result= -1; char log_file_name[FN_REFLEN+1]; @@ -525,7 +526,8 @@ int ReplSemiSyncMaster::reportReplyPacket(uint32 server_id, const uchar *packet, DBUG_ENTER("Repl_semi_sync_master::report_reply_packet"); - if (unlikely(packet[REPLY_MAGIC_NUM_OFFSET] != ReplSemiSyncMaster::kPacketMagicNum)) + if (unlikely(packet[REPLY_MAGIC_NUM_OFFSET] != + Repl_semi_sync_master::k_packet_magic_num)) { sql_print_error("Read semi-sync reply magic number error"); goto l_end; @@ -554,16 +556,16 @@ int ReplSemiSyncMaster::reportReplyPacket(uint32 server_id, const uchar *packet, log_file_name, (ulong)log_file_pos, server_id)); rpl_semi_sync_master_get_ack++; - reportReplyBinlog(server_id, log_file_name, log_file_pos); + report_reply_binlog(server_id, log_file_name, log_file_pos); l_end: DBUG_RETURN(result); } -int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, - const char *log_file_name, - my_off_t log_file_pos) +int Repl_semi_sync_master::report_reply_binlog(uint32 server_id, + const char *log_file_name, + my_off_t log_file_pos) { int cmp; bool can_release_threads = false; @@ -571,13 +573,13 @@ int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, DBUG_ENTER("Repl_semi_sync_master::report_reply_binlog"); - if (!(getMasterEnabled())) + if (!(get_master_enabled())) DBUG_RETURN(0); lock(); /* This is the real check inside the mutex. */ - if (!getMasterEnabled()) + if (!get_master_enabled()) goto l_end; if (!is_on()) @@ -592,7 +594,7 @@ int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, */ if (reply_file_name_inited_) { - cmp = ActiveTranx::compare(log_file_name, log_file_pos, + cmp = Active_tranx::compare(log_file_name, log_file_pos, reply_file_name_, reply_file_pos_); /* If the requested position is behind the sending binlog position, @@ -630,8 +632,8 @@ int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, /* Let us check if some of the waiting threads doing a trx * commit can now proceed. */ - cmp = ActiveTranx::compare(reply_file_name_, reply_file_pos_, - wait_file_name_, wait_file_pos_); + cmp = Active_tranx::compare(reply_file_name_, reply_file_pos_, + wait_file_name_, wait_file_pos_); if (cmp >= 0) { /* Yes, at least one waiting thread can now proceed: @@ -656,22 +658,22 @@ int ReplSemiSyncMaster::reportReplyBinlog(uint32 server_id, DBUG_RETURN(0); } -int ReplSemiSyncMaster::waitAfterSync(const char *log_file, my_off_t log_pos) +int Repl_semi_sync_master::wait_after_sync(const char *log_file, my_off_t log_pos) { - if (!getMasterEnabled()) + if (!get_master_enabled()) return 0; int ret= 0; if(log_pos && - waitPoint() == SEMI_SYNC_MASTER_WAIT_POINT_AFTER_BINLOG_SYNC) - ret= commitTrx(log_file + dirname_length(log_file), log_pos); + wait_point() == SEMI_SYNC_MASTER_WAIT_POINT_AFTER_BINLOG_SYNC) + ret= commit_trx(log_file + dirname_length(log_file), log_pos); return ret; } -int ReplSemiSyncMaster::waitAfterCommit(THD* thd, bool all) +int Repl_semi_sync_master::wait_after_commit(THD* thd, bool all) { - if (!getMasterEnabled()) + if (!get_master_enabled()) return 0; int ret= 0; @@ -682,7 +684,7 @@ int ReplSemiSyncMaster::waitAfterCommit(THD* thd, bool all) (all || thd->transaction.all.ha_list == 0); /* The coordinates are propagated to this point having been computed - in reportBinlogUpdate + in report_binlog_update */ Trans_binlog_info *log_info= thd->semisync_info; log_file= log_info && log_info->log_file[0] ? log_info->log_file : 0; @@ -692,8 +694,8 @@ int ReplSemiSyncMaster::waitAfterCommit(THD* thd, bool all) if (is_real_trans && log_pos && - waitPoint() == SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT) - ret= commitTrx(log_file, log_pos); + wait_point() == SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT) + ret= commit_trx(log_file, log_pos); if (is_real_trans && log_info) { @@ -704,18 +706,18 @@ int ReplSemiSyncMaster::waitAfterCommit(THD* thd, bool all) return ret; } -int ReplSemiSyncMaster::waitAfterRollback(THD *thd, bool all) +int Repl_semi_sync_master::wait_after_rollback(THD *thd, bool all) { - return waitAfterCommit(thd, all); + return wait_after_commit(thd, all); } /** The method runs after flush to binary log is done. */ -int ReplSemiSyncMaster::reportBinlogUpdate(THD* thd, const char *log_file, - my_off_t log_pos) +int Repl_semi_sync_master::report_binlog_update(THD* thd, const char *log_file, + my_off_t log_pos) { - if (getMasterEnabled()) + if (get_master_enabled()) { Trans_binlog_info *log_info; @@ -729,13 +731,13 @@ int ReplSemiSyncMaster::reportBinlogUpdate(THD* thd, const char *log_file, strcpy(log_info->log_file, log_file + dirname_length(log_file)); log_info->log_pos = log_pos; - return writeTranxInBinlog(log_info->log_file, log_pos); + return write_tranx_in_binlog(log_info->log_file, log_pos); } return 0; } -int ReplSemiSyncMaster::dump_start(THD* thd, +int Repl_semi_sync_master::dump_start(THD* thd, const char *log_file, my_off_t log_pos) { @@ -751,14 +753,17 @@ int ReplSemiSyncMaster::dump_start(THD* thd, } add_slave(); - reportReplyBinlog(thd->variables.server_id, log_file + dirname_length(log_file), log_pos); - sql_print_information("Start semi-sync binlog_dump to slave (server_id: %d), pos(%s, %lu", - thd->variables.server_id, log_file, (unsigned long)log_pos); + report_reply_binlog(thd->variables.server_id, + log_file + dirname_length(log_file), log_pos); + sql_print_information("Start semi-sync binlog_dump to slave (server_id: %d), " + "pos(%s, %lu", + thd->variables.server_id, log_file, + (unsigned long)log_pos); return 0; } -void ReplSemiSyncMaster::dump_end(THD* thd) +void Repl_semi_sync_master::dump_end(THD* thd) { if (!thd->semi_sync_slave) return; @@ -771,13 +776,13 @@ void ReplSemiSyncMaster::dump_end(THD* thd) return; } -int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, - my_off_t trx_wait_binlog_pos) +int Repl_semi_sync_master::commit_trx(const char* trx_wait_binlog_name, + my_off_t trx_wait_binlog_pos) { DBUG_ENTER("Repl_semi_sync_master::commit_trx"); - if (getMasterEnabled() && trx_wait_binlog_name) + if (get_master_enabled() && trx_wait_binlog_name) { struct timespec start_ts; struct timespec abstime; @@ -796,7 +801,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, & old_stage); /* This is the real check inside the mutex. */ - if (!getMasterEnabled() || !is_on()) + if (!get_master_enabled() || !is_on()) goto l_end; DBUG_PRINT("semisync", ("%s: wait pos (%s, %lu), repl(%d)\n", @@ -808,8 +813,9 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, { if (reply_file_name_inited_) { - int cmp = ActiveTranx::compare(reply_file_name_, reply_file_pos_, - trx_wait_binlog_name, trx_wait_binlog_pos); + int cmp = Active_tranx::compare(reply_file_name_, reply_file_pos_, + trx_wait_binlog_name, + trx_wait_binlog_pos); if (cmp >= 0) { /* We have already sent the relevant binlog to the slave: no need to @@ -828,8 +834,9 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, */ if (wait_file_name_inited_) { - int cmp = ActiveTranx::compare(trx_wait_binlog_name, trx_wait_binlog_pos, - wait_file_name_, wait_file_pos_); + int cmp = Active_tranx::compare(trx_wait_binlog_name, + trx_wait_binlog_pos, + wait_file_name_, wait_file_pos_); if (cmp <= 0) { /* This thd has a lower position, let's update the minimum info. */ @@ -894,7 +901,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, { int wait_time; - wait_time = getWaitTime(start_ts); + wait_time = get_wait_time(start_ts); if (wait_time < 0) { DBUG_PRINT("semisync", ("Replication semi-sync getWaitTime fail at " @@ -913,7 +920,7 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, /* At this point, the binlog file and position of this transaction - must have been removed from ActiveTranx. + must have been removed from Active_tranx. active_tranxs_ may be NULL if someone disabled semi sync during cond_timewait() */ @@ -950,11 +957,11 @@ int ReplSemiSyncMaster::commitTrx(const char* trx_wait_binlog_name, * If semi-sync is disabled, all transactions still update the wait * position with the last position in binlog. But no transactions will * wait for confirmations and the active transaction list would not be - * maintained. In binlog dump thread, updateSyncHeader() checks whether + * maintained. In binlog dump thread, update_sync_header() checks whether * the current sending event catches up with last wait position. If it * does match, semi-sync will be switched on again. */ -int ReplSemiSyncMaster::switch_off() +int Repl_semi_sync_master::switch_off() { int result; @@ -975,9 +982,9 @@ int ReplSemiSyncMaster::switch_off() DBUG_RETURN(result); } -int ReplSemiSyncMaster::try_switch_on(int server_id, - const char *log_file_name, - my_off_t log_file_pos) +int Repl_semi_sync_master::try_switch_on(int server_id, + const char *log_file_name, + my_off_t log_file_pos) { bool semi_sync_on = false; @@ -991,7 +998,7 @@ int ReplSemiSyncMaster::try_switch_on(int server_id, */ if (commit_file_name_inited_) { - int cmp = ActiveTranx::compare(log_file_name, log_file_pos, + int cmp = Active_tranx::compare(log_file_name, log_file_pos, commit_file_name_, commit_file_pos_); semi_sync_on = (cmp >= 0); } @@ -1014,22 +1021,22 @@ int ReplSemiSyncMaster::try_switch_on(int server_id, DBUG_RETURN(0); } -int ReplSemiSyncMaster::reserveSyncHeader(String* packet) +int Repl_semi_sync_master::reserve_sync_header(String* packet) { DBUG_ENTER("Repl_semi_sync_master::reserve_sync_header"); /* Set the magic number and the sync status. By default, no sync * is required. */ - packet->append(reinterpret_cast(kSyncHeader), - sizeof(kSyncHeader)); + packet->append(reinterpret_cast(k_sync_header), + sizeof(k_sync_header)); DBUG_RETURN(0); } -int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, - const char *log_file_name, - my_off_t log_file_pos, - bool* need_sync) +int Repl_semi_sync_master::update_sync_header(THD* thd, unsigned char *packet, + const char *log_file_name, + my_off_t log_file_pos, + bool* need_sync) { int cmp = 0; bool sync = false; @@ -1039,7 +1046,7 @@ int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, /* If the semi-sync master is not enabled, or the slave is not a semi-sync * target, do not request replies from the slave. */ - if (!getMasterEnabled() || !thd->semi_sync_slave) + if (!get_master_enabled() || !thd->semi_sync_slave) { *need_sync = false; DBUG_RETURN(0); @@ -1048,7 +1055,7 @@ int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, lock(); /* This is the real check inside the mutex. */ - if (!getMasterEnabled()) + if (!get_master_enabled()) { assert(sync == false); goto l_end; @@ -1061,7 +1068,7 @@ int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, if (reply_file_name_inited_) { - cmp = ActiveTranx::compare(log_file_name, log_file_pos, + cmp = Active_tranx::compare(log_file_name, log_file_pos, reply_file_name_, reply_file_pos_); if (cmp <= 0) { @@ -1074,7 +1081,7 @@ int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, if (wait_file_name_inited_) { - cmp = ActiveTranx::compare(log_file_name, log_file_pos, + cmp = Active_tranx::compare(log_file_name, log_file_pos, wait_file_name_, wait_file_pos_); } else @@ -1099,7 +1106,7 @@ int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, { if (commit_file_name_inited_) { - int cmp = ActiveTranx::compare(log_file_name, log_file_pos, + int cmp = Active_tranx::compare(log_file_name, log_file_pos, commit_file_name_, commit_file_pos_); sync = (cmp >= 0); } @@ -1123,14 +1130,14 @@ int ReplSemiSyncMaster::updateSyncHeader(THD* thd, unsigned char *packet, */ if (sync) { - (packet)[2] = kPacketFlagSync; + (packet)[2] = k_packet_flag_sync; } DBUG_RETURN(0); } -int ReplSemiSyncMaster::writeTranxInBinlog(const char* log_file_name, - my_off_t log_file_pos) +int Repl_semi_sync_master::write_tranx_in_binlog(const char* log_file_name, + my_off_t log_file_pos) { int result = 0; @@ -1139,20 +1146,20 @@ int ReplSemiSyncMaster::writeTranxInBinlog(const char* log_file_name, lock(); /* This is the real check inside the mutex. */ - if (!getMasterEnabled()) + if (!get_master_enabled()) goto l_end; /* Update the 'largest' transaction commit position seen so far even * though semi-sync is switched off. * It is much better that we update commit_file_* here, instead of - * inside commitTrx(). This is mostly because updateSyncHeader() + * inside commit_trx(). This is mostly because update_sync_header() * will watch for commit_file_* to decide whether to switch semi-sync - * on. The detailed reason is explained in function updateSyncHeader(). + * on. The detailed reason is explained in function update_sync_header(). */ if (commit_file_name_inited_) { - int cmp = ActiveTranx::compare(log_file_name, log_file_pos, - commit_file_name_, commit_file_pos_); + int cmp = Active_tranx::compare(log_file_name, log_file_pos, + commit_file_name_, commit_file_pos_); if (cmp > 0) { /* This is a larger position, let's update the maximum info. */ @@ -1194,16 +1201,16 @@ int ReplSemiSyncMaster::writeTranxInBinlog(const char* log_file_name, DBUG_RETURN(result); } -int ReplSemiSyncMaster::flushNet(THD *thd, - const char *event_buf) +int Repl_semi_sync_master::flush_net(THD *thd, + const char *event_buf) { int result = -1; NET* net= &thd->net; DBUG_ENTER("Repl_semi_sync_master::flush_net"); - assert((unsigned char)event_buf[1] == kPacketMagicNum); - if ((unsigned char)event_buf[2] != kPacketFlagSync) + assert((unsigned char)event_buf[1] == k_packet_magic_num); + if ((unsigned char)event_buf[2] != k_packet_flag_sync) { /* current event does not require reply */ result = 0; @@ -1231,7 +1238,7 @@ int ReplSemiSyncMaster::flushNet(THD *thd, DBUG_RETURN(result); } -int ReplSemiSyncMaster::afterResetMaster() +int Repl_semi_sync_master::after_reset_master() { int result = 0; @@ -1240,7 +1247,7 @@ int ReplSemiSyncMaster::afterResetMaster() if (rpl_semi_sync_master_enabled) { sql_print_information("Enable Semi-sync Master after reset master"); - enableMaster(); + enable_master(); } lock(); @@ -1249,7 +1256,7 @@ int ReplSemiSyncMaster::afterResetMaster() !rpl_semi_sync_master_wait_no_slave) state_ = 0; else - state_ = getMasterEnabled()? 1 : 0; + state_ = get_master_enabled()? 1 : 0; wait_file_name_inited_ = false; reply_file_name_inited_ = false; @@ -1271,22 +1278,22 @@ int ReplSemiSyncMaster::afterResetMaster() DBUG_RETURN(result); } -int ReplSemiSyncMaster::beforeResetMaster() +int Repl_semi_sync_master::before_reset_master() { int result = 0; DBUG_ENTER("Repl_semi_sync_master::before_reset_master"); if (rpl_semi_sync_master_enabled) - disableMaster(); + disable_master(); DBUG_RETURN(result); } -void ReplSemiSyncMaster::checkAndSwitch() +void Repl_semi_sync_master::check_and_switch() { lock(); - if (getMasterEnabled() && is_on()) + if (get_master_enabled() && is_on()) { if (!rpl_semi_sync_master_wait_no_slave && rpl_semi_sync_master_clients == 0) @@ -1295,7 +1302,7 @@ void ReplSemiSyncMaster::checkAndSwitch() unlock(); } -void ReplSemiSyncMaster::setExportStats() +void Repl_semi_sync_master::set_export_stats() { lock(); @@ -1318,7 +1325,7 @@ void ReplSemiSyncMaster::setExportStats() * >= 0: the waiting time in microsecons(us) * < 0: error in get time or time back traverse */ -static int getWaitTime(const struct timespec& start_ts) +static int get_wait_time(const struct timespec& start_ts) { ulonglong start_usecs, end_usecs; struct timespec end_ts; diff --git a/sql/semisync_master.h b/sql/semisync_master.h index 737ad46dd27..97c5d01a1d5 100644 --- a/sql/semisync_master.h +++ b/sql/semisync_master.h @@ -27,18 +27,18 @@ extern PSI_mutex_key key_LOCK_binlog; extern PSI_cond_key key_COND_binlog_send; #endif -struct TranxNode { +struct Tranx_node { char log_name_[FN_REFLEN]; my_off_t log_pos_; - struct TranxNode *next_; /* the next node in the sorted list */ - struct TranxNode *hash_next_; /* the next node during hash collision */ + struct Tranx_node *next_; /* the next node in the sorted list */ + struct Tranx_node *hash_next_; /* the next node during hash collision */ }; /** - @class TranxNodeAllocator + @class Tranx_node_allocator This class provides memory allocating and freeing methods for - TranxNode. The main target is performance. + Tranx_node. The main target is performance. @section ALLOCATE How to allocate a node The pointer of the first node after 'last_node' in current_block is @@ -51,7 +51,7 @@ struct TranxNode { After some nodes are freed, there probably are some free nodes before the sequence of the allocated nodes, but we do not reuse it. It is better to keep the allocated nodes are in the sequence, for it is more efficient - for allocating and freeing TranxNode. + for allocating and freeing Tranx_node. @section FREENODE How to free nodes There are two methods for freeing nodes. They are free_all_nodes and @@ -68,23 +68,23 @@ struct TranxNode { more efficient. */ #define BLOCK_TRANX_NODES 16 -class TranxNodeAllocator +class Tranx_node_allocator { public: /** @param reserved_nodes - The number of reserved TranxNodes. It is used to set 'reserved_blocks' - which can contain at least 'reserved_nodes' number of TranxNodes. When + The number of reserved Tranx_nodes. It is used to set 'reserved_blocks' + which can contain at least 'reserved_nodes' number of Tranx_nodes. When freeing memory, we will reserve at least reserved_blocks of Blocks not freed. */ - TranxNodeAllocator(uint reserved_nodes) : + Tranx_node_allocator(uint reserved_nodes) : reserved_blocks(reserved_nodes/BLOCK_TRANX_NODES + (reserved_nodes%BLOCK_TRANX_NODES > 1 ? 2 : 1)), first_block(NULL), last_block(NULL), current_block(NULL), last_node(-1), block_num(0) {} - ~TranxNodeAllocator() + ~Tranx_node_allocator() { Block *block= first_block; while (block != NULL) @@ -101,11 +101,11 @@ public: it are in use. A new Block is allocated and is put into the rear of the Block link table if no Block is free. - @return Return a TranxNode *, or NULL if an error occurred. + @return Return a Tranx_node *, or NULL if an error occurred. */ - TranxNode *allocate_node() + Tranx_node *allocate_node() { - TranxNode *trx_node; + Tranx_node *trx_node; Block *block= current_block; if (last_node == BLOCK_TRANX_NODES-1) @@ -151,7 +151,7 @@ public: @return Return 0, or 1 if an error occurred. */ - int free_nodes_before(TranxNode* node) + int free_nodes_before(Tranx_node* node) { Block *block; Block *prev_block= NULL; @@ -186,16 +186,16 @@ private: uint reserved_blocks; /** - A sequence memory which contains BLOCK_TRANX_NODES TranxNodes. + A sequence memory which contains BLOCK_TRANX_NODES Tranx_nodes. - BLOCK_TRANX_NODES The number of TranxNodes which are in a Block. + BLOCK_TRANX_NODES The number of Tranx_nodes which are in a Block. next Every Block has a 'next' pointer which points to the next Block. These linking Blocks constitute a Block link table. */ struct Block { Block *next; - TranxNode nodes[BLOCK_TRANX_NODES]; + Tranx_node nodes[BLOCK_TRANX_NODES]; }; /** @@ -290,20 +290,20 @@ private: /** This class manages memory for active transaction list. - We record each active transaction with a TranxNode, each session + We record each active transaction with a Tranx_node, each session can have only one open transaction. Because of EVENT, the total active transaction nodes can exceed the maximum allowed connections. */ -class ActiveTranx +class Active_tranx :public Trace { private: - TranxNodeAllocator allocator_; + Tranx_node_allocator allocator_; /* These two record the active transaction list in sort order. */ - TranxNode *trx_front_, *trx_rear_; + Tranx_node *trx_front_, *trx_rear_; - TranxNode **trx_htb_; /* A hash table on active transactions. */ + Tranx_node **trx_htb_; /* A hash table on active transactions. */ int num_entries_; /* maximum hash table entries */ mysql_mutex_t *lock_; /* mutex lock */ @@ -314,23 +314,23 @@ private: unsigned int get_hash_value(const char *log_file_name, my_off_t log_file_pos); int compare(const char *log_file_name1, my_off_t log_file_pos1, - const TranxNode *node2) { + const Tranx_node *node2) { return compare(log_file_name1, log_file_pos1, node2->log_name_, node2->log_pos_); } - int compare(const TranxNode *node1, + int compare(const Tranx_node *node1, const char *log_file_name2, my_off_t log_file_pos2) { return compare(node1->log_name_, node1->log_pos_, log_file_name2, log_file_pos2); } - int compare(const TranxNode *node1, const TranxNode *node2) { + int compare(const Tranx_node *node1, const Tranx_node *node2) { return compare(node1->log_name_, node1->log_pos_, node2->log_name_, node2->log_pos_); } public: - ActiveTranx(mysql_mutex_t *lock, unsigned long trace_level); - ~ActiveTranx(); + Active_tranx(mysql_mutex_t *lock, unsigned long trace_level); + ~Active_tranx(); /* Insert an active transaction node with the specified position. * @@ -366,13 +366,13 @@ public: /** The extension class for the master of semi-synchronous replication */ -class ReplSemiSyncMaster - :public ReplSemiSyncBase { +class Repl_semi_sync_master + :public Repl_semi_sync_base { private: - ActiveTranx *active_tranxs_; /* active transaction list: the list will + Active_tranx *active_tranxs_; /* active transaction list: the list will be cleared when semi-sync switches off. */ - /* True when initObject has been called */ + /* True when init_object has been called */ bool init_done_; /* This cond variable is signaled when enough binlog has been sent to slave, @@ -456,32 +456,32 @@ class ReplSemiSyncMaster const char *log_file_name, my_off_t log_file_pos); public: - ReplSemiSyncMaster(); - ~ReplSemiSyncMaster() {} + Repl_semi_sync_master(); + ~Repl_semi_sync_master() {} void cleanup(); - bool getMasterEnabled() { + bool get_master_enabled() { return master_enabled_; } - void setTraceLevel(unsigned long trace_level) { + void set_trace_level(unsigned long trace_level) { trace_level_ = trace_level; if (active_tranxs_) active_tranxs_->trace_level_ = trace_level; } /* Set the transaction wait timeout period, in milliseconds. */ - void setWaitTimeout(unsigned long wait_timeout) { + void set_wait_timeout(unsigned long wait_timeout) { wait_timeout_ = wait_timeout; } /*set the ACK point, after binlog sync or after transaction commit*/ - void setWaitPoint(unsigned long ack_point) + void set_wait_point(unsigned long ack_point) { wait_point_ = ack_point; } - ulong waitPoint() //no cover line + ulong wait_point() //no cover line { return wait_point_; //no cover line } @@ -489,13 +489,13 @@ class ReplSemiSyncMaster /* Initialize this class after MySQL parameters are initialized. this * function should be called once at bootstrap time. */ - int initObject(); + int init_object(); /* Enable the object to enable semi-sync replication inside the master. */ - int enableMaster(); + int enable_master(); /* Enable the object to enable semi-sync replication inside the master. */ - int disableMaster(); + int disable_master(); /* Add a semi-sync replication slave */ void add_slave(); @@ -503,8 +503,8 @@ class ReplSemiSyncMaster /* Remove a semi-sync replication slave */ void remove_slave(); - /* It parses a reply packet and call reportReplyBinlog to handle it. */ - int reportReplyPacket(uint32 server_id, const uchar *packet, + /* It parses a reply packet and call report_reply_binlog to handle it. */ + int report_reply_packet(uint32 server_id, const uchar *packet, ulong packet_len); /* In semi-sync replication, reports up to which binlog position we have @@ -519,9 +519,9 @@ class ReplSemiSyncMaster * Return: * 0: success; non-zero: error */ - int reportReplyBinlog(uint32 server_id, - const char* log_file_name, - my_off_t end_offset); + int report_reply_binlog(uint32 server_id, + const char* log_file_name, + my_off_t end_offset); /* Commit a transaction in the final step. This function is called from * InnoDB before returning from the low commit. If semi-sync is switch on, @@ -538,20 +538,20 @@ class ReplSemiSyncMaster * Return: * 0: success; non-zero: error */ - int commitTrx(const char* trx_wait_binlog_name, - my_off_t trx_wait_binlog_pos); + int commit_trx(const char* trx_wait_binlog_name, + my_off_t trx_wait_binlog_pos); /*Wait for ACK after writing/sync binlog to file*/ - int waitAfterSync(const char* log_file, my_off_t log_pos); + int wait_after_sync(const char* log_file, my_off_t log_pos); /*Wait for ACK after commting the transaction*/ - int waitAfterCommit(THD* thd, bool all); + int wait_after_commit(THD* thd, bool all); /*Wait after the transaction is rollback*/ - int waitAfterRollback(THD *thd, bool all); + int wait_after_rollback(THD *thd, bool all); /*Store the current binlog position in active_tranxs_. This position should * be acked by slave*/ - int reportBinlogUpdate(THD *thd, const char *log_file,my_off_t log_pos); + int report_binlog_update(THD *thd, const char *log_file,my_off_t log_pos); int dump_start(THD* thd, const char *log_file, @@ -569,7 +569,7 @@ class ReplSemiSyncMaster * Return: * size of the bytes reserved for header */ - int reserveSyncHeader(String* packet); + int reserve_sync_header(String* packet); /* Update the sync bit in the packet header to indicate to the slave whether * the master will wait for the reply of the event. If semi-sync is switched @@ -580,16 +580,16 @@ class ReplSemiSyncMaster * packet - (IN) the packet containing the replication event * log_file_name - (IN) the event ending position's file name * log_file_pos - (IN) the event ending position's file offset - * need_sync - (IN) identify if flushNet is needed to call. + * need_sync - (IN) identify if flush_net is needed to call. * server_id - (IN) master server id number * * Return: * 0: success; non-zero: error */ - int updateSyncHeader(THD* thd, unsigned char *packet, - const char *log_file_name, - my_off_t log_file_pos, - bool* need_sync); + int update_sync_header(THD* thd, unsigned char *packet, + const char *log_file_name, + my_off_t log_file_pos, + bool* need_sync); /* Called when a transaction finished writing binlog events. * . update the 'largest' transactions' binlog event position @@ -603,25 +603,25 @@ class ReplSemiSyncMaster * Return: * 0: success; non-zero: error */ - int writeTranxInBinlog(const char* log_file_name, my_off_t log_file_pos); + int write_tranx_in_binlog(const char* log_file_name, my_off_t log_file_pos); /* Read the slave's reply so that we know how much progress the slave makes * on receive replication events. */ - int flushNet(THD* thd, const char *event_buf); + int flush_net(THD* thd, const char *event_buf); /* Export internal statistics for semi-sync replication. */ - void setExportStats(); + void set_export_stats(); /* 'reset master' command is issued from the user and semi-sync need to * go off for that. */ - int afterResetMaster(); + int after_reset_master(); /*called before reset master*/ - int beforeResetMaster(); + int before_reset_master(); - void checkAndSwitch(); + void check_and_switch(); }; enum rpl_semi_sync_master_wait_point_t { @@ -629,7 +629,7 @@ enum rpl_semi_sync_master_wait_point_t { SEMI_SYNC_MASTER_WAIT_POINT_AFTER_STORAGE_COMMIT, }; -extern ReplSemiSyncMaster repl_semisync_master; +extern Repl_semi_sync_master repl_semisync_master; extern Ack_receiver ack_receiver; /* System and status variables for the master component */ @@ -663,7 +663,7 @@ extern unsigned long long rpl_semi_sync_master_get_ack; 1 (default) : keep waiting until timeout even no available semi-sync slave. */ extern char rpl_semi_sync_master_wait_no_slave; -extern ReplSemiSyncMaster repl_semisync_master; +extern Repl_semi_sync_master repl_semisync_master; extern PSI_stage_info stage_waiting_for_semi_sync_ack_from_slave; extern PSI_stage_info stage_reading_semi_sync_ack; diff --git a/sql/semisync_master_ack_receiver.cc b/sql/semisync_master_ack_receiver.cc index 2ad48ac1b1c..f986c629f65 100644 --- a/sql/semisync_master_ack_receiver.cc +++ b/sql/semisync_master_ack_receiver.cc @@ -17,10 +17,10 @@ #include "semisync_master.h" #include "semisync_master_ack_receiver.h" -extern PSI_mutex_key key_ss_mutex_Ack_receiver_mutex; -extern PSI_cond_key key_ss_cond_Ack_receiver_cond; -extern PSI_thread_key key_ss_thread_Ack_receiver_thread; -extern ReplSemiSyncMaster repl_semisync; +extern PSI_mutex_key key_LOCK_ack_receiver; +extern PSI_cond_key key_COND_ack_receiver; +extern PSI_thread_key key_thread_ack_receiver; +extern Repl_semi_sync_master repl_semisync; /* Callback function of ack receive thread */ pthread_handler_t ack_receive_handler(void *arg) @@ -39,9 +39,9 @@ Ack_receiver::Ack_receiver() DBUG_ENTER("Ack_receiver::Ack_receiver"); m_status= ST_DOWN; - mysql_mutex_init(key_ss_mutex_Ack_receiver_mutex, &m_mutex, + mysql_mutex_init(key_LOCK_ack_receiver, &m_mutex, MY_MUTEX_INIT_FAST); - mysql_cond_init(key_ss_cond_Ack_receiver_cond, &m_cond, NULL); + mysql_cond_init(key_COND_ack_receiver, &m_cond, NULL); m_pid= 0; DBUG_VOID_RETURN; @@ -75,7 +75,7 @@ bool Ack_receiver::start() #ifndef _WIN32 pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM) != 0 || #endif - mysql_thread_create(key_ss_thread_Ack_receiver_thread, &m_pid, + mysql_thread_create(key_thread_ack_receiver, &m_pid, &attr, ack_receive_handler, this)) { sql_print_error("Failed to start semi-sync ACK receiver thread, " @@ -283,8 +283,8 @@ void Ack_receiver::run() len= my_net_read(&net); if (likely(len != packet_error)) - repl_semisync_master.reportReplyPacket(slave->server_id(), - net.read_pos, len); + repl_semisync_master.report_reply_packet(slave->server_id(), + net.read_pos, len); else if (net.last_errno == ER_NET_READ_ERROR) FD_CLR(slave->sock_fd(), &read_fds); } diff --git a/sql/semisync_master_ack_receiver.h b/sql/semisync_master_ack_receiver.h index 25307131bad..9e876150f58 100644 --- a/sql/semisync_master_ack_receiver.h +++ b/sql/semisync_master_ack_receiver.h @@ -30,7 +30,7 @@ add_slave: maintain a new semisync slave's information remove_slave: remove a semisync slave's information */ -class Ack_receiver : public ReplSemiSyncBase +class Ack_receiver : public Repl_semi_sync_base { public: Ack_receiver(); @@ -76,7 +76,7 @@ public: */ void run(); - void setTraceLevel(unsigned long trace_level) + void set_trace_level(unsigned long trace_level) { trace_level_= trace_level; } diff --git a/sql/semisync_slave.cc b/sql/semisync_slave.cc index 7cfc1879dd5..fbec0c13b96 100644 --- a/sql/semisync_slave.cc +++ b/sql/semisync_slave.cc @@ -18,7 +18,7 @@ #include #include "semisync_slave.h" -ReplSemiSyncSlave repl_semisync_slave; +Repl_semi_sync_slave repl_semisync_slave; my_bool rpl_semi_sync_slave_enabled= 0; @@ -37,26 +37,26 @@ bool semi_sync_need_reply= false; unsigned int rpl_semi_sync_slave_kill_conn_timeout; unsigned long long rpl_semi_sync_slave_send_ack = 0; -int ReplSemiSyncSlave::initObject() +int Repl_semi_sync_slave::init_object() { int result= 0; init_done_ = true; /* References to the parameter works after set_options(). */ - setSlaveEnabled(rpl_semi_sync_slave_enabled); - setTraceLevel(rpl_semi_sync_slave_trace_level); - setDelayMaster(rpl_semi_sync_slave_delay_master); - setKillConnTimeout(rpl_semi_sync_slave_kill_conn_timeout); + set_slave_enabled(rpl_semi_sync_slave_enabled); + set_trace_level(rpl_semi_sync_slave_trace_level); + set_delay_master(rpl_semi_sync_slave_delay_master); + set_kill_conn_timeout(rpl_semi_sync_slave_kill_conn_timeout); return result; } -int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, - unsigned long total_len, - int *semi_flags, - const char **payload, - unsigned long *payload_len) +int Repl_semi_sync_slave::slave_read_sync_header(const char *header, + unsigned long total_len, + int *semi_flags, + const char **payload, + unsigned long *payload_len) { int read_res = 0; DBUG_ENTER("Repl_semi_sync_slave::slave_read_sync_header"); @@ -64,9 +64,9 @@ int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, if (rpl_semi_sync_slave_status) { if (DBUG_EVALUATE_IF("semislave_corrupt_log", 0, 1) - && (unsigned char)(header[0]) == kPacketMagicNum) + && (unsigned char)(header[0]) == k_packet_magic_num) { - semi_sync_need_reply = (header[1] & kPacketFlagSync); + semi_sync_need_reply = (header[1] & k_packet_flag_sync); *payload_len = total_len - 2; *payload = header + 2; @@ -76,7 +76,7 @@ int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, if (semi_sync_need_reply) *semi_flags |= SEMI_SYNC_NEED_ACK; - if (isDelayMaster()) + if (is_delay_master()) *semi_flags |= SEMI_SYNC_SLAVE_DELAY_SYNC; } else @@ -93,9 +93,9 @@ int ReplSemiSyncSlave::slaveReadSyncHeader(const char *header, DBUG_RETURN(read_res); } -int ReplSemiSyncSlave::slaveStart(Master_info *mi) +int Repl_semi_sync_slave::slave_start(Master_info *mi) { - bool semi_sync= getSlaveEnabled(); + bool semi_sync= get_slave_enabled(); sql_print_information("Slave I/O thread: Start %s replication to\ master '%s@%s:%d' in log '%s' at position %lu", @@ -112,21 +112,21 @@ int ReplSemiSyncSlave::slaveStart(Master_info *mi) return 0; } -int ReplSemiSyncSlave::slaveStop(Master_info *mi) +int Repl_semi_sync_slave::slave_stop(Master_info *mi) { if (rpl_semi_sync_slave_status) rpl_semi_sync_slave_status= 0; - if (getSlaveEnabled()) - killConnection(mi->mysql); + if (get_slave_enabled()) + kill_connection(mi->mysql); return 0; } -int ReplSemiSyncSlave::resetSlave(Master_info *mi) +int Repl_semi_sync_slave::reset_slave(Master_info *mi) { return 0; } -void ReplSemiSyncSlave::killConnection(MYSQL *mysql) +void Repl_semi_sync_slave::kill_connection(MYSQL *mysql) { if (!mysql) return; @@ -154,14 +154,14 @@ void ReplSemiSyncSlave::killConnection(MYSQL *mysql) mysql_close(kill_mysql); } -int ReplSemiSyncSlave::requestTransmit(Master_info *mi) +int Repl_semi_sync_slave::request_transmit(Master_info *mi) { MYSQL *mysql= mi->mysql; MYSQL_RES *res= 0; MYSQL_ROW row; const char *query; - if (!getSlaveEnabled()) + if (!get_slave_enabled()) return 0; query= "SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled'"; @@ -201,7 +201,7 @@ int ReplSemiSyncSlave::requestTransmit(Master_info *mi) return 0; } -int ReplSemiSyncSlave::slaveReply(Master_info *mi) +int Repl_semi_sync_slave::slave_reply(Master_info *mi) { MYSQL* mysql= mi->mysql; const char *binlog_filename= const_cast(mi->master_log_name); @@ -219,7 +219,7 @@ int ReplSemiSyncSlave::slaveReply(Master_info *mi) if (rpl_semi_sync_slave_status && semi_sync_need_reply) { /* Prepare the buffer of the reply. */ - reply_buffer[REPLY_MAGIC_NUM_OFFSET] = kPacketMagicNum; + reply_buffer[REPLY_MAGIC_NUM_OFFSET] = k_packet_magic_num; int8store(reply_buffer + REPLY_BINLOG_POS_OFFSET, binlog_filepos); memcpy(reply_buffer + REPLY_BINLOG_NAME_OFFSET, binlog_filename, diff --git a/sql/semisync_slave.h b/sql/semisync_slave.h index d67e1a05070..e781077a3a9 100644 --- a/sql/semisync_slave.h +++ b/sql/semisync_slave.h @@ -29,39 +29,38 @@ class Master_info; /** The extension class for the slave of semi-synchronous replication */ -class ReplSemiSyncSlave - :public ReplSemiSyncBase { +class Repl_semi_sync_slave + :public Repl_semi_sync_base { public: - ReplSemiSyncSlave() - :slave_enabled_(false) - {} - ~ReplSemiSyncSlave() {} + Repl_semi_sync_slave() :slave_enabled_(false) {} + ~Repl_semi_sync_slave() {} - void setTraceLevel(unsigned long trace_level) { + void set_trace_level(unsigned long trace_level) { trace_level_ = trace_level; } /* Initialize this class after MySQL parameters are initialized. this * function should be called once at bootstrap time. */ - int initObject(); + int init_object(); - bool getSlaveEnabled() { + bool get_slave_enabled() { return slave_enabled_; } - void setSlaveEnabled(bool enabled) { + + void set_slave_enabled(bool enabled) { slave_enabled_ = enabled; } - - bool isDelayMaster(){ + + bool is_delay_master(){ return delay_master_; } - void setDelayMaster(bool enabled) { + void set_delay_master(bool enabled) { delay_master_ = enabled; } - void setKillConnTimeout(unsigned int timeout) { + void set_kill_conn_timeout(unsigned int timeout) { kill_conn_timeout_ = timeout; } @@ -71,29 +70,31 @@ public: * Input: * header - (IN) packet header pointer * total_len - (IN) total packet length: metadata + payload - * semi_flags - (IN) store flags: SEMI_SYNC_SLAVE_DELAY_SYNC and SEMI_SYNC_NEED_ACK + * semi_flags - (IN) store flags: SEMI_SYNC_SLAVE_DELAY_SYNC and + SEMI_SYNC_NEED_ACK * payload - (IN) payload: the replication event * payload_len - (IN) payload length * * Return: * 0: success; non-zero: error */ - int slaveReadSyncHeader(const char *header, unsigned long total_len, int *semi_flags, - const char **payload, unsigned long *payload_len); + int slave_read_sync_header(const char *header, unsigned long total_len, + int *semi_flags, + const char **payload, unsigned long *payload_len); /* A slave replies to the master indicating its replication process. It * indicates that the slave has received all events before the specified * binlog position. */ - int slaveReply(Master_info* mi); - int slaveStart(Master_info *mi); - int slaveStop(Master_info *mi); - int requestTransmit(Master_info*); - void killConnection(MYSQL *mysql); - int resetSlave(Master_info *mi); + int slave_reply(Master_info* mi); + int slave_start(Master_info *mi); + int slave_stop(Master_info *mi); + int request_transmit(Master_info*); + void kill_connection(MYSQL *mysql); + int reset_slave(Master_info *mi); private: - /* True when initObject has been called */ + /* True when init_object has been called */ bool init_done_; bool slave_enabled_; /* semi-sycn is enabled on the slave */ bool delay_master_; @@ -105,7 +106,7 @@ private: extern my_bool rpl_semi_sync_slave_enabled; extern my_bool rpl_semi_sync_slave_status; extern ulong rpl_semi_sync_slave_trace_level; -extern ReplSemiSyncSlave repl_semisync_slave; +extern Repl_semi_sync_slave repl_semisync_slave; extern char rpl_semi_sync_slave_delay_master; extern unsigned int rpl_semi_sync_slave_kill_conn_timeout; diff --git a/sql/slave.cc b/sql/slave.cc index acca99481e8..f36af66f780 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3586,7 +3586,7 @@ static int request_dump(THD *thd, MYSQL* mysql, Master_info* mi, if (opt_log_slave_updates && opt_replicate_annotate_row_events) binlog_flags|= BINLOG_SEND_ANNOTATE_ROWS_EVENT; - if (repl_semisync_slave.requestTransmit(mi)) + if (repl_semisync_slave.request_transmit(mi)) DBUG_RETURN(1); // TODO if big log files: Change next to int8store() @@ -4614,7 +4614,7 @@ pthread_handler_t handle_slave_io(void *arg) if (DBUG_EVALUATE_IF("failed_slave_start", 1, 0) - || repl_semisync_slave.slaveStart(mi)) + || repl_semisync_slave.slave_start(mi)) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, ER_THD(thd, ER_SLAVE_FATAL_ERROR), @@ -4806,8 +4806,8 @@ Stopping slave I/O thread due to out-of-memory error from master"); event_buf= (const char*)mysql->net.read_pos + 1; mi->semi_ack= 0; if (repl_semisync_slave. - slaveReadSyncHeader((const char*)mysql->net.read_pos + 1, event_len, - &(mi->semi_ack), &event_buf, &event_len)) + slave_read_sync_header((const char*)mysql->net.read_pos + 1, event_len, + &(mi->semi_ack), &event_buf, &event_len)) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, ER_THD(thd, ER_SLAVE_FATAL_ERROR), @@ -4865,7 +4865,7 @@ Stopping slave I/O thread due to out-of-memory error from master"); } if (rpl_semi_sync_slave_status && (mi->semi_ack & SEMI_SYNC_NEED_ACK) && - repl_semisync_slave.slaveReply(mi)) + repl_semisync_slave.slave_reply(mi)) { mi->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, ER_THD(thd, ER_SLAVE_FATAL_ERROR), @@ -4879,7 +4879,7 @@ Stopping slave I/O thread due to out-of-memory error from master"); master info only when ack is needed. This may lead to at least one group transaction delay but affords better performance improvement. */ - (!repl_semisync_slave.getSlaveEnabled() || + (!repl_semisync_slave.get_slave_enabled() || (!(mi->semi_ack & SEMI_SYNC_SLAVE_DELAY_SYNC) || (mi->semi_ack & (SEMI_SYNC_NEED_ACK)))) && (DBUG_EVALUATE_IF("failed_flush_master_info", 1, 0) || @@ -4937,7 +4937,7 @@ err: IO_RPL_LOG_NAME, mi->master_log_pos, tmp.c_ptr_safe()); } - repl_semisync_slave.slaveStop(mi); + repl_semisync_slave.slave_stop(mi); thd->reset_query(); thd->reset_db(NULL, 0); if (mysql) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index c9e804135a5..1d6aa0aaab1 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -316,7 +316,7 @@ static int reset_transmit_packet(binlog_send_info *info, ushort flags, if (info->thd->semi_sync_slave) { - if (repl_semisync_master.reserveSyncHeader(packet)) + if (repl_semisync_master.reserve_sync_header(packet)) { info->error= ER_UNKNOWN_ERROR; *errmsg= "Failed to run hook 'reserve_header'"; @@ -1945,9 +1945,10 @@ send_event_to_slave(binlog_send_info *info, Log_event_type event_type, THD_STAGE_INFO(info->thd, stage_sending_binlog_event_to_slave); pos= my_b_tell(log); - if (repl_semisync_master.updateSyncHeader(info->thd, (uchar *)packet->c_ptr(), - info->log_file_name + info->dirlen, - pos, &need_sync)) + if (repl_semisync_master.update_sync_header(info->thd, + (uchar*) packet->c_ptr(), + info->log_file_name + info->dirlen, + pos, &need_sync)) { info->error= ER_UNKNOWN_ERROR; return "run 'before_send_event' hook failed"; @@ -1969,7 +1970,7 @@ send_event_to_slave(binlog_send_info *info, Log_event_type event_type, } } - if (need_sync && repl_semisync_master.flushNet(info->thd, packet->c_ptr())) + if (need_sync && repl_semisync_master.flush_net(info->thd, packet->c_ptr())) { info->error= ER_UNKNOWN_ERROR; return "Failed to run hook 'after_send_event'"; @@ -3341,7 +3342,7 @@ int reset_slave(THD *thd, Master_info* mi) sql_print_information("Deleted Master_info file '%s'.", fname); if (rpl_semi_sync_slave_enabled) - repl_semisync_slave.resetSlave(mi); + repl_semisync_slave.reset_slave(mi); err: mi->unlock_slave_threads(); if (error) @@ -3845,10 +3846,10 @@ int reset_master(THD* thd, rpl_gtid *init_state, uint32 init_state_len, bool ret= 0; /* Temporarily disable master semisync before reseting master. */ - repl_semisync_master.beforeResetMaster(); + repl_semisync_master.before_reset_master(); ret= mysql_bin_log.reset_logs(thd, 1, init_state, init_state_len, next_log_number); - repl_semisync_master.afterResetMaster(); + repl_semisync_master.after_reset_master(); return ret; } diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 9eac6017c7c..79cef6a5681 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -3049,17 +3049,17 @@ static bool fix_rpl_semi_sync_master_enabled(sys_var *self, THD *thd, { if (rpl_semi_sync_master_enabled) { - if (repl_semisync_master.enableMaster() != 0) + if (repl_semisync_master.enable_master() != 0) rpl_semi_sync_master_enabled= false; else if (ack_receiver.start()) { - repl_semisync_master.disableMaster(); + repl_semisync_master.disable_master(); rpl_semi_sync_master_enabled= false; } } else { - if (repl_semisync_master.disableMaster() != 0) + if (repl_semisync_master.disable_master() != 0) rpl_semi_sync_master_enabled= true; if (!rpl_semi_sync_master_enabled) ack_receiver.stop(); @@ -3070,29 +3070,29 @@ static bool fix_rpl_semi_sync_master_enabled(sys_var *self, THD *thd, static bool fix_rpl_semi_sync_master_timeout(sys_var *self, THD *thd, enum_var_type type) { - repl_semisync_master.setWaitTimeout(rpl_semi_sync_master_timeout); + repl_semisync_master.set_wait_timeout(rpl_semi_sync_master_timeout); return false; } static bool fix_rpl_semi_sync_master_trace_level(sys_var *self, THD *thd, enum_var_type type) { - repl_semisync_master.setTraceLevel(rpl_semi_sync_master_trace_level); - ack_receiver.setTraceLevel(rpl_semi_sync_master_trace_level); + repl_semisync_master.set_trace_level(rpl_semi_sync_master_trace_level); + ack_receiver.set_trace_level(rpl_semi_sync_master_trace_level); return false; } static bool fix_rpl_semi_sync_master_wait_point(sys_var *self, THD *thd, enum_var_type type) { - repl_semisync_master.setWaitPoint(rpl_semi_sync_master_wait_point); + repl_semisync_master.set_wait_point(rpl_semi_sync_master_wait_point); return false; } static bool fix_rpl_semi_sync_master_wait_no_slave(sys_var *self, THD *thd, enum_var_type type) { - repl_semisync_master.checkAndSwitch(); + repl_semisync_master.check_and_switch(); return false; } @@ -3147,28 +3147,29 @@ static Sys_var_enum Sys_semisync_master_wait_point( static bool fix_rpl_semi_sync_slave_enabled(sys_var *self, THD *thd, enum_var_type type) { - repl_semisync_slave.setSlaveEnabled(rpl_semi_sync_slave_enabled != 0); + repl_semisync_slave.set_slave_enabled(rpl_semi_sync_slave_enabled != 0); return false; } static bool fix_rpl_semi_sync_slave_trace_level(sys_var *self, THD *thd, enum_var_type type) { - repl_semisync_slave.setTraceLevel(rpl_semi_sync_slave_trace_level); + repl_semisync_slave.set_trace_level(rpl_semi_sync_slave_trace_level); return false; } static bool fix_rpl_semi_sync_slave_delay_master(sys_var *self, THD *thd, enum_var_type type) { - repl_semisync_slave.setDelayMaster(rpl_semi_sync_slave_delay_master); + repl_semisync_slave.set_delay_master(rpl_semi_sync_slave_delay_master); return false; } static bool fix_rpl_semi_sync_slave_kill_conn_timeout(sys_var *self, THD *thd, enum_var_type type) { - repl_semisync_slave.setKillConnTimeout(rpl_semi_sync_slave_kill_conn_timeout); + repl_semisync_slave. + set_kill_conn_timeout(rpl_semi_sync_slave_kill_conn_timeout); return false; } diff --git a/sql/transaction.cc b/sql/transaction.cc index 349f4549d31..ec277e9c9c4 100644 --- a/sql/transaction.cc +++ b/sql/transaction.cc @@ -319,13 +319,13 @@ bool trans_commit(THD *thd) if (res) { #ifdef HAVE_REPLICATION - repl_semisync_master.waitAfterRollback(thd, FALSE); + repl_semisync_master.wait_after_rollback(thd, FALSE); #endif } else { #ifdef HAVE_REPLICATION - repl_semisync_master.waitAfterCommit(thd, FALSE); + repl_semisync_master.wait_after_commit(thd, FALSE); #endif } thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_KEEP_LOG); @@ -421,7 +421,7 @@ bool trans_rollback(THD *thd) DBUG_PRINT("info", ("clearing SERVER_STATUS_IN_TRANS")); res= ha_rollback_trans(thd, TRUE); #ifdef HAVE_REPLICATION - repl_semisync_master.waitAfterRollback(thd, FALSE); + repl_semisync_master.wait_after_rollback(thd, FALSE); #endif thd->variables.option_bits&= ~(OPTION_BEGIN | OPTION_KEEP_LOG); /* Reset the binlog transaction marker */ @@ -537,13 +537,13 @@ bool trans_commit_stmt(THD *thd) if (res) { #ifdef HAVE_REPLICATION - repl_semisync_master.waitAfterRollback(thd, FALSE); + repl_semisync_master.wait_after_rollback(thd, FALSE); #endif } else { #ifdef HAVE_REPLICATION - repl_semisync_master.waitAfterCommit(thd, FALSE); + repl_semisync_master.wait_after_commit(thd, FALSE); #endif } @@ -585,7 +585,7 @@ bool trans_rollback_stmt(THD *thd) } #ifdef HAVE_REPLICATION - repl_semisync_master.waitAfterRollback(thd, FALSE); + repl_semisync_master.wait_after_rollback(thd, FALSE); #endif thd->transaction.stmt.reset(); From 529120e1cb5ca2dc2192b1e68b76538e0d1db145 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Mon, 27 Nov 2017 21:06:17 +0200 Subject: [PATCH 26/51] MDEV-13073. This patch is a followup of the previous one to convert the trailing underscore identifier to mariadb standard. For identifier representing class private members the underscore is replaced with a `m_` prefix. Otherwise `_` is just removed. --- sql/semisync.h | 6 +- sql/semisync_master.cc | 282 ++++++++++++++--------------- sql/semisync_master.h | 88 ++++----- sql/semisync_master_ack_receiver.h | 2 +- sql/semisync_slave.cc | 8 +- sql/semisync_slave.h | 22 +-- 6 files changed, 204 insertions(+), 204 deletions(-) diff --git a/sql/semisync.h b/sql/semisync.h index bf43ba5c8d8..9deb6c5fd01 100644 --- a/sql/semisync.h +++ b/sql/semisync.h @@ -33,13 +33,13 @@ public: static const unsigned long k_trace_detail; static const unsigned long k_trace_net_wait; - unsigned long trace_level_; /* the level for tracing */ + unsigned long m_trace_level; /* the level for tracing */ Trace() - :trace_level_(0L) + :m_trace_level(0L) {} Trace(unsigned long trace_level) - :trace_level_(trace_level) + :m_trace_level(trace_level) {} }; diff --git a/sql/semisync_master.cc b/sql/semisync_master.cc index 5b594d20c71..99d8b75ece8 100644 --- a/sql/semisync_master.cc +++ b/sql/semisync_master.cc @@ -74,29 +74,29 @@ static ulonglong timespec_to_usec(const struct timespec *ts) Active_tranx::Active_tranx(mysql_mutex_t *lock, ulong trace_level) - : Trace(trace_level), allocator_(max_connections), - num_entries_(max_connections << 1), /* Transaction hash table size + : Trace(trace_level), m_allocator(max_connections), + m_num_entries(max_connections << 1), /* Transaction hash table size * is set to double the size * of max_connections */ - lock_(lock) + m_lock(lock) { /* No transactions are in the list initially. */ - trx_front_ = NULL; - trx_rear_ = NULL; + m_trx_front = NULL; + m_trx_rear = NULL; /* Create the hash table to find a transaction's ending event. */ - trx_htb_ = new Tranx_node *[num_entries_]; - for (int idx = 0; idx < num_entries_; ++idx) - trx_htb_[idx] = NULL; + m_trx_htb = new Tranx_node *[m_num_entries]; + for (int idx = 0; idx < m_num_entries; ++idx) + m_trx_htb[idx] = NULL; sql_print_information("Semi-sync replication initialized for transactions."); } Active_tranx::~Active_tranx() { - delete [] trx_htb_; - trx_htb_ = NULL; - num_entries_ = 0; + delete [] m_trx_htb; + m_trx_htb = NULL; + m_num_entries = 0; } unsigned int Active_tranx::calc_hash(const unsigned char *key, @@ -121,7 +121,7 @@ unsigned int Active_tranx::get_hash_value(const char *log_file_name, unsigned int hash2 = calc_hash((const unsigned char *)(&log_file_pos), sizeof(log_file_pos)); - return (hash1 + hash2) % num_entries_; + return (hash1 + hash2) % m_num_entries; } int Active_tranx::compare(const char *log_file_name1, my_off_t log_file_pos1, @@ -148,7 +148,7 @@ int Active_tranx::insert_tranx_node(const char *log_file_name, DBUG_ENTER("Active_tranx:insert_tranx_node"); - ins_node = allocator_.allocate_node(); + ins_node = m_allocator.allocate_node(); if (!ins_node) { sql_print_error("%s: transaction node allocation failed for: (%s, %lu)", @@ -159,25 +159,25 @@ int Active_tranx::insert_tranx_node(const char *log_file_name, } /* insert the binlog position in the active transaction list. */ - strncpy(ins_node->log_name_, log_file_name, FN_REFLEN-1); - ins_node->log_name_[FN_REFLEN-1] = 0; /* make sure it ends properly */ - ins_node->log_pos_ = log_file_pos; + strncpy(ins_node->log_name, log_file_name, FN_REFLEN-1); + ins_node->log_name[FN_REFLEN-1] = 0; /* make sure it ends properly */ + ins_node->log_pos = log_file_pos; - if (!trx_front_) + if (!m_trx_front) { /* The list is empty. */ - trx_front_ = trx_rear_ = ins_node; + m_trx_front = m_trx_rear = ins_node; } else { - int cmp = compare(ins_node, trx_rear_); + int cmp = compare(ins_node, m_trx_rear); if (cmp > 0) { /* Compare with the tail first. If the transaction happens later in * binlog, then make it the new tail. */ - trx_rear_->next_ = ins_node; - trx_rear_ = ins_node; + m_trx_rear->next = ins_node; + m_trx_rear = ins_node; } else { @@ -186,20 +186,20 @@ int Active_tranx::insert_tranx_node(const char *log_file_name, */ sql_print_error("%s: binlog write out-of-order, tail (%s, %lu), " "new node (%s, %lu)", "Active_tranx:insert_tranx_node", - trx_rear_->log_name_, (ulong)trx_rear_->log_pos_, - ins_node->log_name_, (ulong)ins_node->log_pos_); + m_trx_rear->log_name, (ulong)m_trx_rear->log_pos, + ins_node->log_name, (ulong)ins_node->log_pos); result = -1; goto l_end; } } - hash_val = get_hash_value(ins_node->log_name_, ins_node->log_pos_); - ins_node->hash_next_ = trx_htb_[hash_val]; - trx_htb_[hash_val] = ins_node; + hash_val = get_hash_value(ins_node->log_name, ins_node->log_pos); + ins_node->hash_next = m_trx_htb[hash_val]; + m_trx_htb[hash_val] = ins_node; DBUG_PRINT("semisync", ("%s: insert (%s, %lu) in entry(%u)", "Active_tranx:insert_tranx_node", - ins_node->log_name_, (ulong)ins_node->log_pos_, + ins_node->log_name, (ulong)ins_node->log_pos, hash_val)); l_end: @@ -212,14 +212,14 @@ bool Active_tranx::is_tranx_end_pos(const char *log_file_name, DBUG_ENTER("Active_tranx::is_tranx_end_pos"); unsigned int hash_val = get_hash_value(log_file_name, log_file_pos); - Tranx_node *entry = trx_htb_[hash_val]; + Tranx_node *entry = m_trx_htb[hash_val]; while (entry != NULL) { if (compare(entry, log_file_name, log_file_pos) == 0) break; - entry = entry->hash_next_; + entry = entry->hash_next; } DBUG_PRINT("semisync", ("%s: probe (%s, %lu) in entry(%u)", @@ -238,13 +238,13 @@ int Active_tranx::clear_active_tranx_nodes(const char *log_file_name, if (log_file_name != NULL) { - new_front = trx_front_; + new_front = m_trx_front; while (new_front) { if (compare(new_front, log_file_name, log_file_pos) > 0) break; - new_front = new_front->next_; + new_front = new_front->next; } } else @@ -258,54 +258,54 @@ int Active_tranx::clear_active_tranx_nodes(const char *log_file_name, /* No active transaction nodes after the call. */ /* Clear the hash table. */ - memset(trx_htb_, 0, num_entries_ * sizeof(Tranx_node *)); - allocator_.free_all_nodes(); + memset(m_trx_htb, 0, m_num_entries * sizeof(Tranx_node *)); + m_allocator.free_all_nodes(); /* Clear the active transaction list. */ - if (trx_front_ != NULL) + if (m_trx_front != NULL) { - trx_front_ = NULL; - trx_rear_ = NULL; + m_trx_front = NULL; + m_trx_rear = NULL; } DBUG_PRINT("semisync", ("%s: cleared all nodes", "Active_tranx::::clear_active_tranx_nodes")); } - else if (new_front != trx_front_) + else if (new_front != m_trx_front) { Tranx_node *curr_node, *next_node; /* Delete all transaction nodes before the confirmation point. */ int n_frees = 0; - curr_node = trx_front_; + curr_node = m_trx_front; while (curr_node != new_front) { - next_node = curr_node->next_; + next_node = curr_node->next; n_frees++; /* Remove the node from the hash table. */ - unsigned int hash_val = get_hash_value(curr_node->log_name_, curr_node->log_pos_); - Tranx_node **hash_ptr = &(trx_htb_[hash_val]); + unsigned int hash_val = get_hash_value(curr_node->log_name, curr_node->log_pos); + Tranx_node **hash_ptr = &(m_trx_htb[hash_val]); while ((*hash_ptr) != NULL) { if ((*hash_ptr) == curr_node) { - (*hash_ptr) = curr_node->hash_next_; + (*hash_ptr) = curr_node->hash_next; break; } - hash_ptr = &((*hash_ptr)->hash_next_); + hash_ptr = &((*hash_ptr)->hash_next); } curr_node = next_node; } - trx_front_ = new_front; - allocator_.free_nodes_before(trx_front_); + m_trx_front = new_front; + m_allocator.free_nodes_before(m_trx_front); DBUG_PRINT("semisync", ("%s: cleared %d nodes back until pos (%s, %lu)", "Active_tranx::::clear_active_tranx_nodes", n_frees, - trx_front_->log_name_, (ulong)trx_front_->log_pos_)); + m_trx_front->log_name, (ulong)m_trx_front->log_pos)); } DBUG_RETURN(0); @@ -336,26 +336,26 @@ int Active_tranx::clear_active_tranx_nodes(const char *log_file_name, ******************************************************************************/ Repl_semi_sync_master::Repl_semi_sync_master() - : active_tranxs_(NULL), - init_done_(false), - reply_file_name_inited_(false), - reply_file_pos_(0L), - wait_file_name_inited_(false), - wait_file_pos_(0), - master_enabled_(false), - wait_timeout_(0L), - state_(0), - wait_point_(0) + : m_active_tranxs(NULL), + m_init_done(false), + m_reply_file_name_inited(false), + m_reply_file_pos(0L), + m_wait_file_name_inited(false), + m_wait_file_pos(0), + m_master_enabled(false), + m_wait_timeout(0L), + m_state(0), + m_wait_point(0) { - strcpy(reply_file_name_, ""); - strcpy(wait_file_name_, ""); + strcpy(m_reply_file_name, ""); + strcpy(m_wait_file_name, ""); } int Repl_semi_sync_master::init_object() { int result; - init_done_ = true; + m_init_done = true; /* References to the parameter works after set_options(). */ set_wait_timeout(rpl_semi_sync_master_timeout); @@ -398,15 +398,15 @@ int Repl_semi_sync_master::enable_master() if (!get_master_enabled()) { - active_tranxs_ = new Active_tranx(&LOCK_binlog, trace_level_); - if (active_tranxs_ != NULL) + m_active_tranxs = new Active_tranx(&LOCK_binlog, m_trace_level); + if (m_active_tranxs != NULL) { - commit_file_name_inited_ = false; - reply_file_name_inited_ = false; - wait_file_name_inited_ = false; + m_commit_file_name_inited = false; + m_reply_file_name_inited = false; + m_wait_file_name_inited = false; set_master_enabled(true); - state_ = true; + m_state = true; sql_print_information("Semi-sync replication enabled on the master."); } else @@ -433,13 +433,13 @@ int Repl_semi_sync_master::disable_master() */ switch_off(); - assert(active_tranxs_ != NULL); - delete active_tranxs_; - active_tranxs_ = NULL; + assert(m_active_tranxs != NULL); + delete m_active_tranxs; + m_active_tranxs = NULL; - reply_file_name_inited_ = false; - wait_file_name_inited_ = false; - commit_file_name_inited_ = false; + m_reply_file_name_inited = false; + m_wait_file_name_inited = false; + m_commit_file_name_inited = false; set_master_enabled(false); sql_print_information("Semi-sync replication disabled on the master."); @@ -452,14 +452,14 @@ int Repl_semi_sync_master::disable_master() void Repl_semi_sync_master::cleanup() { - if (init_done_) + if (m_init_done) { mysql_mutex_destroy(&LOCK_binlog); mysql_cond_destroy(&COND_binlog_send); - init_done_= 0; + m_init_done= 0; } - delete active_tranxs_; + delete m_active_tranxs; } void Repl_semi_sync_master::lock() @@ -592,10 +592,10 @@ int Repl_semi_sync_master::report_reply_binlog(uint32 server_id, * sync replication slaves. So, if any one of them get the transaction, * the transaction session in the primary can move forward. */ - if (reply_file_name_inited_) + if (m_reply_file_name_inited) { cmp = Active_tranx::compare(log_file_name, log_file_pos, - reply_file_name_, reply_file_pos_); + m_reply_file_name, m_reply_file_pos); /* If the requested position is behind the sending binlog position, * would not adjust sending binlog position. @@ -614,13 +614,13 @@ int Repl_semi_sync_master::report_reply_binlog(uint32 server_id, if (need_copy_send_pos) { - strmake_buf(reply_file_name_, log_file_name); - reply_file_pos_ = log_file_pos; - reply_file_name_inited_ = true; + strmake_buf(m_reply_file_name, log_file_name); + m_reply_file_pos = log_file_pos; + m_reply_file_name_inited = true; /* Remove all active transaction nodes before this point. */ - assert(active_tranxs_ != NULL); - active_tranxs_->clear_active_tranx_nodes(log_file_name, log_file_pos); + assert(m_active_tranxs != NULL); + m_active_tranxs->clear_active_tranx_nodes(log_file_name, log_file_pos); DBUG_PRINT("semisync", ("%s: Got reply at (%s, %lu)", "Repl_semi_sync_master::report_reply_binlog", @@ -632,15 +632,15 @@ int Repl_semi_sync_master::report_reply_binlog(uint32 server_id, /* Let us check if some of the waiting threads doing a trx * commit can now proceed. */ - cmp = Active_tranx::compare(reply_file_name_, reply_file_pos_, - wait_file_name_, wait_file_pos_); + cmp = Active_tranx::compare(m_reply_file_name, m_reply_file_pos, + m_wait_file_name, m_wait_file_pos); if (cmp >= 0) { /* Yes, at least one waiting thread can now proceed: * let us release all waiting threads with a broadcast */ can_release_threads = true; - wait_file_name_inited_ = false; + m_wait_file_name_inited = false; } } @@ -811,9 +811,9 @@ int Repl_semi_sync_master::commit_trx(const char* trx_wait_binlog_name, while (is_on() && !thd_killed(current_thd)) { - if (reply_file_name_inited_) + if (m_reply_file_name_inited) { - int cmp = Active_tranx::compare(reply_file_name_, reply_file_pos_, + int cmp = Active_tranx::compare(m_reply_file_name, m_reply_file_pos, trx_wait_binlog_name, trx_wait_binlog_pos); if (cmp >= 0) @@ -823,8 +823,8 @@ int Repl_semi_sync_master::commit_trx(const char* trx_wait_binlog_name, */ DBUG_PRINT("semisync", ("%s: Binlog reply is ahead (%s, %lu),", "Repl_semi_sync_master::commit_trx", - reply_file_name_, - (ulong)reply_file_pos_)); + m_reply_file_name, + (ulong)m_reply_file_pos)); break; } } @@ -832,37 +832,37 @@ int Repl_semi_sync_master::commit_trx(const char* trx_wait_binlog_name, /* Let us update the info about the minimum binlog position of waiting * threads. */ - if (wait_file_name_inited_) + if (m_wait_file_name_inited) { int cmp = Active_tranx::compare(trx_wait_binlog_name, trx_wait_binlog_pos, - wait_file_name_, wait_file_pos_); + m_wait_file_name, m_wait_file_pos); if (cmp <= 0) { /* This thd has a lower position, let's update the minimum info. */ - strmake_buf(wait_file_name_, trx_wait_binlog_name); - wait_file_pos_ = trx_wait_binlog_pos; + strmake_buf(m_wait_file_name, trx_wait_binlog_name); + m_wait_file_pos = trx_wait_binlog_pos; rpl_semi_sync_master_wait_pos_backtraverse++; DBUG_PRINT("semisync", ("%s: move back wait position (%s, %lu),", "Repl_semi_sync_master::commit_trx", - wait_file_name_, (ulong)wait_file_pos_)); + m_wait_file_name, (ulong)m_wait_file_pos)); } } else { - strmake_buf(wait_file_name_, trx_wait_binlog_name); - wait_file_pos_ = trx_wait_binlog_pos; - wait_file_name_inited_ = true; + strmake_buf(m_wait_file_name, trx_wait_binlog_name); + m_wait_file_pos = trx_wait_binlog_pos; + m_wait_file_name_inited = true; DBUG_PRINT("semisync", ("%s: init wait position (%s, %lu),", "Repl_semi_sync_master::commit_trx", - wait_file_name_, (ulong)wait_file_pos_)); + m_wait_file_name, (ulong)m_wait_file_pos)); } /* Calcuate the waiting period. */ - long diff_secs = (long) (wait_timeout_ / TIME_THOUSAND); - long diff_nsecs = (long) ((wait_timeout_ % TIME_THOUSAND) * TIME_MILLION); + long diff_secs = (long) (m_wait_timeout / TIME_THOUSAND); + long diff_nsecs = (long) ((m_wait_timeout % TIME_THOUSAND) * TIME_MILLION); long nsecs = start_ts.tv_nsec + diff_nsecs; abstime.tv_sec = start_ts.tv_sec + diff_secs + nsecs/TIME_BILLION; abstime.tv_nsec = nsecs % TIME_BILLION; @@ -879,8 +879,8 @@ int Repl_semi_sync_master::commit_trx(const char* trx_wait_binlog_name, DBUG_PRINT("semisync", ("%s: wait %lu ms for binlog sent (%s, %lu)", "Repl_semi_sync_master::commit_trx", - wait_timeout_, - wait_file_name_, (ulong)wait_file_pos_)); + m_wait_timeout, + m_wait_file_name, (ulong)m_wait_file_pos)); wait_result = cond_timewait(&abstime); rpl_semi_sync_master_wait_sessions--; @@ -891,7 +891,7 @@ int Repl_semi_sync_master::commit_trx(const char* trx_wait_binlog_name, sql_print_warning("Timeout waiting for reply of binlog (file: %s, pos: %lu), " "semi-sync up to file %s, position %lu.", trx_wait_binlog_name, (ulong)trx_wait_binlog_pos, - reply_file_name_, (ulong)reply_file_pos_); + m_reply_file_name, (ulong)m_reply_file_pos); rpl_semi_sync_master_wait_timeouts++; /* switch semi-sync off */ @@ -921,11 +921,11 @@ int Repl_semi_sync_master::commit_trx(const char* trx_wait_binlog_name, /* At this point, the binlog file and position of this transaction must have been removed from Active_tranx. - active_tranxs_ may be NULL if someone disabled semi sync during + m_active_tranxs may be NULL if someone disabled semi sync during cond_timewait() */ - assert(thd_killed(current_thd) || !active_tranxs_ || - !active_tranxs_->is_tranx_end_pos(trx_wait_binlog_name, + assert(thd_killed(current_thd) || !m_active_tranxs || + !m_active_tranxs->is_tranx_end_pos(trx_wait_binlog_name, trx_wait_binlog_pos)); l_end: @@ -967,15 +967,15 @@ int Repl_semi_sync_master::switch_off() DBUG_ENTER("Repl_semi_sync_master::switch_off"); - state_ = false; + m_state = false; /* Clear the active transaction list. */ - assert(active_tranxs_ != NULL); - result = active_tranxs_->clear_active_tranx_nodes(NULL, 0); + assert(m_active_tranxs != NULL); + result = m_active_tranxs->clear_active_tranx_nodes(NULL, 0); rpl_semi_sync_master_off_times++; - wait_file_name_inited_ = false; - reply_file_name_inited_ = false; + m_wait_file_name_inited = false; + m_reply_file_name_inited = false; sql_print_information("Semi-sync replication switched OFF."); cond_broadcast(); /* wake up all waiting threads */ @@ -993,13 +993,13 @@ int Repl_semi_sync_master::try_switch_on(int server_id, /* If the current sending event's position is larger than or equal to the * 'largest' commit transaction binlog position, the slave is already * catching up now and we can switch semi-sync on here. - * If commit_file_name_inited_ indicates there are no recent transactions, + * If m_commit_file_name_inited indicates there are no recent transactions, * we can enable semi-sync immediately. */ - if (commit_file_name_inited_) + if (m_commit_file_name_inited) { int cmp = Active_tranx::compare(log_file_name, log_file_pos, - commit_file_name_, commit_file_pos_); + m_commit_file_name, m_commit_file_pos); semi_sync_on = (cmp >= 0); } else @@ -1010,7 +1010,7 @@ int Repl_semi_sync_master::try_switch_on(int server_id, if (semi_sync_on) { /* Switch semi-sync replication on. */ - state_ = true; + m_state = true; sql_print_information("Semi-sync replication switched ON with slave (server_id: %d) " "at (%s, %lu)", @@ -1066,10 +1066,10 @@ int Repl_semi_sync_master::update_sync_header(THD* thd, unsigned char *packet, /* semi-sync is ON */ sync = false; /* No sync unless a transaction is involved. */ - if (reply_file_name_inited_) + if (m_reply_file_name_inited) { cmp = Active_tranx::compare(log_file_name, log_file_pos, - reply_file_name_, reply_file_pos_); + m_reply_file_name, m_reply_file_pos); if (cmp <= 0) { /* If we have already got the reply for the event, then we do @@ -1079,10 +1079,10 @@ int Repl_semi_sync_master::update_sync_header(THD* thd, unsigned char *packet, } } - if (wait_file_name_inited_) + if (m_wait_file_name_inited) { cmp = Active_tranx::compare(log_file_name, log_file_pos, - wait_file_name_, wait_file_pos_); + m_wait_file_name, m_wait_file_pos); } else { @@ -1097,17 +1097,17 @@ int Repl_semi_sync_master::update_sync_header(THD* thd, unsigned char *packet, /* * We only wait if the event is a transaction's ending event. */ - assert(active_tranxs_ != NULL); - sync = active_tranxs_->is_tranx_end_pos(log_file_name, + assert(m_active_tranxs != NULL); + sync = m_active_tranxs->is_tranx_end_pos(log_file_name, log_file_pos); } } else { - if (commit_file_name_inited_) + if (m_commit_file_name_inited) { int cmp = Active_tranx::compare(log_file_name, log_file_pos, - commit_file_name_, commit_file_pos_); + m_commit_file_name, m_commit_file_pos); sync = (cmp >= 0); } else @@ -1151,35 +1151,35 @@ int Repl_semi_sync_master::write_tranx_in_binlog(const char* log_file_name, /* Update the 'largest' transaction commit position seen so far even * though semi-sync is switched off. - * It is much better that we update commit_file_* here, instead of + * It is much better that we update m_commit_file* here, instead of * inside commit_trx(). This is mostly because update_sync_header() - * will watch for commit_file_* to decide whether to switch semi-sync + * will watch for m_commit_file* to decide whether to switch semi-sync * on. The detailed reason is explained in function update_sync_header(). */ - if (commit_file_name_inited_) + if (m_commit_file_name_inited) { int cmp = Active_tranx::compare(log_file_name, log_file_pos, - commit_file_name_, commit_file_pos_); + m_commit_file_name, m_commit_file_pos); if (cmp > 0) { /* This is a larger position, let's update the maximum info. */ - strncpy(commit_file_name_, log_file_name, FN_REFLEN-1); - commit_file_name_[FN_REFLEN-1] = 0; /* make sure it ends properly */ - commit_file_pos_ = log_file_pos; + strncpy(m_commit_file_name, log_file_name, FN_REFLEN-1); + m_commit_file_name[FN_REFLEN-1] = 0; /* make sure it ends properly */ + m_commit_file_pos = log_file_pos; } } else { - strncpy(commit_file_name_, log_file_name, FN_REFLEN-1); - commit_file_name_[FN_REFLEN-1] = 0; /* make sure it ends properly */ - commit_file_pos_ = log_file_pos; - commit_file_name_inited_ = true; + strncpy(m_commit_file_name, log_file_name, FN_REFLEN-1); + m_commit_file_name[FN_REFLEN-1] = 0; /* make sure it ends properly */ + m_commit_file_pos = log_file_pos; + m_commit_file_name_inited = true; } if (is_on()) { - assert(active_tranxs_ != NULL); - if(active_tranxs_->insert_tranx_node(log_file_name, log_file_pos)) + assert(m_active_tranxs != NULL); + if(m_active_tranxs->insert_tranx_node(log_file_name, log_file_pos)) { /* if insert tranx_node failed, print a warning message @@ -1254,13 +1254,13 @@ int Repl_semi_sync_master::after_reset_master() if (rpl_semi_sync_master_clients == 0 && !rpl_semi_sync_master_wait_no_slave) - state_ = 0; + m_state = 0; else - state_ = get_master_enabled()? 1 : 0; + m_state = get_master_enabled()? 1 : 0; - wait_file_name_inited_ = false; - reply_file_name_inited_ = false; - commit_file_name_inited_ = false; + m_wait_file_name_inited = false; + m_reply_file_name_inited = false; + m_commit_file_name_inited = false; rpl_semi_sync_master_yes_transactions = 0; rpl_semi_sync_master_no_transactions = 0; @@ -1306,7 +1306,7 @@ void Repl_semi_sync_master::set_export_stats() { lock(); - rpl_semi_sync_master_status = state_; + rpl_semi_sync_master_status = m_state; rpl_semi_sync_master_avg_trx_wait_time= ((rpl_semi_sync_master_trx_wait_num) ? (ulong)((double)rpl_semi_sync_master_trx_wait_time / diff --git a/sql/semisync_master.h b/sql/semisync_master.h index 97c5d01a1d5..a58c1a7ae6e 100644 --- a/sql/semisync_master.h +++ b/sql/semisync_master.h @@ -28,10 +28,10 @@ extern PSI_cond_key key_COND_binlog_send; #endif struct Tranx_node { - char log_name_[FN_REFLEN]; - my_off_t log_pos_; - struct Tranx_node *next_; /* the next node in the sorted list */ - struct Tranx_node *hash_next_; /* the next node during hash collision */ + char log_name[FN_REFLEN]; + my_off_t log_pos; + struct Tranx_node *next; /* the next node in the sorted list */ + struct Tranx_node *hash_next; /* the next node during hash collision */ }; /** @@ -123,10 +123,10 @@ public: } trx_node= &(current_block->nodes[++last_node]); - trx_node->log_name_[0] = '\0'; - trx_node->log_pos_= 0; - trx_node->next_= 0; - trx_node->hash_next_= 0; + trx_node->log_name[0] = '\0'; + trx_node->log_pos= 0; + trx_node->next= 0; + trx_node->hash_next= 0; return trx_node; } @@ -299,14 +299,14 @@ class Active_tranx :public Trace { private: - Tranx_node_allocator allocator_; + Tranx_node_allocator m_allocator; /* These two record the active transaction list in sort order. */ - Tranx_node *trx_front_, *trx_rear_; + Tranx_node *m_trx_front, *m_trx_rear; - Tranx_node **trx_htb_; /* A hash table on active transactions. */ + Tranx_node **m_trx_htb; /* A hash table on active transactions. */ - int num_entries_; /* maximum hash table entries */ - mysql_mutex_t *lock_; /* mutex lock */ + int m_num_entries; /* maximum hash table entries */ + mysql_mutex_t *m_lock; /* mutex lock */ inline void assert_lock_owner(); @@ -316,16 +316,16 @@ private: int compare(const char *log_file_name1, my_off_t log_file_pos1, const Tranx_node *node2) { return compare(log_file_name1, log_file_pos1, - node2->log_name_, node2->log_pos_); + node2->log_name, node2->log_pos); } int compare(const Tranx_node *node1, const char *log_file_name2, my_off_t log_file_pos2) { - return compare(node1->log_name_, node1->log_pos_, + return compare(node1->log_name, node1->log_pos, log_file_name2, log_file_pos2); } int compare(const Tranx_node *node1, const Tranx_node *node2) { - return compare(node1->log_name_, node1->log_pos_, - node2->log_name_, node2->log_pos_); + return compare(node1->log_name, node1->log_pos, + node2->log_name, node2->log_pos); } public: @@ -369,11 +369,11 @@ public: class Repl_semi_sync_master :public Repl_semi_sync_base { private: - Active_tranx *active_tranxs_; /* active transaction list: the list will + Active_tranx *m_active_tranxs; /* active transaction list: the list will be cleared when semi-sync switches off. */ /* True when init_object has been called */ - bool init_done_; + bool m_init_done; /* This cond variable is signaled when enough binlog has been sent to slave, * so that a waiting trx can return the 'ok' to the client for a commit. @@ -383,32 +383,32 @@ class Repl_semi_sync_master /* Mutex that protects the following state variables and the active * transaction list. * Under no cirumstances we can acquire mysql_bin_log.LOCK_log if we are - * already holding LOCK_binlog_ because it can cause deadlocks. + * already holding m_LOCK_binlog because it can cause deadlocks. */ mysql_mutex_t LOCK_binlog; - /* This is set to true when reply_file_name_ contains meaningful data. */ - bool reply_file_name_inited_; + /* This is set to true when m_reply_file_name contains meaningful data. */ + bool m_reply_file_name_inited; /* The binlog name up to which we have received replies from any slaves. */ - char reply_file_name_[FN_REFLEN]; + char m_reply_file_name[FN_REFLEN]; /* The position in that file up to which we have the reply from any slaves. */ - my_off_t reply_file_pos_; + my_off_t m_reply_file_pos; /* This is set to true when we know the 'smallest' wait position. */ - bool wait_file_name_inited_; + bool m_wait_file_name_inited; /* NULL, or the 'smallest' filename that a transaction is waiting for * slave replies. */ - char wait_file_name_[FN_REFLEN]; + char m_wait_file_name[FN_REFLEN]; /* The smallest position in that file that a trx is waiting for: the trx * can proceed and send an 'ok' to the client when the master has got the * reply from the slave indicating that it already got the binlog events. */ - my_off_t wait_file_pos_; + my_off_t m_wait_file_pos; /* This is set to true when we know the 'largest' transaction commit * position in the binlog file. @@ -417,22 +417,22 @@ class Repl_semi_sync_master * switch off. Binlog-dump thread can use the three fields to detect when * slaves catch up on replication so that semi-sync can switch on again. */ - bool commit_file_name_inited_; + bool m_commit_file_name_inited; /* The 'largest' binlog filename that a commit transaction is seeing. */ - char commit_file_name_[FN_REFLEN]; + char m_commit_file_name[FN_REFLEN]; /* The 'largest' position in that file that a commit transaction is seeing. */ - my_off_t commit_file_pos_; + my_off_t m_commit_file_pos; /* All global variables which can be set by parameters. */ - volatile bool master_enabled_; /* semi-sync is enabled on the master */ - unsigned long wait_timeout_; /* timeout period(ms) during tranx wait */ + volatile bool m_master_enabled; /* semi-sync is enabled on the master */ + unsigned long m_wait_timeout; /* timeout period(ms) during tranx wait */ - bool state_; /* whether semi-sync is switched */ + bool m_state; /* whether semi-sync is switched */ /*Waiting for ACK before/after innodb commit*/ - ulong wait_point_; + ulong m_wait_point; void lock(); void unlock(); @@ -441,11 +441,11 @@ class Repl_semi_sync_master /* Is semi-sync replication on? */ bool is_on() { - return (state_); + return (m_state); } void set_master_enabled(bool enabled) { - master_enabled_ = enabled; + m_master_enabled = enabled; } /* Switch semi-sync off because of timeout in transaction waiting. */ @@ -462,28 +462,28 @@ class Repl_semi_sync_master void cleanup(); bool get_master_enabled() { - return master_enabled_; + return m_master_enabled; } void set_trace_level(unsigned long trace_level) { - trace_level_ = trace_level; - if (active_tranxs_) - active_tranxs_->trace_level_ = trace_level; + m_trace_level = trace_level; + if (m_active_tranxs) + m_active_tranxs->m_trace_level = trace_level; } /* Set the transaction wait timeout period, in milliseconds. */ void set_wait_timeout(unsigned long wait_timeout) { - wait_timeout_ = wait_timeout; + m_wait_timeout = wait_timeout; } /*set the ACK point, after binlog sync or after transaction commit*/ void set_wait_point(unsigned long ack_point) { - wait_point_ = ack_point; + m_wait_point = ack_point; } ulong wait_point() //no cover line { - return wait_point_; //no cover line + return m_wait_point; //no cover line } /* Initialize this class after MySQL parameters are initialized. this @@ -549,7 +549,7 @@ class Repl_semi_sync_master /*Wait after the transaction is rollback*/ int wait_after_rollback(THD *thd, bool all); - /*Store the current binlog position in active_tranxs_. This position should + /*Store the current binlog position in m_active_tranxs. This position should * be acked by slave*/ int report_binlog_update(THD *thd, const char *log_file,my_off_t log_pos); diff --git a/sql/semisync_master_ack_receiver.h b/sql/semisync_master_ack_receiver.h index 9e876150f58..619748a2159 100644 --- a/sql/semisync_master_ack_receiver.h +++ b/sql/semisync_master_ack_receiver.h @@ -78,7 +78,7 @@ public: void set_trace_level(unsigned long trace_level) { - trace_level_= trace_level; + m_trace_level= trace_level; } private: enum status {ST_UP, ST_DOWN, ST_STOPPING}; diff --git a/sql/semisync_slave.cc b/sql/semisync_slave.cc index fbec0c13b96..2d77ee7b10c 100644 --- a/sql/semisync_slave.cc +++ b/sql/semisync_slave.cc @@ -41,7 +41,7 @@ int Repl_semi_sync_slave::init_object() { int result= 0; - init_done_ = true; + m_init_done = true; /* References to the parameter works after set_options(). */ set_slave_enabled(rpl_semi_sync_slave_enabled); @@ -134,9 +134,9 @@ void Repl_semi_sync_slave::kill_connection(MYSQL *mysql) char kill_buffer[30]; MYSQL *kill_mysql = NULL; kill_mysql = mysql_init(kill_mysql); - mysql_options(kill_mysql, MYSQL_OPT_CONNECT_TIMEOUT, &kill_conn_timeout_); - mysql_options(kill_mysql, MYSQL_OPT_READ_TIMEOUT, &kill_conn_timeout_); - mysql_options(kill_mysql, MYSQL_OPT_WRITE_TIMEOUT, &kill_conn_timeout_); + mysql_options(kill_mysql, MYSQL_OPT_CONNECT_TIMEOUT, &m_kill_conn_timeout); + mysql_options(kill_mysql, MYSQL_OPT_READ_TIMEOUT, &m_kill_conn_timeout); + mysql_options(kill_mysql, MYSQL_OPT_WRITE_TIMEOUT, &m_kill_conn_timeout); bool ret= (!mysql_real_connect(kill_mysql, mysql->host, mysql->user, mysql->passwd,0, mysql->port, mysql->unix_socket, 0)); diff --git a/sql/semisync_slave.h b/sql/semisync_slave.h index e781077a3a9..d65262f151d 100644 --- a/sql/semisync_slave.h +++ b/sql/semisync_slave.h @@ -32,11 +32,11 @@ class Master_info; class Repl_semi_sync_slave :public Repl_semi_sync_base { public: - Repl_semi_sync_slave() :slave_enabled_(false) {} + Repl_semi_sync_slave() :m_slave_enabled(false) {} ~Repl_semi_sync_slave() {} void set_trace_level(unsigned long trace_level) { - trace_level_ = trace_level; + m_trace_level = trace_level; } /* Initialize this class after MySQL parameters are initialized. this @@ -45,23 +45,23 @@ public: int init_object(); bool get_slave_enabled() { - return slave_enabled_; + return m_slave_enabled; } void set_slave_enabled(bool enabled) { - slave_enabled_ = enabled; + m_slave_enabled = enabled; } bool is_delay_master(){ - return delay_master_; + return m_delay_master; } void set_delay_master(bool enabled) { - delay_master_ = enabled; + m_delay_master = enabled; } void set_kill_conn_timeout(unsigned int timeout) { - kill_conn_timeout_ = timeout; + m_kill_conn_timeout = timeout; } /* A slave reads the semi-sync packet header and separate the metadata @@ -95,10 +95,10 @@ public: private: /* True when init_object has been called */ - bool init_done_; - bool slave_enabled_; /* semi-sycn is enabled on the slave */ - bool delay_master_; - unsigned int kill_conn_timeout_; + bool m_init_done; + bool m_slave_enabled; /* semi-sycn is enabled on the slave */ + bool m_delay_master; + unsigned int m_kill_conn_timeout; }; From 86308aa9953a72d94efde2381de98cb10f18ca09 Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Wed, 13 Dec 2017 02:23:57 +0200 Subject: [PATCH 27/51] MDEV-14579: New tests for condition pushdown into materialized views/defined tables defined with INTERSECT/EXCEPT added --- mysql-test/r/derived_cond_pushdown.result | 3716 +++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 578 +++- 2 files changed, 4292 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index d8d03d6d062..f7a0430f070 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -6999,6 +6999,3722 @@ drop view v_union,v2_union,v3_union,v4_union; drop view v_double,v_char,v_decimal; drop table t1,t2,t1_double,t2_double,t1_char,t2_char,t1_decimal,t2_decimal; # +# MDEV-14579: pushdown conditions into materialized views/derived tables +# that are defined with EXIST or/and INTERSECT +# +create table t1 (a int, b int, c int); +create table t2 (a int, b int, c int); +insert into t1 values +(1,21,345), (1,33,7), (8,33,114), (1,21,500), (1,19,117), (5,14,787), +(8,33,123), (9,10,211), (5,16,207), (1,33,988), (5,27,132), (1,21,104), +(6,20,309), (6,20,315), (1,21,101), (4,33,404), (9,10,800), (1,21,123); +insert into t2 values +(2,3,207), (1,16,909), (5,14,312), +(5,33,207), (6,20,211), (1,19,132), +(8,33,117), (3,21,231), (6,23,303); +create view v1 as +select a, b, min(c) as c from t1 +where t1.a<9 group by a,b having c < 300 +intersect +select a, b, min(c) as c from t1 +where t1.b>10 group by a,b having c > 100; +# using intersect in view definition +# conjunctive subformulas : pushing into WHERE +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +a b c a b c +1 21 101 1 16 909 +1 19 117 1 16 909 +1 21 101 1 19 132 +1 19 117 1 19 132 +select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +a b c a b c +1 21 101 1 16 909 +1 19 117 1 16 909 +1 21 101 1 19 132 +1 19 117 1 19 132 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a < 5 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a < 5" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 100", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a < 5" + } + } + } + } + } + ] + } + } + } + } + } +} +# using intersect in view definition +# conjunctive subformulas : pushing into WHERE +# pushing equalities +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a=8); +a b c a b c +8 33 114 8 33 117 +select * from v1,t2 where (v1.a=t2.a) and (v1.a=8); +a b c a b c +8 33 114 8 33 117 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a=8); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ALL NULL NULL NULL NULL 18 Using where; Using join buffer (flat, BNL join) +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a=8); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a = 8" + }, + "block-nl-join": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "v1.a = 8" + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a = 8" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 100", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a = 8 and t1.b > 10" + } + } + } + } + } + ] + } + } + } + } + } +} +# using intersect in view definition +# conjunctive subformulas : pushing into WHERE using equalities +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (t2.a=8); +a b c a b c +8 33 114 8 33 117 +select * from v1,t2 where (v1.a=t2.a) and (t2.a=8); +a b c a b c +8 33 114 8 33 117 +explain select * from v1,t2 where (v1.a=t2.a) and (t2.a=8); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ALL NULL NULL NULL NULL 18 Using where; Using join buffer (flat, BNL join) +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (t2.a=8); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a = 8" + }, + "block-nl-join": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "v1.a = 8" + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a = 8" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 100", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a = 8 and t1.b > 10" + } + } + } + } + } + ] + } + } + } + } + } +} +# using intersect in view definition +# conjunctive subformulas : pushing into HAVING +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.c>200); +a b c a b c +5 16 207 5 14 312 +5 16 207 5 33 207 +select * from v1,t2 where (v1.a=t2.a) and (v1.c>200); +a b c a b c +5 16 207 5 14 312 +5 16 207 5 33 207 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.c>200); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.c>200); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.c > 200", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300 and c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 100 and c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10" + } + } + } + } + } + ] + } + } + } + } + } +} +# using intersect in view definition +# conjunctive subformulas : pushing into WHERE +# conjunctive subformulas : pushing into HAVING +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>110); +a b c a b c +1 19 117 1 16 909 +1 19 117 1 19 132 +select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>110); +a b c a b c +1 19 117 1 16 909 +1 19 117 1 19 132 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>110); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>110); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a < 5 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.c > 110", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300 and c > 110", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a < 5" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 100 and c > 110", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a < 5" + } + } + } + } + } + ] + } + } + } + } + } +} +# using intersect in view definition +# extracted or formula : pushing into WHERE +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +a b c a b c +5 16 207 5 14 312 +5 16 207 5 33 207 +8 33 114 8 33 117 +select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +a b c a b c +5 16 207 5 14 312 +5 16 207 5 33 207 +8 33 114 8 33 117 +explain select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.b > 27 or v1.b < 19", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and (t1.b > 27 or t1.b < 19)" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 100", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and (t1.b > 27 or t1.b < 19)" + } + } + } + } + } + ] + } + } + } + } + } +} +# using intersect in view definition +# extracted or formula : pushing into HAVING +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +(v1.a=t2.a) and ((v1.c>200) or (v1.c<105)); +a b c a b c +1 21 101 1 16 909 +5 16 207 5 14 312 +5 16 207 5 33 207 +1 21 101 1 19 132 +select * from v1,t2 where +(v1.a=t2.a) and ((v1.c>200) or (v1.c<105)); +a b c a b c +1 21 101 1 16 909 +5 16 207 5 14 312 +5 16 207 5 33 207 +1 21 101 1 19 132 +explain select * from v1,t2 where +(v1.a=t2.a) and ((v1.c>200) or (v1.c<105)); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where +(v1.a=t2.a) and ((v1.c>200) or (v1.c<105)); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.c > 200 or v1.c < 105", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300 and (c > 200 or c < 105)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 100 and (c > 200 or c < 105)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10" + } + } + } + } + } + ] + } + } + } + } + } +} +# using intersect in view definition +# extracted or formula : pushing into WHERE +# extracted or formula : pushing into HAVING using equalities +# pushing equalities +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +((v1.a>3) and (t2.c>110) and (v1.c=t2.c)) or +((v1.a=1) and (v1.c<110)); +a b c a b c +1 21 101 2 3 207 +1 21 101 1 16 909 +1 21 101 5 14 312 +1 21 101 5 33 207 +1 21 101 6 20 211 +1 21 101 1 19 132 +1 21 101 8 33 117 +1 21 101 3 21 231 +1 21 101 6 23 303 +5 16 207 2 3 207 +5 16 207 5 33 207 +5 27 132 1 19 132 +select * from v1,t2 where +((v1.a>3) and (t2.c>110) and (v1.c=t2.c)) or +((v1.a=1) and (v1.c<110)); +a b c a b c +1 21 101 2 3 207 +1 21 101 1 16 909 +1 21 101 5 14 312 +1 21 101 5 33 207 +1 21 101 6 20 211 +1 21 101 1 19 132 +1 21 101 8 33 117 +1 21 101 3 21 231 +1 21 101 6 23 303 +5 16 207 2 3 207 +5 16 207 5 33 207 +5 27 132 1 19 132 +explain select * from v1,t2 where +((v1.a>3) and (t2.c>110) and (v1.c=t2.c)) or +((v1.a=1) and (v1.c<110)); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 +1 PRIMARY ALL NULL NULL NULL NULL 18 Using where; Using join buffer (flat, BNL join) +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where +((v1.a>3) and (t2.c>110) and (v1.c=t2.c)) or +((v1.a=1) and (v1.c<110)); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100 + }, + "block-nl-join": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "v1.a > 3 or v1.a = 1 and v1.c < 110" + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "v1.c = t2.c and v1.a > 3 and t2.c > 110 or v1.a = 1 and v1.c < 110", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300 and (t1.a > 3 and c > 110 or c < 110 and t1.a = 1)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and (t1.a > 3 or t1.a = 1)" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 100 and (t1.a > 3 and c > 110 or c < 110 and t1.a = 1)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and (t1.a > 3 or t1.a = 1)" + } + } + } + } + } + ] + } + } + } + } + } +} +# using intersect in view definition +# prepare of a query +# conjunctive subformulas : pushing into WHERE +# conjunctive subformulas : pushing into HAVING +prepare stmt from "select * from v1,t2 + where (v1.a=t2.a) and (v1.a<5) and (v1.c>110);"; +execute stmt; +a b c a b c +1 19 117 1 16 909 +1 19 117 1 19 132 +execute stmt; +a b c a b c +1 19 117 1 16 909 +1 19 117 1 19 132 +deallocate prepare stmt; +# using intersect in derived table definition +# extracted or formula : pushing into WHERE using equalities +# extracted or formula : pushing into HAVING +# pushing equalities +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * +from t2, +(select a, b, min(c) as c from t1 +where t1.a<9 group by a,b having c < 300 +intersect +select a, b, min(c) as c from t1 +where t1.b>10 group by a,b having c > 100) as d1 +where +(d1.b=t2.b) and +(((t2.b>13) and (t2.c=909)) or +((d1.a<4) and (d1.c<200))); +a b c a b c +1 16 909 5 16 207 +1 19 132 1 19 117 +3 21 231 1 21 101 +select * +from t2, +(select a, b, min(c) as c from t1 +where t1.a<9 group by a,b having c < 300 +intersect +select a, b, min(c) as c from t1 +where t1.b>10 group by a,b having c > 100) as d1 +where +(d1.b=t2.b) and +(((t2.b>13) and (t2.c=909)) or +((d1.a<4) and (d1.c<200))); +a b c a b c +1 16 909 5 16 207 +1 19 132 1 19 117 +3 21 231 1 21 101 +explain select * +from t2, +(select a, b, min(c) as c from t1 +where t1.a<9 group by a,b having c < 300 +intersect +select a, b, min(c) as c from t1 +where t1.b>10 group by a,b having c > 100) as d1 +where +(d1.b=t2.b) and +(((t2.b>13) and (t2.c=909)) or +((d1.a<4) and (d1.c<200))); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.b 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * +from t2, +(select a, b, min(c) as c from t1 +where t1.a<9 group by a,b having c < 300 +intersect +select a, b, min(c) as c from t1 +where t1.b>10 group by a,b having c > 100) as d1 +where +(d1.b=t2.b) and +(((t2.b>13) and (t2.c=909)) or +((d1.a<4) and (d1.c<200))); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.b is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["b"], + "ref": ["test.t2.b"], + "rows": 2, + "filtered": 100, + "attached_condition": "t2.c = 909 and t2.b > 13 or d1.a < 4 and d1.c < 200", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300 and (t1.b > 13 or t1.a < 4 and c < 200)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and (t1.b > 13 or t1.a < 4)" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 100 and (t1.b > 13 or t1.a < 4 and c < 200)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and (t1.b > 13 or t1.a < 4)" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +create view v1 as +select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 200 +except +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300; +# using except in view definition +# conjunctive subformulas : pushing into WHERE +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +a b c a b c +1 33 988 1 16 909 +1 21 500 1 16 909 +1 33 988 1 19 132 +1 21 500 1 19 132 +select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +a b c a b c +1 33 988 1 16 909 +1 21 500 1 16 909 +1 33 988 1 19 132 +1 21 500 1 19 132 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a < 5 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a < 5" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a < 5" + } + } + } + } + } + ] + } + } + } + } + } +} +# using except in view definition +# conjunctive subformulas : pushing into WHERE +# pushing equalities +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a=6); +a b c a b c +6 20 315 6 20 211 +6 20 315 6 23 303 +select * from v1,t2 where (v1.a=t2.a) and (v1.a=6); +a b c a b c +6 20 315 6 20 211 +6 20 315 6 23 303 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a=6); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ALL NULL NULL NULL NULL 18 Using where; Using join buffer (flat, BNL join) +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a=6); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a = 6" + }, + "block-nl-join": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "v1.a = 6" + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a = 6" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a = 6 and t1.b > 10" + } + } + } + } + } + ] + } + } + } + } + } +} +# using except in view definition +# conjunctive subformulas : pushing into WHERE using equalities +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (t2.a=6); +a b c a b c +6 20 315 6 20 211 +6 20 315 6 23 303 +select * from v1,t2 where (v1.a=t2.a) and (t2.a=6); +a b c a b c +6 20 315 6 20 211 +6 20 315 6 23 303 +explain select * from v1,t2 where (v1.a=t2.a) and (t2.a=6); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ALL NULL NULL NULL NULL 18 Using where; Using join buffer (flat, BNL join) +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (t2.a=6); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a = 6" + }, + "block-nl-join": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "v1.a = 6" + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a = 6" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a = 6 and t1.b > 10" + } + } + } + } + } + ] + } + } + } + } + } +} +# using except in view definition +# conjunctive subformulas : pushing into HAVING +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.c>500); +a b c a b c +1 33 988 1 16 909 +5 14 787 5 14 312 +5 14 787 5 33 207 +1 33 988 1 19 132 +select * from v1,t2 where (v1.a=t2.a) and (v1.c>500); +a b c a b c +1 33 988 1 16 909 +5 14 787 5 14 312 +5 14 787 5 33 207 +1 33 988 1 19 132 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.c>500); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.c>500); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.c > 500", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200 and c > 500", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 300 and c > 500", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10" + } + } + } + } + } + ] + } + } + } + } + } +} +# using except in view definition +# conjunctive subformulas : pushing into WHERE +# conjunctive subformulas : pushing into HAVING +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>500); +a b c a b c +1 33 988 1 16 909 +1 33 988 1 19 132 +select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>500); +a b c a b c +1 33 988 1 16 909 +1 33 988 1 19 132 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>500); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>500); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a < 5 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.c > 500", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200 and c > 500", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a < 5" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 300 and c > 500", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a < 5" + } + } + } + } + } + ] + } + } + } + } + } +} +# using except in view definition +# extracted or formula : pushing into WHERE +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +a b c a b c +1 33 988 1 16 909 +5 14 787 5 14 312 +5 14 787 5 33 207 +1 33 988 1 19 132 +select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +a b c a b c +1 33 988 1 16 909 +5 14 787 5 14 312 +5 14 787 5 33 207 +1 33 988 1 19 132 +explain select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.b > 27 or v1.b < 19", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and (t1.b > 27 or t1.b < 19)" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and (t1.b > 27 or t1.b < 19)" + } + } + } + } + } + ] + } + } + } + } + } +} +# using except in view definition +# extracted or formula : pushing into HAVING +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +(v1.a=t2.a) and ((v1.c<400) or (v1.c>800)); +a b c a b c +1 33 988 1 16 909 +6 20 315 6 20 211 +1 33 988 1 19 132 +6 20 315 6 23 303 +select * from v1,t2 where +(v1.a=t2.a) and ((v1.c<400) or (v1.c>800)); +a b c a b c +1 33 988 1 16 909 +6 20 315 6 20 211 +1 33 988 1 19 132 +6 20 315 6 23 303 +explain select * from v1,t2 where +(v1.a=t2.a) and ((v1.c<400) or (v1.c>800)); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where +(v1.a=t2.a) and ((v1.c<400) or (v1.c>800)); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.c < 400 or v1.c > 800", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200 and (c < 400 or c > 800)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 300 and (c < 400 or c > 800)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10" + } + } + } + } + } + ] + } + } + } + } + } +} +# using except in view definition +# extracted or formula : pushing into WHERE +# extracted or formula : pushing into HAVING using equalities +# pushing equalities +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where +(v1.c=t2.c) and +((v1.a>1) and (t2.c<500)) or +((v1.a=1) and (v1.c>500)); +a b c a b c +1 33 988 2 3 207 +1 33 988 1 16 909 +1 33 988 5 14 312 +1 33 988 5 33 207 +1 33 988 6 20 211 +1 33 988 1 19 132 +1 33 988 8 33 117 +1 33 988 3 21 231 +1 33 988 6 23 303 +select * from v1,t2 where +(v1.c=t2.c) and +((v1.a>1) and (t2.c<500)) or +((v1.a=1) and (v1.c>500)); +a b c a b c +1 33 988 2 3 207 +1 33 988 1 16 909 +1 33 988 5 14 312 +1 33 988 5 33 207 +1 33 988 6 20 211 +1 33 988 1 19 132 +1 33 988 8 33 117 +1 33 988 3 21 231 +1 33 988 6 23 303 +explain select * from v1,t2 where +(v1.c=t2.c) and +((v1.a>1) and (t2.c<500)) or +((v1.a=1) and (v1.c>500)); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 +1 PRIMARY ALL NULL NULL NULL NULL 18 Using where; Using join buffer (flat, BNL join) +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where +(v1.c=t2.c) and +((v1.a>1) and (t2.c<500)) or +((v1.a=1) and (v1.c>500)); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100 + }, + "block-nl-join": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "v1.a > 1 or v1.a = 1 and v1.c > 500" + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "v1.c = t2.c and v1.a > 1 and t2.c < 500 or v1.a = 1 and v1.c > 500", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200 and (t1.a > 1 and c < 500 or c > 500 and t1.a = 1)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and (t1.a > 1 or t1.a = 1)" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 300 and (t1.a > 1 and c < 500 or c > 500 and t1.a = 1)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and (t1.a > 1 or t1.a = 1)" + } + } + } + } + } + ] + } + } + } + } + } +} +# using except in view definition +# prepare of a query +# conjunctive subformulas : pushing into WHERE +# conjunctive subformulas : pushing into HAVING +prepare stmt from "select * from v1,t2 + where (v1.a=t2.a) and (v1.a<5) and (v1.c>500);"; +execute stmt; +a b c a b c +1 33 988 1 16 909 +1 33 988 1 19 132 +execute stmt; +a b c a b c +1 33 988 1 16 909 +1 33 988 1 19 132 +deallocate prepare stmt; +# using except in view definition +# extracted or formula : pushing into WHERE using equalities +# extracted or formula : pushing into HAVING +# pushing equalities +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * +from t2, +(select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 200 +except +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300) as d1 +where +(d1.b=t2.b) and +(((t2.b>13) and (t2.c=988)) or +((d1.a>4) and (d1.c>500))); +a b c a b c +5 14 312 5 14 787 +select * +from t2, +(select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 200 +except +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300) as d1 +where +(d1.b=t2.b) and +(((t2.b>13) and (t2.c=988)) or +((d1.a>4) and (d1.c>500))); +a b c a b c +5 14 312 5 14 787 +explain select * +from t2, +(select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 200 +except +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300) as d1 +where +(d1.b=t2.b) and +(((t2.b>13) and (t2.c=988)) or +((d1.a>4) and (d1.c>500))); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.b 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * +from t2, +(select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 200 +except +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300) as d1 +where +(d1.b=t2.b) and +(((t2.b>13) and (t2.c=988)) or +((d1.a>4) and (d1.c>500))); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.b is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["b"], + "ref": ["test.t2.b"], + "rows": 2, + "filtered": 100, + "attached_condition": "t2.c = 988 and t2.b > 13 or d1.a > 4 and d1.c > 500", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200 and (t1.b > 13 or t1.a > 4 and c > 500)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and (t1.b > 13 or t1.a > 4)" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 300 and (t1.b > 13 or t1.a > 4 and c > 500)", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and (t1.b > 13 or t1.a > 4)" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using union and intersect in view definition +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, min(c) as c from t1 +where t1.a<9 group by a,b having c > 200 +union +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300 +intersect +select a, b, max(c) as c from t1 +where t1.a>3 group by a,b having c < 530; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +a b c a b c +6 20 309 6 20 211 +6 20 309 6 23 303 +select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +a b c a b c +6 20 309 6 20 211 +6 20 309 6 23 303 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 3 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 UNION ALL NULL NULL NULL NULL 18 Using where +4 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +5 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +NULL UNION RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 5 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 3, + "filtered": 100, + "attached_condition": "v1.c > 200", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200 and c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a > 5" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "UNION", + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "__3.a > 5 and __3.c > 200", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 4, + "having_condition": "c < 300 and c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a > 5" + } + } + } + } + }, + { + "query_block": { + "select_id": 5, + "operation": "INTERSECT", + "having_condition": "c < 530 and c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a > 3 and t1.a > 5" + } + } + } + } + } + ] + } + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using union and intersect in view definition +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, min(c) as c from t1 +where t1.a<9 group by a,b having c > 200 +intersect +select a, b, max(c) as c from t1 +where t1.a>3 group by a,b having c < 500 +union +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +8 33 123 8 33 117 +select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +8 33 123 8 33 117 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 3 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +4 UNION t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 4 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 3, + "filtered": 100, + "attached_condition": "v1.c < 200", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200 and c < 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c < 500 and c < 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a > 3 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 4, + "operation": "UNION", + "having_condition": "c < 300 and c < 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a > 4" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using union and except in view definition +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, min(c) as c from t1 +where t1.a<9 group by a,b having c > 200 +union +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300 +except +select a, b, max(c) as c from t1 +where t1.a>3 group by a,b having c < 530; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +a b c a b c +6 20 309 6 20 211 +6 20 309 6 23 303 +select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +a b c a b c +6 20 309 6 20 211 +6 20 309 6 23 303 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 3 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 UNION t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +4 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 5 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 3, + "filtered": 100, + "attached_condition": "v1.c > 200", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200 and c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a > 5" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "UNION", + "having_condition": "c < 300 and c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a > 5" + } + } + } + } + }, + { + "query_block": { + "select_id": 4, + "operation": "EXCEPT", + "having_condition": "c < 530 and c > 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a > 3 and t1.a > 5" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using union and except in view definition +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, min(c) as c from t1 +where t1.a<9 group by a,b having c > 200 +except +select a, b, max(c) as c from t1 +where t1.a>3 group by a,b having c < 500 +union +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +8 33 123 8 33 117 +select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +8 33 123 8 33 117 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 3 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +4 UNION t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 4 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 3, + "filtered": 100, + "attached_condition": "v1.c < 200", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 200 and c < 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c < 500 and c < 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a > 3 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 4, + "operation": "UNION", + "having_condition": "c < 300 and c < 200", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a > 4" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using except and intersect in view definition +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300 +intersect +select a, b, max(c) as c from t1 +where t1.a<7 group by a,b having c < 500 +except +select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 150; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<150); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<150); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<150); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +4 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<150); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 4 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.c < 150", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300 and c < 150", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c < 500 and c < 150", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 7 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 4, + "operation": "EXCEPT", + "having_condition": "c > 150 and c < 150", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a > 4" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using except and intersect in view definition +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300 +except +select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 150 +intersect +select a, b, max(c) as c from t1 +where t1.a<7 group by a,b having c < 500; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +a b c a b c +8 33 123 8 33 117 +select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +a b c a b c +8 33 123 8 33 117 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT ALL NULL NULL NULL NULL 18 Using where +4 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +5 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 4 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.c < 130", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300 and c < 130", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "__3.a > 4 and __3.c < 130", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 4, + "having_condition": "c > 150 and c < 130", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 5, + "operation": "INTERSECT", + "having_condition": "c < 500 and c < 130", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 7 and t1.a > 4" + } + } + } + } + } + ] + } + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using except, intersect and union in view definition +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300 +except +select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 150 +intersect +select a, b, max(c) as c from t1 +where t1.a<7 group by a,b having c < 500 +union +select a, b, max(c) as c from t1 +where t1.a<7 group by a,b having c < 120; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +a b c a b c +8 33 123 8 33 117 +select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +a b c a b c +8 33 123 8 33 117 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 3 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT ALL NULL NULL NULL NULL 18 Using where +4 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +5 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +6 UNION t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 4 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 3, + "filtered": 100, + "attached_condition": "v1.c < 130", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c < 300 and c < 130", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "__3.a > 4 and __3.c < 130", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 4, + "having_condition": "c > 150 and c < 130", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 5, + "operation": "INTERSECT", + "having_condition": "c < 500 and c < 130", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 7 and t1.a > 4" + } + } + } + } + } + ] + } + } + } + } + } + }, + { + "query_block": { + "select_id": 6, + "operation": "UNION", + "having_condition": "c < 120 and c < 130", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 7 and t1.a > 4" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using intersect in view definition +# using embedded view +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300 +intersect +select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 120; +create view v2 as +select a, b, max(c) as c from v1 +where v1.a<7 group by a,b; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +explain select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +4 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 4 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v2.c < 150", + "materialized": { + "query_block": { + "select_id": 2, + "having_condition": "c < 150", + "filesort": { + "sort_key": "v1.a, v1.b", + "temporary_table": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "v1.a < 7 and v1.a > 4", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 3, + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a < 7 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 4, + "operation": "INTERSECT", + "having_condition": "c > 120", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a < 7 and t1.a > 4" + } + } + } + } + } + ] + } + } + } + } + } + } + } + } + } + } +} +drop view v1,v2; +# using except in view definition +# using embedded view +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c < 300 +except +select a, b, max(c) as c from t1 +where t1.a<9 group by a,b having c > 150; +create view v2 as +select a, b, max(c) as c from v1 +where v1.a<7 group by a,b; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +a b c a b c +5 27 132 5 14 312 +5 27 132 5 33 207 +explain select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +4 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 4 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v2.c < 150", + "materialized": { + "query_block": { + "select_id": 2, + "having_condition": "c < 150", + "filesort": { + "sort_key": "v1.a, v1.b", + "temporary_table": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "v1.a < 7 and v1.a > 4", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 3, + "having_condition": "c < 300", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a < 7 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 4, + "operation": "EXCEPT", + "having_condition": "c > 150", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a < 7 and t1.a > 4" + } + } + } + } + } + ] + } + } + } + } + } + } + } + } + } + } +} +drop view v1,v2; +# using intersect in view definition +# conditions are pushed in different parts of selects +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, max(c) as c from t1 +where t1.a<9 group by a having c > 300 +intersect +select a, b, max(c) as c from t1 +where t1.b<21 group by b having c > 200; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.b>12) and (v1.c<450); +a b c a b c +6 20 315 6 20 211 +6 20 315 6 23 303 +select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.b>12) and (v1.c<450); +a b c a b c +6 20 315 6 20 211 +6 20 315 6 23 303 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.b>12) and (v1.c<450); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL INTERSECT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.b>12) and (v1.c<450); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 4 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.b > 12 and v1.c < 450", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 300 and t1.b > 12 and c < 450", + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a > 4" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c > 200 and t1.a > 4 and c < 450", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b < 21 and t1.b > 12" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using except in view definition +# conditions are pushed in different parts of selects +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, max(c) as c from t1 +where t1.b>20 group by a having c > 300 +except +select a, b, max(c) as c from t1 +where t1.a<7 group by b having c > 150; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a<2) and (v1.b<30) and (v1.c>450); +a b c a b c +1 21 988 1 16 909 +1 21 988 1 19 132 +select * from v1,t2 where (v1.a=t2.a) and (v1.a<2) and (v1.b<30) and (v1.c>450); +a b c a b c +1 21 988 1 16 909 +1 21 988 1 19 132 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a<2) and (v1.b<30) and (v1.c>450); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a<2) and (v1.b<30) and (v1.c>450); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a < 2 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.b < 30 and v1.c > 450", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 300 and t1.b < 30 and c > 450", + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 20 and t1.a < 2" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c > 150 and t1.a < 2 and c > 450", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 7 and t1.b < 30" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using except and union in view definition +# conditions are pushed in different parts of selects +# conjunctive subformulas : pushing into HAVING +# extracted or formula : pushing into WHERE +# extracted or formula : pushing into HAVING +create view v1 as +select a, b, max(c) as c from t1 +where t1.b>20 group by a having c > 300 +except +select a, b, max(c) as c from t1 +where t1.a<7 group by b having c > 150; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and ((v1.a<2) or (v1.a<5)) and (v1.c>450); +a b c a b c +1 21 988 1 16 909 +1 21 988 1 19 132 +select * from v1,t2 where (v1.a=t2.a) and ((v1.a<2) or (v1.a<5)) and (v1.c>450); +a b c a b c +1 21 988 1 16 909 +1 21 988 1 19 132 +explain select * from v1,t2 where (v1.a=t2.a) and ((v1.a<2) or (v1.a<5)) and (v1.c>450); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 2 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL EXCEPT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and ((v1.a<2) or (v1.a<5)) and (v1.c>450); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "(t2.a < 2 or t2.a < 5) and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.c > 450", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 300 and c > 450", + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 20 and (t1.a < 2 or t1.a < 5)" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "EXCEPT", + "having_condition": "c > 150 and (t1.a < 2 or t1.a < 5) and c > 450", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 7" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +# using union and intersect in view definition +# conditions are pushed in different parts of selects +# conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as +select a, b, max(c) as c from t1 +where t1.a<9 group by a having c > 100 +intersect +select a, b, max(c) as c from t1 +where t1.a>3 group by b having c < 800 +union +select a, b, max(c) as c from t1 +where t1.b>10 group by a,b having c > 300; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.a=t2.a) and (v1.a>1) and (v1.b > 12) and (v1.c>400); +a b c a b c +5 14 787 5 14 312 +5 14 787 5 33 207 +select * from v1,t2 where (v1.a=t2.a) and (v1.a>1) and (v1.b > 12) and (v1.c>400); +a b c a b c +5 14 787 5 14 312 +5 14 787 5 33 207 +explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>1) and (v1.b > 12) and (v1.c>400); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.a 3 Using where +2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +3 INTERSECT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +4 UNION t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort +NULL UNIT RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.a=t2.a) and (v1.a>1) and (v1.b > 12) and (v1.c>400); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.a > 1 and t2.a is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["a"], + "ref": ["test.t2.a"], + "rows": 3, + "filtered": 100, + "attached_condition": "v1.b > 12 and v1.c > 400", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "having_condition": "c > 100 and t1.b > 12 and c > 400", + "filesort": { + "sort_key": "t1.a", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a < 9 and t1.a > 1" + } + } + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "INTERSECT", + "having_condition": "c < 800 and t1.a > 1 and c > 400", + "filesort": { + "sort_key": "t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.a > 3 and t1.b > 12" + } + } + } + } + }, + { + "query_block": { + "select_id": 4, + "operation": "UNION", + "having_condition": "c > 300 and c > 400", + "filesort": { + "sort_key": "t1.a, t1.b", + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 18, + "filtered": 100, + "attached_condition": "t1.b > 10 and t1.a > 1 and t1.b > 12" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +create table t3 (a int, b int, c int); +insert into t3 values +(1,21,345), (2,33,7), (8,33,114), (3,21,500), (1,19,107), (5,14,787), +(4,33,123), (9,10,211), (11,16,207), (10,33,988), (5,27,132), (12,21,104), +(6,20,309), (16,20,315), (16,21,101), (18,33,404), (19,10,800), (10,21,123), +(17,11,708), (6,20,214); +create index i1 on t3(a); +# conjunctive subformulas : pushing into WHERE +# pushed condition gives range access +create view v1 as +select a, b, max(c) as max_c from t3 +where a>0 group by a; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.b=t2.b) and (v1.a<5); +a b max_c a b c +4 33 123 5 33 207 +2 33 7 5 33 207 +4 33 123 8 33 117 +2 33 7 8 33 117 +3 21 500 3 21 231 +1 21 345 3 21 231 +select * from v1,t2 where (v1.b=t2.b) and (v1.a<5); +a b max_c a b c +1 21 345 3 21 231 +2 33 7 5 33 207 +2 33 7 8 33 117 +3 21 500 3 21 231 +4 33 123 5 33 207 +4 33 123 8 33 117 +explain select * from v1,t2 where (v1.b=t2.b) and (v1.a<5); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 +1 PRIMARY ALL NULL NULL NULL NULL 5 Using where; Using join buffer (flat, BNL join) +2 DERIVED t3 range i1 i1 5 NULL 5 Using index condition +explain format=json select * from v1,t2 where (v1.b=t2.b) and (v1.a<5); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100 + }, + "block-nl-join": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 5, + "filtered": 80, + "attached_condition": "v1.a < 5" + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "v1.b = t2.b", + "materialized": { + "query_block": { + "select_id": 2, + "table": { + "table_name": "t3", + "access_type": "range", + "possible_keys": ["i1"], + "key": "i1", + "key_length": "5", + "used_key_parts": ["a"], + "rows": 5, + "filtered": 100, + "index_condition": "t3.a > 0 and t3.a < 5" + } + } + } + } + } +} +drop view v1; +# using union in view definition +# conjunctive subformulas : pushing into WHERE +# pushed condition gives range access +create view v1 as +select a, b, max(c) as c from t3 +where t3.a>1 group by a +union +select a, b, max(c) as c from t3 +where t3.a>2 group by a; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.b=t2.b) and (v1.a<4); +a b c a b c +2 33 7 5 33 207 +2 33 7 8 33 117 +3 21 500 3 21 231 +select * from v1,t2 where (v1.b=t2.b) and (v1.a<4); +a b c a b c +2 33 7 5 33 207 +2 33 7 8 33 117 +3 21 500 3 21 231 +explain select * from v1,t2 where (v1.b=t2.b) and (v1.a<4); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY ALL NULL NULL NULL NULL 3 Using where +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where; Using join buffer (flat, BNL join) +2 DERIVED t3 range i1 i1 5 NULL 2 Using index condition +3 UNION t3 range i1 i1 5 NULL 1 Using index condition +NULL UNION RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.b=t2.b) and (v1.a<4); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "v1.a < 4", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "table": { + "table_name": "t3", + "access_type": "range", + "possible_keys": ["i1"], + "key": "i1", + "key_length": "5", + "used_key_parts": ["a"], + "rows": 2, + "filtered": 100, + "index_condition": "t3.a > 1 and t3.a < 4" + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "UNION", + "table": { + "table_name": "t3", + "access_type": "range", + "possible_keys": ["i1"], + "key": "i1", + "key_length": "5", + "used_key_parts": ["a"], + "rows": 1, + "filtered": 100, + "index_condition": "t3.a > 2 and t3.a < 4" + } + } + } + ] + } + } + } + }, + "block-nl-join": { + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "t2.b = v1.b" + } + } +} +drop view v1; +# using union in view definition +# conjunctive subformulas : pushing into WHERE +# pushed condition gives range access in one of the selects +create view v1 as +select a, b, max(c) as c from t3 +where t3.a>1 group by a +union +select a, b, max(c) as c from t3 +where t3.b<21 group by b; +set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.b=t2.b) and (v1.a<3); +a b c a b c +2 33 7 5 33 207 +1 19 107 1 19 132 +2 33 7 8 33 117 +select * from v1,t2 where (v1.b=t2.b) and (v1.a<3); +a b c a b c +2 33 7 5 33 207 +1 19 107 1 19 132 +2 33 7 8 33 117 +explain select * from v1,t2 where (v1.b=t2.b) and (v1.a<3); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where +1 PRIMARY ref key0 key0 5 test.t2.b 2 Using where +2 DERIVED t3 range i1 i1 5 NULL 1 Using index condition +3 UNION t3 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort +NULL UNION RESULT ALL NULL NULL NULL NULL NULL +explain format=json select * from v1,t2 where (v1.b=t2.b) and (v1.a<3); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 9, + "filtered": 100, + "attached_condition": "t2.b is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "5", + "used_key_parts": ["b"], + "ref": ["test.t2.b"], + "rows": 2, + "filtered": 100, + "attached_condition": "v1.a < 3", + "materialized": { + "query_block": { + "union_result": { + "table_name": "", + "access_type": "ALL", + "query_specifications": [ + { + "query_block": { + "select_id": 2, + "table": { + "table_name": "t3", + "access_type": "range", + "possible_keys": ["i1"], + "key": "i1", + "key_length": "5", + "used_key_parts": ["a"], + "rows": 1, + "filtered": 100, + "index_condition": "t3.a > 1 and t3.a < 3" + } + } + }, + { + "query_block": { + "select_id": 3, + "operation": "UNION", + "having_condition": "t3.a < 3", + "filesort": { + "sort_key": "t3.b", + "temporary_table": { + "table": { + "table_name": "t3", + "access_type": "ALL", + "rows": 20, + "filtered": 100, + "attached_condition": "t3.b < 21" + } + } + } + } + } + ] + } + } + } + } + } +} +drop view v1; +alter table t3 drop index i1; +drop table t1,t2,t3; +# # MDEV-10782: condition extracted from a multiple equality # pushed into HAVING # diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index a9bb998bc33..2a3da09a3d7 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -873,11 +873,585 @@ eval $no_pushdown $query; eval $query; eval explain $query; eval explain format=json $query; - + drop view v1,v2,v3,v4; drop view v_union,v2_union,v3_union,v4_union; drop view v_double,v_char,v_decimal; -drop table t1,t2,t1_double,t2_double,t1_char,t2_char,t1_decimal,t2_decimal; +drop table t1,t2,t1_double,t2_double,t1_char,t2_char,t1_decimal,t2_decimal; + +--echo # +--echo # MDEV-14579: pushdown conditions into materialized views/derived tables +--echo # that are defined with EXIST or/and INTERSECT +--echo # + +create table t1 (a int, b int, c int); +create table t2 (a int, b int, c int); + +insert into t1 values + (1,21,345), (1,33,7), (8,33,114), (1,21,500), (1,19,117), (5,14,787), + (8,33,123), (9,10,211), (5,16,207), (1,33,988), (5,27,132), (1,21,104), + (6,20,309), (6,20,315), (1,21,101), (4,33,404), (9,10,800), (1,21,123); + +insert into t2 values + (2,3,207), (1,16,909), (5,14,312), + (5,33,207), (6,20,211), (1,19,132), + (8,33,117), (3,21,231), (6,23,303); + +create view v1 as + select a, b, min(c) as c from t1 + where t1.a<9 group by a,b having c < 300 + intersect + select a, b, min(c) as c from t1 + where t1.b>10 group by a,b having c > 100; + +--echo # using intersect in view definition +--echo # conjunctive subformulas : pushing into WHERE +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using intersect in view definition +--echo # conjunctive subformulas : pushing into WHERE +--echo # pushing equalities +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a=8); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using intersect in view definition +--echo # conjunctive subformulas : pushing into WHERE using equalities +let $query= select * from v1,t2 where (v1.a=t2.a) and (t2.a=8); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using intersect in view definition +--echo # conjunctive subformulas : pushing into HAVING +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.c>200); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using intersect in view definition +--echo # conjunctive subformulas : pushing into WHERE +--echo # conjunctive subformulas : pushing into HAVING +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>110); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using intersect in view definition +--echo # extracted or formula : pushing into WHERE +let $query= + select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using intersect in view definition +--echo # extracted or formula : pushing into HAVING +let $query= + select * from v1,t2 where + (v1.a=t2.a) and ((v1.c>200) or (v1.c<105)); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using intersect in view definition +--echo # extracted or formula : pushing into WHERE +--echo # extracted or formula : pushing into HAVING using equalities +--echo # pushing equalities +let $query= + select * from v1,t2 where + ((v1.a>3) and (t2.c>110) and (v1.c=t2.c)) or + ((v1.a=1) and (v1.c<110)); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using intersect in view definition +--echo # prepare of a query +--echo # conjunctive subformulas : pushing into WHERE +--echo # conjunctive subformulas : pushing into HAVING +prepare stmt from "select * from v1,t2 + where (v1.a=t2.a) and (v1.a<5) and (v1.c>110);"; +execute stmt; +execute stmt; +deallocate prepare stmt; + +--echo # using intersect in derived table definition +--echo # extracted or formula : pushing into WHERE using equalities +--echo # extracted or formula : pushing into HAVING +--echo # pushing equalities +let $query= + select * + from t2, + (select a, b, min(c) as c from t1 + where t1.a<9 group by a,b having c < 300 + intersect + select a, b, min(c) as c from t1 + where t1.b>10 group by a,b having c > 100) as d1 + where + (d1.b=t2.b) and + (((t2.b>13) and (t2.c=909)) or + ((d1.a<4) and (d1.c<200))); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +create view v1 as + select a, b, max(c) as c from t1 + where t1.a<9 group by a,b having c > 200 + except + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300; + +--echo # using except in view definition +--echo # conjunctive subformulas : pushing into WHERE +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using except in view definition +--echo # conjunctive subformulas : pushing into WHERE +--echo # pushing equalities +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a=6); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using except in view definition +--echo # conjunctive subformulas : pushing into WHERE using equalities +let $query= select * from v1,t2 where (v1.a=t2.a) and (t2.a=6); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using except in view definition +--echo # conjunctive subformulas : pushing into HAVING +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.c>500); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using except in view definition +--echo # conjunctive subformulas : pushing into WHERE +--echo # conjunctive subformulas : pushing into HAVING +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>500); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using except in view definition +--echo # extracted or formula : pushing into WHERE +let $query= + select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using except in view definition +--echo # extracted or formula : pushing into HAVING +let $query= + select * from v1,t2 where + (v1.a=t2.a) and ((v1.c<400) or (v1.c>800)); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using except in view definition +--echo # extracted or formula : pushing into WHERE +--echo # extracted or formula : pushing into HAVING using equalities +--echo # pushing equalities +let $query= + select * from v1,t2 where + (v1.c=t2.c) and + ((v1.a>1) and (t2.c<500)) or + ((v1.a=1) and (v1.c>500)); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +--echo # using except in view definition +--echo # prepare of a query +--echo # conjunctive subformulas : pushing into WHERE +--echo # conjunctive subformulas : pushing into HAVING +prepare stmt from "select * from v1,t2 + where (v1.a=t2.a) and (v1.a<5) and (v1.c>500);"; +execute stmt; +execute stmt; +deallocate prepare stmt; + +--echo # using except in view definition +--echo # extracted or formula : pushing into WHERE using equalities +--echo # extracted or formula : pushing into HAVING +--echo # pushing equalities +let $query= + select * + from t2, + (select a, b, max(c) as c from t1 + where t1.a<9 group by a,b having c > 200 + except + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300) as d1 + where + (d1.b=t2.b) and + (((t2.b>13) and (t2.c=988)) or + ((d1.a>4) and (d1.c>500))); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using union and intersect in view definition +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, min(c) as c from t1 + where t1.a<9 group by a,b having c > 200 + union + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300 + intersect + select a, b, max(c) as c from t1 + where t1.a>3 group by a,b having c < 530; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using union and intersect in view definition +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, min(c) as c from t1 + where t1.a<9 group by a,b having c > 200 + intersect + select a, b, max(c) as c from t1 + where t1.a>3 group by a,b having c < 500 + union + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using union and except in view definition +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, min(c) as c from t1 + where t1.a<9 group by a,b having c > 200 + union + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300 + except + select a, b, max(c) as c from t1 + where t1.a>3 group by a,b having c < 530; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using union and except in view definition +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, min(c) as c from t1 + where t1.a<9 group by a,b having c > 200 + except + select a, b, max(c) as c from t1 + where t1.a>3 group by a,b having c < 500 + union + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using except and intersect in view definition +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300 + intersect + select a, b, max(c) as c from t1 + where t1.a<7 group by a,b having c < 500 + except + select a, b, max(c) as c from t1 + where t1.a<9 group by a,b having c > 150; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<150); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using except and intersect in view definition +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300 + except + select a, b, max(c) as c from t1 + where t1.a<9 group by a,b having c > 150 + intersect + select a, b, max(c) as c from t1 + where t1.a<7 group by a,b having c < 500; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using except, intersect and union in view definition +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300 + except + select a, b, max(c) as c from t1 + where t1.a<9 group by a,b having c > 150 + intersect + select a, b, max(c) as c from t1 + where t1.a<7 group by a,b having c < 500 + union + select a, b, max(c) as c from t1 + where t1.a<7 group by a,b having c < 120; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using intersect in view definition +--echo # using embedded view +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300 + intersect + select a, b, max(c) as c from t1 + where t1.a<9 group by a,b having c > 120; + +create view v2 as + select a, b, max(c) as c from v1 + where v1.a<7 group by a,b; + +let $query= select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1,v2; + +--echo # using except in view definition +--echo # using embedded view +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c < 300 + except + select a, b, max(c) as c from t1 + where t1.a<9 group by a,b having c > 150; + +create view v2 as + select a, b, max(c) as c from v1 + where v1.a<7 group by a,b; + +let $query= select * from v2,t2 where (v2.a=t2.a) and (v2.a>4) and (v2.c<150); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1,v2; + +--echo # using intersect in view definition +--echo # conditions are pushed in different parts of selects +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, max(c) as c from t1 + where t1.a<9 group by a having c > 300 + intersect + select a, b, max(c) as c from t1 + where t1.b<21 group by b having c > 200; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.b>12) and (v1.c<450); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using except in view definition +--echo # conditions are pushed in different parts of selects +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, max(c) as c from t1 + where t1.b>20 group by a having c > 300 + except + select a, b, max(c) as c from t1 + where t1.a<7 group by b having c > 150; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a<2) and (v1.b<30) and (v1.c>450); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using except and union in view definition +--echo # conditions are pushed in different parts of selects +--echo # conjunctive subformulas : pushing into HAVING +--echo # extracted or formula : pushing into WHERE +--echo # extracted or formula : pushing into HAVING +create view v1 as + select a, b, max(c) as c from t1 + where t1.b>20 group by a having c > 300 + except + select a, b, max(c) as c from t1 + where t1.a<7 group by b having c > 150; + +let $query= select * from v1,t2 where (v1.a=t2.a) and ((v1.a<2) or (v1.a<5)) and (v1.c>450); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using union and intersect in view definition +--echo # conditions are pushed in different parts of selects +--echo # conjunctive subformulas : pushing into WHERE and HAVING +create view v1 as + select a, b, max(c) as c from t1 + where t1.a<9 group by a having c > 100 + intersect + select a, b, max(c) as c from t1 + where t1.a>3 group by b having c < 800 + union + select a, b, max(c) as c from t1 + where t1.b>10 group by a,b having c > 300; + +let $query= select * from v1,t2 where (v1.a=t2.a) and (v1.a>1) and (v1.b > 12) and (v1.c>400); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +create table t3 (a int, b int, c int); +insert into t3 values + (1,21,345), (2,33,7), (8,33,114), (3,21,500), (1,19,107), (5,14,787), + (4,33,123), (9,10,211), (11,16,207), (10,33,988), (5,27,132), (12,21,104), + (6,20,309), (16,20,315), (16,21,101), (18,33,404), (19,10,800), (10,21,123), + (17,11,708), (6,20,214); + +create index i1 on t3(a); + +--echo # conjunctive subformulas : pushing into WHERE +--echo # pushed condition gives range access +create view v1 as + select a, b, max(c) as max_c from t3 + where a>0 group by a; + +let $query= select * from v1,t2 where (v1.b=t2.b) and (v1.a<5); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using union in view definition +--echo # conjunctive subformulas : pushing into WHERE +--echo # pushed condition gives range access +create view v1 as + select a, b, max(c) as c from t3 + where t3.a>1 group by a + union + select a, b, max(c) as c from t3 + where t3.a>2 group by a; + +let $query= select * from v1,t2 where (v1.b=t2.b) and (v1.a<4); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +--echo # using union in view definition +--echo # conjunctive subformulas : pushing into WHERE +--echo # pushed condition gives range access in one of the selects +create view v1 as + select a, b, max(c) as c from t3 + where t3.a>1 group by a + union + select a, b, max(c) as c from t3 + where t3.b<21 group by b; + +let $query= select * from v1,t2 where (v1.b=t2.b) and (v1.a<3); +eval $no_pushdown $query; +eval $query; +eval explain $query; +eval explain format=json $query; + +drop view v1; + +alter table t3 drop index i1; + +drop table t1,t2,t3; --echo # --echo # MDEV-10782: condition extracted from a multiple equality From 3791d0cfcf62b49b6f6f1f5aae7cd52f2bf9adcd Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Sun, 17 Dec 2017 21:25:48 +0200 Subject: [PATCH 28/51] Test result changed --- mysql-test/r/derived_cond_pushdown.result | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index f7a0430f070..41c8c51a09b 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -10443,7 +10443,7 @@ EXPLAIN } drop view v1; create table t3 (a int, b int, c int); -insert into t3 values +insert into t3 values (1,21,345), (2,33,7), (8,33,114), (3,21,500), (1,19,107), (5,14,787), (4,33,123), (9,10,211), (11,16,207), (10,33,988), (5,27,132), (12,21,104), (6,20,309), (16,20,315), (16,21,101), (18,33,404), (19,10,800), (10,21,123), @@ -10451,7 +10451,7 @@ insert into t3 values create index i1 on t3(a); # conjunctive subformulas : pushing into WHERE # pushed condition gives range access -create view v1 as +create view v1 as select a, b, max(c) as max_c from t3 where a>0 group by a; set statement optimizer_switch='condition_pushdown_for_derived=off' for select * from v1,t2 where (v1.b=t2.b) and (v1.a<5); From 03e91ce324465e465468021edd050c3b73d4cee3 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 15 Dec 2017 16:38:46 +0100 Subject: [PATCH 29/51] MDEV-14641 Incompatible key or row definition between the MariaDB .frm file and the information in the storage engine make sure that mysql_create_frm_image() and fast_alter_partition_table() use the same code to derive HA_OPTION_PACK_RECORD from create_info->row_type. --- .../suite/parts/r/partition_alter_myisam.result | 10 ++++++++++ .../suite/parts/t/partition_alter_myisam.test | 17 +++++++++++++++++ sql/handler.h | 7 +++++++ sql/sql_partition.cc | 5 +---- sql/sql_table.cc | 5 +---- 5 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/parts/r/partition_alter_myisam.result create mode 100644 mysql-test/suite/parts/t/partition_alter_myisam.test diff --git a/mysql-test/suite/parts/r/partition_alter_myisam.result b/mysql-test/suite/parts/r/partition_alter_myisam.result new file mode 100644 index 00000000000..514593fd4ef --- /dev/null +++ b/mysql-test/suite/parts/r/partition_alter_myisam.result @@ -0,0 +1,10 @@ +CREATE TABLE t1 (i INT) ENGINE=MYISAM +PARTITION BY LIST(i) ( +PARTITION p0 VALUES IN (1), +PARTITION p1 VALUES IN (2) +); +ALTER TABLE t1 ROW_FORMAT=COMPRESSED; +ALTER TABLE t1 DROP PARTITION p1; +SELECT * FROM t1; +i +DROP TABLE t1; diff --git a/mysql-test/suite/parts/t/partition_alter_myisam.test b/mysql-test/suite/parts/t/partition_alter_myisam.test new file mode 100644 index 00000000000..91ce8d21327 --- /dev/null +++ b/mysql-test/suite/parts/t/partition_alter_myisam.test @@ -0,0 +1,17 @@ +# +# MDEV-14641 Incompatible key or row definition between the MariaDB .frm file and the information in the storage engine +# + +--source include/have_partition.inc + +CREATE TABLE t1 (i INT) ENGINE=MYISAM +PARTITION BY LIST(i) ( + PARTITION p0 VALUES IN (1), + PARTITION p1 VALUES IN (2) +); +ALTER TABLE t1 ROW_FORMAT=COMPRESSED; +ALTER TABLE t1 DROP PARTITION p1; +SELECT * FROM t1; + +# Cleanup +DROP TABLE t1; diff --git a/sql/handler.h b/sql/handler.h index 772f2e68dab..c422094b4d5 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1666,6 +1666,13 @@ struct HA_CREATE_INFO used_fields|= (HA_CREATE_USED_CHARSET | HA_CREATE_USED_DEFAULT_CHARSET); return false; } + ulong table_options_with_row_type() + { + if (row_type == ROW_TYPE_DYNAMIC || row_type == ROW_TYPE_PAGE) + return table_options | HA_OPTION_PACK_RECORD; + else + return table_options; + } }; diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index 02109b22898..a675d325a8b 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -6834,10 +6834,7 @@ uint fast_alter_partition_table(THD *thd, TABLE *table, lpt->part_info= part_info; lpt->alter_info= alter_info; lpt->create_info= create_info; - lpt->db_options= create_info->table_options; - if (create_info->row_type != ROW_TYPE_FIXED && - create_info->row_type != ROW_TYPE_DEFAULT) - lpt->db_options|= HA_OPTION_PACK_RECORD; + lpt->db_options= create_info->table_options_with_row_type(); lpt->table= table; lpt->key_info_buffer= 0; lpt->key_count= 0; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 9549884bc4e..7d37c559e20 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -4405,10 +4405,7 @@ handler *mysql_create_frm_image(THD *thd, set_table_default_charset(thd, create_info, (char*) db); - db_options= create_info->table_options; - if (create_info->row_type == ROW_TYPE_DYNAMIC || - create_info->row_type == ROW_TYPE_PAGE) - db_options|= HA_OPTION_PACK_RECORD; + db_options= create_info->table_options_with_row_type(); if (!(file= get_new_handler((TABLE_SHARE*) 0, thd->mem_root, create_info->db_type))) From f32063c5137adc028a9b038b9e99f9384e583d0b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 18 Dec 2017 15:37:06 +0000 Subject: [PATCH 30/51] MDEV-13620 - improve help message for 'plugin-dir' and 'plugin-load' options. --- extra/mariabackup/xtrabackup.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 4d179961d1d..eb1cbf4748d 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1103,11 +1103,13 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, (G_PTR*) &defaults_group, (G_PTR*) &defaults_group, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"plugin-dir", OPT_PLUGIN_DIR, "Server plugin directory", + {"plugin-dir", OPT_PLUGIN_DIR, + "Server plugin directory. Used to load encryption plugin during 'prepare' phase." + "Has no effect in the 'backup' phase (plugin directory during backup is the same as server's)", &xb_plugin_dir, &xb_plugin_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, - { "plugin-load", OPT_PLUGIN_LOAD, "encrypton plugin to load", + { "plugin-load", OPT_PLUGIN_LOAD, "encrypton plugin to load during 'prepare' phase.", &xb_plugin_load, &xb_plugin_load, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 }, From 1c2f59f7fbacc0fa7c4037bf03630f849859ad45 Mon Sep 17 00:00:00 2001 From: Martynas Bendorius Date: Sun, 1 Oct 2017 15:45:51 +0200 Subject: [PATCH 31/51] MDEV-13969 sst mysqldump and xtrabackup-v2 handle WSREP_SST_OPT_CONF incorrectly wrep_sst_common: Setting "-c ''" for my_print_defaults just takes no values from config at all. $MY_PRINT_DEFAULTS is already set at the top of the script to have --defaults-file and --defaults-extra-file. If WSREP_SST_OPT_CONF if set to "--defaults-file=/etc/my.cnf --defaults-extra-file=/etc/my.extra.cnf", then "my_print_defaults -c "" --defaults-file=/etc/my.cnf" succeeds, but if WSREP_SST_OPT_CONF is empty - no default values are taken at all. wsrep_sst_xtrabackup-v2: innobackupex does not support --defaults-extra-file, so ${WSREP_SST_OPT_CONF} cannot be used as an argument, it has been changed to ${WSREP_SST_OPT_DEFAULT}. Removed --defaults-file= from INNOMOVE line, because WSREP_SST_OPT_CONF already includes it (INNOBACKUP was fine, INNOMOVE - not). --- scripts/wsrep_sst_common.sh | 4 ++-- scripts/wsrep_sst_xtrabackup-v2.sh | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 1ef4830661b..788e1a12f37 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -257,12 +257,12 @@ parse_cnf() # look in group+suffix if [ -n $WSREP_SST_OPT_CONF_SUFFIX ]; then - reval=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF "${group}${WSREP_SST_OPT_CONF_SUFFIX}" | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) + reval=$($MY_PRINT_DEFAULTS "${group}${WSREP_SST_OPT_CONF_SUFFIX}" | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) fi # look in group if [ -z $reval ]; then - reval=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) + reval=$($MY_PRINT_DEFAULTS $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) fi # use default if we haven't found a value diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index faa7bc5e815..40e686d4d6b 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -492,7 +492,7 @@ read_cnf() ssystag+="-" if [[ $ssyslog -ne -1 ]];then - if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then + if $MY_PRINT_DEFAULTS mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then ssyslog=1 fi fi @@ -669,7 +669,7 @@ check_extra() local use_socket=1 if [[ $uextra -eq 1 ]];then if $MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then - local eport=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + local eport=$($MY_PRINT_DEFAULTS mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) if [[ -n $eport ]];then # Xtrabackup works only locally. # Hence, setting host to 127.0.0.1 unconditionally. @@ -865,14 +865,14 @@ if [[ $ssyslog -eq 1 ]];then } INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply " - INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " - INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" + INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_DEFAULT} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " + INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_DEFAULT} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" fi else INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" - INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" - INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" + INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_DEFAULT} --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" + INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_DEFAULT} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" fi get_stream From beabe6b2167ee7e8c2659da9507e33ddd56a9c29 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 16 Oct 2017 19:33:06 +0200 Subject: [PATCH 32/51] MDEV-13969 sst mysqldump and xtrabackup-v2 handle WSREP_SST_OPT_CONF incorrectly $WSREP_SST_OPT_CONF already includes --defaults-extra-file= prefix. --- scripts/wsrep_sst_mysqldump.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 59d9a3c5c9c..f086d9873d0 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -117,7 +117,7 @@ GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" |\ $MYSQL_CLIENT $AUTH -S$WSREP_SST_OPT_SOCKET --disable-reconnect --connect_timeout=10 |\ tail -1 | awk -F ' ' '{ print $2 }') -MYSQL="$MYSQL_CLIENT --defaults-extra-file=$WSREP_SST_OPT_CONF "\ +MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\ "$AUTH -h${WSREP_SST_OPT_HOST_UNESCAPED:-$WSREP_SST_OPT_HOST} "\ "-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10" From 91daf8819cf84828b4c96be6f0b35caca97bb61c Mon Sep 17 00:00:00 2001 From: sjaakola Date: Tue, 12 Dec 2017 17:47:06 +0100 Subject: [PATCH 33/51] MW-416 Moved TOI replication to happen after ACL checking for commands: SQLCOM_CREATE_EVENT SQLCOM_ALTER_EVENT SQLCOM_DROP_EVENT SQLCOM_CREATE_VIEW SQLCOM_CREATE_TRIGGER SQLCOM_DROP_TRIGGER SQLCOM_INSTALL_PLUGIN SQLCOM_UNINSTALL_PLUGIN --- sql/events.cc | 18 ++++++++++++++++++ sql/sql_parse.cc | 7 ------- sql/sql_plugin.cc | 20 ++++++++++++++++---- sql/sql_trigger.cc | 5 +++++ sql/sql_view.cc | 5 +++++ 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/sql/events.cc b/sql/events.cc index 51f68ca4c03..cdc15f41716 100644 --- a/sql/events.cc +++ b/sql/events.cc @@ -335,6 +335,7 @@ Events::create_event(THD *thd, Event_parse_data *parse_data) if (check_access(thd, EVENT_ACL, parse_data->dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (lock_object_name(thd, MDL_key::EVENT, parse_data->dbname.str, parse_data->name.str)) @@ -417,6 +418,10 @@ Events::create_event(THD *thd, Event_parse_data *parse_data) thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); +#ifdef WITH_WSREP + error: + DBUG_RETURN(true); +#endif /* WITH_WSREP */ } @@ -457,6 +462,9 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, if (check_access(thd, EVENT_ACL, parse_data->dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + if (lock_object_name(thd, MDL_key::EVENT, parse_data->dbname.str, parse_data->name.str)) DBUG_RETURN(TRUE); @@ -541,6 +549,10 @@ Events::update_event(THD *thd, Event_parse_data *parse_data, thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); +#ifdef WITH_WSREP +error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -581,6 +593,8 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) if (check_access(thd, EVENT_ACL, dbname.str, NULL, NULL, 0, 0)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + /* Turn off row binlogging of this statement and use statement-based so that all supporting tables are updated for DROP EVENT command. @@ -602,6 +616,10 @@ Events::drop_event(THD *thd, LEX_STRING dbname, LEX_STRING name, bool if_exists) thd->restore_stmt_binlog_format(save_binlog_format); DBUG_RETURN(ret); +#ifdef WITH_WSREP +error: + DBUG_RETURN(TRUE); +#endif } diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 3121ee99834..6084c59a257 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4565,7 +4565,6 @@ end_with_restore_list: if (res) break; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) switch (lex->sql_command) { case SQLCOM_CREATE_EVENT: { @@ -4599,7 +4598,6 @@ end_with_restore_list: lex->spname->m_name); break; case SQLCOM_DROP_EVENT: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (!(res= Events::drop_event(thd, lex->spname->m_db, lex->spname->m_name, lex->if_exists()))) @@ -5505,7 +5503,6 @@ end_with_restore_list: Note: SQLCOM_CREATE_VIEW also handles 'ALTER VIEW' commands as specified through the thd->lex->create_view_mode flag. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_view(thd, first_table, thd->lex->create_view_mode); break; } @@ -5521,7 +5518,6 @@ end_with_restore_list: case SQLCOM_CREATE_TRIGGER: { /* Conditionally writes to binlog. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_or_drop_trigger(thd, all_tables, 1); break; @@ -5529,7 +5525,6 @@ end_with_restore_list: case SQLCOM_DROP_TRIGGER: { /* Conditionally writes to binlog. */ - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) res= mysql_create_or_drop_trigger(thd, all_tables, 0); break; } @@ -5594,13 +5589,11 @@ end_with_restore_list: my_ok(thd); break; case SQLCOM_INSTALL_PLUGIN: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (! (res= mysql_install_plugin(thd, &thd->lex->comment, &thd->lex->ident))) my_ok(thd); break; case SQLCOM_UNINSTALL_PLUGIN: - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (! (res= mysql_uninstall_plugin(thd, &thd->lex->comment, &thd->lex->ident))) my_ok(thd); diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc index f41d1e0fdbf..3a4097734ec 100644 --- a/sql/sql_plugin.cc +++ b/sql/sql_plugin.cc @@ -2108,12 +2108,16 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, bool error; int argc=orig_argc; char **argv=orig_argv; + unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = + { MYSQL_AUDIT_GENERAL_CLASSMASK }; DBUG_ENTER("mysql_install_plugin"); tables.init_one_table("mysql", 5, "plugin", 6, "plugin", TL_WRITE); if (!opt_noacl && check_table_access(thd, INSERT_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table = open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT))) @@ -2146,8 +2150,6 @@ bool mysql_install_plugin(THD *thd, const LEX_STRING *name, See also mysql_uninstall_plugin() and initialize_audit_plugin() */ - unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = - { MYSQL_AUDIT_GENERAL_CLASSMASK }; mysql_audit_acquire_plugins(thd, event_class_mask); mysql_mutex_lock(&LOCK_plugin); @@ -2178,6 +2180,10 @@ err: if (argv) free_defaults(argv); DBUG_RETURN(error); +#ifdef WITH_WSREP +error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } @@ -2244,6 +2250,8 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, TABLE_LIST tables; LEX_STRING dl= *dl_arg; bool error= false; + unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = + { MYSQL_AUDIT_GENERAL_CLASSMASK }; DBUG_ENTER("mysql_uninstall_plugin"); tables.init_one_table("mysql", 5, "plugin", 6, "plugin", TL_WRITE); @@ -2251,6 +2259,8 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, if (!opt_noacl && check_table_access(thd, DELETE_ACL, &tables, FALSE, 1, FALSE)) DBUG_RETURN(TRUE); + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + /* need to open before acquiring LOCK_plugin or it will deadlock */ if (! (table= open_ltable(thd, &tables, TL_WRITE, MYSQL_LOCK_IGNORE_TIMEOUT))) DBUG_RETURN(TRUE); @@ -2276,8 +2286,6 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, See also mysql_install_plugin() and initialize_audit_plugin() */ - unsigned long event_class_mask[MYSQL_AUDIT_CLASS_MASK_SIZE] = - { MYSQL_AUDIT_GENERAL_CLASSMASK }; mysql_audit_acquire_plugins(thd, event_class_mask); mysql_mutex_lock(&LOCK_plugin); @@ -2307,6 +2315,10 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name, mysql_mutex_unlock(&LOCK_plugin); DBUG_RETURN(error); +#ifdef WITH_WSREP +error: + DBUG_RETURN(TRUE); +#endif /* WITH_WSREP */ } diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index 28e59319a50..0b4978b2862 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -441,6 +441,7 @@ bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create) my_error(ER_BINLOG_CREATE_ROUTINE_NEED_SUPER, MYF(0)); DBUG_RETURN(TRUE); } + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) if (!create) { @@ -606,6 +607,10 @@ end: my_ok(thd); DBUG_RETURN(result); +#ifdef WITH_WSREP + error: + DBUG_RETURN(true); +#endif /* WITH_WSREP */ } /** diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 90c94e6a503..629dd865b0e 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -425,6 +425,7 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, if ((res= create_view_precheck(thd, tables, view, mode))) goto err; + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; @@ -695,6 +696,10 @@ err: lex->link_first_table_back(view, link_to_local); unit->cleanup(); DBUG_RETURN(res || thd->is_error()); +#ifdef WITH_WSREP + error: + DBUG_RETURN(true); +#endif /* WITH_WSREP */ } From 682c3bfd259b3184232e5329558094faa0bc8562 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 21 Nov 2016 16:20:10 -0500 Subject: [PATCH 34/51] MDEV-10442: "Address already in use" on restart SST processes should inherit mysqld's process group. --- sql/wsrep_utils.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/wsrep_utils.cc b/sql/wsrep_utils.cc index 580c8bbd55c..8a72d754a43 100644 --- a/sql/wsrep_utils.cc +++ b/sql/wsrep_utils.cc @@ -264,7 +264,6 @@ process::process (const char* cmd, const char* type, char** env) err_ = posix_spawnattr_setflags (&attr, POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK | - /* start a new process group */ POSIX_SPAWN_SETPGROUP | POSIX_SPAWN_USEVFORK); if (err_) { From 40088bfc7e100294d622ab32a6e7cbcb79c12eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 18 Dec 2017 19:46:23 +0200 Subject: [PATCH 35/51] MDEV-13407 innodb.drop_table_background failed in buildbot with "Tablespace for table exists" The InnoDB background DROP TABLE queue is something that we should really remove, but are unable to until we remove dict_operation_lock so that DDL and DML operations can be combined in a single transaction. Because the queue is not persistent, it is not crash-safe. In stable versions of MariaDB, we can only try harder to drop all enqueued tables before server shutdown. row_mysql_drop_t::table_id: Replaces table_name. row_drop_tables_for_mysql_in_background(): Do not remove the entry from the list as long as the table exists. In this way, the table should eventually be dropped. --- storage/innobase/row/row0mysql.cc | 142 +++++++++--------------------- storage/xtradb/row/row0mysql.cc | 142 +++++++++--------------------- 2 files changed, 80 insertions(+), 204 deletions(-) diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 74a17b9a1c3..4a39fc311b0 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -73,7 +73,7 @@ UNIV_INTERN ibool row_rollback_on_timeout = FALSE; /** Chain node of the list of tables to drop in the background. */ struct row_mysql_drop_t{ - char* table_name; /*!< table name */ + table_id_t table_id; /*!< table id */ UT_LIST_NODE_T(row_mysql_drop_t)row_mysql_drop_list; /*!< list chain node */ }; @@ -136,19 +136,6 @@ row_mysql_is_system_table( || 0 == strcmp(name + 6, "db")); } -/*********************************************************************//** -If a table is not yet in the drop list, adds the table to the list of tables -which the master thread drops in background. We need this on Unix because in -ALTER TABLE MySQL may call drop table even if the table has running queries on -it. Also, if there are running foreign key checks on the table, we drop the -table lazily. -@return TRUE if the table was not yet in the drop list, and was added there */ -static -ibool -row_add_table_to_background_drop_list( -/*==================================*/ - const char* name); /*!< in: table name */ - /*******************************************************************//** Delays an INSERT, DELETE or UPDATE operation if the purge is lagging. */ static @@ -2727,7 +2714,7 @@ loop: mutex_enter(&row_drop_list_mutex); ut_a(row_mysql_drop_list_inited); - +next: drop = UT_LIST_GET_FIRST(row_mysql_drop_list); n_tables = UT_LIST_GET_LEN(row_mysql_drop_list); @@ -2740,62 +2727,39 @@ loop: return(n_tables + n_tables_dropped); } - DBUG_EXECUTE_IF("row_drop_tables_in_background_sleep", - os_thread_sleep(5000000); - ); + table = dict_table_open_on_id(drop->table_id, FALSE, + DICT_TABLE_OP_NORMAL); - table = dict_table_open_on_name(drop->table_name, FALSE, FALSE, - DICT_ERR_IGNORE_NONE); - - if (table == NULL) { - /* If for some reason the table has already been dropped - through some other mechanism, do not try to drop it */ - - goto already_dropped; - } - - if (!table->to_be_dropped) { - /* There is a scenario: the old table is dropped - just after it's added into drop list, and new - table with the same name is created, then we try - to drop the new table in background. */ - dict_table_close(table, FALSE, FALSE); - - goto already_dropped; + if (!table) { + n_tables_dropped++; + mutex_enter(&row_drop_list_mutex); + UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); + MONITOR_DEC(MONITOR_BACKGROUND_DROP_TABLE); + ut_free(drop); + goto next; } ut_a(!table->can_be_evicted); + if (!table->to_be_dropped) { + dict_table_close(table, FALSE, FALSE); + + mutex_enter(&row_drop_list_mutex); + UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); + UT_LIST_ADD_LAST(row_mysql_drop_list, row_mysql_drop_list, + drop); + goto next; + } + dict_table_close(table, FALSE, FALSE); if (DB_SUCCESS != row_drop_table_for_mysql_in_background( - drop->table_name)) { + table->name)) { /* If the DROP fails for some table, we return, and let the main thread retry later */ - return(n_tables + n_tables_dropped); } - n_tables_dropped++; - -already_dropped: - mutex_enter(&row_drop_list_mutex); - - UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); - - MONITOR_DEC(MONITOR_BACKGROUND_DROP_TABLE); - - ut_print_timestamp(stderr); - fputs(" InnoDB: Dropped table ", stderr); - ut_print_name(stderr, NULL, TRUE, drop->table_name); - fputs(" in background drop queue.\n", stderr); - - mem_free(drop->table_name); - - mem_free(drop); - - mutex_exit(&row_drop_list_mutex); - goto loop; } @@ -2827,14 +2791,13 @@ which the master thread drops in background. We need this on Unix because in ALTER TABLE MySQL may call drop table even if the table has running queries on it. Also, if there are running foreign key checks on the table, we drop the table lazily. -@return TRUE if the table was not yet in the drop list, and was added there */ +@return whether background DROP TABLE was scheduled for the first time */ static -ibool -row_add_table_to_background_drop_list( -/*==================================*/ - const char* name) /*!< in: table name */ +bool +row_add_table_to_background_drop_list(table_id_t table_id) { row_mysql_drop_t* drop; + bool added = true; mutex_enter(&row_drop_list_mutex); @@ -2845,31 +2808,21 @@ row_add_table_to_background_drop_list( drop != NULL; drop = UT_LIST_GET_NEXT(row_mysql_drop_list, drop)) { - if (strcmp(drop->table_name, name) == 0) { - /* Already in the list */ - - mutex_exit(&row_drop_list_mutex); - - return(FALSE); + if (drop->table_id == table_id) { + added = false; + goto func_exit; } } - drop = static_cast( - mem_alloc(sizeof(row_mysql_drop_t))); - - drop->table_name = mem_strdup(name); + drop = static_cast(ut_malloc(sizeof *drop)); + drop->table_id = table_id; UT_LIST_ADD_LAST(row_mysql_drop_list, row_mysql_drop_list, drop); MONITOR_INC(MONITOR_BACKGROUND_DROP_TABLE); - - /* fputs("InnoDB: Adding table ", stderr); - ut_print_name(stderr, trx, TRUE, drop->table_name); - fputs(" to background drop list\n", stderr); */ - +func_exit: mutex_exit(&row_drop_list_mutex); - - return(TRUE); + return added; } /*********************************************************************//** @@ -4043,7 +3996,7 @@ row_drop_table_for_mysql( DBUG_EXECUTE_IF("row_drop_table_add_to_background", - row_add_table_to_background_drop_list(table->name); + row_add_table_to_background_drop_list(table->id); err = DB_SUCCESS; goto funct_exit; ); @@ -4055,33 +4008,22 @@ row_drop_table_for_mysql( checks take an IS or IX lock on the table. */ if (table->n_foreign_key_checks_running > 0) { - - const char* save_tablename = table->name; - ibool added; - - added = row_add_table_to_background_drop_list(save_tablename); - - if (added) { + if (row_add_table_to_background_drop_list(table->id)) { ut_print_timestamp(stderr); fputs(" InnoDB: You are trying to drop table ", stderr); - ut_print_name(stderr, trx, TRUE, save_tablename); + ut_print_name(stderr, trx, TRUE, table->name); fputs("\n" "InnoDB: though there is a" " foreign key check running on it.\n" "InnoDB: Adding the table to" " the background drop queue.\n", stderr); - - /* We return DB_SUCCESS to MySQL though the drop will - happen lazily later */ - - err = DB_SUCCESS; - } else { - /* The table is already in the background drop list */ - err = DB_ERROR; } + /* We return DB_SUCCESS to MySQL though the drop will + happen lazily later */ + err = DB_SUCCESS; goto funct_exit; } @@ -4103,11 +4045,7 @@ row_drop_table_for_mysql( lock_remove_all_on_table(table, TRUE); ut_a(table->n_rec_locks == 0); } else if (table->n_ref_count > 0 || table->n_rec_locks > 0) { - ibool added; - - added = row_add_table_to_background_drop_list(table->name); - - if (added) { + if (row_add_table_to_background_drop_list(table->id)) { ut_print_timestamp(stderr); fputs(" InnoDB: Warning: MySQL is" " trying to drop table ", stderr); diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index ebdee381713..8a6e2728342 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -72,7 +72,7 @@ UNIV_INTERN ibool row_rollback_on_timeout = FALSE; /** Chain node of the list of tables to drop in the background. */ struct row_mysql_drop_t{ - char* table_name; /*!< table name */ + table_id_t table_id; /*!< table id */ UT_LIST_NODE_T(row_mysql_drop_t)row_mysql_drop_list; /*!< list chain node */ }; @@ -135,19 +135,6 @@ row_mysql_is_system_table( || 0 == strcmp(name + 6, "db")); } -/*********************************************************************//** -If a table is not yet in the drop list, adds the table to the list of tables -which the master thread drops in background. We need this on Unix because in -ALTER TABLE MySQL may call drop table even if the table has running queries on -it. Also, if there are running foreign key checks on the table, we drop the -table lazily. -@return TRUE if the table was not yet in the drop list, and was added there */ -static -ibool -row_add_table_to_background_drop_list( -/*==================================*/ - const char* name); /*!< in: table name */ - /*******************************************************************//** Delays an INSERT, DELETE or UPDATE operation if the purge is lagging. */ static @@ -2739,7 +2726,7 @@ loop: mutex_enter(&row_drop_list_mutex); ut_a(row_mysql_drop_list_inited); - +next: drop = UT_LIST_GET_FIRST(row_mysql_drop_list); n_tables = UT_LIST_GET_LEN(row_mysql_drop_list); @@ -2752,62 +2739,39 @@ loop: return(n_tables + n_tables_dropped); } - DBUG_EXECUTE_IF("row_drop_tables_in_background_sleep", - os_thread_sleep(5000000); - ); + table = dict_table_open_on_id(drop->table_id, FALSE, + DICT_TABLE_OP_NORMAL); - table = dict_table_open_on_name(drop->table_name, FALSE, FALSE, - DICT_ERR_IGNORE_NONE); - - if (table == NULL) { - /* If for some reason the table has already been dropped - through some other mechanism, do not try to drop it */ - - goto already_dropped; - } - - if (!table->to_be_dropped) { - /* There is a scenario: the old table is dropped - just after it's added into drop list, and new - table with the same name is created, then we try - to drop the new table in background. */ - dict_table_close(table, FALSE, FALSE); - - goto already_dropped; + if (!table) { + n_tables_dropped++; + mutex_enter(&row_drop_list_mutex); + UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); + MONITOR_DEC(MONITOR_BACKGROUND_DROP_TABLE); + ut_free(drop); + goto next; } ut_a(!table->can_be_evicted); + if (!table->to_be_dropped) { + dict_table_close(table, FALSE, FALSE); + + mutex_enter(&row_drop_list_mutex); + UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); + UT_LIST_ADD_LAST(row_mysql_drop_list, row_mysql_drop_list, + drop); + goto next; + } + dict_table_close(table, FALSE, FALSE); if (DB_SUCCESS != row_drop_table_for_mysql_in_background( - drop->table_name)) { + table->name)) { /* If the DROP fails for some table, we return, and let the main thread retry later */ - return(n_tables + n_tables_dropped); } - n_tables_dropped++; - -already_dropped: - mutex_enter(&row_drop_list_mutex); - - UT_LIST_REMOVE(row_mysql_drop_list, row_mysql_drop_list, drop); - - MONITOR_DEC(MONITOR_BACKGROUND_DROP_TABLE); - - ut_print_timestamp(stderr); - fputs(" InnoDB: Dropped table ", stderr); - ut_print_name(stderr, NULL, TRUE, drop->table_name); - fputs(" in background drop queue.\n", stderr); - - mem_free(drop->table_name); - - mem_free(drop); - - mutex_exit(&row_drop_list_mutex); - goto loop; } @@ -2839,14 +2803,13 @@ which the master thread drops in background. We need this on Unix because in ALTER TABLE MySQL may call drop table even if the table has running queries on it. Also, if there are running foreign key checks on the table, we drop the table lazily. -@return TRUE if the table was not yet in the drop list, and was added there */ +@return whether background DROP TABLE was scheduled for the first time */ static -ibool -row_add_table_to_background_drop_list( -/*==================================*/ - const char* name) /*!< in: table name */ +bool +row_add_table_to_background_drop_list(table_id_t table_id) { row_mysql_drop_t* drop; + bool added = true; mutex_enter(&row_drop_list_mutex); @@ -2857,31 +2820,21 @@ row_add_table_to_background_drop_list( drop != NULL; drop = UT_LIST_GET_NEXT(row_mysql_drop_list, drop)) { - if (strcmp(drop->table_name, name) == 0) { - /* Already in the list */ - - mutex_exit(&row_drop_list_mutex); - - return(FALSE); + if (drop->table_id == table_id) { + added = false; + goto func_exit; } } - drop = static_cast( - mem_alloc(sizeof(row_mysql_drop_t))); - - drop->table_name = mem_strdup(name); + drop = static_cast(ut_malloc(sizeof *drop)); + drop->table_id = table_id; UT_LIST_ADD_LAST(row_mysql_drop_list, row_mysql_drop_list, drop); MONITOR_INC(MONITOR_BACKGROUND_DROP_TABLE); - - /* fputs("InnoDB: Adding table ", stderr); - ut_print_name(stderr, trx, TRUE, drop->table_name); - fputs(" to background drop list\n", stderr); */ - +func_exit: mutex_exit(&row_drop_list_mutex); - - return(TRUE); + return added; } /*********************************************************************//** @@ -4057,7 +4010,7 @@ row_drop_table_for_mysql( DBUG_EXECUTE_IF("row_drop_table_add_to_background", - row_add_table_to_background_drop_list(table->name); + row_add_table_to_background_drop_list(table->id); err = DB_SUCCESS; goto funct_exit; ); @@ -4069,33 +4022,22 @@ row_drop_table_for_mysql( checks take an IS or IX lock on the table. */ if (table->n_foreign_key_checks_running > 0) { - - const char* save_tablename = table->name; - ibool added; - - added = row_add_table_to_background_drop_list(save_tablename); - - if (added) { + if (row_add_table_to_background_drop_list(table->id)) { ut_print_timestamp(stderr); fputs(" InnoDB: You are trying to drop table ", stderr); - ut_print_name(stderr, trx, TRUE, save_tablename); + ut_print_name(stderr, trx, TRUE, table->name); fputs("\n" "InnoDB: though there is a" " foreign key check running on it.\n" "InnoDB: Adding the table to" " the background drop queue.\n", stderr); - - /* We return DB_SUCCESS to MySQL though the drop will - happen lazily later */ - - err = DB_SUCCESS; - } else { - /* The table is already in the background drop list */ - err = DB_ERROR; } + /* We return DB_SUCCESS to MySQL though the drop will + happen lazily later */ + err = DB_SUCCESS; goto funct_exit; } @@ -4117,11 +4059,7 @@ row_drop_table_for_mysql( lock_remove_all_on_table(table, TRUE); ut_a(table->n_rec_locks == 0); } else if (table->n_ref_count > 0 || table->n_rec_locks > 0) { - ibool added; - - added = row_add_table_to_background_drop_list(table->name); - - if (added) { + if (row_add_table_to_background_drop_list(table->id)) { ut_print_timestamp(stderr); fputs(" InnoDB: Warning: MySQL is" " trying to drop table ", stderr); From ca9ed393ef56b963425d8d9dc101641ae7f1fd3a Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Mon, 18 Dec 2017 22:59:05 +0200 Subject: [PATCH 36/51] MDEV-13073. rpl.perf_buildin_semisync_issue40 is corrected to expect the Rpl_semi_sync_master_clients value of 1 (ll.307..). Explicit sleeps are converted to wait_xyz. --- .../r/perf_buildin_semisync_issue40.result | 14 --------- .../rpl/t/perf_buildin_semisync_issue40.test | 31 +++++++++++++------ 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result b/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result index 96b07f665d4..9607e8a7998 100644 --- a/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result +++ b/mysql-test/suite/rpl/r/perf_buildin_semisync_issue40.result @@ -277,14 +277,7 @@ SET GLOBAL debug = 'd,rpl_semisync_simulate_create_thread_failure'; Warnings: Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead SET GLOBAL rpl_semi_sync_master_enabled= ON; -SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled'; -Variable_name Value -rpl_semi_sync_master_enabled OFF # Test failure of pthread_join -SET GLOBAL DEBUG = 'd,rpl_semisync_simulate_thread_join_failure'; -Warnings: -Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead -SET GLOBAL rpl_semi_sync_master_enabled= ON; SET GLOBAL rpl_semi_sync_master_enabled= OFF; # # Failure on registering semisync slave @@ -299,10 +292,6 @@ include/wait_for_slave_io_to_stop.inc START SLAVE IO_THREAD; include/wait_for_slave_io_to_start.inc connection con1; -# Should be Zero. -show status like 'Rpl_semi_sync_master_clients'; -Variable_name Value -Rpl_semi_sync_master_clients 0 SET GLOBAL debug=''; Warnings: Warning 1287 '@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead @@ -325,9 +314,6 @@ connection slave; connection con1; INSERT INTO t2 values (1); connection slave; -SELECT * FROM t2; -c1 -1 connection con2; connection con1; connection slave; diff --git a/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test index 113e691e0af..a12730fd8c9 100644 --- a/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test +++ b/mysql-test/suite/rpl/t/perf_buildin_semisync_issue40.test @@ -272,15 +272,18 @@ connection con1; SET GLOBAL rpl_semi_sync_master_enabled = 0; SET GLOBAL debug = 'd,rpl_semisync_simulate_create_thread_failure'; SET GLOBAL rpl_semi_sync_master_enabled= ON; ---sleep 3 -SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled'; +--let $wait_condition= SELECT @@global.rpl_semi_sync_master_enabled = 0 +--source include/wait_condition.inc + +# Todo: implement the thread join failure simulation --echo # Test failure of pthread_join -SET GLOBAL DEBUG = 'd,rpl_semisync_simulate_thread_join_failure'; -SET GLOBAL rpl_semi_sync_master_enabled= ON; ---sleep 3 +#SET GLOBAL DEBUG = 'd,rpl_semisync_simulate_thread_join_failure'; +#SET GLOBAL rpl_semi_sync_master_enabled= ON; +# +#--let $wait_condition= SELECT @@global.rpl_semi_sync_master_enabled = 0 +#--source include/wait_condition.inc SET GLOBAL rpl_semi_sync_master_enabled= OFF; ---sleep 3 --echo # --echo # Failure on registering semisync slave @@ -295,10 +298,17 @@ START SLAVE IO_THREAD; --source include/wait_for_slave_io_to_start.inc connection con1; ---echo # Should be Zero. -show status like 'Rpl_semi_sync_master_clients'; +#--echo # Should be Zero. +# Todo: implement the add_slave_failure simulation. Meanwhile +# the status will be 1. +# show status like 'Rpl_semi_sync_master_clients'; SET GLOBAL debug=''; +--let $status_var= Rpl_semi_sync_master_clients +--let $status_var_value= 1 +--let $status_type= GLOBAL +--source include/wait_for_status_var.inc + connection slave; --disable_warnings START SLAVE IO_THREAD; @@ -324,9 +334,10 @@ connection con1; #SET DEBUG_SYNC = 'before_sync_binlog_file SIGNAL before_sync_done WAIT_FOR continue'; send INSERT INTO t2 values (1); ---sleep 3 connection slave; -SELECT * FROM t2; +--let $table= t2 +--let $count= 1 +--source include/wait_until_rows_count.inc connection con2; #SET DEBUG_SYNC= "now WAIT_FOR before_sync_done"; From 64f1fab068582a82ac54ce4793542655d5fc58ab Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 19 Dec 2017 10:24:50 +1100 Subject: [PATCH 37/51] MDEV-12128: systemd - add Documentation= directives --- support-files/mariadb.service.in | 4 +++- support-files/mariadb@.service.in | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in index 6a307b2c41f..2b02a1d0db9 100644 --- a/support-files/mariadb.service.in +++ b/support-files/mariadb.service.in @@ -13,7 +13,9 @@ # and probably others [Unit] -Description=MariaDB database server +Description=MariaDB @VERSION@ database server +Documentation=man:mysqld(8) +Documentation=https://mariadb.com/kb/en/library/systemd/ After=network.target After=syslog.target diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index 410e7433b2b..a4a6a0ffef6 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -18,7 +18,9 @@ # Inspired from https://gitweb.gentoo.org/repo/gentoo.git/tree/dev-db/mysql-init-scripts/files/mysqld_at.service [Unit] -Description=MariaDB database server +Description=MariaDB @VERSION@ database server (multi-instance) +Documentation=man:mysqld(8) +Documentation=https://mariadb.com/kb/en/library/systemd/ After=network.target After=syslog.target From ce4cdfa0f8691f47366d50acd6049b8cd7520e8a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 19 Dec 2017 08:56:31 +1100 Subject: [PATCH 38/51] MDEV-13809: [service] should [Service] in systemd service files --- support-files/mariadb.service.in | 5 +++-- support-files/mariadb@.service.in | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in index 2b02a1d0db9..fe00f160d28 100644 --- a/support-files/mariadb.service.in +++ b/support-files/mariadb.service.in @@ -78,7 +78,7 @@ ExecStartPre=/bin/sh -c "[ ! -e @bindir@/galera_recovery ] && VAR= || \ # Start main service # MYSQLD_OPTS here is for users to set in /etc/systemd/system/mariadb.service.d/MY_SPECIAL.conf -# Use the [service] section and Environment="MYSQLD_OPTS=...". +# Use the [Service] section and Environment="MYSQLD_OPTS=...". # This isn't a replacement for my.cnf. # _WSREP_NEW_CLUSTER is for the exclusive use of the script galera_new_cluster ExecStart=@sbindir@/mysqld $MYSQLD_OPTS $_WSREP_NEW_CLUSTER $_WSREP_START_POSITION @@ -105,7 +105,8 @@ UMask=007 ## ## ## by creating a file in /etc/systemd/system/mariadb.service.d/MY_SPECIAL.conf -## and adding/setting the following will override this file's settings. +## and adding/setting the following under [Service] will override this file's +## settings. # Useful options not previously available in [mysqld_safe] diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index a4a6a0ffef6..000724d7fe2 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -91,7 +91,7 @@ ExecStartPre=/bin/sh -c "[ ! -e @bindir@/galera_recovery ] && VAR= || \ # Start main service # MYSQLD_OPTS here is for users to set in /etc/systemd/system/mariadb@.service.d/MY_SPECIAL.conf -# Use the [service] section and Environment="MYSQLD_OPTS=...". +# Use the [Service] section and Environment="MYSQLD_OPTS=...". # This isn't a replacement for my.cnf. # _WSREP_NEW_CLUSTER is for the exclusive use of the script galera_new_cluster @@ -126,7 +126,8 @@ UMask=007 ## ## ## by creating a file in /etc/systemd/system/mariadb.service.d/MY_SPECIAL.conf -## and adding/setting the following will override this file's settings. +## and adding/setting the following below [Service] will override this file's +## settings. # Useful options not previously available in [mysqld_safe] From 22f2b39c147a46193c5a616a8df84d78ce3db99c Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 6 Dec 2017 00:36:17 +0300 Subject: [PATCH 39/51] fix data races in rwlock --- .../r/innodb_skip_innodb_is_tables.result | 2 +- storage/innobase/handler/i_s.cc | 24 +++---------------- storage/innobase/handler/i_s.h | 8 +++---- storage/innobase/include/sync0rw.h | 6 ----- storage/innobase/include/sync0rw.ic | 5 ---- storage/innobase/sync/sync0arr.cc | 6 ----- storage/innobase/sync/sync0rw.cc | 2 -- 7 files changed, 7 insertions(+), 46 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result index 103937fa408..79f0e73e745 100644 --- a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result +++ b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result @@ -395,6 +395,6 @@ NAME CREATE_FILE CREATE_LINE OS_WAITS Warnings: Warning 1012 InnoDB: SELECTing from INFORMATION_SCHEMA.innodb_mutexes but the InnoDB storage engine is not installed select * from information_schema.innodb_sys_semaphore_waits; -THREAD_ID OBJECT_NAME FILE LINE WAIT_TIME WAIT_OBJECT WAIT_TYPE HOLDER_THREAD_ID HOLDER_FILE HOLDER_LINE CREATED_FILE CREATED_LINE WRITER_THREAD RESERVATION_MODE READERS WAITERS_FLAG LOCK_WORD LAST_READER_FILE LAST_READER_LINE LAST_WRITER_FILE LAST_WRITER_LINE OS_WAIT_COUNT +THREAD_ID OBJECT_NAME FILE LINE WAIT_TIME WAIT_OBJECT WAIT_TYPE HOLDER_THREAD_ID HOLDER_FILE HOLDER_LINE CREATED_FILE CREATED_LINE WRITER_THREAD RESERVATION_MODE READERS WAITERS_FLAG LOCK_WORD LAST_WRITER_FILE LAST_WRITER_LINE OS_WAIT_COUNT Warnings: Warning 1012 InnoDB: SELECTing from INFORMATION_SCHEMA.innodb_sys_semaphore_waits but the InnoDB storage engine is not installed diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 8f03620b556..e737404c600 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -9462,25 +9462,7 @@ static ST_FIELD_INFO innodb_sys_semaphore_waits_fields_info[] = STRUCT_FLD(old_name, ""), STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - // SYS_SEMAPHORE_WAITS_LAST_READER_FILE 17 - {STRUCT_FLD(field_name, "LAST_READER_FILE"), - STRUCT_FLD(field_length, OS_FILE_MAX_PATH), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_MAYBE_NULL), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - - // SYS_SEMAPHORE_WAITS_LAST_READER_LINE 18 - {STRUCT_FLD(field_name, "LAST_READER_LINE"), - STRUCT_FLD(field_length, MY_INT32_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - - // SYS_SEMAPHORE_WAITS_LAST_WRITER_FILE 19 + // SYS_SEMAPHORE_WAITS_LAST_WRITER_FILE 17 {STRUCT_FLD(field_name, "LAST_WRITER_FILE"), STRUCT_FLD(field_length, OS_FILE_MAX_PATH), STRUCT_FLD(field_type, MYSQL_TYPE_STRING), @@ -9489,7 +9471,7 @@ static ST_FIELD_INFO innodb_sys_semaphore_waits_fields_info[] = STRUCT_FLD(old_name, ""), STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - // SYS_SEMAPHORE_WAITS_LAST_WRITER_LINE 20 + // SYS_SEMAPHORE_WAITS_LAST_WRITER_LINE 18 {STRUCT_FLD(field_name, "LAST_WRITER_LINE"), STRUCT_FLD(field_length, MY_INT32_NUM_DECIMAL_DIGITS), STRUCT_FLD(field_type, MYSQL_TYPE_LONG), @@ -9498,7 +9480,7 @@ static ST_FIELD_INFO innodb_sys_semaphore_waits_fields_info[] = STRUCT_FLD(old_name, ""), STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - // SYS_SEMAPHORE_WAITS_OS_WAIT_COUNT 21 + // SYS_SEMAPHORE_WAITS_OS_WAIT_COUNT 19 {STRUCT_FLD(field_name, "OS_WAIT_COUNT"), STRUCT_FLD(field_length, MY_INT32_NUM_DECIMAL_DIGITS), STRUCT_FLD(field_type, MYSQL_TYPE_LONG), diff --git a/storage/innobase/handler/i_s.h b/storage/innobase/handler/i_s.h index e07fe49f7fa..4ff2248c28e 100644 --- a/storage/innobase/handler/i_s.h +++ b/storage/innobase/handler/i_s.h @@ -127,11 +127,9 @@ HPUX aCC: HP ANSI C++ B3910B A.03.65) can't handle it. */ #define SYS_SEMAPHORE_WAITS_READERS 14 #define SYS_SEMAPHORE_WAITS_WAITERS_FLAG 15 #define SYS_SEMAPHORE_WAITS_LOCK_WORD 16 -#define SYS_SEMAPHORE_WAITS_LAST_READER_FILE 17 -#define SYS_SEMAPHORE_WAITS_LAST_READER_LINE 18 -#define SYS_SEMAPHORE_WAITS_LAST_WRITER_FILE 19 -#define SYS_SEMAPHORE_WAITS_LAST_WRITER_LINE 20 -#define SYS_SEMAPHORE_WAITS_OS_WAIT_COUNT 21 +#define SYS_SEMAPHORE_WAITS_LAST_WRITER_FILE 17 +#define SYS_SEMAPHORE_WAITS_LAST_WRITER_LINE 18 +#define SYS_SEMAPHORE_WAITS_OS_WAIT_COUNT 19 /*******************************************************************//** Auxiliary function to store ulint value in MYSQL_TYPE_LONGLONG field. diff --git a/storage/innobase/include/sync0rw.h b/storage/innobase/include/sync0rw.h index ae5f410e810..c9bc443fc55 100644 --- a/storage/innobase/include/sync0rw.h +++ b/storage/innobase/include/sync0rw.h @@ -603,9 +603,6 @@ struct rw_lock_t /** File name where lock created */ const char* cfile_name; - /** last s-lock file/line is not guaranteed to be correct */ - const char* last_s_file_name; - /** File name where last x-locked */ const char* last_x_file_name; @@ -615,9 +612,6 @@ struct rw_lock_t /** If 1 then the rw-lock is a block lock */ unsigned is_block_lock:1; - /** Line number where last time s-locked */ - unsigned last_s_line:14; - /** Line number where last time x-locked */ unsigned last_x_line:14; diff --git a/storage/innobase/include/sync0rw.ic b/storage/innobase/include/sync0rw.ic index cbbf421d9f2..8a1a3741b47 100644 --- a/storage/innobase/include/sync0rw.ic +++ b/storage/innobase/include/sync0rw.ic @@ -249,11 +249,6 @@ rw_lock_s_lock_low( ut_d(rw_lock_add_debug_info(lock, pass, RW_LOCK_S, file_name, line)); - /* These debugging values are not set safely: they may be incorrect - or even refer to a line that is invalid for the file name. */ - lock->last_s_file_name = file_name; - lock->last_s_line = line; - return(TRUE); /* locking succeeded */ } diff --git a/storage/innobase/sync/sync0arr.cc b/storage/innobase/sync/sync0arr.cc index 858b8c02e73..d03efd7653d 100644 --- a/storage/innobase/sync/sync0arr.cc +++ b/storage/innobase/sync/sync0arr.cc @@ -590,7 +590,6 @@ sync_array_cell_print( "number of readers " ULINTPF ", waiters flag %d, " "lock_word: %x\n" - "Last time read locked in file %s line %u\n" "Last time write locked in file %s line %u" #if 0 /* JAN: TODO: FIX LATER */ "\nHolder thread " ULINTPF @@ -600,8 +599,6 @@ sync_array_cell_print( rw_lock_get_reader_count(rwlock), my_atomic_load32_explicit(&rwlock->waiters, MY_MEMORY_ORDER_RELAXED), my_atomic_load32_explicit(&rwlock->lock_word, MY_MEMORY_ORDER_RELAXED), - innobase_basename(rwlock->last_s_file_name), - rwlock->last_s_line, innobase_basename(rwlock->last_x_file_name), rwlock->last_x_line #if 0 /* JAN: TODO: FIX LATER */ @@ -1401,9 +1398,6 @@ sync_arr_fill_sys_semphore_waits_table( my_atomic_load32_explicit(&rwlock->waiters, MY_MEMORY_ORDER_RELAXED))); OK(field_store_ulint(fields[SYS_SEMAPHORE_WAITS_LOCK_WORD], my_atomic_load32_explicit(&rwlock->lock_word, MY_MEMORY_ORDER_RELAXED))); - OK(field_store_string(fields[SYS_SEMAPHORE_WAITS_LAST_READER_FILE], innobase_basename(rwlock->last_s_file_name))); - OK(fields[SYS_SEMAPHORE_WAITS_LAST_READER_LINE]->store(rwlock->last_s_line, true)); - fields[SYS_SEMAPHORE_WAITS_LAST_READER_LINE]->set_notnull(); OK(field_store_string(fields[SYS_SEMAPHORE_WAITS_LAST_WRITER_FILE], innobase_basename(rwlock->last_x_file_name))); OK(fields[SYS_SEMAPHORE_WAITS_LAST_WRITER_LINE]->store(rwlock->last_x_line, true)); fields[SYS_SEMAPHORE_WAITS_LAST_WRITER_LINE]->set_notnull(); diff --git a/storage/innobase/sync/sync0rw.cc b/storage/innobase/sync/sync0rw.cc index 509fd9cf19b..13c81d7333d 100644 --- a/storage/innobase/sync/sync0rw.cc +++ b/storage/innobase/sync/sync0rw.cc @@ -239,9 +239,7 @@ rw_lock_create_func( ut_ad(cline <= 8192); lock->cline = cline; lock->count_os_wait = 0; - lock->last_s_file_name = "not yet reserved"; lock->last_x_file_name = "not yet reserved"; - lock->last_s_line = 0; lock->last_x_line = 0; lock->event = os_event_create(0); lock->wait_ex_event = os_event_create(0); From 3464b670257e954cd6ee88c95da7b4a217d0ea82 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 5 Dec 2017 12:39:37 +1100 Subject: [PATCH 40/51] mysys: handle T, P, E suffixs Signed-off-by: Daniel Black --- mysys/my_getopt.c | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index c9bfe2e1eaa..0ea062988f6 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -969,31 +969,43 @@ my_bool getopt_compare_strings(register const char *s, register const char *t, /* function: eval_num_suffix - Transforms suffix like k/m/g to their real value. + Transforms suffix like k/m/g/t/p/e to their real value. */ -static inline long eval_num_suffix(char *suffix, int *error) +static inline ulonglong eval_num_suffix(char *suffix, int *error) { - long num= 1; - if (*suffix == 'k' || *suffix == 'K') - num*= 1024L; - else if (*suffix == 'm' || *suffix == 'M') - num*= 1024L * 1024L; - else if (*suffix == 'g' || *suffix == 'G') - num*= 1024L * 1024L * 1024L; - else if (*suffix) - { + switch (*suffix) { + case '\0': + return 1ULL; + case 'k': + case 'K': + return 1ULL << 10; + case 'm': + case 'M': + return 1ULL << 20; + case 'g': + case 'G': + return 1ULL << 30; + case 't': + case 'T': + return 1ULL << 40; + case 'p': + case 'P': + return 1ULL << 50; + case 'e': + case 'E': + return 1ULL << 60; + default: *error= 1; - return 0; + return 0ULL; } - return num; } /* function: eval_num_suffix_ll Transforms a number with a suffix to real number. Suffix can - be k|K for kilo, m|M for mega or g|G for giga. + be k|K for kilo, m|M for mega, etc. */ static longlong eval_num_suffix_ll(char *argument, @@ -1026,7 +1038,7 @@ static longlong eval_num_suffix_ll(char *argument, function: eval_num_suffix_ull Transforms a number with a suffix to positive Integer. Suffix can - be k|K for kilo, m|M for mega or g|G for giga. + be k|K for kilo, m|M for mega, etc. */ static ulonglong eval_num_suffix_ull(char *argument, From 079c3599711b9cbd3ce323f32cf99693cc3d5e3b Mon Sep 17 00:00:00 2001 From: Galina Shalygina Date: Tue, 19 Dec 2017 11:49:40 +0200 Subject: [PATCH 41/51] MDEV-14629: failing assertion when a user-defined variable is defined by the recursive CTE During the user-defined variable defined by the recursive CTE handling procedure check_dependencies_in_with_clauses that checks dependencies between the tables that are defined in the CTE and find recursive definitions wasn't called. --- mysql-test/r/cte_recursive.result | 47 ++++++++++++++++++++++++++++ mysql-test/t/cte_recursive.test | 51 +++++++++++++++++++++++++++++++ sql/sql_parse.cc | 3 ++ 3 files changed, 101 insertions(+) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index b744f24bb57..ece5421e279 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -2897,3 +2897,50 @@ n 1 2 3 +# +# mdev-14629: a user-defined variable is defined by the recursive CTE +# +set @var= +( +with recursive cte_tab(a) as ( +select 1 +union +select a+1 from cte_tab +where a<3) +select count(*) from cte_tab +); +select @var; +@var +3 +create table t1(a int, b int); +insert into t1 values (3,8),(1,5),(5,7),(7,4),(4,3); +set @var= +( +with recursive summ(a,s) as ( +select 1, 0 union +select t1.b, t1.b+summ.s from summ, t1 +where summ.a=t1.a) +select s from summ +order by a desc +limit 1 +); +select @var; +@var +27 +set @var= +( +with recursive +cte_1 as ( +select 1 +union +select * from cte_2), +cte_2 as ( +select * from cte_1 +union +select a from t1, cte_2 +where t1.a=cte_2.a) +select * from cte_2 +limit 1 +); +ERROR HY000: Unacceptable mutual recursion with anchored table 'cte_1' +drop table t1; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 36d948251c3..928f8a4334b 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -1947,3 +1947,54 @@ cte2 as ( ) SELECT * FROM cte1; + +--echo # +--echo # mdev-14629: a user-defined variable is defined by the recursive CTE +--echo # + +set @var= +( + with recursive cte_tab(a) as ( + select 1 + union + select a+1 from cte_tab + where a<3) + select count(*) from cte_tab +); + +select @var; + +create table t1(a int, b int); +insert into t1 values (3,8),(1,5),(5,7),(7,4),(4,3); + +set @var= +( + with recursive summ(a,s) as ( + select 1, 0 union + select t1.b, t1.b+summ.s from summ, t1 + where summ.a=t1.a) + select s from summ + order by a desc + limit 1 +); + +select @var; + +--ERROR ER_UNACCEPTABLE_MUTUAL_RECURSION +set @var= +( + with recursive + cte_1 as ( + select 1 + union + select * from cte_2), + cte_2 as ( + select * from cte_1 + union + select a from t1, cte_2 + where t1.a=cte_2.a) + select * from cte_2 + limit 1 +); + +drop table t1; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index d964831a098..7a4530082eb 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4818,6 +4818,9 @@ end_with_restore_list: { List *lex_var_list= &lex->var_list; + if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) + goto error; + if ((check_table_access(thd, SELECT_ACL, all_tables, FALSE, UINT_MAX, FALSE) || open_and_lock_tables(thd, all_tables, TRUE, 0))) goto error; From ccb3550221497e7e652883b79e4a01451a55c4d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 19 Dec 2017 14:37:48 +0200 Subject: [PATCH 42/51] Replace MLOG_UNDO_INIT with MLOG_2BYTES, MLOG_4BYTES --- storage/innobase/include/mtr0types.h | 3 -- storage/innobase/include/trx0undo.h | 10 ----- storage/innobase/log/log0recv.cc | 7 ---- storage/innobase/trx/trx0undo.cc | 63 +++++----------------------- 4 files changed, 11 insertions(+), 72 deletions(-) diff --git a/storage/innobase/include/mtr0types.h b/storage/innobase/include/mtr0types.h index ac24812cdfc..7bf162e90a9 100644 --- a/storage/innobase/include/mtr0types.h +++ b/storage/innobase/include/mtr0types.h @@ -106,9 +106,6 @@ enum mlog_id_t { /** erase an undo log page end */ MLOG_UNDO_ERASE_END = 21, - /** initialize a page in an undo log */ - MLOG_UNDO_INIT = 22, - /** create an undo log header */ MLOG_UNDO_HDR_CREATE = 25, diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h index 51f8035d886..b51484a61cb 100644 --- a/storage/innobase/include/trx0undo.h +++ b/storage/innobase/include/trx0undo.h @@ -315,16 +315,6 @@ bool trx_undo_truncate_tablespace( undo::Truncate* undo_trunc); -/***********************************************************//** -Parses the redo log entry of an undo log page initialization. -@return end of log record or NULL */ -byte* -trx_undo_parse_page_init( -/*=====================*/ - const byte* ptr, /*!< in: buffer */ - const byte* end_ptr,/*!< in: buffer end */ - page_t* page, /*!< in: page or NULL */ - mtr_t* mtr); /*!< in: mtr or NULL */ /** Parse the redo log entry of an undo log page header create. @param[in] ptr redo log record @param[in] end_ptr end of log buffer diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 1097dafb9b6..63edfdd63e8 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1402,10 +1402,6 @@ parse_log: ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); ptr = trx_undo_parse_erase_page_end(ptr, end_ptr, page, mtr); break; - case MLOG_UNDO_INIT: - /* Allow anything in page_type when creating a page. */ - ptr = trx_undo_parse_page_init(ptr, end_ptr, page, mtr); - break; case MLOG_UNDO_HDR_CREATE: ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); ptr = trx_undo_parse_page_header(ptr, end_ptr, page, mtr); @@ -3636,9 +3632,6 @@ get_mlog_string(mlog_id_t type) case MLOG_UNDO_ERASE_END: return("MLOG_UNDO_ERASE_END"); - case MLOG_UNDO_INIT: - return("MLOG_UNDO_INIT"); - case MLOG_UNDO_HDR_CREATE: return("MLOG_UNDO_HDR_CREATE"); diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 6dcdf4c51a6..1340d19aee8 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -309,47 +309,6 @@ trx_undo_get_first_rec( /*============== UNDO LOG FILE COPY CREATION AND FREEING ==================*/ -/**********************************************************************//** -Writes the mtr log entry of an undo log page initialization. */ -UNIV_INLINE -void -trx_undo_page_init_log( -/*===================*/ - page_t* undo_page, /*!< in: undo log page */ - mtr_t* mtr) /*!< in: mtr */ -{ - mlog_write_initial_log_record(undo_page, MLOG_UNDO_INIT, mtr); - - mlog_catenate_ulint_compressed(mtr, 0); -} - -/***********************************************************//** -Parses the redo log entry of an undo log page initialization. -@return end of log record or NULL */ -byte* -trx_undo_parse_page_init( -/*=====================*/ - const byte* ptr, /*!< in: buffer */ - const byte* end_ptr,/*!< in: buffer end */ - page_t* page, /*!< in: page or NULL */ - mtr_t* mtr) /*!< in: mtr or NULL */ -{ - if (mach_parse_compressed(&ptr, end_ptr)) { - recv_sys->found_corrupt_log = true; - } - - if (ptr == NULL) { - - return(NULL); - } - - if (page) { - trx_undo_page_init(page, mtr); - } - - return(const_cast(ptr)); -} - /********************************************************************//** Initializes the fields in an undo log segment page. */ static @@ -361,18 +320,18 @@ trx_undo_page_init( { trx_upagef_t* page_hdr; + mlog_write_ulint(undo_page + FIL_PAGE_TYPE, + FIL_PAGE_UNDO_LOG, MLOG_2BYTES, mtr); + compile_time_assert(TRX_UNDO_PAGE_TYPE == 0); + compile_time_assert(TRX_UNDO_PAGE_START == 2); + compile_time_assert(TRX_UNDO_PAGE_NODE == TRX_UNDO_PAGE_FREE + 2); + page_hdr = undo_page + TRX_UNDO_PAGE_HDR; - - *reinterpret_cast(page_hdr + TRX_UNDO_PAGE_TYPE) = 0; - - mach_write_to_2(page_hdr + TRX_UNDO_PAGE_START, - TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE); - mach_write_to_2(page_hdr + TRX_UNDO_PAGE_FREE, - TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE); - - fil_page_set_type(undo_page, FIL_PAGE_UNDO_LOG); - - trx_undo_page_init_log(undo_page, mtr); + mlog_write_ulint(page_hdr, TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE, + MLOG_4BYTES, mtr); + mlog_write_ulint(page_hdr + TRX_UNDO_PAGE_FREE, + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE, + MLOG_2BYTES, mtr); } /***************************************************************//** From 9ee8917dfd4b6c9d342cbc030ca71bad637fa24a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 19 Dec 2017 15:34:02 +0200 Subject: [PATCH 43/51] Replace MLOG_UNDO_INSERT with MLOG_WRITE_STRING, MLOG_2BYTES --- storage/innobase/include/mtr0types.h | 3 -- storage/innobase/include/trx0rec.h | 9 ---- storage/innobase/log/log0recv.cc | 7 --- storage/innobase/trx/trx0rec.cc | 81 ++++------------------------ 4 files changed, 9 insertions(+), 91 deletions(-) diff --git a/storage/innobase/include/mtr0types.h b/storage/innobase/include/mtr0types.h index 7bf162e90a9..bdb16706f73 100644 --- a/storage/innobase/include/mtr0types.h +++ b/storage/innobase/include/mtr0types.h @@ -100,9 +100,6 @@ enum mlog_id_t { /** Create an index page */ MLOG_PAGE_CREATE = 19, - /** Insert entry in an undo log */ - MLOG_UNDO_INSERT = 20, - /** erase an undo log page end */ MLOG_UNDO_ERASE_END = 21, diff --git a/storage/innobase/include/trx0rec.h b/storage/innobase/include/trx0rec.h index 6712f28a6e7..5c245d23abc 100644 --- a/storage/innobase/include/trx0rec.h +++ b/storage/innobase/include/trx0rec.h @@ -237,15 +237,6 @@ trx_undo_prev_version_build( And if we read "after image" of undo log */ /***********************************************************//** -Parses a redo log record of adding an undo log record. -@return end of log record or NULL */ -byte* -trx_undo_parse_add_undo_rec( -/*========================*/ - byte* ptr, /*!< in: buffer */ - byte* end_ptr,/*!< in: buffer end */ - page_t* page); /*!< in: page or NULL */ -/***********************************************************//** Parses a redo log record of erasing of an undo page end. @return end of log record or NULL */ byte* diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 63edfdd63e8..6e52cebfb9f 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1394,10 +1394,6 @@ parse_log: page_parse_create(block, type == MLOG_COMP_PAGE_CREATE_RTREE, true); break; - case MLOG_UNDO_INSERT: - ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); - ptr = trx_undo_parse_add_undo_rec(ptr, end_ptr, page); - break; case MLOG_UNDO_ERASE_END: ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); ptr = trx_undo_parse_erase_page_end(ptr, end_ptr, page, mtr); @@ -3626,9 +3622,6 @@ get_mlog_string(mlog_id_t type) case MLOG_PAGE_CREATE: return("MLOG_PAGE_CREATE"); - case MLOG_UNDO_INSERT: - return("MLOG_UNDO_INSERT"); - case MLOG_UNDO_ERASE_END: return("MLOG_UNDO_ERASE_END"); diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index ac116a572a8..9b3619fbdba 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -65,78 +65,15 @@ trx_undof_page_add_undo_rec_log( ulint new_free, /*!< in: end offset of the entry */ mtr_t* mtr) /*!< in: mtr */ { - byte* log_ptr; - const byte* log_end; - ulint len; - - log_ptr = mlog_open(mtr, 11 + 13 + MLOG_BUF_MARGIN); - - if (log_ptr == NULL) { - - return; - } - - log_end = &log_ptr[11 + 13 + MLOG_BUF_MARGIN]; - log_ptr = mlog_write_initial_log_record_fast( - undo_page, MLOG_UNDO_INSERT, log_ptr, mtr); - len = new_free - old_free - 4; - - mach_write_to_2(log_ptr, len); - log_ptr += 2; - - if (log_ptr + len <= log_end) { - memcpy(log_ptr, undo_page + old_free + 2, len); - mlog_close(mtr, log_ptr + len); - } else { - mlog_close(mtr, log_ptr); - mlog_catenate_string(mtr, undo_page + old_free + 2, len); - } -} - -/***********************************************************//** -Parses a redo log record of adding an undo log record. -@return end of log record or NULL */ -byte* -trx_undo_parse_add_undo_rec( -/*========================*/ - byte* ptr, /*!< in: buffer */ - byte* end_ptr,/*!< in: buffer end */ - page_t* page) /*!< in: page or NULL */ -{ - ulint len; - byte* rec; - ulint first_free; - - if (end_ptr < ptr + 2) { - - return(NULL); - } - - len = mach_read_from_2(ptr); - ptr += 2; - - if (end_ptr < ptr + len) { - - return(NULL); - } - - if (page == NULL) { - - return(ptr + len); - } - - first_free = mach_read_from_2(page + TRX_UNDO_PAGE_HDR - + TRX_UNDO_PAGE_FREE); - rec = page + first_free; - - mach_write_to_2(rec, first_free + 4 + len); - mach_write_to_2(rec + 2 + len, first_free); - - mach_write_to_2(page + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_FREE, - first_free + 4 + len); - ut_memcpy(rec + 2, ptr, len); - - return(ptr + len); + ut_ad(old_free >= TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE); + ut_ad(new_free >= old_free); + ut_ad(new_free < UNIV_PAGE_SIZE); + ut_ad(mach_read_from_2(undo_page + + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_FREE) + == new_free); + mlog_write_ulint(undo_page + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_FREE, + new_free, MLOG_2BYTES, mtr); + mlog_log_string(undo_page + old_free, new_free - old_free, mtr); } /**********************************************************************//** From 0fd3def284b78dff71590686e8f82571fc3808e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 19 Dec 2017 15:36:36 +0200 Subject: [PATCH 44/51] Remove MLOG_UNDO_ERASE_END --- storage/innobase/include/mtr0types.h | 3 --- storage/innobase/include/trx0rec.h | 11 ----------- storage/innobase/log/log0recv.cc | 7 ------- storage/innobase/trx/trx0rec.cc | 27 +-------------------------- 4 files changed, 1 insertion(+), 47 deletions(-) diff --git a/storage/innobase/include/mtr0types.h b/storage/innobase/include/mtr0types.h index bdb16706f73..9b0e19d100f 100644 --- a/storage/innobase/include/mtr0types.h +++ b/storage/innobase/include/mtr0types.h @@ -100,9 +100,6 @@ enum mlog_id_t { /** Create an index page */ MLOG_PAGE_CREATE = 19, - /** erase an undo log page end */ - MLOG_UNDO_ERASE_END = 21, - /** create an undo log header */ MLOG_UNDO_HDR_CREATE = 25, diff --git a/storage/innobase/include/trx0rec.h b/storage/innobase/include/trx0rec.h index 5c245d23abc..d0c88b00a35 100644 --- a/storage/innobase/include/trx0rec.h +++ b/storage/innobase/include/trx0rec.h @@ -236,17 +236,6 @@ trx_undo_prev_version_build( into this function by purge thread or not. And if we read "after image" of undo log */ -/***********************************************************//** -Parses a redo log record of erasing of an undo page end. -@return end of log record or NULL */ -byte* -trx_undo_parse_erase_page_end( -/*==========================*/ - byte* ptr, /*!< in: buffer */ - byte* end_ptr,/*!< in: buffer end */ - page_t* page, /*!< in: page or NULL */ - mtr_t* mtr); /*!< in: mtr or NULL */ - /** Read from an undo log record a non-virtual column value. @param[in,out] ptr pointer to remaining part of the undo record @param[in,out] field stored field diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 6e52cebfb9f..b681df255f4 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1394,10 +1394,6 @@ parse_log: page_parse_create(block, type == MLOG_COMP_PAGE_CREATE_RTREE, true); break; - case MLOG_UNDO_ERASE_END: - ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); - ptr = trx_undo_parse_erase_page_end(ptr, end_ptr, page, mtr); - break; case MLOG_UNDO_HDR_CREATE: ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); ptr = trx_undo_parse_page_header(ptr, end_ptr, page, mtr); @@ -3622,9 +3618,6 @@ get_mlog_string(mlog_id_t type) case MLOG_PAGE_CREATE: return("MLOG_PAGE_CREATE"); - case MLOG_UNDO_ERASE_END: - return("MLOG_UNDO_ERASE_END"); - case MLOG_UNDO_HDR_CREATE: return("MLOG_UNDO_HDR_CREATE"); diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index 9b3619fbdba..90625bf98e5 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -1772,37 +1772,12 @@ trx_undo_erase_page_end( first_free = mach_read_from_2(undo_page + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_FREE); - memset(undo_page + first_free, 0xff, + memset(undo_page + first_free, 0, (UNIV_PAGE_SIZE - FIL_PAGE_DATA_END) - first_free); - mlog_write_initial_log_record(undo_page, MLOG_UNDO_ERASE_END, mtr); return(first_free != TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE); } -/***********************************************************//** -Parses a redo log record of erasing of an undo page end. -@return end of log record or NULL */ -byte* -trx_undo_parse_erase_page_end( -/*==========================*/ - byte* ptr, /*!< in: buffer */ - byte* end_ptr MY_ATTRIBUTE((unused)), /*!< in: buffer end */ - page_t* page, /*!< in: page or NULL */ - mtr_t* mtr) /*!< in: mtr or NULL */ -{ - ut_ad(ptr != NULL); - ut_ad(end_ptr != NULL); - - if (page == NULL) { - - return(ptr); - } - - trx_undo_erase_page_end(page, mtr); - - return(ptr); -} - /***********************************************************************//** Writes information to an undo log about an insert, update, or a delete marking of a clustered index record. This information is used in a rollback of the From 252e690c859b53c51c7eecf072f33dbaaa01bcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 19 Dec 2017 16:13:35 +0200 Subject: [PATCH 45/51] Fix galera.view test case crash. WSREP_TO_ISOLATION_BEGIN() call must be after view name is back on tables list. --- sql/sql_view.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 629dd865b0e..0f08883639a 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -425,11 +425,12 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, if ((res= create_view_precheck(thd, tables, view, mode))) goto err; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) + /* ignore lock specs for CREATE statement */ From 88aff5f471d3d9ae8ecc2f909bcf5bd0ddd6aa7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 18 Dec 2017 13:38:36 +0200 Subject: [PATCH 46/51] Follow-up to MDEV-13407 innodb.drop_table_background failed in buildbot with "Tablespace for table exists" The InnoDB background DROP TABLE queue is something that we should really remove, but are unable to until we remove dict_operation_lock so that DDL and DML operations can be combined in a single transaction. Because the queue is not persistent, it is not crash-safe. We should in some way ensure that the deferred-dropped tables will be dropped after server restart. The existence of two separate transactions complicates the error handling of CREATE TABLE...SELECT. We should really not break locks in DROP TABLE. Our solution to these problems is to rename the table to a temporary name, and to drop such-named tables on InnoDB startup. Also, the queue will use table IDs instead of names from now on. check-testcase.test: Ignore #sql-ib*.ibd files, because tables may enter the background DROP TABLE queue shortly before the test finishes. innodb.drop_table_background: Test CREATE...SELECT and the creation of tables whose file name starts with #sql-ib. innodb.alter_crash: Adjust the recovery, now that the #sql-ib tables will be dropped on InnoDB startup. row_mysql_drop_garbage_tables(): New function, to drop all #sql-ib tables on InnoDB startup. row_drop_table_for_mysql_in_background(): Remove an unnecessary and misplaced call to log_buffer_flush_to_disk(). (The call should have been after the transaction commit. We do not care about flushing the redo log here, because the table would be dropped again at server startup.) Remove the entry from the list after the table no longer exists. If server shutdown has been initiated, empty the list without actually dropping any tables. They will be dropped again on startup. row_drop_table_for_mysql(): Do not call lock_remove_all_on_table(). Instead, if locks exist, defer the DROP TABLE until they do not exist. If the table name does not start with #sql-ib, rename it to that prefix before adding it to the background DROP TABLE queue. --- mysql-test/include/check-testcase.test | 5 +- mysql-test/suite/innodb/r/alter_crash.result | 21 ++- .../innodb/r/drop_table_background.result | 15 ++ mysql-test/suite/innodb/t/alter_crash.test | 69 ++++----- .../suite/innodb/t/drop_table_background.test | 14 ++ storage/innobase/include/row0mysql.h | 4 + storage/innobase/log/log0recv.cc | 2 + storage/innobase/row/row0mysql.cc | 141 ++++++++++++------ 8 files changed, 166 insertions(+), 105 deletions(-) diff --git a/mysql-test/include/check-testcase.test b/mysql-test/include/check-testcase.test index a282201857e..4ca53989d06 100644 --- a/mysql-test/include/check-testcase.test +++ b/mysql-test/include/check-testcase.test @@ -82,7 +82,10 @@ call mtr.check_testcase(); let $datadir=`select @@datadir`; list_files $datadir mysql_upgrade_info; -list_files $datadir/test #sql*; +list_files_write_file $datadir.tempfiles.txt $datadir/test #sql*; +--replace_regex /#sql-ib[0-9a-f]+-[0-9a-f]+\.ibd\n// +cat_file $datadir.tempfiles.txt; +remove_file $datadir.tempfiles.txt; list_files $datadir/mysql #sql*; --enable_query_log diff --git a/mysql-test/suite/innodb/r/alter_crash.result b/mysql-test/suite/innodb/r/alter_crash.result index 8de02cc5fbd..df1645a4ef6 100644 --- a/mysql-test/suite/innodb/r/alter_crash.result +++ b/mysql-test/suite/innodb/r/alter_crash.result @@ -44,10 +44,9 @@ SET DEBUG_DBUG='+d,innodb_alter_commit_crash_after_commit'; ALTER TABLE t1 ADD PRIMARY KEY (f2, f1); ERROR HY000: Lost connection to MySQL server during query # Restart mysqld after the crash and reconnect. -# Manual *.frm recovery begin. -# Manual recovery end -FLUSH TABLES; -# Drop the orphaned original table. +SELECT * FROM information_schema.innodb_sys_tables +WHERE table_id = ID; +TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE # Files in datadir after manual recovery. t1.frm t1.ibd @@ -83,11 +82,9 @@ SET DEBUG_DBUG='+d,innodb_alter_commit_crash_before_commit'; ALTER TABLE t2 ADD PRIMARY KEY (f2, f1); ERROR HY000: Lost connection to MySQL server during query # Startup the server after the crash -# Read and remember the temporary table name -# Manual *.frm recovery begin. The dictionary was not updated -# and the files were not renamed. The rebuilt table -# was left behind on purpose, to faciliate data recovery. -# Manual recovery end +SELECT * FROM information_schema.innodb_sys_tables +WHERE name LIKE 'test/#sql-ib%'; +TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE # Drop the orphaned rebuilt table. SHOW TABLES; Tables_in_test @@ -123,10 +120,10 @@ SET DEBUG_DBUG='+d,innodb_alter_commit_crash_after_commit'; ALTER TABLE t1 ADD INDEX (b), CHANGE c d int, ALGORITHM=INPLACE; ERROR HY000: Lost connection to MySQL server during query # Restart mysqld after the crash and reconnect. -# Manual *.frm recovery begin. -# Manual recovery end +SELECT * FROM information_schema.innodb_sys_tables +WHERE table_id = ID; +TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE FLUSH TABLES; -# Drop the orphaned original table. # Files in datadir after manual recovery. t1.frm t1.ibd diff --git a/mysql-test/suite/innodb/r/drop_table_background.result b/mysql-test/suite/innodb/r/drop_table_background.result index a6f5672ba7f..e74bcd5e780 100644 --- a/mysql-test/suite/innodb/r/drop_table_background.result +++ b/mysql-test/suite/innodb/r/drop_table_background.result @@ -3,7 +3,22 @@ KEY(c1), KEY(c2), KEY(c2,c1), KEY(c3), KEY(c3,c1), KEY(c3,c2), KEY(c3,c2,c1), KEY(c4), KEY(c4,c1), KEY(c4,c2), KEY(c4,c2,c1), KEY(c4,c3), KEY(c4,c3,c1), KEY(c4,c3,c2), KEY(c4,c3,c2,c1)) ENGINE=InnoDB; +CREATE TABLE `#mysql50##sql-ib-foo`(a SERIAL) ENGINE=InnoDB; +INSERT INTO t (c1) VALUES (1),(2),(1); SET DEBUG_DBUG='+d,row_drop_table_add_to_background'; +CREATE TABLE target (PRIMARY KEY(c1)) ENGINE=InnoDB SELECT * FROM t; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +SELECT * from target; +ERROR 42S02: Table 'test.target' doesn't exist DROP TABLE t; CREATE TABLE t (a INT) ENGINE=InnoDB; DROP TABLE t; +DROP TABLE target; +ERROR 42S02: Unknown table 'test.target' +CREATE TABLE target (a INT) ENGINE=InnoDB; +DROP TABLE target; +SELECT * FROM `#mysql50##sql-ib-foo`; +ERROR 42S02: Table 'test.#mysql50##sql-ib-foo' doesn't exist in engine +DROP TABLE `#mysql50##sql-ib-foo`; +Warnings: +Warning 1932 Table 'test.#mysql50##sql-ib-foo' doesn't exist in engine diff --git a/mysql-test/suite/innodb/t/alter_crash.test b/mysql-test/suite/innodb/t/alter_crash.test index b4fdfd2f2d5..c4ee895d192 100644 --- a/mysql-test/suite/innodb/t/alter_crash.test +++ b/mysql-test/suite/innodb/t/alter_crash.test @@ -75,28 +75,22 @@ ALTER TABLE t1 ADD PRIMARY KEY (f2, f1); --echo # Restart mysqld after the crash and reconnect. --source include/start_mysqld.inc -let $temp_table_name = `SELECT SUBSTR(name, 6) - FROM information_schema.innodb_sys_tables - WHERE table_id = $orig_table_id`; - ---echo # Manual *.frm recovery begin. - ---move_file $MYSQLD_DATADIR/test/t1.frm $MYSQLD_DATADIR/test/$temp_table_name.frm - +let TABLENAME_INC= $MYSQLTEST_VARDIR/tmp/tablename.inc; perl; -my @frm_file = glob "$ENV{'datadir'}/test/#sql-*.frm"; -my $t1_frm = "$ENV{'datadir'}/test/t1.frm"; -rename($frm_file[0], $t1_frm); +die unless open OUT, ">$ENV{TABLENAME_INC}"; +chdir "$ENV{'datadir'}/test"; +my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm"; +print OUT 'let $tablename=', $frm_file[0], ';'; +close OUT or die; EOF +source $TABLENAME_INC; +remove_file $TABLENAME_INC; ---echo # Manual recovery end +--replace_result $orig_table_id ID +eval SELECT * FROM information_schema.innodb_sys_tables +WHERE table_id = $orig_table_id; -FLUSH TABLES; - ---echo # Drop the orphaned original table. ---disable_query_log -eval DROP TABLE `#mysql50#$temp_table_name`; ---enable_query_log +move_file $datadir/test/$tablename.frm $datadir/test/t1.frm; --echo # Files in datadir after manual recovery. --list_files $MYSQLD_DATADIR/test @@ -134,16 +128,9 @@ ALTER TABLE t2 ADD PRIMARY KEY (f2, f1); --echo # Startup the server after the crash --source include/start_mysqld.inc ---echo # Read and remember the temporary table name -let $temp_table_name = `SELECT SUBSTRING(name,6) - FROM information_schema.innodb_sys_tables - WHERE name LIKE "test/#sql-ib$orig_table_id%"`; +SELECT * FROM information_schema.innodb_sys_tables +WHERE name LIKE 'test/#sql-ib%'; ---echo # Manual *.frm recovery begin. The dictionary was not updated ---echo # and the files were not renamed. The rebuilt table ---echo # was left behind on purpose, to faciliate data recovery. - -let TABLENAME_INC= $MYSQLTEST_VARDIR/tmp/tablename.inc; perl; die unless open OUT, ">$ENV{TABLENAME_INC}"; chdir "$ENV{'datadir'}/test"; @@ -154,8 +141,6 @@ EOF source $TABLENAME_INC; remove_file $TABLENAME_INC; ---echo # Manual recovery end - --echo # Drop the orphaned rebuilt table. --disable_query_log eval DROP TABLE `#mysql50#$tablename`; @@ -198,28 +183,24 @@ ALTER TABLE t1 ADD INDEX (b), CHANGE c d int, ALGORITHM=INPLACE; --echo # Restart mysqld after the crash and reconnect. --source include/start_mysqld.inc -let $temp_table_name = `SELECT SUBSTR(name, 6) - FROM information_schema.innodb_sys_tables - WHERE table_id = $orig_table_id`; - ---echo # Manual *.frm recovery begin. ---move_file $MYSQLD_DATADIR/test/t1.frm $MYSQLD_DATADIR/test/$temp_table_name.frm +--replace_result $orig_table_id ID +eval SELECT * FROM information_schema.innodb_sys_tables +WHERE table_id = $orig_table_id; perl; -my @frm_file = glob "$ENV{'datadir'}/test/#sql-*.frm"; -my $t1_frm = "$ENV{'datadir'}/test/t1.frm"; -rename($frm_file[0], $t1_frm); +die unless open OUT, ">$ENV{TABLENAME_INC}"; +chdir "$ENV{'datadir'}/test"; +my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm"; +print OUT 'let $tablename=', $frm_file[0], ';'; +close OUT or die; EOF ---echo # Manual recovery end +source $TABLENAME_INC; +remove_file $TABLENAME_INC; +move_file $datadir/test/$tablename.frm $datadir/test/t1.frm; FLUSH TABLES; ---echo # Drop the orphaned original table. ---disable_query_log -eval DROP TABLE `#mysql50#$temp_table_name`; ---enable_query_log - --echo # Files in datadir after manual recovery. --list_files $MYSQLD_DATADIR/test diff --git a/mysql-test/suite/innodb/t/drop_table_background.test b/mysql-test/suite/innodb/t/drop_table_background.test index 0f596dec574..8d82bea9675 100644 --- a/mysql-test/suite/innodb/t/drop_table_background.test +++ b/mysql-test/suite/innodb/t/drop_table_background.test @@ -9,6 +9,9 @@ KEY(c3), KEY(c3,c1), KEY(c3,c2), KEY(c3,c2,c1), KEY(c4), KEY(c4,c1), KEY(c4,c2), KEY(c4,c2,c1), KEY(c4,c3), KEY(c4,c3,c1), KEY(c4,c3,c2), KEY(c4,c3,c2,c1)) ENGINE=InnoDB; +CREATE TABLE `#mysql50##sql-ib-foo`(a SERIAL) ENGINE=InnoDB; +INSERT INTO t (c1) VALUES (1),(2),(1); + let $n= 10; SET DEBUG_DBUG='+d,row_drop_table_add_to_background'; @@ -24,7 +27,18 @@ while ($i) { dec $i; } --enable_query_log +--error ER_DUP_ENTRY +CREATE TABLE target (PRIMARY KEY(c1)) ENGINE=InnoDB SELECT * FROM t; +--error ER_NO_SUCH_TABLE +SELECT * from target; DROP TABLE t; --source include/restart_mysqld.inc CREATE TABLE t (a INT) ENGINE=InnoDB; DROP TABLE t; +--error ER_BAD_TABLE_ERROR +DROP TABLE target; +CREATE TABLE target (a INT) ENGINE=InnoDB; +DROP TABLE target; +--error ER_NO_SUCH_TABLE_IN_ENGINE +SELECT * FROM `#mysql50##sql-ib-foo`; +DROP TABLE `#mysql50##sql-ib-foo`; diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index a7a55d202e8..044e732c22d 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -420,6 +420,10 @@ ulint row_get_background_drop_list_len_low(void); /*======================================*/ +/** Drop garbage tables during recovery. */ +void +row_mysql_drop_garbage_tables(); + /*********************************************************************//** Sets an exclusive lock on a table. @return error code or DB_SUCCESS */ diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 2e967a99121..93943620ecf 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3410,6 +3410,8 @@ recv_recovery_rollback_active(void) /* Drop partially created indexes. */ row_merge_drop_temp_indexes(); + /* Drop garbage tables. */ + row_mysql_drop_garbage_tables(); /* Drop any auxiliary tables that were not dropped when the parent table was dropped. This can happen if the parent table diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index e6ef1caa7ff..6d8b0b968c6 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -64,6 +64,7 @@ Created 9/17/2000 Heikki Tuuri #include "trx0roll.h" #include "trx0undo.h" #include "row0ext.h" +#include "srv0start.h" #include "ut0new.h" #include @@ -2776,12 +2777,6 @@ row_drop_table_for_mysql_in_background( error = row_drop_table_for_mysql(name, trx, FALSE, FALSE); - /* Flush the log to reduce probability that the .frm files and - the InnoDB data dictionary get out-of-sync if the user runs - with innodb_flush_log_at_trx_commit = 0 */ - - log_buffer_flush_to_disk(); - trx_commit_for_mysql(trx); trx_free_for_background(trx); @@ -2819,8 +2814,11 @@ next: return(n_tables + n_tables_dropped); } - table = dict_table_open_on_id(drop->table_id, FALSE, - DICT_TABLE_OP_OPEN_ONLY_IF_CACHED); + /* On fast shutdown, just empty the list without dropping tables. */ + table = srv_shutdown_state == SRV_SHUTDOWN_NONE || !srv_fast_shutdown + ? dict_table_open_on_id(drop->table_id, FALSE, + DICT_TABLE_OP_OPEN_ONLY_IF_CACHED) + : NULL; if (!table) { n_tables_dropped++; @@ -2875,6 +2873,74 @@ row_get_background_drop_list_len_low(void) return(len); } +/** Drop garbage tables during recovery. */ +void +row_mysql_drop_garbage_tables() +{ + mem_heap_t* heap = mem_heap_create(FN_REFLEN); + btr_pcur_t pcur; + mtr_t mtr; + trx_t* trx = trx_allocate_for_background(); + trx->op_info = "dropping garbage tables"; + row_mysql_lock_data_dictionary(trx); + + mtr.start(); + btr_pcur_open_at_index_side( + true, dict_table_get_first_index(dict_sys->sys_tables), + BTR_SEARCH_LEAF, &pcur, true, 0, &mtr); + + for (;;) { + const rec_t* rec; + const byte* field; + ulint len; + const char* table_name; + + btr_pcur_move_to_next_user_rec(&pcur, &mtr); + + if (!btr_pcur_is_on_user_rec(&pcur)) { + break; + } + + rec = btr_pcur_get_rec(&pcur); + if (rec_get_deleted_flag(rec, 0)) { + continue; + } + + field = rec_get_nth_field_old(rec, 0/*NAME*/, &len); + if (len == UNIV_SQL_NULL || len == 0) { + /* Corrupted SYS_TABLES.NAME */ + continue; + } + + table_name = mem_heap_strdupl( + heap, + reinterpret_cast(field), len); + if (strstr(table_name, "/" TEMP_FILE_PREFIX_INNODB)) { + btr_pcur_store_position(&pcur, &mtr); + btr_pcur_commit_specify_mtr(&pcur, &mtr); + + if (dict_load_table(table_name, true, + DICT_ERR_IGNORE_ALL)) { + row_drop_table_for_mysql( + table_name, trx, FALSE, FALSE); + trx_commit_for_mysql(trx); + } + + mtr.start(); + btr_pcur_restore_position(BTR_SEARCH_LEAF, + &pcur, &mtr); + } + + mem_heap_empty(heap); + } + + btr_pcur_close(&pcur); + mtr.commit(); + row_mysql_unlock_data_dictionary(trx); + trx_free_for_background(trx); + mem_heap_free(heap); +} + /*********************************************************************//** If a table is not yet in the drop list, adds the table to the list of tables which the master thread drops in background. We need this on Unix because in @@ -3645,11 +3711,7 @@ row_drop_table_for_mysql( } - DBUG_EXECUTE_IF("row_drop_table_add_to_background", - row_add_table_to_background_drop_list(table->id); - err = DB_SUCCESS; - goto funct_exit; - ); + DBUG_EXECUTE_IF("row_drop_table_add_to_background", goto defer;); /* TODO: could we replace the counter n_foreign_key_checks_running with lock checks on the table? Acquire here an exclusive lock on the @@ -3658,17 +3720,22 @@ row_drop_table_for_mysql( checks take an IS or IX lock on the table. */ if (table->n_foreign_key_checks_running > 0) { - if (row_add_table_to_background_drop_list(table->id)) { - ib::info() << "You are trying to drop table " - << table->name - << " though there is a foreign key check" - " running on it. Adding the table to the" - " background drop queue."; +defer: + if (!strstr(table->name.m_name, "/" TEMP_FILE_PREFIX_INNODB)) { + heap = mem_heap_create(FN_REFLEN); + const char* tmp_name + = dict_mem_create_temporary_tablename( + heap, table->name.m_name, table->id); + ib::info() << "Deferring DROP TABLE " << table->name + << "; renaming to " << tmp_name; + err = row_rename_table_for_mysql( + table->name.m_name, tmp_name, trx, false); + } else { + err = DB_SUCCESS; + } + if (err == DB_SUCCESS) { + row_add_table_to_background_drop_list(table->id); } - - /* We return DB_SUCCESS to MySQL though the drop will - happen lazily later */ - err = DB_SUCCESS; goto funct_exit; } @@ -3689,26 +3756,9 @@ row_drop_table_for_mysql( /* Wait on background threads to stop using table */ fil_wait_crypt_bg_threads(table); - if (table->get_ref_count() == 0) { - lock_remove_all_on_table(table, TRUE); - ut_a(table->n_rec_locks == 0); - } else if (table->get_ref_count() > 0 || table->n_rec_locks > 0) { - if (row_add_table_to_background_drop_list(table->id)) { - ib::info() << "MySQL is trying to drop table " - << table->name - << " though there are still open handles to" - " it. Adding the table to the background drop" - " queue."; - - /* We return DB_SUCCESS to MySQL though the drop will - happen lazily later */ - err = DB_SUCCESS; - } else { - /* The table is already in the background drop list */ - err = DB_ERROR; - } - - goto funct_exit; + if (table->get_ref_count() > 0 || table->n_rec_locks > 0 + || lock_table_has_locks(table)) { + goto defer; } /* The "to_be_dropped" marks table that is to be dropped, but @@ -3718,11 +3768,6 @@ row_drop_table_for_mysql( and it is free to be dropped */ table->to_be_dropped = false; - /* If we get this far then the table to be dropped must not have - any table or record locks on it. */ - - ut_a(!lock_table_has_locks(table)); - switch (trx_get_dict_operation(trx)) { case TRX_DICT_OP_NONE: trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); From b4e5d5e2db19967e1e20547a13f388af64879d2d Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 20 Dec 2017 01:30:12 +0200 Subject: [PATCH 47/51] Updated result for alter_crash --- mysql-test/suite/innodb/r/alter_crash.result | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/innodb/r/alter_crash.result b/mysql-test/suite/innodb/r/alter_crash.result index df1645a4ef6..4a96d907a26 100644 --- a/mysql-test/suite/innodb/r/alter_crash.result +++ b/mysql-test/suite/innodb/r/alter_crash.result @@ -46,7 +46,7 @@ ERROR HY000: Lost connection to MySQL server during query # Restart mysqld after the crash and reconnect. SELECT * FROM information_schema.innodb_sys_tables WHERE table_id = ID; -TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE +TABLE_ID NAME FLAG N_COLS SPACE ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE # Files in datadir after manual recovery. t1.frm t1.ibd @@ -84,7 +84,7 @@ ERROR HY000: Lost connection to MySQL server during query # Startup the server after the crash SELECT * FROM information_schema.innodb_sys_tables WHERE name LIKE 'test/#sql-ib%'; -TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE +TABLE_ID NAME FLAG N_COLS SPACE ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE # Drop the orphaned rebuilt table. SHOW TABLES; Tables_in_test @@ -122,7 +122,7 @@ ERROR HY000: Lost connection to MySQL server during query # Restart mysqld after the crash and reconnect. SELECT * FROM information_schema.innodb_sys_tables WHERE table_id = ID; -TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE +TABLE_ID NAME FLAG N_COLS SPACE ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE FLUSH TABLES; # Files in datadir after manual recovery. t1.frm From 0bc36758ba08ddeea6f7896a0fb815a13a48895a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 20 Dec 2017 22:19:47 +0200 Subject: [PATCH 48/51] MDEV-14717 RENAME TABLE in InnoDB is not crash-safe InnoDB in MariaDB 10.2 appears to only write MLOG_FILE_RENAME2 redo log records during table-rebuilding ALGORITHM=INPLACE operations. We must write the records for any .ibd file renames, so that the operations are crash-safe. If InnoDB is killed during a RENAME TABLE operation, it can happen that the transaction for updating the data dictionary will be rolled back. But, nothing will roll back the renaming of the .ibd file (the MLOG_FILE_RENAME2 only guarantees roll-forward), or for that matter, the renaming of the dict_table_t::name in the dict_sys cache. We introduce the undo log record TRX_UNDO_RENAME_TABLE to fix this. fil_space_for_table_exists_in_mem(): Remove the parameters adjust_space, table_id and some code that was trying to work around these deficiencies. fil_name_write_rename(): Write a MLOG_FILE_RENAME2 record. dict_table_rename_in_cache(): Invoke fil_name_write_rename(). trx_undo_rec_copy(): Set the first 2 bytes to the length of the copied undo log record. trx_undo_page_report_rename(), trx_undo_report_rename(): Write a TRX_UNDO_RENAME_TABLE record with the old table name. row_rename_table_for_mysql(): Invoke trx_undo_report_rename() before modifying any data dictionary tables. row_undo_ins_parse_undo_rec(): Roll back TRX_UNDO_RENAME_TABLE by invoking dict_table_rename_in_cache(), which will take care of both renaming the table and the file. --- .../suite/innodb/r/rename_table_debug.result | 12 ++ .../suite/innodb/t/rename_table_debug.test | 19 +++ storage/innobase/dict/dict0dict.cc | 2 + storage/innobase/dict/dict0load.cc | 5 +- storage/innobase/fil/fil0fil.cc | 69 ++++------- storage/innobase/handler/ha_innodb.cc | 7 +- storage/innobase/include/dict0dict.h | 2 +- storage/innobase/include/fil0fil.h | 40 ++++--- storage/innobase/include/trx0rec.h | 8 ++ storage/innobase/include/trx0rec.ic | 5 +- storage/innobase/row/row0mysql.cc | 14 ++- storage/innobase/row/row0uins.cc | 26 +++- storage/innobase/trx/trx0rec.cc | 113 ++++++++++++++++++ storage/innobase/trx/trx0roll.cc | 10 +- 14 files changed, 248 insertions(+), 84 deletions(-) create mode 100644 mysql-test/suite/innodb/r/rename_table_debug.result create mode 100644 mysql-test/suite/innodb/t/rename_table_debug.test diff --git a/mysql-test/suite/innodb/r/rename_table_debug.result b/mysql-test/suite/innodb/r/rename_table_debug.result new file mode 100644 index 00000000000..912ed9de48b --- /dev/null +++ b/mysql-test/suite/innodb/r/rename_table_debug.result @@ -0,0 +1,12 @@ +CREATE TABLE t1 (a INT UNSIGNED PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES(42); +connect con1,localhost,root,,test; +SET DEBUG_SYNC='before_rename_table_commit SIGNAL renamed WAIT_FOR ever'; +RENAME TABLE t1 TO t2; +connection default; +SET DEBUG_SYNC='now WAIT_FOR renamed'; +disconnect con1; +SELECT * FROM t1; +a +42 +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/rename_table_debug.test b/mysql-test/suite/innodb/t/rename_table_debug.test new file mode 100644 index 00000000000..4620f7bef22 --- /dev/null +++ b/mysql-test/suite/innodb/t/rename_table_debug.test @@ -0,0 +1,19 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/not_embedded.inc + +CREATE TABLE t1 (a INT UNSIGNED PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES(42); + +--connect (con1,localhost,root,,test) +SET DEBUG_SYNC='before_rename_table_commit SIGNAL renamed WAIT_FOR ever'; +--send +RENAME TABLE t1 TO t2; +--connection default +SET DEBUG_SYNC='now WAIT_FOR renamed'; +--let $shutdown_timeout=0 +--source include/restart_mysqld.inc +--disconnect con1 +SELECT * FROM t1; +DROP TABLE t1; diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index d228f8ccc10..f93a71a48c5 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1684,6 +1684,8 @@ dict_table_rename_in_cache( return(err); } + fil_name_write_rename(table->space, old_path, new_path); + bool success = fil_rename_tablespace( table->space, old_path, new_name, new_path); diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 6d149d6c7d0..da838f83918 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1444,7 +1444,7 @@ dict_check_sys_tables( look to see if it is already in the tablespace cache. */ if (fil_space_for_table_exists_in_mem( space_id, table_name.m_name, - false, true, NULL, 0, flags)) { + false, NULL, flags)) { /* Recovery can open a datafile that does not match SYS_DATAFILES. If they don't match, update SYS_DATAFILES. */ @@ -2852,8 +2852,7 @@ dict_load_tablespace( /* The tablespace may already be open. */ if (fil_space_for_table_exists_in_mem( - table->space, space_name, false, - true, heap, table->id, table->flags)) { + table->space, space_name, false, heap, table->flags)) { return; } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 24bdaf2c263..8c86e272853 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2331,7 +2331,7 @@ fil_op_write_log( @param[in,out] mtr mini-transaction */ static void -fil_name_write_rename( +fil_name_write_rename_low( ulint space_id, ulint first_page_no, const char* old_name, @@ -2345,6 +2345,23 @@ fil_name_write_rename( space_id, first_page_no, old_name, new_name, 0, mtr); } +/** Write redo log for renaming a file. +@param[in] space_id tablespace id +@param[in] old_name tablespace file name +@param[in] new_name tablespace file name after renaming */ +void +fil_name_write_rename( + ulint space_id, + const char* old_name, + const char* new_name) +{ + mtr_t mtr; + mtr.start(); + fil_name_write_rename_low(space_id, 0, old_name, new_name, &mtr); + mtr.commit(); + log_write_up_to(mtr.commit_lsn(), true); +} + /** Write MLOG_FILE_NAME for a file. @param[in] space_id tablespace id @param[in] first_page_no first page number in the file @@ -3583,12 +3600,7 @@ func_exit: ut_ad(strchr(new_file_name, OS_PATH_SEPARATOR) != NULL); if (!recv_recovery_on) { - mtr_t mtr; - - mtr.start(); - fil_name_write_rename( - id, 0, old_file_name, new_file_name, &mtr); - mtr.commit(); + fil_name_write_rename(id, old_file_name, new_file_name); log_mutex_enter(); } @@ -4651,9 +4663,7 @@ startup, there may be many tablespaces which are not yet in the memory cache. @param[in] print_error_if_does_not_exist Print detailed error information to the error log if a matching tablespace is not found from memory. -@param[in] adjust_space Whether to adjust space id on mismatch @param[in] heap Heap memory -@param[in] table_id table id @param[in] table_flags table flags @return true if a matching tablespace exists in the memory cache */ bool @@ -4661,9 +4671,7 @@ fil_space_for_table_exists_in_mem( ulint id, const char* name, bool print_error_if_does_not_exist, - bool adjust_space, mem_heap_t* heap, - table_id_t table_id, ulint table_flags) { fil_space_t* fnamespace; @@ -4688,41 +4696,6 @@ fil_space_for_table_exists_in_mem( } else if (!valid || space == fnamespace) { /* Found with the same file name, or got a flag mismatch. */ goto func_exit; - } else if (adjust_space - && row_is_mysql_tmp_table_name(space->name) - && !row_is_mysql_tmp_table_name(name)) { - /* Info from fnamespace comes from the ibd file - itself, it can be different from data obtained from - System tables since renaming files is not - transactional. We shall adjust the ibd file name - according to system table info. */ - mutex_exit(&fil_system->mutex); - - DBUG_EXECUTE_IF("ib_crash_before_adjust_fil_space", - DBUG_SUICIDE();); - - const char* tmp_name = dict_mem_create_temporary_tablename( - heap, name, table_id); - - fil_rename_tablespace( - fnamespace->id, - UT_LIST_GET_FIRST(fnamespace->chain)->name, - tmp_name, NULL); - - DBUG_EXECUTE_IF("ib_crash_after_adjust_one_fil_space", - DBUG_SUICIDE();); - - fil_rename_tablespace( - id, UT_LIST_GET_FIRST(space->chain)->name, - name, NULL); - - DBUG_EXECUTE_IF("ib_crash_after_adjust_fil_space", - DBUG_SUICIDE();); - - mutex_enter(&fil_system->mutex); - fnamespace = fil_space_get_by_name(name); - ut_ad(space == fnamespace); - goto func_exit; } if (!print_error_if_does_not_exist) { @@ -6215,7 +6188,7 @@ fil_mtr_rename_log( return(err); } - fil_name_write_rename( + fil_name_write_rename_low( old_table->space, 0, old_path, tmp_path, mtr); ut_free(tmp_path); @@ -6246,7 +6219,7 @@ fil_mtr_rename_log( } } - fil_name_write_rename( + fil_name_write_rename_low( new_table->space, 0, new_path, old_path, mtr); ut_free(new_path); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index b509d7262af..7310e602645 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -13910,17 +13910,13 @@ innobase_rename_table( TrxInInnoDB trx_in_innodb(trx); trx_start_if_not_started(trx, true); + ut_ad(trx->will_lock > 0); /* Serialize data dictionary operations with dictionary mutex: no deadlocks can occur then in these operations. */ row_mysql_lock_data_dictionary(trx); - /* Transaction must be flagged as a locking transaction or it hasn't - been started yet. */ - - ut_a(trx->will_lock > 0); - error = row_rename_table_for_mysql(norm_from, norm_to, trx, TRUE); if (error == DB_TABLE_NOT_FOUND) { @@ -13929,7 +13925,6 @@ innobase_rename_table( We are doing a DDL operation. */ ++trx->will_lock; - trx_set_dict_operation(trx, TRX_DICT_OP_INDEX); trx_start_if_not_started(trx, true); error = row_rename_partitions_for_mysql(norm_from, norm_to, trx); diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index a0fd78e4e0d..60c100e80ca 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -423,7 +423,7 @@ dict_table_rename_in_cache( /*!< in: in ALTER TABLE we want to preserve the original table name in constraints which reference it */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((nonnull)); /** Removes an index from the dictionary cache. @param[in,out] table table whose index to remove diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 1027cf5cbb7..65d8b019da2 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -869,6 +869,15 @@ fil_create_directory_for_tablename( /*===============================*/ const char* name); /*!< in: name in the standard 'databasename/tablename' format */ +/** Write redo log for renaming a file. +@param[in] space_id tablespace id +@param[in] old_name tablespace file name +@param[in] new_name tablespace file name after renaming */ +void +fil_name_write_rename( + ulint space_id, + const char* old_name, + const char* new_name); /********************************************************//** Recreates table indexes by applying TRUNCATE log record during recovery. @@ -1144,27 +1153,24 @@ fil_file_readdir_next_file( os_file_dir_t dir, /*!< in: directory stream */ os_file_stat_t* info); /*!< in/out: buffer where the info is returned */ -/*******************************************************************//** -Returns true if a matching tablespace exists in the InnoDB tablespace memory -cache. Note that if we have not done a crash recovery at the database startup, -there may be many tablespaces which are not yet in the memory cache. +/** Determine if a matching tablespace exists in the InnoDB tablespace +memory cache. Note that if we have not done a crash recovery at the database +startup, there may be many tablespaces which are not yet in the memory cache. +@param[in] id Tablespace ID +@param[in] name Tablespace name used in fil_space_create(). +@param[in] print_error_if_does_not_exist + Print detailed error information to the +error log if a matching tablespace is not found from memory. +@param[in] heap Heap memory +@param[in] table_flags table flags @return true if a matching tablespace exists in the memory cache */ bool fil_space_for_table_exists_in_mem( -/*==============================*/ - ulint id, /*!< in: space id */ - const char* name, /*!< in: table name in the standard - 'databasename/tablename' format */ + ulint id, + const char* name, bool print_error_if_does_not_exist, - /*!< in: print detailed error - information to the .err log if a - matching tablespace is not found from - memory */ - bool adjust_space, /*!< in: whether to adjust space id - when find table space mismatch */ - mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id, /*!< in: table id */ - ulint table_flags); /*!< in: table flags */ + mem_heap_t* heap, + ulint table_flags); /** Try to extend a tablespace if it is smaller than the specified size. @param[in,out] space tablespace diff --git a/storage/innobase/include/trx0rec.h b/storage/innobase/include/trx0rec.h index f2c30a20f7f..ada98b776f0 100644 --- a/storage/innobase/include/trx0rec.h +++ b/storage/innobase/include/trx0rec.h @@ -178,6 +178,13 @@ trx_undo_rec_get_partial_row( mem_heap_t* heap) /*!< in: memory heap from which the memory needed is allocated */ MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Report a RENAME TABLE operation. +@param[in,out] trx transaction +@param[in] table table that is being renamed +@return DB_SUCCESS or error code */ +dberr_t +trx_undo_report_rename(trx_t* trx, const dict_table_t* table) + MY_ATTRIBUTE((nonnull, warn_unused_result)); /***********************************************************************//** Writes information to an undo log about an insert, update, or a delete marking of a clustered index record. This information is used in a rollback of the @@ -323,6 +330,7 @@ trx_undo_read_v_idx( compilation info multiplied by 16 is ORed to this value in an undo log record */ +#define TRX_UNDO_RENAME_TABLE 9 /*!< RENAME TABLE */ #define TRX_UNDO_INSERT_REC 11 /* fresh insert into clustered index */ #define TRX_UNDO_UPD_EXIST_REC 12 /* update of a non-delete-marked record */ diff --git a/storage/innobase/include/trx0rec.ic b/storage/innobase/include/trx0rec.ic index c2c756484b2..d0771a94b05 100644 --- a/storage/innobase/include/trx0rec.ic +++ b/storage/innobase/include/trx0rec.ic @@ -95,5 +95,8 @@ trx_undo_rec_copy( len = mach_read_from_2(undo_rec) - ut_align_offset(undo_rec, UNIV_PAGE_SIZE); ut_ad(len < UNIV_PAGE_SIZE); - return((trx_undo_rec_t*) mem_heap_dup(heap, undo_rec, len)); + trx_undo_rec_t* rec = static_cast( + mem_heap_dup(heap, undo_rec, len)); + mach_write_to_2(rec, len); + return rec; } diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 6d8b0b968c6..d3ff0011881 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -3509,7 +3509,7 @@ row_drop_single_table_tablespace( /* If the tablespace is not in the cache, just delete the file. */ if (!fil_space_for_table_exists_in_mem( - space_id, tablename, true, false, NULL, 0, table_flags)) { + space_id, tablename, true, NULL, table_flags)) { /* Force a delete of any discarded or temporary files. */ fil_delete_file(filepath); @@ -4521,6 +4521,14 @@ row_rename_table_for_mysql( goto funct_exit; } + if (!table->is_temporary()) { + err = trx_undo_report_rename(trx, table); + + if (err != DB_SUCCESS) { + goto funct_exit; + } + } + /* We use the private SQL parser of Innobase to generate the query graphs needed in updating the dictionary data from system tables. */ @@ -4706,7 +4714,8 @@ row_rename_table_for_mysql( } } - if (dict_table_has_fts_index(table) + if (err == DB_SUCCESS + && dict_table_has_fts_index(table) && !dict_tables_have_same_db(old_name, new_name)) { err = fts_rename_aux_tables(table, new_name, trx); if (err != DB_TABLE_NOT_FOUND) { @@ -4861,6 +4870,7 @@ funct_exit: } if (commit) { + DEBUG_SYNC(trx->mysql_thd, "before_rename_table_commit"); trx_commit_for_mysql(trx); } diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc index 09c22cdcfd2..898eb289257 100644 --- a/storage/innobase/row/row0uins.cc +++ b/storage/innobase/row/row0uins.cc @@ -332,16 +332,13 @@ row_undo_ins_parse_undo_rec( byte* ptr; undo_no_t undo_no; table_id_t table_id; - ulint type; ulint dummy; bool dummy_extern; ut_ad(node); - ptr = trx_undo_rec_get_pars(node->undo_rec, &type, &dummy, + ptr = trx_undo_rec_get_pars(node->undo_rec, &node->rec_type, &dummy, &dummy_extern, &undo_no, &table_id); - ut_ad(type == TRX_UNDO_INSERT_REC); - node->rec_type = type; node->update = NULL; node->table = dict_table_open_on_id( @@ -352,6 +349,27 @@ row_undo_ins_parse_undo_rec( return; } + switch (node->rec_type) { + default: + ut_ad(!"wrong undo record type"); + goto close_table; + case TRX_UNDO_INSERT_REC: + break; + case TRX_UNDO_RENAME_TABLE: + dict_table_t* table = node->table; + ut_ad(!table->is_temporary()); + ut_ad(dict_table_is_file_per_table(table) + == (table->space != TRX_SYS_SPACE)); + size_t len = mach_read_from_2(node->undo_rec) + + node->undo_rec - ptr - 2; + ptr[len] = 0; + const char* name = reinterpret_cast(ptr); + if (strcmp(table->name.m_name, name)) { + dict_table_rename_in_cache(table, name, false); + } + goto close_table; + } + if (UNIV_UNLIKELY(!fil_table_accessible(node->table))) { close_table: /* Normally, tables should not disappear or become diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index 6ed554c1810..6a6b4b2f1b6 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -1854,6 +1854,119 @@ trx_undo_parse_erase_page_end( return(ptr); } +/** Report a RENAME TABLE operation. +@param[in,out] trx transaction +@param[in] table table that is being renamed +@param[in,out] block undo page +@param[in,out] mtr mini-transaction +@return byte offset of the undo log record +@retval 0 in case of failure */ +static +ulint +trx_undo_page_report_rename(trx_t* trx, const dict_table_t* table, + buf_block_t* block, mtr_t* mtr) +{ + ulint first_free = mach_read_from_2(block->frame + TRX_UNDO_PAGE_HDR + + TRX_UNDO_PAGE_FREE); + ut_ad(first_free >= TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE); + ut_ad(first_free <= UNIV_PAGE_SIZE); + byte* start = block->frame + first_free; + size_t len = strlen(table->name.m_name); + const size_t fixed = 2 + 1 + 11 + 11 + 2; + ut_ad(len <= NAME_LEN * 2 + 1); + /* The -10 is used in trx_undo_left() */ + compile_time_assert((NAME_LEN * 1) * 2 + fixed + + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE + < UNIV_PAGE_SIZE_MIN - 10 - FIL_PAGE_DATA_END); + + if (trx_undo_left(block->frame, start) < fixed + len) { + ut_ad(first_free > TRX_UNDO_PAGE_HDR + + TRX_UNDO_PAGE_HDR_SIZE); + return 0; + } + + byte* ptr = start + 2; + *ptr++ = TRX_UNDO_RENAME_TABLE; + ptr += mach_u64_write_much_compressed(ptr, trx->undo_no); + ptr += mach_u64_write_much_compressed(ptr, table->id); + memcpy(ptr, table->name.m_name, len); + ptr += len; + mach_write_to_2(ptr, first_free); + ptr += 2; + ulint offset = page_offset(ptr); + mach_write_to_2(start, offset); + mach_write_to_2(block->frame + TRX_UNDO_PAGE_HDR + + TRX_UNDO_PAGE_FREE, offset); + + trx_undof_page_add_undo_rec_log(block->frame, first_free, offset, mtr); + return offset; +} + +/** Report a RENAME TABLE operation. +@param[in,out] trx transaction +@param[in] table table that is being renamed +@return DB_SUCCESS or error code */ +dberr_t +trx_undo_report_rename(trx_t* trx, const dict_table_t* table) +{ + ut_ad(!trx->read_only); + ut_ad(trx->id); + ut_ad(!table->is_temporary()); + + trx_rseg_t* rseg = trx->rsegs.m_redo.rseg; + trx_undo_t** pundo = &trx->rsegs.m_redo.insert_undo; + mutex_enter(&trx->undo_mutex); + dberr_t err = *pundo + ? DB_SUCCESS + : trx_undo_assign_undo(trx, rseg, pundo, TRX_UNDO_INSERT); + ut_ad((err == DB_SUCCESS) == (*pundo != NULL)); + if (trx_undo_t* undo = *pundo) { + mtr_t mtr; + mtr.start(trx); + + buf_block_t* block = buf_page_get_gen( + page_id_t(undo->space, undo->last_page_no), + univ_page_size, RW_X_LATCH, + buf_pool_is_obsolete(undo->withdraw_clock) + ? NULL : undo->guess_block, + BUF_GET, __FILE__, __LINE__, &mtr, &err); + ut_ad((err == DB_SUCCESS) == !!block); + + for (ut_d(int loop_count = 0); block;) { + ut_ad(++loop_count < 2); + buf_block_dbg_add_level(block, SYNC_TRX_UNDO_PAGE); + ut_ad(undo->last_page_no == block->page.id.page_no()); + + if (ulint offset = trx_undo_page_report_rename( + trx, table, block, &mtr)) { + undo->withdraw_clock = buf_withdraw_clock; + undo->empty = FALSE; + undo->top_page_no = undo->last_page_no; + undo->top_offset = offset; + undo->top_undo_no = trx->undo_no++; + undo->guess_block = block; + + trx->undo_rseg_space = rseg->space; + err = DB_SUCCESS; + break; + } else { + mtr.commit(); + mtr.start(trx); + block = trx_undo_add_page(trx, undo, &mtr); + if (!block) { + err = DB_OUT_OF_FILE_SPACE; + break; + } + } + } + + mtr.commit(); + } + + mutex_exit(&trx->undo_mutex); + return err; +} + /***********************************************************************//** Writes information to an undo log about an insert, update, or a delete marking of a clustered index record. This information is used in a rollback of the diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index c9c77acba11..585de7c3ec2 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -1074,11 +1074,17 @@ trx_roll_pop_top_rec_of_trx(trx_t* trx, roll_ptr_t* roll_ptr, mem_heap_t* heap) trx_undo_rec_t* undo_rec = trx_roll_pop_top_rec(trx, undo, &mtr); const undo_no_t undo_no = trx_undo_rec_get_undo_no(undo_rec); - if (trx_undo_rec_get_type(undo_rec) == TRX_UNDO_INSERT_REC) { + switch (trx_undo_rec_get_type(undo_rec)) { + case TRX_UNDO_RENAME_TABLE: + ut_ad(undo == insert); + /* fall through */ + case TRX_UNDO_INSERT_REC: ut_ad(undo == insert || undo == temp); *roll_ptr |= 1ULL << ROLL_PTR_INSERT_FLAG_POS; - } else { + break; + default: ut_ad(undo == update || undo == temp); + break; } ut_ad(trx_roll_check_undo_rec_ordering( From b4165985c97a4133e19dd99b459dea27f87fbb1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 20 Dec 2017 22:47:01 +0200 Subject: [PATCH 49/51] MDEV-14585 Automatically remove #sql- tables in InnoDB dictionary during recovery Now that MDEV-14717 made RENAME TABLE crash-safe within InnoDB, it should be safe to drop the #sql- tables within InnoDB during crash recovery. These tables can be one of two things: (1) #sql-ib related to deferred DROP TABLE (follow-up to MDEV-13407) or to table-rebuilding ALTER TABLE...ALGORITHM=INPLACE (since MDEV-14378, only related to the intermediate copy of a table), (2) #sql- related to the intermediate copy of a table during ALTER TABLE...ALGORITHM=COPY We will not drop tables whose name starts with #sql2, because the server can be killed during an ALGORITHM=COPY operation at a point where the original table was renamed to #sql2 but the finished intermediate copy was not yet renamed from #sql- to the original table name. --- storage/innobase/row/row0mysql.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index d3ff0011881..4b06c81ff6c 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2915,7 +2915,7 @@ row_mysql_drop_garbage_tables() table_name = mem_heap_strdupl( heap, reinterpret_cast(field), len); - if (strstr(table_name, "/" TEMP_FILE_PREFIX_INNODB)) { + if (strstr(table_name, "/" TEMP_FILE_PREFIX "-")) { btr_pcur_store_position(&pcur, &mtr); btr_pcur_commit_specify_mtr(&pcur, &mtr); From 69e88de0fe6dc5312f5d6e7a179a5ab73d60dc43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 20 Dec 2017 23:15:44 +0200 Subject: [PATCH 50/51] MDEV-14585: Adjust the innodb.alter_crash test case --- mysql-test/suite/innodb/r/alter_crash.result | 4 +- mysql-test/suite/innodb/t/alter_crash.test | 45 +++++++------------- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/mysql-test/suite/innodb/r/alter_crash.result b/mysql-test/suite/innodb/r/alter_crash.result index df1645a4ef6..3b6a5d1b39a 100644 --- a/mysql-test/suite/innodb/r/alter_crash.result +++ b/mysql-test/suite/innodb/r/alter_crash.result @@ -83,9 +83,8 @@ ALTER TABLE t2 ADD PRIMARY KEY (f2, f1); ERROR HY000: Lost connection to MySQL server during query # Startup the server after the crash SELECT * FROM information_schema.innodb_sys_tables -WHERE name LIKE 'test/#sql-ib%'; +WHERE name LIKE 'test/#sql-%'; TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE -# Drop the orphaned rebuilt table. SHOW TABLES; Tables_in_test t2 @@ -123,7 +122,6 @@ ERROR HY000: Lost connection to MySQL server during query SELECT * FROM information_schema.innodb_sys_tables WHERE table_id = ID; TABLE_ID NAME FLAG N_COLS SPACE FILE_FORMAT ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE -FLUSH TABLES; # Files in datadir after manual recovery. t1.frm t1.ibd diff --git a/mysql-test/suite/innodb/t/alter_crash.test b/mysql-test/suite/innodb/t/alter_crash.test index c4ee895d192..101e0fa44c2 100644 --- a/mysql-test/suite/innodb/t/alter_crash.test +++ b/mysql-test/suite/innodb/t/alter_crash.test @@ -72,9 +72,6 @@ let $orig_table_id = `SELECT table_id --error 2013 ALTER TABLE t1 ADD PRIMARY KEY (f2, f1); ---echo # Restart mysqld after the crash and reconnect. ---source include/start_mysqld.inc - let TABLENAME_INC= $MYSQLTEST_VARDIR/tmp/tablename.inc; perl; die unless open OUT, ">$ENV{TABLENAME_INC}"; @@ -86,12 +83,15 @@ EOF source $TABLENAME_INC; remove_file $TABLENAME_INC; +move_file $datadir/test/$tablename.frm $datadir/test/t1.frm; + +--echo # Restart mysqld after the crash and reconnect. +--source include/start_mysqld.inc + --replace_result $orig_table_id ID eval SELECT * FROM information_schema.innodb_sys_tables WHERE table_id = $orig_table_id; -move_file $datadir/test/$tablename.frm $datadir/test/t1.frm; - --echo # Files in datadir after manual recovery. --list_files $MYSQLD_DATADIR/test @@ -125,26 +125,13 @@ let $orig_table_id = `SELECT table_id --error 2013 ALTER TABLE t2 ADD PRIMARY KEY (f2, f1); +remove_files_wildcard $datadir/test #sql-*.frm; + --echo # Startup the server after the crash --source include/start_mysqld.inc SELECT * FROM information_schema.innodb_sys_tables -WHERE name LIKE 'test/#sql-ib%'; - -perl; -die unless open OUT, ">$ENV{TABLENAME_INC}"; -chdir "$ENV{'datadir'}/test"; -my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm"; -print OUT 'let $tablename=', $frm_file[0], ';'; -close OUT or die; -EOF -source $TABLENAME_INC; -remove_file $TABLENAME_INC; - ---echo # Drop the orphaned rebuilt table. ---disable_query_log -eval DROP TABLE `#mysql50#$tablename`; ---enable_query_log +WHERE name LIKE 'test/#sql-%'; SHOW TABLES; INSERT INTO t2 VALUES (5,6),(7,8); @@ -180,13 +167,6 @@ let $orig_table_id = `select table_id from --error 2013 ALTER TABLE t1 ADD INDEX (b), CHANGE c d int, ALGORITHM=INPLACE; ---echo # Restart mysqld after the crash and reconnect. ---source include/start_mysqld.inc - ---replace_result $orig_table_id ID -eval SELECT * FROM information_schema.innodb_sys_tables -WHERE table_id = $orig_table_id; - perl; die unless open OUT, ">$ENV{TABLENAME_INC}"; chdir "$ENV{'datadir'}/test"; @@ -194,12 +174,17 @@ my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm"; print OUT 'let $tablename=', $frm_file[0], ';'; close OUT or die; EOF - source $TABLENAME_INC; remove_file $TABLENAME_INC; + move_file $datadir/test/$tablename.frm $datadir/test/t1.frm; -FLUSH TABLES; +--echo # Restart mysqld after the crash and reconnect. +--source include/start_mysqld.inc + +--replace_result $orig_table_id ID +eval SELECT * FROM information_schema.innodb_sys_tables +WHERE table_id = $orig_table_id; --echo # Files in datadir after manual recovery. --list_files $MYSQLD_DATADIR/test From 40f4525f43aba5d579cf55bae2df504001cd04f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 21 Dec 2017 08:20:31 +0200 Subject: [PATCH 51/51] MDEV-14585: Adjust the innodb.innodb-alter-tempfile test case --- .../suite/innodb/t/innodb-alter-tempfile.test | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/mysql-test/suite/innodb/t/innodb-alter-tempfile.test b/mysql-test/suite/innodb/t/innodb-alter-tempfile.test index f2038da8c4c..f7635e96d50 100644 --- a/mysql-test/suite/innodb/t/innodb-alter-tempfile.test +++ b/mysql-test/suite/innodb/t/innodb-alter-tempfile.test @@ -32,24 +32,12 @@ SET debug_dbug='+d,innodb_alter_commit_crash_before_commit'; --error 2013 ALTER TABLE t1 ADD PRIMARY KEY (f2, f1); -let TABLENAME_INC= $MYSQLTEST_VARDIR/tmp/tablename.inc; -perl; -die unless open OUT, ">$ENV{TABLENAME_INC}"; -chdir "$ENV{'datadir'}/test"; -my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm"; -print OUT 'let $temp_table_name=', $frm_file[0], ';'; -close OUT or die; -EOF -source $TABLENAME_INC; -remove_file $TABLENAME_INC; +remove_files_wildcard $datadir/test #sql-*.frm; --source include/start_mysqld.inc show create table t1; --echo # Consecutive Alter table does not create same temporary file name ALTER TABLE t1 ADD PRIMARY KEY (f2, f1); ---disable_query_log -eval DROP TABLE `#mysql50#$temp_table_name`; ---enable_query_log show create table t1; drop table t1;