From 5111df08146bcc9f233edada7eecdb1457b09d59 Mon Sep 17 00:00:00 2001 From: Luis Soares Date: Thu, 26 Jun 2014 12:54:27 +0100 Subject: [PATCH] BUG#13874553: rpl.rpl_stop_slave fails sporadically on pb2 The test case makes use of the fine DEBUG_SYNC facility. Furthermore, since it needs synchronization on internal threads (dump and SQL threads) the server code has DEBUG_SYNC commands internally deployed and activated through the DBUG_EXECUTE_IF macro. The internal DBUG_SYNC commands are then controlled from the test case through the DEBUG variable. There were three problems around the DEBUG + DEBUG_SYNC facility usage: 1. When signaling the SQL thread to continue, the test would reset immediately the DEBUG_SYNC variable. This could mean that the SQL thread might loose the signal and continue to wait forever; 2. A similar scenario was happening with the dump thread on the master. This thread was instructed to wait, and later it would be signaled to continue, but immediately after the DEBUG_SYNC would be reset. This could lead to the dump thread missing the signal and wait forever; 3. The test was not cleaning itself up with respect to the instrumentation of the dump thread. This would leave the conditional execution of an internal DEBUG_SYNC command active (through the usage of DBUG_EXECUTE_IF). We fix #1 and #2 by waiting for the threads to receive the signal and only then issue the reset. We fix #3 by reseting the DEBUG variable, thus deactivating the dump thread internal DEBUG_SYNC command. --- .../extra/rpl_tests/rpl_stop_slave.test | 4 +++ mysql-test/suite/rpl/r/rpl_stop_slave.result | 9 ++++-- mysql-test/suite/rpl/t/rpl_stop_slave.test | 29 +++++++++++++++++-- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/mysql-test/extra/rpl_tests/rpl_stop_slave.test b/mysql-test/extra/rpl_tests/rpl_stop_slave.test index 64e69cebc32..3aaec3ad4d8 100644 --- a/mysql-test/extra/rpl_tests/rpl_stop_slave.test +++ b/mysql-test/extra/rpl_tests/rpl_stop_slave.test @@ -42,6 +42,10 @@ send STOP SLAVE SQL_THREAD; connection slave1; --echo # To resume slave SQL thread SET DEBUG_SYNC= 'now SIGNAL signal.continue'; +# wait for the sql thread to come out of the +# waiting phase before resetting the signals +--let $wait_condition= select count(*)=0 from information_schema.processlist where state LIKE '%debug sync point%' and info LIKE '%INSERT INTO t1 VALUES (1)%' +--source include/wait_condition.inc SET DEBUG_SYNC= 'RESET'; --echo diff --git a/mysql-test/suite/rpl/r/rpl_stop_slave.result b/mysql-test/suite/rpl/r/rpl_stop_slave.result index 588d9bbabf5..c1f4caffa28 100644 --- a/mysql-test/suite/rpl/r/rpl_stop_slave.result +++ b/mysql-test/suite/rpl/r/rpl_stop_slave.result @@ -93,10 +93,12 @@ DROP TABLE t1, t2; CREATE TABLE t1 (c1 INT KEY, c2 INT) ENGINE=InnoDB; CREATE TABLE t2 (c1 INT) ENGINE=MyISAM; INSERT INTO t1 VALUES(1, 1); +include/stop_slave.inc [connection master] +include/stop_dump_threads.inc SET GLOBAL debug= 'd,dump_thread_wait_before_send_xid'; [connection slave] -include/restart_slave.inc +include/start_slave.inc BEGIN; UPDATE t1 SET c2 = 2 WHERE c1 = 1; [connection master] @@ -115,8 +117,11 @@ SET DEBUG_SYNC= 'RESET'; [connection slave] include/wait_for_slave_to_stop.inc [connection slave1] +[connection master] +SET GLOBAL debug= '$debug_save'; +include/stop_dump_threads.inc +[connection slave1] include/start_slave.inc [connection master] DROP TABLE t1, t2; -SET GLOBAL debug= $debug_save; include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_stop_slave.test b/mysql-test/suite/rpl/t/rpl_stop_slave.test index 296b002dbb7..9dc2bce5603 100644 --- a/mysql-test/suite/rpl/t/rpl_stop_slave.test +++ b/mysql-test/suite/rpl/t/rpl_stop_slave.test @@ -72,14 +72,17 @@ CREATE TABLE t2 (c1 INT) ENGINE=MyISAM; INSERT INTO t1 VALUES(1, 1); sync_slave_with_master; +--source include/stop_slave.inc --source include/rpl_connection_master.inc +# make sure that there are no zombie threads +--source include/stop_dump_threads.inc let $debug_save= `SELECT @@GLOBAL.debug`; SET GLOBAL debug= 'd,dump_thread_wait_before_send_xid'; --source include/rpl_connection_slave.inc -source include/restart_slave_sql.inc; +--source include/start_slave.inc BEGIN; UPDATE t1 SET c2 = 2 WHERE c1 = 1; @@ -91,6 +94,10 @@ INSERT INTO t2 VALUES(1); UPDATE t1 SET c2 = 3 WHERE c1 = 1; COMMIT; +# wait for the dump thread reach the sync point +--let $wait_condition= select count(*)=1 from information_schema.processlist where state LIKE '%debug sync point%' and command='Binlog Dump' +--source include/wait_condition.inc + --source include/rpl_connection_slave1.inc let $show_statement= SHOW PROCESSLIST; let $field= Info; @@ -103,7 +110,12 @@ send STOP SLAVE; ROLLBACK; --source include/rpl_connection_master.inc + SET DEBUG_SYNC= 'now SIGNAL signal.continue'; +# wait for the dump thread to come out of the +# waiting phase before resetting the signals +--let $wait_condition= select count(*)=0 from information_schema.processlist where state LIKE '%debug sync point%' and command='Binlog Dump' +--source include/wait_condition.inc SET DEBUG_SYNC= 'RESET'; --source include/rpl_connection_slave.inc @@ -111,9 +123,22 @@ source include/wait_for_slave_to_stop.inc; --source include/rpl_connection_slave1.inc reap; + +# Slave has stopped, thence lets make sure that +# we kill the zombie dump threads. Also, make +# sure that we disable the DBUG_EXECUTE_IF +# that would set the dump thread to wait +--source include/rpl_connection_master.inc +SET GLOBAL debug= '$debug_save'; +# make sure that there are no zombie threads +--source include/stop_dump_threads.inc + +--source include/rpl_connection_slave1.inc +# now the dump thread on the master will start +# from a clean slate, i.e. without the +# DBUG_EXECUTE_IF set source include/start_slave.inc; --source include/rpl_connection_master.inc DROP TABLE t1, t2; -SET GLOBAL debug= $debug_save; --source include/rpl_end.inc