From b55c18a1fbdcdf7bb43b759714f0aed78214d49f Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 20 Jul 2006 13:23:45 +0200 Subject: [PATCH] Bug#19535 mysql-test-run cannot handle crashing test cases - Make mysql-test-run.pl restart a crashed/stopped process if it was expected it should crash. - Added testcase for "crash_commit_before", which tests behaviour when server crashes just before commit. * The testcase first write a small var/tmp/master0.expect file indicating expected crash * Then sets "DEBUG" variable to for example "d,crash_commit_before" and then executes a commit. * The server will crash and be brought back up by mysql-test-run.pl * Test case will then wait for the process to come back online before continuing. client/mysqltest.c: Add "die" command used to abort a failing test case Remove unused "exit" command mysql-test/lib/mtr_process.pl: When a process crash during execution of mysqltest, check if it was expected and restart the process in that case. mysql-test/mysql-test-run.pl: Remmeber options used when ndbd's and mysqld's was started, to be used by restart code mysql-test/r/crash_commit_before.result: New BitKeeper file ``mysql-test/r/crash_commit_before.result'' mysql-test/include/wait_until_connected_again.inc: New BitKeeper file ``mysql-test/include/wait_until_connected_again.inc'' mysql-test/t/crash_commit_before.test: New BitKeeper file ``mysql-test/t/crash_commit_before.test'' --- client/mysqltest.c | 15 ++-- .../include/wait_until_connected_again.inc | 20 +++++ mysql-test/lib/mtr_process.pl | 81 ++++++++++++++++++- mysql-test/mysql-test-run.pl | 5 ++ mysql-test/r/crash_commit_before.result | 13 +++ mysql-test/t/crash_commit_before.test | 22 +++++ 6 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 mysql-test/include/wait_until_connected_again.inc create mode 100644 mysql-test/r/crash_commit_before.result create mode 100644 mysql-test/t/crash_commit_before.test diff --git a/client/mysqltest.c b/client/mysqltest.c index 1b82936dfea..0db78bab92b 100644 --- a/client/mysqltest.c +++ b/client/mysqltest.c @@ -338,11 +338,10 @@ Q_DISPLAY_VERTICAL_RESULTS, Q_DISPLAY_HORIZONTAL_RESULTS, Q_QUERY_VERTICAL, Q_QUERY_HORIZONTAL, Q_START_TIMER, Q_END_TIMER, Q_CHARACTER_SET, Q_DISABLE_PS_PROTOCOL, Q_ENABLE_PS_PROTOCOL, -Q_EXIT, Q_DISABLE_RECONNECT, Q_ENABLE_RECONNECT, Q_IF, Q_DISABLE_PARSING, Q_ENABLE_PARSING, -Q_REPLACE_REGEX, +Q_REPLACE_REGEX, Q_DIE, Q_UNKNOWN, /* Unknown command. */ Q_COMMENT, /* Comments, ignored. */ @@ -424,13 +423,13 @@ const char *command_names[]= "character_set", "disable_ps_protocol", "enable_ps_protocol", - "exit", "disable_reconnect", "enable_reconnect", "if", "disable_parsing", "enable_parsing", "replace_regex", + "die", 0 }; @@ -5081,7 +5080,7 @@ static void init_var_hash(MYSQL *mysql) test run completes */ -static void mark_progress(struct st_query* q, int line) +static void mark_progress(struct st_query* q __attribute__((unused)), int line) { char buf[32], *end; ulonglong timer= timer_now(); @@ -5116,7 +5115,7 @@ static void mark_progress(struct st_query* q, int line) int main(int argc, char **argv) { struct st_query *q; - my_bool require_file=0, q_send_flag=0, abort_flag= 0, + my_bool require_file=0, q_send_flag=0, query_executed= 0; char save_file[FN_REFLEN]; MY_STAT res_info; @@ -5219,7 +5218,7 @@ int main(int argc, char **argv) open_file(opt_include); } - while (!abort_flag && !read_query(&q)) + while (!read_query(&q)) { int current_line_inc = 1, processed = 0; if (q->type == Q_UNKNOWN || q->type == Q_COMMENT_WITH_COMMAND) @@ -5434,8 +5433,8 @@ int main(int argc, char **argv) parsing_disabled--; break; - case Q_EXIT: - abort_flag= 1; + case Q_DIE: + die("%s", q->first_argument); break; default: diff --git a/mysql-test/include/wait_until_connected_again.inc b/mysql-test/include/wait_until_connected_again.inc new file mode 100644 index 00000000000..5ae722e6db7 --- /dev/null +++ b/mysql-test/include/wait_until_connected_again.inc @@ -0,0 +1,20 @@ +# +# Include this script to wait until the connection to the +# server has been restored or timeout occurs +--disable_result_log +--disable_query_log +let $counter= 100; +while ($mysql_errno) +{ + --error 0,2002,2006 + show status; + + dec $counter; + if (!$counter) + { + --die Server failed to restart + } + --sleep 0.1 +} +--enable_query_log +--enable_result_log diff --git a/mysql-test/lib/mtr_process.pl b/mysql-test/lib/mtr_process.pl index b593d80a5e8..2d94fd9adb1 100644 --- a/mysql-test/lib/mtr_process.pl +++ b/mysql-test/lib/mtr_process.pl @@ -272,10 +272,10 @@ sub spawn_parent_impl { last; } - # If one of the processes died, we want to - # mark this, and kill the mysqltest process. + # One of the child processes died, unless this was expected + # mysqltest should be killed and test aborted - mark_process_dead($ret_pid); + check_expected_crash_and_restart($ret_pid); } if ( $ret_pid != $pid ) @@ -809,6 +809,81 @@ sub mark_process_dead($) } +# +# Loop through our list of processes and look for and entry +# with the provided pid, if found check for the file indicating +# expected crash and restart it. +# +sub check_expected_crash_and_restart($) +{ + my $ret_pid= shift; + + foreach my $mysqld (@{$::master}, @{$::slave}) + { + if ( $mysqld->{'pid'} eq $ret_pid ) + { + mtr_verbose("$mysqld->{'type'} $mysqld->{'idx'} exited, pid: $ret_pid"); + $mysqld->{'pid'}= 0; + + # Check if crash expected and restart if it was + my $expect_file= "$::opt_vardir/tmp/" . "$mysqld->{'type'}" . + "$mysqld->{'idx'}" . ".expect"; + if ( -f $expect_file ) + { + mtr_verbose("Crash was expected, file $expect_file exists"); + mysqld_start($mysqld, $mysqld->{'start_opts'}, + $mysqld->{'start_slave_master_info'}); + unlink($expect_file); + } + + return; + } + } + + foreach my $cluster (@{$::clusters}) + { + if ( $cluster->{'pid'} eq $ret_pid ) + { + mtr_verbose("$cluster->{'name'} cluster ndb_mgmd exited, pid: $ret_pid"); + $cluster->{'pid'}= 0; + + # Check if crash expected and restart if it was + my $expect_file= "$::opt_vardir/tmp/ndb_mgmd_" . "$cluster->{'type'}" . + ".expect"; + if ( -f $expect_file ) + { + mtr_verbose("Crash was expected, file $expect_file exists"); + ndbmgmd_start($cluster); + unlink($expect_file); + } + return; + } + + foreach my $ndbd (@{$cluster->{'ndbds'}}) + { + if ( $ndbd->{'pid'} eq $ret_pid ) + { + mtr_verbose("$cluster->{'name'} cluster ndbd exited, pid: $ret_pid"); + $ndbd->{'pid'}= 0; + + # Check if crash expected and restart if it was + my $expect_file= "$::opt_vardir/tmp/ndbd_" . "$cluster->{'type'}" . + "$ndbd->{'idx'}" . ".expect"; + if ( -f $expect_file ) + { + mtr_verbose("Crash was expected, file $expect_file exists"); + ndbd_start($cluster, $ndbd->{'idx'}, + $ndbd->{'start_extra_args'}); + unlink($expect_file); + } + return; + } + } + } + mtr_warning("check_expected_crash_and_restart couldn't find an entry for pid: $ret_pid"); + +} + ############################################################################## # # The operating system will keep information about dead children, diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 8b48f87c138..588392f2af5 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1793,6 +1793,10 @@ sub ndbd_start ($$$) { # Add pid to list of pids for this cluster $cluster->{'ndbds'}->[$idx]->{'pid'}= $pid; + # Rememeber options used when starting + $cluster->{'ndbds'}->[$idx]->{'start_extra_args'}= $extra_args; + $cluster->{'ndbds'}->[$idx]->{'idx'}= $idx; + mtr_verbose("ndbd_start, pid: $pid"); return $pid; @@ -2894,6 +2898,7 @@ sub mysqld_start ($$$) { # Remember options used when starting $mysqld->{'start_opts'}= $extra_opt; + $mysqld->{'start_slave_master_info'}= $slave_master_info; mtr_verbose("mysqld pid: $pid"); return $pid; diff --git a/mysql-test/r/crash_commit_before.result b/mysql-test/r/crash_commit_before.result new file mode 100644 index 00000000000..8eba584c539 --- /dev/null +++ b/mysql-test/r/crash_commit_before.result @@ -0,0 +1,13 @@ +CREATE TABLE t1(a int) engine=innodb; +START TRANSACTION; +insert into t1 values(9); +SET SESSION debug="d,crash_commit_before"; +COMMIT; +ERROR HY000: Lost connection to MySQL server during query +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +SELECT * FROM t1; +a diff --git a/mysql-test/t/crash_commit_before.test b/mysql-test/t/crash_commit_before.test new file mode 100644 index 00000000000..4a0453901bc --- /dev/null +++ b/mysql-test/t/crash_commit_before.test @@ -0,0 +1,22 @@ +--source include/have_debug.inc + +CREATE TABLE t1(a int) engine=innodb; +START TRANSACTION; +insert into t1 values(9); + +# Setup the mysqld to crash at certain point +SET SESSION debug="d,crash_commit_before"; + +# Write file to make mysql-test-run.pl expect crash and restart +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/master0.expect + +# Run the crashing query +--error 2013 +COMMIT; + +# Call script that will poll the server waiting for it to be back online again +--source include/wait_until_connected_again.inc + +SHOW CREATE TABLE t1; + +SELECT * FROM t1;