From 089663f9a7effab6198213e48daaea725c6d4267 Mon Sep 17 00:00:00 2001 From: Bjorn Munch Date: Wed, 21 Jan 2009 10:34:01 +0100 Subject: [PATCH] Bug #40399 Please make mtr print stack trace after every failure SIGABRT is sent to relevant processes after a timeout client/mysqltest.cc: Fixed signal handlers to mysqltest actually dumps core mysql-test/lib/My/CoreDump.pm: Added support for dbx mysql-test/lib/My/SafeProcess.pm: Added dump_core to force process to dump core mysql-test/lib/My/SafeProcess/safe_process.cc: Traps SIGABRT and sends this on to child mysql-test/mysql-test-run.pl: When test times out, force core dumps on mysqltest and servers --- client/mysqltest.cc | 7 +++ mysql-test/lib/My/CoreDump.pm | 50 +++++++++++++++++++ mysql-test/lib/My/SafeProcess.pm | 13 ++++- mysql-test/lib/My/SafeProcess/safe_process.cc | 11 ++++ mysql-test/mysql-test-run.pl | 36 ++++++++++--- 5 files changed, 108 insertions(+), 9 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index adc7a9be161..b4b338eca72 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -7326,6 +7326,13 @@ static sig_handler signal_handler(int sig) { fprintf(stderr, "mysqltest got " SIGNAL_FMT "\n", sig); dump_backtrace(); + + fprintf(stderr, "Writing a core file...\n"); + fflush(stderr); + my_write_core(sig); +#ifndef __WIN__ + exit(1); // Shouldn't get here but just in case +#endif } #ifdef __WIN__ diff --git a/mysql-test/lib/My/CoreDump.pm b/mysql-test/lib/My/CoreDump.pm index ab00d15feb3..599f9ccbfca 100644 --- a/mysql-test/lib/My/CoreDump.pm +++ b/mysql-test/lib/My/CoreDump.pm @@ -49,12 +49,56 @@ sub _gdb { unlink $tmp_name or die "Error removing $tmp_name: $!"; + return if $? >> 8; return unless $gdb_output; print <&1` =~ + /Corefile specified executable: "([^"]+)"/; + my $binary= $1 or return; + print "Core generated by '$binary'\n"; + + # Find all threads + my @thr_ids = `echo threads | dbx '$binary' '$core_name' 2>&1` =~ /t@\d+/g; + + # Create tempfile containing dbx commands + my ($tmp, $tmp_name) = tempfile(); + foreach my $thread (@thr_ids) { + print $tmp "where $thread\n"; + } + print $tmp "exit\n"; + close $tmp or die "Error closing $tmp_name: $!"; + + # Run dbx + my $dbx_output= + `cat '$tmp_name' | dbx '$binary' '$core_name' 2>&1`; + + unlink $tmp_name or die "Error removing $tmp_name: $!"; + + return if $? >> 8; + return unless $dbx_output; + + print <($core_name)){ return; diff --git a/mysql-test/lib/My/SafeProcess.pm b/mysql-test/lib/My/SafeProcess.pm index cb68b323d5e..0e3aa968052 100644 --- a/mysql-test/lib/My/SafeProcess.pm +++ b/mysql-test/lib/My/SafeProcess.pm @@ -349,13 +349,24 @@ sub start_kill { { $pid= $self->{SAFE_PID}; die "Can't kill not started process" unless defined $pid; - $ret= kill(15, $pid); + $ret= kill("TERM", $pid); } return $ret; } +sub dump_core { + my ($self)= @_; + return if IS_WINDOWS; + my $pid= $self->{SAFE_PID}; + die "Can't cet core from not started process" unless defined $pid; + _verbose("Sending ABRT to $self"); + kill ("ABRT", $pid); + return 1; +} + + # # Kill the process as fast as possible # and wait for it to return diff --git a/mysql-test/lib/My/SafeProcess/safe_process.cc b/mysql-test/lib/My/SafeProcess/safe_process.cc index 85aaf970c6d..7932f3fd2d6 100644 --- a/mysql-test/lib/My/SafeProcess/safe_process.cc +++ b/mysql-test/lib/My/SafeProcess/safe_process.cc @@ -117,6 +117,16 @@ static void kill_child (void) } +static void handle_abort (int sig) +{ + message("Got signal %d, child_pid: %d, sending ABRT", sig, child_pid); + + if (child_pid > 0) { + kill (-child_pid, SIGABRT); // Don't wait for it to terminate + } +} + + static void handle_signal (int sig) { message("Got signal %d, child_pid: %d", sig, child_pid); @@ -144,6 +154,7 @@ int main(int argc, char* const argv[] ) signal(SIGTERM, handle_signal); signal(SIGINT, handle_signal); signal(SIGCHLD, handle_signal); + signal(SIGABRT, handle_abort); sprintf(safe_process_name, "safe_process[%d]", own_pid); diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index cdb1352e7f6..857dda62c0a 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -464,7 +464,11 @@ sub run_test_server ($$$) { else { mtr_report(" - saving '$worker_savedir/' to '$savedir/'"); rename($worker_savedir, $savedir); - + # Move any core files from e.g. mysqltest + foreach my $coref (glob("core*")) + { + move($coref, $savedir); + } if ($opt_max_save_core > 0) { # Limit number of core files saved find({ no_chdir => 1, @@ -2381,7 +2385,7 @@ sub kill_leftovers ($) { } mtr_report(" - found old pid $pid in '$elem', killing it..."); - my $ret= kill(9, $pid); + my $ret= kill("KILL", $pid); if ($ret == 0) { mtr_report(" process did not exist!"); next; @@ -3283,11 +3287,6 @@ sub run_testcase ($) { # ---------------------------------------------------- $test_timeout_proc->kill(); - # ---------------------------------------------------- - # It's not mysqltest that has exited, kill it - # ---------------------------------------------------- - $test->kill(); - # ---------------------------------------------------- # Check if it was a server that died # ---------------------------------------------------- @@ -3297,10 +3296,30 @@ sub run_testcase ($) { $tinfo->{comment}= "Server $proc failed during test run"; + # ---------------------------------------------------- + # It's not mysqltest that has exited, kill it + # ---------------------------------------------------- + $test->kill(); + report_failure_and_restart($tinfo); return 1; } + # Try to dump core for mysqltest and all servers + foreach my $proc ($test, started(all_servers())) + { + mtr_print("Trying to dump core for $proc"); + if ($proc->dump_core()) + { + $proc->wait_one(20); + } + } + + # ---------------------------------------------------- + # It's not mysqltest that has exited, kill it + # ---------------------------------------------------- + $test->kill(); + # ---------------------------------------------------- # Check if testcase timer expired # ---------------------------------------------------- @@ -3319,6 +3338,7 @@ sub run_testcase ($) { } $tinfo->{'timeout'}= testcase_timeout(); # Mark as timeout run_on_all($tinfo, 'analyze-timeout'); + report_failure_and_restart($tinfo); return 1; } @@ -4222,7 +4242,7 @@ sub start_servers($) { } else { - mysql_install_db($mysqld); + mysql_install_db($mysqld); # For versional testing mtr_error("Failed to install system db to '$datadir'") unless -d $datadir;