Bug #42804 --parallel option does not work for MTR under ActiveState
perl The problem here was the method how MTR gets its unique thread ids. Prior to this patch, the method to do it was to maintain a global table of pid,mtr_unique_id) pairs. The table was backed by a text file. The table was cleaned up one in a while and dead processes leaking unique_ids were determined with with kill(0) or with scripting tasklist on Windows. This method is flawed specifically on native Windows Perl. fork() is implemented with starting a new thread, give it a syntetic negative PID (threadID*(-1)), until this thread creates a new process with exec() However, neither tasklist nor any other native Windows tool can cope with negative perl PIDs. This lead to incorrect determination of dead process and reusing already used mtr_unique_id. The patch introduces alternative portable method of solving unique-id problem. When a process needs a unique id in range [min...max], it just starts to open files named min, min+1,...max in a loop . After file is opened, we do non-blocking flock(). When flock() succeeds, process has allocated the ID. When process dies, file is unlocked . Checks for zombies are not necessary. Since the change would create a co-existence problems with older version of MTR, because of different way to calculate IDs, the default ID range is changed from 250-299 to 300-349. Another fix that was necessary enable --parallel option was to serialize spawn() calls on Windows. specifically, IO redirects needed to be protected. This patch also fixes hanging CRTL-C (as described in Bug #38629) for the "new" MTR. The fix was already in 6.0 and is now downported.
This commit is contained in:
Vladislav Vaintroub
parent
0128b4fed3
commit
2943d2b7e9
@ -83,6 +83,13 @@ sub exit_status {
|
||||
};
|
||||
}
|
||||
|
||||
# threads.pm may not exist everywhere, so use only on Windows.
|
||||
|
||||
use if $^O eq "MSWin32", "threads";
|
||||
use if $^O eq "MSWin32", "threads::shared";
|
||||
|
||||
my $win32_spawn_lock :shared;
|
||||
|
||||
|
||||
#
|
||||
# Create a new process
|
||||
@ -104,6 +111,8 @@ sub create_process {
|
||||
|
||||
if ($^O eq "MSWin32"){
|
||||
|
||||
lock($win32_spawn_lock);
|
||||
|
||||
#printf STDERR "stdin %d, stdout %d, stderr %d\n",
|
||||
# fileno STDIN, fileno STDOUT, fileno STDERR;
|
||||
|
||||
|
||||
@ -30,6 +30,8 @@ our @EXPORT= qw(report_option mtr_print_line mtr_print_thick_line
|
||||
mtr_report_test);
|
||||
|
||||
use mtr_match;
|
||||
use My::Platform;
|
||||
use POSIX qw[ _exit ];
|
||||
require "mtr_io.pl";
|
||||
|
||||
my $tot_real_time= 0;
|
||||
@ -470,8 +472,15 @@ sub mtr_warning (@) {
|
||||
sub mtr_error (@) {
|
||||
print STDERR _name(), _timestamp(),
|
||||
"mysql-test-run: *** ERROR: ", join(" ", @_), "\n";
|
||||
if (IS_WINDOWS)
|
||||
{
|
||||
POSIX::_exit(1);
|
||||
}
|
||||
else
|
||||
{
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub mtr_debug (@) {
|
||||
|
||||
@ -28,32 +28,36 @@ sub msg {
|
||||
# print "### unique($$) - ", join(" ", @_), "\n";
|
||||
}
|
||||
|
||||
my $file;
|
||||
my $dir;
|
||||
|
||||
if(!IS_WINDOWS)
|
||||
{
|
||||
$file= "/tmp/mysql-test-ports";
|
||||
$dir= "/tmp/mysql-unique-ids";
|
||||
}
|
||||
else
|
||||
{
|
||||
$file= $ENV{'TEMP'}."/mysql-test-ports";
|
||||
}
|
||||
|
||||
|
||||
my %mtr_unique_ids;
|
||||
|
||||
END {
|
||||
my $allocated_id= $mtr_unique_ids{$$};
|
||||
if (defined $allocated_id)
|
||||
# Try to use machine-wide directory location for unique IDs,
|
||||
# $ALLUSERSPROFILE . IF it is not available, fallback to $TEMP
|
||||
# which is typically a per-user temporary directory
|
||||
if (exists $ENV{'ALLUSERSPROFILE'} && -w $ENV{'ALLUSERSPROFILE'})
|
||||
{
|
||||
mtr_release_unique_id($allocated_id);
|
||||
$dir= $ENV{'ALLUSERSPROFILE'}."/mysql-unique-ids";
|
||||
}
|
||||
delete $mtr_unique_ids{$$};
|
||||
else
|
||||
{
|
||||
$dir= $ENV{'TEMP'}."/mysql-unique-ids";
|
||||
}
|
||||
}
|
||||
|
||||
my $mtr_unique_fh = undef;
|
||||
|
||||
END
|
||||
{
|
||||
mtr_release_unique_id();
|
||||
}
|
||||
|
||||
#
|
||||
# Get a unique, numerical ID, given a file name (where all
|
||||
# requested IDs are stored), a minimum and a maximum value.
|
||||
# Get a unique, numerical ID in a specified range.
|
||||
#
|
||||
# If no unique ID within the specified parameters can be
|
||||
# obtained, return undef.
|
||||
@ -61,135 +65,63 @@ END {
|
||||
sub mtr_get_unique_id($$) {
|
||||
my ($min, $max)= @_;;
|
||||
|
||||
msg("get, '$file', $min-$max");
|
||||
msg("get $min-$max, $$");
|
||||
|
||||
die "Can only get one unique id per process!" if $mtr_unique_ids{$$};
|
||||
die "Can only get one unique id per process!" if defined $mtr_unique_fh;
|
||||
|
||||
my $ret = undef;
|
||||
my $changed = 0;
|
||||
|
||||
if(eval("readlink '$file'") || eval("readlink '$file.sem'")) {
|
||||
die 'lock file is a symbolic link';
|
||||
# Make sure our ID directory exists
|
||||
if (! -d $dir)
|
||||
{
|
||||
# If there is a file with the reserved
|
||||
# directory name, just delete the file.
|
||||
if (-e $dir)
|
||||
{
|
||||
unlink($dir);
|
||||
}
|
||||
|
||||
chmod 0777, "$file.sem";
|
||||
open SEM, ">", "$file.sem" or die "can't write to $file.sem";
|
||||
flock SEM, LOCK_EX or die "can't lock $file.sem";
|
||||
if(! -e $file) {
|
||||
open FILE, ">", $file or die "can't create $file";
|
||||
close FILE;
|
||||
mkdir $dir;
|
||||
chmod 0777, $dir;
|
||||
|
||||
if(! -d $dir)
|
||||
{
|
||||
die "can't make directory $dir";
|
||||
}
|
||||
}
|
||||
|
||||
msg("HAVE THE LOCK");
|
||||
|
||||
if(eval("readlink '$file'") || eval("readlink '$file.sem'")) {
|
||||
die 'lock file is a symbolic link';
|
||||
my $fh;
|
||||
for(my $id = $min; $id <= $max; $id++)
|
||||
{
|
||||
open( $fh, ">$dir/$id");
|
||||
chmod 0666, "$dir/$id";
|
||||
# Try to lock the file exclusively. If lock succeeds, we're done.
|
||||
if (flock($fh, LOCK_EX|LOCK_NB))
|
||||
{
|
||||
# Store file handle - we would need it to release the ID (==unlock the file)
|
||||
$mtr_unique_fh = $fh;
|
||||
return $id;
|
||||
}
|
||||
|
||||
chmod 0777, $file;
|
||||
open FILE, "+<", $file or die "can't open $file";
|
||||
#select undef,undef,undef,0.2;
|
||||
seek FILE, 0, 0;
|
||||
my %taken = ();
|
||||
while(<FILE>) {
|
||||
chomp;
|
||||
my ($id, $pid) = split / /;
|
||||
$taken{$id} = $pid;
|
||||
msg("taken: $id, $pid");
|
||||
# Check if process with given pid is alive
|
||||
if(!process_alive($pid)) {
|
||||
print "Removing slot $id used by missing process $pid\n";
|
||||
msg("Removing slot $id used by missing process $pid");
|
||||
delete $taken{$id};
|
||||
$changed++;
|
||||
else
|
||||
{
|
||||
close $fh;
|
||||
}
|
||||
}
|
||||
for(my $i=$min; $i<=$max; ++$i) {
|
||||
if(! exists $taken{$i}) {
|
||||
$ret = $i;
|
||||
$taken{$i} = $$;
|
||||
$changed++;
|
||||
# Remember the id this process got
|
||||
$mtr_unique_ids{$$}= $i;
|
||||
msg(" got $i");
|
||||
last;
|
||||
}
|
||||
}
|
||||
if($changed) {
|
||||
seek FILE, 0, 0;
|
||||
truncate FILE, 0 or die "can't truncate $file";
|
||||
for my $k (keys %taken) {
|
||||
print FILE $k . ' ' . $taken{$k} . "\n";
|
||||
}
|
||||
}
|
||||
close FILE;
|
||||
|
||||
msg("RELEASING THE LOCK");
|
||||
flock SEM, LOCK_UN or warn "can't unlock $file.sem";
|
||||
close SEM;
|
||||
|
||||
return $ret;
|
||||
return undef;
|
||||
}
|
||||
|
||||
|
||||
#
|
||||
# Release a unique ID.
|
||||
#
|
||||
sub mtr_release_unique_id($) {
|
||||
my ($myid)= @_;
|
||||
|
||||
msg("release, $myid");
|
||||
|
||||
|
||||
if(eval("readlink '$file'") || eval("readlink '$file.sem'")) {
|
||||
die 'lock file is a symbolic link';
|
||||
}
|
||||
|
||||
open SEM, ">", "$file.sem" or die "can't write to $file.sem";
|
||||
flock SEM, LOCK_EX or die "can't lock $file.sem";
|
||||
|
||||
msg("HAVE THE LOCK");
|
||||
|
||||
if(eval("readlink '$file'") || eval("readlink '$file.sem'")) {
|
||||
die 'lock file is a symbolic link';
|
||||
}
|
||||
|
||||
if(! -e $file) {
|
||||
open FILE, ">", $file or die "can't create $file";
|
||||
close FILE;
|
||||
}
|
||||
open FILE, "+<", $file or die "can't open $file";
|
||||
#select undef,undef,undef,0.2;
|
||||
seek FILE, 0, 0;
|
||||
my %taken = ();
|
||||
while(<FILE>) {
|
||||
chomp;
|
||||
my ($id, $pid) = split / /;
|
||||
msg(" taken, $id $pid");
|
||||
$taken{$id} = $pid;
|
||||
}
|
||||
|
||||
if ($taken{$myid} != $$)
|
||||
sub mtr_release_unique_id()
|
||||
{
|
||||
msg(" The unique id for this process does not match pid");
|
||||
msg("release $$");
|
||||
if (defined $mtr_unique_fh)
|
||||
{
|
||||
close $mtr_unique_fh;
|
||||
$mtr_unique_fh = undef;
|
||||
}
|
||||
|
||||
|
||||
msg(" removing $myid");
|
||||
delete $taken{$myid};
|
||||
seek FILE, 0, 0;
|
||||
truncate FILE, 0 or die "can't truncate $file";
|
||||
for my $k (keys %taken) {
|
||||
print FILE $k . ' ' . $taken{$k} . "\n";
|
||||
}
|
||||
close FILE;
|
||||
|
||||
msg("RELEASE THE LOCK");
|
||||
|
||||
flock SEM, LOCK_UN or warn "can't unlock $file.sem";
|
||||
close SEM;
|
||||
|
||||
delete $mtr_unique_ids{$$};
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1326,16 +1326,19 @@ sub set_build_thread_ports($) {
|
||||
|
||||
if ( lc($opt_build_thread) eq 'auto' ) {
|
||||
my $found_free = 0;
|
||||
$build_thread = 250; # Start attempts from here
|
||||
$build_thread = 300; # Start attempts from here
|
||||
while (! $found_free)
|
||||
{
|
||||
$build_thread= mtr_get_unique_id($build_thread, 299);
|
||||
$build_thread= mtr_get_unique_id($build_thread, 349);
|
||||
if ( !defined $build_thread ) {
|
||||
mtr_error("Could not get a unique build thread id");
|
||||
}
|
||||
$found_free= check_ports_free($build_thread);
|
||||
# If not free, release and try from next number
|
||||
mtr_release_unique_id($build_thread++) unless $found_free;
|
||||
if (! $found_free) {
|
||||
mtr_release_unique_id();
|
||||
$build_thread++;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user