Bug#42888 for MySQL 5.1. Add collections of test runs to make it both configurable and transparent what kinds of tests we run during integration testing. Implement filter list in mysql-test-run.pl to filter out failures of experimental test cases, using a new status code "exp-fail", so we can tell regressions ("fail") from failures of test cases that are still in development ("exp-fail").

mysql-test/collections/README

@@ -0,0 +1,30 @@
+This directory contains collections of test runs that we run during our
+integration and release testing. Each file contains zero or more lines,
+with one invocation of mysql-test-run.pl on each. These invocations are
+written so that, with the assumption that perl is in your search path,
+any collection can run as a shell script or a batch file, with the parent
+mysql-test directory being the current working directory.
+
+During integration testing, we choose the collection to run by following
+these steps:
+
+1) We choose the extension to look for, based on these rules:
+   - If we're running a per-push test, we choose ".push" as the extension.
+   - If we're running a daily test, we choose ".daily" as the extension.
+   - If we're running a weekly test, we choose ".weekly" as the extension.
+
+2) If there is a collection that has the same name as the branch we're
+   testing plus the extension as determined in step 1, we choose that
+   collection.
+
+3) If the branch is unknown or we have removed all characters from it
+   and still not found a matching collection, we choose the name "default"
+   plus the extension determined in step 1. If there is no such file,
+   we give up and don't test anything at all.
+
+4) If we haven't found a collection yet, we remove the last character from 
+   the branch name and go back to step 2.
+
+5) The commands from the collection are run line by line via execv() or
+   similar system calls. They are not run as a shell script. Shell 
+   expansions are not guaranteed to work and most likely won't.

mysql-test/collections/README.experimental

@@ -0,0 +1,25 @@
+The .experimental files in this directory contain names of test cases that
+are still in development and whose failures should be considered expected,
+instead of regressions.
+
+These files are to be used with the --experimental option of 
+mysql-test-run.pl. Please look at its help screen for usage information.
+
+The syntax is as follows:
+
+1) One line per test case.
+
+2) Empty lines and lines starting with a hash (#) are ignored.
+
+3) If any other line contains a black followed by a hash (#), the hash 
+   and any subsequent characters are ignored.
+
+4) The full test case name including the suite and execution mode
+   must be specified, for example:
+        main.alias 'row'     # bug#00000
+
+5) As an exception to item 4, the last character of the test case
+   specification may be an asterisk (*). In that case, all test cases that
+   start with the same characters up to the last letter before the asterisk
+   are considered experimental:
+        main.a*  # get rid of main.alias, main.alibaba and main.agliolio

mysql-test/collections/default.daily

@@ -0,0 +1 @@
+perl mysql-test-run.pl --timer --force --comment=rpl_ndb_row --suite=rpl_ndb,ndb --mysqld=--binlog-format=row --experimental=collections/default.experimental

mysql-test/collections/default.experimental

@@ -0,0 +1 @@
+funcs_1.charset_collation_1              # depends on compile-time decisions

mysql-test/collections/default.push

@@ -0,0 +1,5 @@
+perl mysql-test-run.pl --timer --force --comment=n_mix --mysqld=--binlog-format=mixed --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=ps_row --ps-protocol --mysqld=--binlog-format=row --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=embedded --embedded --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=rpl_binlog_row --suite=rpl,binlog --mysqld=--binlog-format=row --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=funcs_1 --suite=funcs_1 --experimental=collections/default.experimental

mysql-test/lib/mtr_report.pm

@@ -69,6 +69,8 @@ sub _mtr_report_test_name ($) {
 
   print _name(), _timestamp();
   printf "%-40s ", $tname;
+
+  return $tname;
 }
 
 
@@ -105,20 +107,41 @@ sub mtr_report_test_passed ($) {
 
 sub mtr_report_test ($) {
   my ($tinfo)= @_;
-  _mtr_report_test_name($tinfo);
+  my $test_name = _mtr_report_test_name($tinfo);
 
-  my $comment=  $tinfo->{'comment'};
+  my $comment=      $tinfo->{'comment'};
-  my $logfile=  $tinfo->{'logfile'};
+  my $logfile=      $tinfo->{'logfile'};
-  my $warnings= $tinfo->{'warnings'};
+  my $warnings=     $tinfo->{'warnings'};
-  my $result=   $tinfo->{'result'};
+  my $result=       $tinfo->{'result'};
 
   if ($result eq 'MTR_RES_FAILED'){
 
     my $timest = format_time();
+    my $fail = "fail";
+
+    if ( $::opt_experimental )
+    {
+      # Find out if this test case is an experimental one, so we can treat
+      # the failure as an expected failure instead of a regression.
+      for my $exp ( @$::opt_experimental ) {
+        if ( $exp ne $test_name ) {
+          if ( $exp ne "" && substr($exp, -1, 1) eq "*" ) {
+            $exp = substr($exp, 0, length($exp) - 1);
+            if ( substr($test_name, 0, length($exp)) ne $exp ) {
+              next;
+            }
+          } else {
+            next;
+          }
+        }
+        $fail = "exp-fail";
+        last;
+      }
+    }
 
     if ( $warnings )
     {
-      mtr_report("[ fail ]  Found warnings/errors in server log file!");
+      mtr_report("[ $fail ]  Found warnings/errors in server log file!");
       mtr_report("        Test ended at $timest");
       mtr_report($warnings);
       return;
@@ -126,14 +149,14 @@ sub mtr_report_test ($) {
     my $timeout= $tinfo->{'timeout'};
     if ( $timeout )
     {
-      mtr_report("[ fail ]  timeout after $timeout seconds");
+      mtr_report("[ $fail ]  timeout after $timeout seconds");
       mtr_report("        Test ended at $timest");
       mtr_report("\n$tinfo->{'comment'}");
       return;
     }
     else
     {
-      mtr_report("[ fail ]\n        Test ended at $timest");
+      mtr_report("[ $fail ]\n        Test ended at $timest");
     }
 
     if ( $logfile )

mysql-test/mysql-test-run.pl

@@ -169,6 +169,8 @@ our $opt_client_debugger;
 my $config; # The currently running config
 my $current_config_name; # The currently running config file template
 
+our $opt_experimental;
+
 my $baseport;
 my $opt_build_thread= $ENV{'MTR_BUILD_THREAD'} || "auto";
 
@@ -792,7 +794,7 @@ sub command_line_setup {
              'big-test'                 => \$opt_big_test,
 	     'combination=s'            => \@opt_combinations,
              'skip-combinations'        => \&collect_option,
-
+             'experimental=s'           => \$opt_experimental,
 	     'skip-im'                  => \&ignore_option,
 
              # Specify ports
@@ -959,6 +961,28 @@ sub command_line_setup {
     mtr_print_thick_line('#');
   }
 
+  if ( $opt_experimental )
+  {
+    if ( open(FILE, "<", $opt_experimental) ) {
+      mtr_report("Using experimental file: $opt_experimental");
+      $opt_experimental = [];
+      while(<FILE>) {
+        chomp;
+        s/( +|^)#.*$//;
+        s/^ +//;
+        s/ +$//;
+        if ( $_ eq "" ) {
+          next;
+        }
+        print " - $_\n";
+        push @$opt_experimental, $_;
+      }
+      close FILE;
+    } else {
+      mtr_error("Can't read experimental file: $opt_experimental");      
+    }
+  }
+
   foreach my $arg ( @ARGV )
   {
     if ( $arg =~ /^--skip-/ )