move ctest handling from mtr to a separate suite

mysql-test/README.suites

@@ -51,16 +51,16 @@ with bless {}; - that is it must return an object of that class.
 It can also return a string - in this case all tests in the suite
 will be skipped, with this string being printed as a reason.
 
-A suite class can define config_files() and servers() methods.
+A suite class can define config_files(), servers(), list_cases(),
+and start_test() methods.
 
-A config_files method returns a list of additional config files (besides
+A config_files() method returns a list of additional config files (besides
 my.cnf), that this suite needs to be created. For every file it specifies
 a function that will create it, when given a My::Config object. For example:
 
-   sub config_files { ( 'config.ini' => \&write_ini,
+   sub config_files { ( 'config.ini' => \&write_ini, 'new.conf' => \&do_new ) }
-                         'new.conf' => \&do_new_conf ) }
 
-A servers method returns a list of processes that needs to be started for
+A servers() method returns a list of processes that needs to be started for
 this suite. A process is specified as a pair (regex, hash). A regex must
 match a section in the my.cnf template (for example, qr/mysqld\./ corresponds
 to all mysqld processes), a hash contains these options:
@@ -80,6 +80,15 @@ example: sub servers { ( qr/^foo$/ => { SORT => 200,
 
 See sphinx suite for an example.
 
+A list_cases() method returns a complete list of cases for this suite.
+By default it will be the list of files that have .test extension,
+but without the extension. This list will be filtered by mtr, subject to
+different mtr options (--big-test, --start-from, etc), the suite does not have
+to do it.
+
+A start_test() method starts one test process, by default it will be mysqltest.
+
+See unit suite for an example of list_cases() and start_test() methods.
 ==========================
 A suite can have my.cnf template file in the suitedir.
 A my.cnf template uses a normal my.cnf syntax - groups, options,
@@ -149,7 +158,7 @@ names. E.g.
 
   RPL_COMBINATIONS=mix:row ./mtr --suite rpl
 
-See innodb_plugin suite for an example of how suite.pm may set this variable
+See innodb suite for an example of how suite.pm may set this variable
 to exclude unsupported configurations.
 ==========================
 

mysql-test/lib/My/SafeProcess.pm

@@ -94,7 +94,8 @@ if(defined $ENV{MTR_BINDIR})
 }
 else
 {
-  $bindir = ".";
+  use Cwd;
+  $bindir = getcwd();
 }
 
 

mysql-test/lib/My/Suite.pm

@@ -6,5 +6,18 @@ package My::Suite;
 sub config_files { () }
 sub servers { () }
 
+sub list_cases {
+  my ($self, $testdir) = @_;
+  opendir(TESTDIR, $testdir) or mtr_error("Can't open dir \"$testdir\": $!");
+  my (@cases) = grep { s/\.test$// } readdir TESTDIR;
+  closedir TESTDIR;
+  @cases;
+}
+
+sub start_test {
+  my ($self, $tinfo)= @_;
+  &::start_mysqltest($tinfo);
+}
+
 bless { };
 

mysql-test/lib/mtr_cases.pm

@@ -125,7 +125,7 @@ sub collect_test_cases ($$$$) {
     foreach my $test_name_spec ( @$opt_cases )
     {
       my $found= 0;
-      my ($sname, $tname, $extension)= split_testname($test_name_spec);
+      my ($sname, $tname)= split_testname($test_name_spec);
       foreach my $test ( @$cases )
       {
 	last unless $opt_reorder;
@@ -205,7 +205,7 @@ sub collect_test_cases ($$$$) {
 }
 
 
-# Returns (suitename, testname, extension)
+# Returns (suitename, testname)
 sub split_testname {
   my ($test_name)= @_;
 
@@ -217,24 +217,19 @@ sub split_testname {
 
   if (@parts == 1){
     # Only testname given, ex: alias
-    return (undef , $parts[0], undef);
+    return (undef , $parts[0]);
   } elsif (@parts == 2) {
     # Either testname.test or suite.testname given
     # Ex. main.alias or alias.test
 
     if ($parts[1] eq "test")
     {
-      return (undef , $parts[0], $parts[1]);
+      return (undef , $parts[0]);
     }
     else
     {
-      return ($parts[0], $parts[1], undef);
+      return ($parts[0], $parts[1]);
     }
-
-  } elsif (@parts == 3) {
-    # Fully specified suitename.testname.test
-    # ex main.alias.test
-    return ( $parts[0], $parts[1], $parts[2]);
   }
 
   mtr_error("Illegal format of test name: $test_name");
@@ -342,80 +337,43 @@ sub collect_one_suite
   my $suite_opts= [ opts_from_file("$testdir/suite.opt") ];
   $suite_opts = [ opts_from_file("$suitedir/suite.opt") ] unless @$suite_opts;
 
+  my (@case_names)= $::suites{$suite}->list_cases($testdir);
+
   if ( @$opt_cases )
   {
+    my (%case_names)= map { $_ => 1 } @case_names;
+    @case_names= ();
+
     # Collect in specified order
     foreach my $test_name_spec ( @$opt_cases )
     {
-      my ($sname, $tname, $extension)= split_testname($test_name_spec);
+      my ($sname, $tname)= split_testname($test_name_spec);
 
-      # The test name parts have now been defined
+      # Check correct suite if suitename is defined
-      #print "  suite_name: $sname\n";
-      #print "  tname:      $tname\n";
-      #print "  extension:  $extension\n";
-
-      # Check cirrect suite if suitename is defined
       next if (defined $sname and $suite ne $sname);
 
-      if ( defined $extension )
+      # Extension was specified, check if the test exists
+      if ( ! $case_names{$tname})
       {
-	my $full_name= "$testdir/$tname.$extension";
+        # This is only an error if suite was specified, otherwise it
-	# Extension was specified, check if the test exists
+        # could exist in another suite
-        if ( ! -f $full_name)
+        mtr_error("Test '$tname' was not found in suite '$sname'")
-        {
+          if $sname;
-	  # This is only an error if suite was specified, otherwise it
-	  # could exist in another suite
-          mtr_error("Test '$full_name' was not found in suite '$sname'")
-	    if $sname;
 
-	  next;
+        next;
-        }
       }
-      else
+      push @case_names, $tname;
-      {
-	# No extension was specified, use default
-	$extension= "test";
-	my $full_name= "$testdir/$tname.$extension";
-
-	# Test not found here, could exist in other suite
-	next if ( ! -f $full_name );
-      }
-
-      push(@cases,
-	   collect_one_test_case($suitedir,
-				 $testdir,
-				 $resdir,
-				 $suite,
-				 $tname,
-				 "$tname.$extension",
-				 \%disabled,
-				 $suite_opts));
     }
   }
-  else
+
+  foreach (@case_names)
   {
-    opendir(TESTDIR, $testdir) or mtr_error("Can't open dir \"$testdir\": $!");
+    # Skip tests that do not match the --do-test= filter
+    next if ($do_test_reg and not $_ =~ /$do_test_reg/o);
 
-    foreach my $elem ( sort readdir(TESTDIR) )
+    push(@cases, collect_one_test_case($suitedir, $testdir, $resdir,
-    {
+                                       $suite, $_, "$_.test", \%disabled,
-      my $tname= mtr_match_extension($elem, 'test');
+                                       $suite_opts));
-
-      next unless defined $tname;
-
-      # Skip tests that does not match the --do-test= filter
-      next if ($do_test_reg and not $tname =~ /$do_test_reg/o);
-
-      push(@cases,
-	   collect_one_test_case($suitedir,
-				 $testdir,
-				 $resdir,
-				 $suite,
-				 $tname,
-				 $elem,
-				 \%disabled,
-				 $suite_opts));
-    }
-    closedir TESTDIR;
   }
 
   #  Return empty list if no testcases found
@@ -748,7 +706,7 @@ sub collect_one_test_case {
   if ( $start_from && 0)
   {
     # start_from can be specified as [suite.].testname_prefix
-    my ($suite, $test, $ext)= split_testname($start_from);
+    my ($suite, $test)= split_testname($start_from);
 
     if ( $suite and $suitename lt $suite){
       return; # Skip silently
@@ -1032,26 +990,6 @@ sub collect_one_test_case {
     $tinfo->{template_path}= $config;
   }
 
-  if ( $tinfo->{'example_plugin_test'} )
-  {
-    if ( !$ENV{'HA_EXAMPLE_SO'} )
-    {
-      $tinfo->{'skip'}= 1;
-      $tinfo->{'comment'}= "Test requires the 'example' plugin";
-      return $tinfo;
-    }
-  }
-
-  if ( $tinfo->{'oqgraph_test'} )
-  {
-    if ( !$ENV{'GRAPH_ENGINE_SO'} )
-    {
-      $tinfo->{'skip'}= 1;
-      $tinfo->{'comment'}= "Test requires the OQGraph storage engine";
-      return $tinfo;
-    }
-  }
-
   if (not ref $::suites{$tinfo->{suite}})
   {
     $tinfo->{'skip'}= 1;
@@ -1093,16 +1031,18 @@ my @tags=
  ["include/ndb_master-slave.inc", "ndb_test", 1],
  ["include/not_embedded.inc", "not_embedded", 1],
  ["include/not_valgrind.inc", "not_valgrind", 1],
- ["include/have_example_plugin.inc", "example_plugin_test", 1],
- ["include/have_oqgraph_engine.inc", "oqgraph_test", 1],
  ["include/have_ssl.inc", "need_ssl", 1],
  ["include/check_ipv6.inc", "need_ipv6", 1],
 );
 
 
 sub tags_from_test_file {
+
   my $tinfo= shift;
   my $file= shift;
+
+  return unless -f $file;
+
   #mtr_verbose("$file");
   my $F= IO::File->new($file) or mtr_error("can't open file \"$file\": $!");
   my @all_files=($file);

mysql-test/mysql-test-run.pl

@@ -162,7 +162,7 @@ our $opt_vs_config = $ENV{'MTR_VS_CONFIG'};
 
 # If you add a new suite, please check TEST_DIRS in Makefile.am.
 #
-my $DEFAULT_SUITES= "main,sys_vars,binlog,federated,rpl,innodb,perfschema,maria,parts,percona,vcol,oqgraph,sphinx,pbxt";
+my $DEFAULT_SUITES= "main,sys_vars,binlog,federated,rpl,innodb,perfschema,maria,parts,percona,vcol,oqgraph,sphinx,pbxt,unit";
 my $opt_suites;
 
 our $opt_verbose= 0;  # Verbose output, enable with --verbose
@@ -196,10 +196,6 @@ my $opt_debug_common;
 our $opt_debug_server;
 our @opt_cases;                  # The test cases names in argv
 our $opt_embedded_server;
-# -1 indicates use default, override with env.var.
-my $opt_ctest= env_or_val(MTR_UNIT_TESTS => -1);
-# Unit test report stored here for delayed printing
-my $ctest_report;
 
 # Options used when connecting to an already running server
 my %opts_extern;
@@ -542,10 +538,6 @@ sub main {
     mtr_report("Only ", int(@$completed), " of $num_tests completed.");
   }
 
-  mark_time_used('init');
-
-  push @$completed, run_ctest() if $opt_ctest;
-
   if ($opt_valgrind) {
     # Create minimalistic "test" for the reporting
     my $tinfo = My::Test->new
@@ -572,11 +564,6 @@ sub main {
 		 $opt_gcov_msg, $opt_gcov_err);
   }
 
-  if ($ctest_report) {
-    print "$ctest_report\n";
-    mtr_print_line();
-  }
-
   print_total_times($opt_parallel) if $opt_report_times;
 
   mtr_report_stats($prefix, $fail, $completed, $extra_warnings);
@@ -1237,7 +1224,6 @@ sub command_line_setup {
 	     'default-myisam!'          => \&collect_option,
 	     'report-times'             => \$opt_report_times,
 	     'result-file'              => \$opt_resfile,
-	     'unit-tests!'              => \$opt_ctest,
 
              'help|h'                   => \$opt_usage,
 	     # list-options is internal, not listed in help
@@ -1680,14 +1666,6 @@ sub command_line_setup {
       if $opt_suites || @opt_cases;
   }
 
-  # --------------------------------------------------------------------------
-  # Don't run ctest if tests or suites named
-  # --------------------------------------------------------------------------
-
-  $opt_ctest= 0 if $opt_ctest == -1 && ($opt_suites || @opt_cases);
-  # Override: disable if running in the PB test environment
-  $opt_ctest= 0 if $opt_ctest == -1 && defined $ENV{PB2WORKDIR};
-
   # --------------------------------------------------------------------------
   # Check use of wait-all
   # --------------------------------------------------------------------------
@@ -3643,16 +3621,6 @@ sub do_before_run_mysqltest($)
     unlink("$base_file.log");
     unlink("$base_file.warnings");
   }
-
-  if ( $mysql_version_id < 50000 ) {
-    # Set environment variable NDB_STATUS_OK to 1
-    # if script decided to run mysqltest cluster _is_ installed ok
-    $ENV{'NDB_STATUS_OK'} = "1";
-  } elsif ( $mysql_version_id < 50100 ) {
-    # Set environment variable NDB_STATUS_OK to YES
-    # if script decided to run mysqltest cluster _is_ installed ok
-    $ENV{'NDB_STATUS_OK'} = "YES";
-  }
 }
 
 
@@ -4163,6 +4131,7 @@ sub run_testcase ($$) {
       # Write config files:
       my %config_files = config_files($tinfo);
       while (my ($file, $generate) = each %config_files) {
+        next unless $generate;
         my ($path) = "$opt_vardir/$file";
         open (F, '>', $path) or die "Could not open '$path': $!";
         print F &$generate($config);
@@ -4262,7 +4231,7 @@ sub run_testcase ($$) {
     return 1;
   }
 
-  my $test= start_mysqltest($tinfo);
+  my $test= $suites{$tinfo->{suite}}->start_test($tinfo);
   # Set only when we have to keep waiting after expectedly died server
   my $keep_waiting_proc = 0;
 
@@ -6181,79 +6150,6 @@ sub valgrind_exit_reports() {
   return $found_err;
 }
 
-sub run_ctest() {
-  my $olddir= getcwd();
-  chdir ($bindir) or die ("Could not chdir to $bindir");
-  my $tinfo;
-  my $no_ctest= (IS_WINDOWS) ? 256 : -1;
-  my $ctest_vs= "";
-
-  # Just ignore if not configured/built to run ctest
-  if (! -f "CTestTestfile.cmake") {
-    chdir($olddir);
-    return;
-  }
-
-  # Add vs-config option if needed
-  $ctest_vs= "-C $opt_vs_config" if $opt_vs_config;
-
-  # Also silently ignore if we don't have ctest and didn't insist
-  # Special override: also ignore in Pushbuild, some platforms may not have it
-  # Now, run ctest and collect output
-  my $ctest_out= `ctest $ctest_vs 2>&1`;
-  if ($? == $no_ctest && $opt_ctest == -1 && ! defined $ENV{PB2WORKDIR}) {
-    chdir($olddir);
-    return;
-  }
-
-  # Create minimalistic "test" for the reporting
-  $tinfo = My::Test->new
-    (
-     name           => 'unit_tests',
-    );
-  # Set dummy worker id to align report with normal tests
-  $tinfo->{worker} = 0 if $opt_parallel > 1;
-
-  my $ctfail= 0;		# Did ctest fail?
-  if ($?) {
-    $ctfail= 1;
-    $tinfo->{result}= 'MTR_RES_FAILED';
-    $tinfo->{comment}= "ctest failed with exit code $?, see result below";
-    $ctest_out= "" unless $ctest_out;
-  }
-  my $ctfile= "$opt_vardir/ctest.log";
-  my $ctres= 0;			# Did ctest produce report summary?
-
-  open (CTEST, " > $ctfile") or die ("Could not open output file $ctfile");
-
-  # Put ctest output in log file, while analyzing results
-  for (split ('\n', $ctest_out)) {
-    print CTEST "$_\n";
-    if (/tests passed/) {
-      $ctres= 1;
-      $ctest_report .= "\nUnit tests: $_\n";
-    }
-    if ( /FAILED/ or /\(Failed\)/ ) {
-      $ctfail= 1;
-      $ctest_report .= "  $_\n";
-    }
-  }
-  close CTEST;
-
-  # Set needed 'attributes' for test reporting
-  $tinfo->{comment}.= "\nctest did not pruduce report summary" if ! $ctres;
-  $tinfo->{result}= ($ctres && !$ctfail)
-    ? 'MTR_RES_PASSED' : 'MTR_RES_FAILED';
-  $ctest_report .= "Report from unit tests in $ctfile";
-  $tinfo->{failures}= ($tinfo->{result} eq 'MTR_RES_FAILED');
-
-  mark_time_used('test');
-  mtr_report_test($tinfo);
-  chdir($olddir);
-  return $tinfo;
-}
-
-
 #
 # Usage
 #
@@ -6500,9 +6396,6 @@ Misc options
                         engine to InnoDB.
   report-times          Report how much time has been spent on different
                         phases of test execution.
-  nounit-tests          Do not run unit tests. Normally run if configured
-                        and if not running named tests/suites
-  unit-tests            Run unit tests even if they would otherwise not be run
 
 Some options that control enabling a feature for normal test runs,
 can be turned off by prepending 'no' to the option, e.g. --notimer.

mysql-test/suite/unit/my.cnf

@@ -0,0 +1,8 @@
+#
+# This file is intentionally left empty
+#
+
+#
+# It has not a single [mysql] section, which means that mtr
+# will not start any mysqld's for tests in this suite
+#

mysql-test/suite/unit/suite.pm

@@ -0,0 +1,43 @@
+package My::Suite::CTest;
+use Cwd;
+
+@ISA = qw(My::Suite);
+
+sub list_cases {
+  my ($self) = @_;
+  keys %{$self->{ctests}}
+}
+
+sub start_test {
+  my ($self, $tinfo)= @_;
+  my $args=[ ];
+
+  my $oldpwd=getcwd();
+  chdir $::opt_vardir;
+  my $proc=My::SafeProcess->new
+           (
+            name          => $tinfo->{shortname},
+            path          => $self->{ctests}->{$tinfo->{shortname}},
+            args          => \$args,
+            append        => 1,
+            output        => $::path_current_testlog,
+            error         => $::path_current_testlog,
+           );
+  chdir $oldpwd;
+  $proc;
+}
+
+{ 
+  return "Not configured to run ctest" unless -f "../CTestTestfile.cmake";
+  my ($ctest_vs)= $opt_vs_config ? "--build-config $opt_vs_config" : "";
+  my (@ctest_list)= `cd .. && ctest $opt_vs_config --show-only --verbose`;
+  return "No ctest" if $?;
+
+  my ($command, %tests);
+  for (@ctest_list) {
+    chomp;
+    $command= $' if /^\d+: Test command: +/;
+    $tests{$'}=$command if /^ +Test +#\d+: +/;
+  }
+  bless { ctests => { %tests } };
+}