| #! @PERL@ |
| ##--------------------------------------------------------------------## |
| ##--- Valgrind regression testing script vg_regtest ---## |
| ##--------------------------------------------------------------------## |
| |
| # This file is part of Valgrind, a dynamic binary instrumentation |
| # framework. |
| # |
| # Copyright (C) 2003 Nicholas Nethercote |
| # njn@valgrind.org |
| # |
| # This program is free software; you can redistribute it and/or |
| # modify it under the terms of the GNU General Public License as |
| # published by the Free Software Foundation; either version 2 of the |
| # License, or (at your option) any later version. |
| # |
| # This program is distributed in the hope that it will be useful, but |
| # WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| # General Public License for more details. |
| # |
| # You should have received a copy of the GNU General Public License |
| # along with this program; if not, write to the Free Software |
| # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA |
| # 02111-1307, USA. |
| # |
| # The GNU General Public License is contained in the file COPYING. |
| |
| #---------------------------------------------------------------------------- |
| # usage: vg_regtest [options] <dirs | files> |
| # |
| # Options: |
| # --all: run tests in all subdirs |
| # --valgrind: valgrind launcher to use. Default is ./coregrind/valgrind. |
| # (This option should probably only be used in conjunction with |
| # --valgrind-lib.) |
| # --valgrind-lib: valgrind libraries to use. Default is $tests_dir/.in_place. |
| # (This option should probably only be used in conjunction with |
| # --valgrind.) |
| # |
| # The easiest way is to run all tests in valgrind/ with (assuming you installed |
| # in $PREFIX): |
| # |
| # $PREFIX/bin/vg_regtest --all |
| # |
| # You can specify individual files to test, or whole directories, or both. |
| # Directories are traversed recursively, except for ones named, for example, |
| # CVS/ or docs/. |
| # |
| # Each test is defined in a file <test>.vgtest, containing one or more of the |
| # following lines, in any order: |
| # - prog: <prog to run> (compulsory) |
| # - args: <args for prog> (default: none) |
| # - vgopts: <Valgrind options> (default: none; |
| # multiple are allowed) |
| # - stdout_filter: <filter to run stdout through> (default: none) |
| # - stderr_filter: <filter to run stderr through> (default: ./filter_stderr) |
| # - prereq: <prerequisite command> (default: none) |
| # - post: <post-test check command> (default: none) |
| # - cleanup: <post-test cleanup cmd> (default: none) |
| # |
| # Note that filters are necessary for stderr results to filter out things that |
| # always change, eg. process id numbers. |
| # |
| # Expected stdout (filtered) is kept in <test>.stdout.exp* (can be more |
| # than one expected output). It can be missing if it would be empty. Expected |
| # stderr (filtered) is kept in <test>.stderr.exp*. There must be at least |
| # one stderr.exp* file. Any .exp* file that ends in '~' or '#' is ignored; |
| # this is because Emacs creates temporary files of these names. |
| # |
| # If results don't match, the output can be found in <test>.std<strm>.out, |
| # and the diff between expected and actual in <test>.std<strm>.diff*. |
| # |
| # The prerequisite command, if present, works like this: |
| # - if it returns 0 the test is run |
| # - if it returns 1 the test is skipped |
| # - if it returns anything else the script aborts. |
| # The idea here is results other than 0 or 1 are likely to be due to |
| # problems with the commands, and you don't want to conflate them with the 1 |
| # case, which would happen if you just tested for zero or non-zero. |
| # |
| # The post-test command, if present, must return 0 and its stdout must match |
| # the expected stdout which is kept in <test>.post.exp*. |
| # |
| # Sometimes it is useful to run all the tests at a high sanity check |
| # level or with arbitrary other flags. To make this simple, extra |
| # options, applied to all tests run, are read from $EXTRA_REGTEST_OPTS, |
| # and handed to valgrind prior to any other flags specified by the |
| # .vgtest file. |
| # |
| # Some more notes on adding regression tests for a new tool are in |
| # docs/xml/manual-writing-tools.xml. |
| #---------------------------------------------------------------------------- |
| |
| use warnings; |
| use strict; |
| |
| #---------------------------------------------------------------------------- |
| # Global vars |
| #---------------------------------------------------------------------------- |
| my $usage="\n" |
| . "Usage:\n" |
| . " vg_regtest [--all, --valgrind, --valgrind-lib]\n" |
| . " Use EXTRA_REGTEST_OPTS to supply extra args for all tests\n" |
| . "\n"; |
| |
| my $tmp="vg_regtest.tmp.$$"; |
| |
| # Test variables |
| my $vgopts; # valgrind options |
| my $prog; # test prog |
| my $args; # test prog args |
| my $stdout_filter; # filter program to run stdout results file through |
| my $stderr_filter; # filter program to run stderr results file through |
| my $prereq; # prerequisite test to satisfy before running test |
| my $post; # check command after running test |
| my $cleanup; # cleanup command to run |
| |
| my @failures; # List of failed tests |
| |
| my $num_tests_done = 0; |
| my %num_failures = (stderr => 0, stdout => 0, post => 0); |
| |
| # Default valgrind to use is this build tree's (uninstalled) one |
| my $valgrind = "./coregrind/valgrind"; |
| |
| chomp(my $tests_dir = `pwd`); |
| |
| my $valgrind_lib = "$tests_dir/.in_place"; |
| |
| # default filter is the one named "filter_stderr" in the test's directory |
| my $default_stderr_filter = "filter_stderr"; |
| |
| |
| #---------------------------------------------------------------------------- |
| # Process command line, setup |
| #---------------------------------------------------------------------------- |
| |
| # If $prog is a relative path, it prepends $dir to it. Useful for two reasons: |
| # |
| # 1. Can prepend "." onto programs to avoid trouble with users who don't have |
| # "." in their path (by making $dir = ".") |
| # 2. Can prepend the current dir to make the command absolute to avoid |
| # subsequent trouble when we change directories. |
| # |
| # Also checks the program exists and is executable. |
| sub validate_program ($$$$) |
| { |
| my ($dir, $prog, $must_exist, $must_be_executable) = @_; |
| |
| # If absolute path, leave it alone. If relative, make it |
| # absolute -- by prepending current dir -- so we can change |
| # dirs and still use it. |
| $prog = "$dir/$prog" if ($prog !~ /^\//); |
| if ($must_exist) { |
| (-f $prog) or die "vg_regtest: `$prog' not found or not a file ($dir)\n"; |
| } |
| if ($must_be_executable) { |
| (-x $prog) or die "vg_regtest: `$prog' not executable ($dir)\n"; |
| } |
| |
| return $prog; |
| } |
| |
| sub process_command_line() |
| { |
| my $alldirs = 0; |
| my @fs; |
| |
| for my $arg (@ARGV) { |
| if ($arg =~ /^-/) { |
| if ($arg =~ /^--all$/) { |
| $alldirs = 1; |
| } elsif ($arg =~ /^--valgrind=(.*)$/) { |
| $valgrind = $1; |
| } elsif ($arg =~ /^--valgrind-lib=(.*)$/) { |
| $valgrind_lib = $1; |
| } else { |
| die $usage; |
| } |
| } else { |
| push(@fs, $arg); |
| } |
| } |
| $valgrind = validate_program($tests_dir, $valgrind, 1, 0); |
| |
| if ($alldirs) { |
| @fs = (); |
| foreach my $f (glob "*") { |
| push(@fs, $f) if (-d $f); |
| } |
| } |
| |
| (0 != @fs) or die "No test files or directories specified\n"; |
| |
| return @fs; |
| } |
| |
| #---------------------------------------------------------------------------- |
| # Read a .vgtest file |
| #---------------------------------------------------------------------------- |
| sub read_vgtest_file($) |
| { |
| my ($f) = @_; |
| |
| # Defaults. |
| ($vgopts, $prog, $args) = ("", undef, ""); |
| ($stdout_filter, $stderr_filter) = (undef, undef); |
| ($prereq, $post, $cleanup) = (undef, undef, undef); |
| |
| # Every test directory must have a "filter_stderr" |
| $stderr_filter = validate_program(".", $default_stderr_filter, 1, 1); |
| |
| open(INPUTFILE, "< $f") || die "File $f not openable\n"; |
| |
| while (my $line = <INPUTFILE>) { |
| if ($line =~ /^\s*#/ || $line =~ /^\s*$/) { |
| next; |
| } elsif ($line =~ /^\s*vgopts:\s*(.*)$/) { |
| $vgopts = $vgopts . " " . $1; # Nb: Make sure there's a space! |
| } elsif ($line =~ /^\s*prog:\s*(.*)$/) { |
| $prog = validate_program(".", $1, 0, 0); |
| } elsif ($line =~ /^\s*args:\s*(.*)$/) { |
| $args = $1; |
| } elsif ($line =~ /^\s*stdout_filter:\s*(.*)$/) { |
| $stdout_filter = validate_program(".", $1, 1, 1); |
| } elsif ($line =~ /^\s*stderr_filter:\s*(.*)$/) { |
| $stderr_filter = validate_program(".", $1, 1, 1); |
| } elsif ($line =~ /^\s*prereq:\s*(.*)$/) { |
| $prereq = $1; |
| } elsif ($line =~ /^\s*post:\s*(.*)$/) { |
| $post = $1; |
| } elsif ($line =~ /^\s*cleanup:\s*(.*)$/) { |
| $cleanup = $1; |
| } else { |
| die "Bad line in $f: $line\n"; |
| } |
| } |
| close(INPUTFILE); |
| |
| if (!defined $prog) { |
| $prog = ""; # allow no prog for testing error and --help cases |
| } |
| } |
| |
| #---------------------------------------------------------------------------- |
| # Do one test |
| #---------------------------------------------------------------------------- |
| # Since most of the program time is spent in system() calls, need this to |
| # propagate a Ctrl-C enabling us to quit. |
| sub mysystem($) |
| { |
| my $exit_code = system($_[0]); |
| ($exit_code == 2) and exit 1; # 2 is SIGINT |
| return $exit_code; |
| } |
| |
| # from a directory name like "/foo/cachesim/tests/" determine the tool name |
| sub determine_tool() |
| { |
| my $dir = `pwd`; |
| $dir =~ /.*\/([^\/]+)\/tests.*/; # foo/tool_name/tests/foo |
| return $1; |
| } |
| |
| # Compare output against expected output; it should match at least one of |
| # them. |
| sub do_diffs($$$$) |
| { |
| my ($fullname, $name, $mid, $f_exps) = @_; |
| |
| for my $f_exp (@$f_exps) { |
| (-r $f_exp) or die "Could not read `$f_exp'\n"; |
| |
| # Emacs produces temporary files that end in '~' and '#'. We ignore |
| # these. |
| if ($f_exp !~ /[~#]$/) { |
| # $n is the (optional) suffix after the ".exp"; we tack it onto |
| # the ".diff" file. |
| my $n = ""; |
| if ($f_exp =~ /.*\.exp(.*)$/) { |
| $n = $1; |
| } else { |
| $n = ""; |
| ($f_exp eq "/dev/null") or die "Unexpected .exp file: $f_exp\n"; |
| } |
| |
| mysystem("@DIFF@ $f_exp $name.$mid.out > $name.$mid.diff$n"); |
| |
| if (not -s "$name.$mid.diff$n") { |
| # A match; remove .out and any previously created .diff files. |
| unlink("$name.$mid.out"); |
| unlink(<$name.$mid.diff*>); |
| return; |
| } |
| } |
| } |
| # If we reach here, none of the .exp files matched. |
| print "*** $name failed ($mid) ***\n"; |
| push(@failures, sprintf("%-40s ($mid)", "$fullname")); |
| $num_failures{$mid}++; |
| } |
| |
| sub do_one_test($$) |
| { |
| my ($dir, $vgtest) = @_; |
| $vgtest =~ /^(.*)\.vgtest/; |
| my $name = $1; |
| my $fullname = "$dir/$name"; |
| |
| # Pull any extra options (for example, --sanity-level=4) |
| # from $EXTRA_REGTEST_OPTS. |
| my $maybe_extraopts = $ENV{"EXTRA_REGTEST_OPTS"}; |
| my $extraopts = $maybe_extraopts ? $maybe_extraopts : ""; |
| |
| read_vgtest_file($vgtest); |
| |
| if (defined $prereq) { |
| my $prereq_res = system("$prereq"); |
| if (0 == $prereq_res) { |
| # Do nothing (ie. continue with the test) |
| } elsif (256 == $prereq_res) { |
| # Nb: weird Perl-ism -- exit code of '1' is seen by Perl as 256... |
| # Prereq failed, skip. |
| printf("%-16s (skipping, prereq failed: $prereq)\n", "$name:"); |
| return; |
| } else { |
| # Bad prereq; abort. |
| $prereq_res /= 256; |
| die "prereq returned $prereq_res: $prereq\n"; |
| } |
| } |
| |
| printf("%-16s valgrind $extraopts $vgopts $prog $args\n", "$name:"); |
| |
| # Pass the appropriate --tool option for the directory (can be overridden |
| # by an "args:" line, though). Set both VALGRIND_LIB and |
| # VALGRIND_LIB_INNER in case this Valgrind was configured with |
| # --enable-inner. |
| my $tool=determine_tool(); |
| mysystem("VALGRIND_LIB=$valgrind_lib VALGRIND_LIB_INNER=$valgrind_lib " |
| . "$valgrind --command-line-only=yes --memcheck:leak-check=no " |
| . "--tool=$tool $extraopts $vgopts " |
| . "$prog $args > $name.stdout.out 2> $name.stderr.out"); |
| |
| # Filter stdout |
| if (defined $stdout_filter) { |
| mysystem("$stdout_filter < $name.stdout.out > $tmp"); |
| rename($tmp, "$name.stdout.out"); |
| } |
| # Find all the .stdout.exp files. If none, use /dev/null. |
| my @stdout_exps = <$name.stdout.exp*>; |
| @stdout_exps = ( "/dev/null" ) if (0 == scalar @stdout_exps); |
| do_diffs($fullname, $name, "stdout", \@stdout_exps); |
| |
| # Filter stderr |
| mysystem("$stderr_filter < $name.stderr.out > $tmp"); |
| rename($tmp, "$name.stderr.out"); |
| # Find all the .stderr.exp files. At least one must exist. |
| my @stderr_exps = <$name.stderr.exp*>; |
| (0 != scalar @stderr_exps) or die "Could not find `$name.stderr.exp*'\n"; |
| do_diffs($fullname, $name, "stderr", \@stderr_exps); |
| |
| # Maybe do post-test check |
| if (defined $post) { |
| if (mysystem("$post > $name.post.out") != 0) { |
| print("post check failed: $post\n"); |
| $num_failures{"post"}++; |
| } else { |
| # Find all the .post.exp files. If none, use /dev/null. |
| my @post_exps = <$name.post.exp*>; |
| @post_exps = ( "/dev/null" ) if (0 == scalar @post_exps); |
| do_diffs($fullname, $name, "post", \@post_exps); |
| } |
| } |
| |
| if (defined $cleanup) { |
| (system("$cleanup") == 0) or |
| print("(cleanup operation failed: $cleanup)\n"); |
| } |
| |
| $num_tests_done++; |
| } |
| |
| #---------------------------------------------------------------------------- |
| # Test one directory (and any subdirs) |
| #---------------------------------------------------------------------------- |
| sub test_one_dir($$); # forward declaration |
| |
| sub test_one_dir($$) |
| { |
| my ($dir, $prev_dirs) = @_; |
| $dir =~ s/\/$//; # trim a trailing '/' |
| |
| # Ignore dirs into which we should not recurse. |
| if ($dir =~ /^(BitKeeper|CVS|SCCS|docs|doc)$/) { return; } |
| |
| (-x "$tests_dir/tests/arch_test") or die |
| "vg_regtest: 'arch_test' is missing. Did you forget to 'make check'?\n"; |
| |
| # Ignore any dir whose name matches that of an architecture which is not |
| # the architecture we are running on. Eg. when running on x86, ignore |
| # ppc/ directories ('arch_test' returns 1 for this case). Likewise for |
| # the OS and platform. |
| # Nb: weird Perl-ism -- exit code of '1' is seen by Perl as 256... |
| if (256 == system("$tests_dir/tests/arch_test $dir")) { return; } |
| if (256 == system("$tests_dir/tests/os_test $dir")) { return; } |
| if ($dir =~ /(\w+)-(\w+)/ && |
| 256 == system("sh $tests_dir/tests/platform_test $1 $2")) { return; } |
| |
| chdir($dir) or die "Could not change into $dir\n"; |
| |
| # Nb: Don't prepend a '/' to the base directory |
| my $full_dir = $prev_dirs . ($prev_dirs eq "" ? "" : "/") . $dir; |
| my $dashes = "-" x (50 - length $full_dir); |
| |
| my @fs = glob "*"; |
| my $found_tests = (0 != (grep { $_ =~ /\.vgtest$/ } @fs)); |
| |
| if ($found_tests) { |
| print "-- Running tests in $full_dir $dashes\n"; |
| } |
| foreach my $f (@fs) { |
| if (-d $f) { |
| test_one_dir($f, $full_dir); |
| } elsif ($f =~ /\.vgtest$/) { |
| do_one_test($full_dir, $f); |
| } |
| } |
| if ($found_tests) { |
| print "-- Finished tests in $full_dir $dashes\n"; |
| } |
| |
| chdir(".."); |
| } |
| |
| #---------------------------------------------------------------------------- |
| # Summarise results |
| #---------------------------------------------------------------------------- |
| sub plural($) |
| { |
| return ( $_[0] == 1 ? "" : "s" ); |
| } |
| |
| sub summarise_results |
| { |
| my $x = ( $num_tests_done == 1 ? "test" : "tests" ); |
| |
| printf("\n== %d test%s, %d stderr failure%s, %d stdout failure%s, " |
| . "%d post failure%s ==\n", |
| $num_tests_done, plural($num_tests_done), |
| $num_failures{"stderr"}, plural($num_failures{"stderr"}), |
| $num_failures{"stdout"}, plural($num_failures{"stdout"}), |
| $num_failures{"post"}, plural($num_failures{"post"})); |
| |
| foreach my $failure (@failures) { |
| print "$failure\n"; |
| } |
| print "\n"; |
| } |
| |
| #---------------------------------------------------------------------------- |
| # main(), sort of |
| #---------------------------------------------------------------------------- |
| sub warn_about_EXTRA_REGTEST_OPTS() |
| { |
| print "WARNING: \$EXTRA_REGTEST_OPTS is set. You probably don't want\n"; |
| print "to run the regression tests with it set, unless you are doing some\n"; |
| print "strange experiment, and/or you really know what you are doing.\n"; |
| print "\n"; |
| } |
| |
| # nuke VALGRIND_OPTS |
| $ENV{"VALGRIND_OPTS"} = ""; |
| |
| if ($ENV{"EXTRA_REGTEST_OPTS"}) { |
| print "\n"; |
| warn_about_EXTRA_REGTEST_OPTS(); |
| } |
| |
| my @fs = process_command_line(); |
| foreach my $f (@fs) { |
| if (-d $f) { |
| test_one_dir($f, ""); |
| } else { |
| # Allow the .vgtest suffix to be given or omitted |
| if ($f =~ /.vgtest$/ && -r $f) { |
| # do nothing |
| } elsif (-r "$f.vgtest") { |
| $f = "$f.vgtest"; |
| } else { |
| die "`$f' neither a directory nor a readable test file/name\n" |
| } |
| my $dir = `dirname $f`; chomp $dir; |
| my $file = `basename $f`; chomp $file; |
| chdir($dir) or die "Could not change into $dir\n"; |
| do_one_test($dir, $file); |
| chdir($tests_dir); |
| } |
| } |
| summarise_results(); |
| |
| if ($ENV{"EXTRA_REGTEST_OPTS"}) { |
| warn_about_EXTRA_REGTEST_OPTS(); |
| } |
| |
| if (0 == $num_failures{"stdout"} && |
| 0 == $num_failures{"stderr"} && |
| 0 == $num_failures{"post"}) { |
| exit 0; |
| } else { |
| exit 1; |
| } |
| |
| ##--------------------------------------------------------------------## |
| ##--- end vg_regtest ---## |
| ##--------------------------------------------------------------------## |