tests/vg_regtest.in

#! @PERL@
##--------------------------------------------------------------------##
##--- Valgrind regression testing script                vg_regtest ---##
##--------------------------------------------------------------------##

#  This file is part of Valgrind, a dynamic binary instrumentation
#  framework.
#
#  Copyright (C) 2003 Nicholas Nethercote
#     njn@valgrind.org
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License as
#  published by the Free Software Foundation; either version 2 of the
#  License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
#  02111-1307, USA.
#
#  The GNU General Public License is contained in the file COPYING.

#----------------------------------------------------------------------------
# usage: vg_regtest [options] <dirs | files>
#
# Options:
#   --all:      run tests in all subdirs
#   --valgrind: valgrind launcher to use.  Default is ./coregrind/valgrind.
#               (This option should probably only be used in conjunction with
#               --valgrind-lib.)
#   --valgrind-lib: valgrind libraries to use.  Default is $tests_dir/.in_place.
#               (This option should probably only be used in conjunction with
#               --valgrind.)
#
# The easiest way is to run all tests in valgrind/ with (assuming you installed
# in $PREFIX):
#
#   $PREFIX/bin/vg_regtest --all
#
# You can specify individual files to test, or whole directories, or both.
# Directories are traversed recursively, except for ones named, for example, 
# CVS/ or docs/.
#
# Each test is defined in a file <test>.vgtest, containing one or more of the
# following lines, in any order:
#   - prog:   <prog to run>                         (compulsory)
#   - args:   <args for prog>                       (default: none)
#   - vgopts: <Valgrind options>                    (default: none;
#                                                    multiple are allowed)
#   - stdout_filter: <filter to run stdout through> (default: none)
#   - stderr_filter: <filter to run stderr through> (default: ./filter_stderr)
#   - prereq: <prerequisite command>                (default: none)
#   - post: <post-test check command>               (default: none)
#   - cleanup: <post-test cleanup cmd>              (default: none)
#
# Note that filters are necessary for stderr results to filter out things that
# always change, eg. process id numbers.
#
# Expected stdout (filtered) is kept in <test>.stdout.exp* (can be more
# than one expected output).  It can be missing if it would be empty.  Expected
# stderr (filtered) is kept in <test>.stderr.exp*.   There must be at least
# one stderr.exp* file.  Any .exp* file that ends in '~' or '#' is ignored;
# this is because Emacs creates temporary files of these names.
#
# If results don't match, the output can be found in <test>.std<strm>.out,
# and the diff between expected and actual in <test>.std<strm>.diff*.
#
# The prerequisite command, if present, works like this:
# - if it returns 0 the test is run
# - if it returns 1 the test is skipped
# - if it returns anything else the script aborts.
# The idea here is results other than 0 or 1 are likely to be due to
# problems with the commands, and you don't want to conflate them with the 1
# case, which would happen if you just tested for zero or non-zero.
#
# The post-test command, if present, must return 0 and its stdout must match
# the expected stdout which is kept in <test>.post.exp*.
#
# Sometimes it is useful to run all the tests at a high sanity check
# level or with arbitrary other flags.  To make this simple, extra 
# options, applied to all tests run, are read from $EXTRA_REGTEST_OPTS,
# and handed to valgrind prior to any other flags specified by the 
# .vgtest file.
#
# Some more notes on adding regression tests for a new tool are in
# docs/xml/manual-writing-tools.xml.
#----------------------------------------------------------------------------

use warnings;
use strict;

#----------------------------------------------------------------------------
# Global vars
#----------------------------------------------------------------------------
my $usage="\n"
          . "Usage:\n"
          . "   vg_regtest [--all, --valgrind, --valgrind-lib]\n"
          . "   Use EXTRA_REGTEST_OPTS to supply extra args for all tests\n"
          . "\n";

my $tmp="vg_regtest.tmp.$$";

# Test variables
my $vgopts;             # valgrind options
my $prog;               # test prog
my $args;               # test prog args
my $stdout_filter;      # filter program to run stdout results file through
my $stderr_filter;      # filter program to run stderr results file through
my $prereq;             # prerequisite test to satisfy before running test
my $post;               # check command after running test
my $cleanup;            # cleanup command to run

my @failures;           # List of failed tests

my $num_tests_done      = 0;
my %num_failures        = (stderr => 0, stdout => 0, post => 0);

# Default valgrind to use is this build tree's (uninstalled) one
my $valgrind = "./coregrind/valgrind";

chomp(my $tests_dir = `pwd`);

my $valgrind_lib = "$tests_dir/.in_place";

# default filter is the one named "filter_stderr" in the test's directory
my $default_stderr_filter = "filter_stderr";


#----------------------------------------------------------------------------
# Process command line, setup
#----------------------------------------------------------------------------

# If $prog is a relative path, it prepends $dir to it.  Useful for two reasons:
#
# 1. Can prepend "." onto programs to avoid trouble with users who don't have
#    "." in their path (by making $dir = ".")
# 2. Can prepend the current dir to make the command absolute to avoid
#    subsequent trouble when we change directories.
#
# Also checks the program exists and is executable.
sub validate_program ($$$$) 
{
    my ($dir, $prog, $must_exist, $must_be_executable) = @_;

    # If absolute path, leave it alone.  If relative, make it
    # absolute -- by prepending current dir -- so we can change
    # dirs and still use it.
    $prog = "$dir/$prog" if ($prog !~ /^\//);
    if ($must_exist) {
        (-f $prog) or die "vg_regtest: `$prog' not found or not a file ($dir)\n";
    }
    if ($must_be_executable) { 
        (-x $prog) or die "vg_regtest: `$prog' not executable ($dir)\n";
    }

    return $prog;
}

sub process_command_line() 
{
    my $alldirs = 0;
    my @fs;
    
    for my $arg (@ARGV) {
        if ($arg =~ /^-/) {
            if      ($arg =~ /^--all$/) {
                $alldirs = 1;
            } elsif ($arg =~ /^--valgrind=(.*)$/) {
                $valgrind = $1;
            } elsif ($arg =~ /^--valgrind-lib=(.*)$/) {
                $valgrind_lib = $1;
            } else {
                die $usage;
            }
        } else {
            push(@fs, $arg);
        }
    }
    $valgrind = validate_program($tests_dir, $valgrind, 1, 0);

    if ($alldirs) {
        @fs = ();
        foreach my $f (glob "*") {
            push(@fs, $f) if (-d $f);
        }
    }

    (0 != @fs) or die "No test files or directories specified\n";

    return @fs;
}

#----------------------------------------------------------------------------
# Read a .vgtest file
#----------------------------------------------------------------------------
sub read_vgtest_file($)
{
    my ($f) = @_;

    # Defaults.
    ($vgopts, $prog, $args)          = ("", undef, "");
    ($stdout_filter, $stderr_filter) = (undef, undef);
    ($prereq, $post, $cleanup)       = (undef, undef, undef);

    # Every test directory must have a "filter_stderr"
    $stderr_filter = validate_program(".", $default_stderr_filter, 1, 1);

    open(INPUTFILE, "< $f") || die "File $f not openable\n";

    while (my $line = <INPUTFILE>) {
        if      ($line =~ /^\s*#/ || $line =~ /^\s*$/) {
	    next;
	} elsif ($line =~ /^\s*vgopts:\s*(.*)$/) {
            $vgopts = $vgopts . " " . $1;   # Nb: Make sure there's a space!
        } elsif ($line =~ /^\s*prog:\s*(.*)$/) {
            $prog = validate_program(".", $1, 0, 0);
        } elsif ($line =~ /^\s*args:\s*(.*)$/) {
            $args = $1;
        } elsif ($line =~ /^\s*stdout_filter:\s*(.*)$/) {
            $stdout_filter = validate_program(".", $1, 1, 1);
        } elsif ($line =~ /^\s*stderr_filter:\s*(.*)$/) {
            $stderr_filter = validate_program(".", $1, 1, 1);
        } elsif ($line =~ /^\s*prereq:\s*(.*)$/) {
            $prereq = $1;
        } elsif ($line =~ /^\s*post:\s*(.*)$/) {
            $post = $1;
        } elsif ($line =~ /^\s*cleanup:\s*(.*)$/) {
            $cleanup = $1;
        } else {
            die "Bad line in $f: $line\n";
        }
    }
    close(INPUTFILE);

    if (!defined $prog) {
        $prog = "";     # allow no prog for testing error and --help cases
    }
}

#----------------------------------------------------------------------------
# Do one test
#----------------------------------------------------------------------------
# Since most of the program time is spent in system() calls, need this to
# propagate a Ctrl-C enabling us to quit.
sub mysystem($) 
{
    my $exit_code = system($_[0]);
    ($exit_code == 2) and exit 1;      # 2 is SIGINT
    return $exit_code;
}

# from a directory name like "/foo/cachesim/tests/" determine the tool name
sub determine_tool()
{
    my $dir = `pwd`;
    $dir =~ /.*\/([^\/]+)\/tests.*/;   # foo/tool_name/tests/foo
    return $1;
}

# Compare output against expected output;  it should match at least one of
# them.
sub do_diffs($$$$)
{
    my ($fullname, $name, $mid, $f_exps) = @_;
    
    for my $f_exp (@$f_exps) {
        (-r $f_exp) or die "Could not read `$f_exp'\n";

        # Emacs produces temporary files that end in '~' and '#'.  We ignore
        # these.
        if ($f_exp !~ /[~#]$/) {
            # $n is the (optional) suffix after the ".exp";  we tack it onto
            # the ".diff" file.
            my $n = "";
            if ($f_exp =~ /.*\.exp(.*)$/) {
                $n = $1;
            } else {
                $n = "";
                ($f_exp eq "/dev/null") or die "Unexpected .exp file: $f_exp\n";
            }

            mysystem("@DIFF@ $f_exp $name.$mid.out > $name.$mid.diff$n");

            if (not -s "$name.$mid.diff$n") {
                # A match;  remove .out and any previously created .diff files.
                unlink("$name.$mid.out");
                unlink(<$name.$mid.diff*>);
                return;
            }
        }
    }
    # If we reach here, none of the .exp files matched.
    print "*** $name failed ($mid) ***\n";
    push(@failures, sprintf("%-40s ($mid)", "$fullname"));
    $num_failures{$mid}++;
}

sub do_one_test($$) 
{
    my ($dir, $vgtest) = @_;
    $vgtest =~ /^(.*)\.vgtest/;
    my $name = $1;
    my $fullname = "$dir/$name"; 

    # Pull any extra options (for example, --sanity-level=4)
    # from $EXTRA_REGTEST_OPTS.
    my $maybe_extraopts = $ENV{"EXTRA_REGTEST_OPTS"};
    my $extraopts = $maybe_extraopts ?  $maybe_extraopts  : "";

    read_vgtest_file($vgtest);

    if (defined $prereq) {
        my $prereq_res = system("$prereq");
        if (0 == $prereq_res) {
            # Do nothing (ie. continue with the test)
        } elsif (256 == $prereq_res) {
            # Nb: weird Perl-ism -- exit code of '1' is seen by Perl as 256...
            # Prereq failed, skip.
            printf("%-16s (skipping, prereq failed: $prereq)\n", "$name:");
            return;
        } else {
            # Bad prereq; abort.
            $prereq_res /= 256;
            die "prereq returned $prereq_res: $prereq\n";
        }
    }

    printf("%-16s valgrind $extraopts $vgopts $prog $args\n", "$name:");

    # Pass the appropriate --tool option for the directory (can be overridden
    # by an "args:" line, though).  Set both VALGRIND_LIB and
    # VALGRIND_LIB_INNER in case this Valgrind was configured with
    # --enable-inner.
    my $tool=determine_tool();
    mysystem("VALGRIND_LIB=$valgrind_lib VALGRIND_LIB_INNER=$valgrind_lib "
           . "$valgrind --command-line-only=yes --memcheck:leak-check=no "
           . "--tool=$tool $extraopts $vgopts "
           . "$prog $args > $name.stdout.out 2> $name.stderr.out");

    # Filter stdout
    if (defined $stdout_filter) {
        mysystem("$stdout_filter < $name.stdout.out > $tmp");
        rename($tmp, "$name.stdout.out");
    }
    # Find all the .stdout.exp files.  If none, use /dev/null.
    my @stdout_exps = <$name.stdout.exp*>;
    @stdout_exps = ( "/dev/null" ) if (0 == scalar @stdout_exps);
    do_diffs($fullname, $name, "stdout", \@stdout_exps); 

    # Filter stderr
    mysystem("$stderr_filter < $name.stderr.out > $tmp");
    rename($tmp, "$name.stderr.out");
    # Find all the .stderr.exp files.  At least one must exist.
    my @stderr_exps = <$name.stderr.exp*>;
    (0 != scalar @stderr_exps) or die "Could not find `$name.stderr.exp*'\n";
    do_diffs($fullname, $name, "stderr", \@stderr_exps); 

    # Maybe do post-test check
    if (defined $post) {
	if (mysystem("$post > $name.post.out") != 0) {
	    print("post check failed: $post\n");
	    $num_failures{"post"}++;
	} else {
	    # Find all the .post.exp files.  If none, use /dev/null.
	    my @post_exps = <$name.post.exp*>;
	    @post_exps = ( "/dev/null" ) if (0 == scalar @post_exps);
	    do_diffs($fullname, $name, "post", \@post_exps);
	}
    }
 
    if (defined $cleanup) {
        (system("$cleanup") == 0) or 
            print("(cleanup operation failed: $cleanup)\n");
    }

    $num_tests_done++;
}

#----------------------------------------------------------------------------
# Test one directory (and any subdirs)
#----------------------------------------------------------------------------
sub test_one_dir($$);    # forward declaration

sub test_one_dir($$) 
{
    my ($dir, $prev_dirs) = @_;
    $dir =~ s/\/$//;    # trim a trailing '/'

    # Ignore dirs into which we should not recurse.
    if ($dir =~ /^(BitKeeper|CVS|SCCS|docs|doc)$/) { return; }

    (-x "$tests_dir/tests/arch_test") or die 
        "vg_regtest: 'arch_test' is missing.  Did you forget to 'make check'?\n";
    
    # Ignore any dir whose name matches that of an architecture which is not
    # the architecture we are running on.  Eg. when running on x86, ignore
    # ppc/ directories ('arch_test' returns 1 for this case).  Likewise for
    # the OS and platform.
    # Nb: weird Perl-ism -- exit code of '1' is seen by Perl as 256...
    if (256 == system("$tests_dir/tests/arch_test $dir"))  { return; }
    if (256 == system("$tests_dir/tests/os_test   $dir"))  { return; }
    if ($dir =~ /(\w+)-(\w+)/ &&
        256 == system("sh $tests_dir/tests/platform_test $1 $2")) { return; }
    
    chdir($dir) or die "Could not change into $dir\n";

    # Nb: Don't prepend a '/' to the base directory
    my $full_dir = $prev_dirs . ($prev_dirs eq "" ? "" : "/") . $dir;
    my $dashes = "-" x (50 - length $full_dir);

    my @fs = glob "*";
    my $found_tests = (0 != (grep { $_ =~ /\.vgtest$/ } @fs));

    if ($found_tests) {
        print "-- Running  tests in $full_dir $dashes\n";
    }
    foreach my $f (@fs) {
        if (-d $f) {
            test_one_dir($f, $full_dir);
        } elsif ($f =~ /\.vgtest$/) {
            do_one_test($full_dir, $f);
        }
    }
    if ($found_tests) {
        print "-- Finished tests in $full_dir $dashes\n";
    }

    chdir("..");
}

#----------------------------------------------------------------------------
# Summarise results
#----------------------------------------------------------------------------
sub plural($)
{
   return ( $_[0] == 1 ? "" : "s" );
}

sub summarise_results 
{
    my $x = ( $num_tests_done == 1 ? "test" : "tests" );
    
    printf("\n== %d test%s, %d stderr failure%s, %d stdout failure%s, "
                         . "%d post failure%s ==\n", 
           $num_tests_done, plural($num_tests_done),
           $num_failures{"stderr"},   plural($num_failures{"stderr"}),
           $num_failures{"stdout"},   plural($num_failures{"stdout"}),
           $num_failures{"post"},     plural($num_failures{"post"}));

    foreach my $failure (@failures) {
        print "$failure\n";
    }
    print "\n";
}

#----------------------------------------------------------------------------
# main(), sort of
#----------------------------------------------------------------------------
sub warn_about_EXTRA_REGTEST_OPTS()
{
    print "WARNING: \$EXTRA_REGTEST_OPTS is set.  You probably don't want\n";
    print "to run the regression tests with it set, unless you are doing some\n";
    print "strange experiment, and/or you really know what you are doing.\n";
    print "\n";
}

# nuke VALGRIND_OPTS
$ENV{"VALGRIND_OPTS"} = "";

if ($ENV{"EXTRA_REGTEST_OPTS"}) {
    print "\n";
    warn_about_EXTRA_REGTEST_OPTS();
}

my @fs = process_command_line();
foreach my $f (@fs) {
    if (-d $f) {
        test_one_dir($f, "");
    } else { 
        # Allow the .vgtest suffix to be given or omitted
        if ($f =~ /.vgtest$/ && -r $f) {
            # do nothing
        } elsif (-r "$f.vgtest") {
            $f = "$f.vgtest";
        } else {
            die "`$f' neither a directory nor a readable test file/name\n"
        }
        my $dir  = `dirname  $f`;   chomp $dir;
        my $file = `basename $f`;   chomp $file;
        chdir($dir) or die "Could not change into $dir\n";
        do_one_test($dir, $file);
        chdir($tests_dir);
    }
}
summarise_results();

if ($ENV{"EXTRA_REGTEST_OPTS"}) {
    warn_about_EXTRA_REGTEST_OPTS();
}

if (0 == $num_failures{"stdout"} &&
    0 == $num_failures{"stderr"} &&
    0 == $num_failures{"post"}) {
    exit 0;
} else {
    exit 1;
}

##--------------------------------------------------------------------##
##--- end                                               vg_regtest ---##
##--------------------------------------------------------------------##