blob: db26e4ebe26f986c7baca88374bd43024c0abf15 [file] [log] [blame]
sewardj2f685952002-12-22 19:32:23 +00001#! @PERL@ -w
njna217f3d2002-09-27 08:26:27 +00002##--------------------------------------------------------------------##
3##--- Valgrind regression testing script vg_regtest ---##
4##--------------------------------------------------------------------##
njnc2e7f482002-09-27 08:44:17 +00005
njnc9539842002-10-02 13:26:35 +00006# This file is part of Valgrind, an extensible x86 protected-mode
7# emulator for monitoring program execution on x86-Unixes.
njnc2e7f482002-09-27 08:44:17 +00008#
9# Copyright (C) 2002 Nicholas Nethercote
10# njn25@cam.ac.uk
11#
12# This program is free software; you can redistribute it and/or
13# modify it under the terms of the GNU General Public License as
14# published by the Free Software Foundation; either version 2 of the
15# License, or (at your option) any later version.
16#
17# This program is distributed in the hope that it will be useful, but
18# WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20# General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License
23# along with this program; if not, write to the Free Software
24# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
25# 02111-1307, USA.
26#
27# The GNU General Public License is contained in the file COPYING.
28
29#----------------------------------------------------------------------------
njna217f3d2002-09-27 08:26:27 +000030# usage: vg_regtest [options] <dirs | files>
njn25e49d8e72002-09-23 09:36:25 +000031#
njna217f3d2002-09-27 08:26:27 +000032# Options:
njna217f3d2002-09-27 08:26:27 +000033# --all: run tests in all subdirs
njnc2e7f482002-09-27 08:44:17 +000034# --valgrind: valgrind to use. Default is one built from this source tree.
njna217f3d2002-09-27 08:26:27 +000035#
njn4ba5a792002-09-30 10:23:54 +000036# The easiest way is to run all tests in valgrind/ with (assuming you installed
37# in $PREFIX):
38#
39# $PREFIX/bin/vg_regtest --all
40#
njna217f3d2002-09-27 08:26:27 +000041# You can specify individual files to test, or whole directories, or both.
njndb3c4692002-10-04 10:03:34 +000042# Directories are traversed recursively, except for ones name CVS/ or docs/.
njn25e49d8e72002-09-23 09:36:25 +000043#
44# Each test is defined in a file <test>.vgtest, containing one or more of the
njna217f3d2002-09-27 08:26:27 +000045# following lines, in any order:
njn25e49d8e72002-09-23 09:36:25 +000046# - prog: <prog to run> (compulsory)
47# - args: <args for prog> (default: none)
48# - vgopts: <Valgrind options> (default: none)
49# - stdout_filter: <filter to run stdout through> (default: none)
njna217f3d2002-09-27 08:26:27 +000050# - stderr_filter: <filter to run stderr through> (default: ./filter_stderr)
njn25e49d8e72002-09-23 09:36:25 +000051#
njna217f3d2002-09-27 08:26:27 +000052# Note that filters are necessary for stderr results to filter out things that
53# always change, eg. process id numbers.
njn25e49d8e72002-09-23 09:36:25 +000054#
njna217f3d2002-09-27 08:26:27 +000055# Expected stdout (filtered) is kept in <test>.stdout.exp. It can be missing
njnd8ced862003-04-08 00:47:05 +000056# if it would be empty. Expected stderr (filtered) is kept in
57# <test>.stderr.exp.
njn25e49d8e72002-09-23 09:36:25 +000058#
59# If results don't match, the output can be found in <test>.std<strm>.out,
60# and the diff between expected and actual in <test>.std<strm>.diff.
61#
njna217f3d2002-09-27 08:26:27 +000062# Notes on adding regression tests for a new skin are in
63# coregrind/docs/skins.html.
njn25e49d8e72002-09-23 09:36:25 +000064#----------------------------------------------------------------------------
65
66use strict;
67
68#----------------------------------------------------------------------------
69# Global vars
70#----------------------------------------------------------------------------
njnd8ced862003-04-08 00:47:05 +000071my $usage="vg_regtest [--all, --valgrind]\n";
njn25e49d8e72002-09-23 09:36:25 +000072
73my $tmp="vg_regtest.tmp.$$";
74
75# Test variables
76my $vgopts; # valgrind options
77my $prog; # test prog
78my $args; # test prog args
79my $stdout_filter; # filter program to run stdout results file through
80my $stderr_filter; # filter program to run stderr results file through
81
82my @failures; # List of failed tests
83
njnd8ced862003-04-08 00:47:05 +000084my $num_tests_done = 0;
njn25e49d8e72002-09-23 09:36:25 +000085
njnc2e7f482002-09-27 08:44:17 +000086# Default valgrind to use is this build tree's one
87my $prefix="@prefix@";
88my $exec_prefix="@exec_prefix@";
89my $valgrind = "@bindir@/valgrind";
njn25e49d8e72002-09-23 09:36:25 +000090
91chomp(my $tests_dir = `pwd`);
92
93# default filter is the one named "filter_stderr" in the test's directory
94my $default_stderr_filter = "filter_stderr";
95
96
97#----------------------------------------------------------------------------
98# Process command line, setup
99#----------------------------------------------------------------------------
100
101# If $prog is a relative path, it prepends $dir to it. Useful for two reasons:
102#
103# 1. Can prepend "." onto programs to avoid trouble with users who don't have
104# "." in their path (by making $dir = ".")
105# 2. Can prepend the current dir to make the command absolute to avoid
106# subsequent trouble when we change directories.
107#
108# Also checks the program exists and is executable.
109sub validate_program ($$)
110{
111 my ($dir, $prog) = @_;
112
113 # If absolute path, leave it alone. If relative, make it
114 # absolute -- by prepending current dir -- so we can change
115 # dirs and still use it.
116 $prog = "$dir/$prog" if ($prog !~ /^\//);
117 (-f $prog) or die "`$prog' not found or not a file ($dir)\n";
118 (-x $prog) or die "`$prog' not found or not executable ($dir)\n";
119
120 return $prog;
121}
122
123sub process_command_line()
124{
125 my $alldirs = 0;
126 my @fs;
127
128 for my $arg (@ARGV) {
129 if ($arg =~ /^-/) {
njnd8ced862003-04-08 00:47:05 +0000130 if ($arg =~ /^--all$/) {
njn25e49d8e72002-09-23 09:36:25 +0000131 $alldirs = 1;
132 } elsif ($arg =~ /^--valgrind=(.*)$/) {
133 $valgrind = $1;
134 } else {
135 die $usage;
136 }
137 } else {
138 push(@fs, $arg);
139 }
140 }
141 $valgrind = validate_program($tests_dir, $valgrind);
142
143 if ($alldirs) {
144 @fs = ();
145 foreach my $f (glob "*") {
146 push(@fs, $f) if (-d $f);
147 }
148 }
149
150 (0 != @fs) or die "No test files or directories specified\n";
151
152 return @fs;
153}
154
155#----------------------------------------------------------------------------
156# Read a .vgtest file
157#----------------------------------------------------------------------------
158sub read_vgtest_file($)
159{
160 my ($f) = @_;
161
162 # Defaults.
163 ($vgopts, $prog, $args, $stdout_filter, $stderr_filter) =
164 ("", undef, "", undef, undef);
165
166 # Every test directory must have a "filter_stderr"
167 $stderr_filter = validate_program(".", $default_stderr_filter);
168
169 open(INPUTFILE, "< $f") || die "File $f not openable\n";
170
171 while (my $line = <INPUTFILE>) {
172 if ($line =~ /^\s*vgopts:\s*(.*)$/) {
173 $vgopts = $1;
174 } elsif ($line =~ /^\s*prog:\s*(.*)$/) {
175 $prog = validate_program(".", $1);
176 } elsif ($line =~ /^\s*args:\s*(.*)$/) {
177 $args = $1;
njn25e49d8e72002-09-23 09:36:25 +0000178 } elsif ($line =~ /^\s*stdout_filter:\s*(.*)$/) {
179 $stdout_filter = validate_program(".", $1);
180 } elsif ($line =~ /^\s*stderr_filter:\s*(.*)$/) {
181 $stderr_filter = validate_program(".", $1);
182 } else {
183 die "Bad line in $f: $line\n";
184 }
185 }
186 close(INPUTFILE);
187
188 if (!defined $prog) {
189 die "no `prog:' line in `$f'\n";
190 }
191}
192
193#----------------------------------------------------------------------------
194# Do one test
195#----------------------------------------------------------------------------
196# Since most of the program time is spent in system() calls, need this to
197# propagate a Ctrl-C enabling us to quit.
198sub mysystem($)
199{
200 (system($_[0]) != 2) or exit 1; # 2 is SIGINT
201}
202
njn25cac76cb2002-09-23 11:21:57 +0000203# from a directory name like "/foo/cachesim/tests/" determine the skin name
204sub determine_skin()
205{
206 my $dir = `pwd`;
207 $dir =~ /.*\/([^\/]+)\/tests.*/; # foo/skin_name/tests/foo
208 return $1;
209}
210
njn25e49d8e72002-09-23 09:36:25 +0000211sub do_one_test($$)
212{
213 my ($dir, $vgtest) = @_;
214 $vgtest =~ /^(.*)\.vgtest/;
215 my $name = $1;
216 my $fullname = "$dir/$name";
217
218 read_vgtest_file($vgtest);
219
njndb3c4692002-10-04 10:03:34 +0000220 printf("%-16s valgrind $vgopts $prog $args\n", "$name:");
njn25e49d8e72002-09-23 09:36:25 +0000221
njnd8ced862003-04-08 00:47:05 +0000222 # Pass the appropriate --skin option for the directory (can be overridden
223 # by an "args:" or "args.dev:" line, though)
224 my $skin=determine_skin();
225 mysystem("$valgrind --skin=$skin $vgopts $prog $args > $name.stdout.out 2> $name.stderr.out");
njn25e49d8e72002-09-23 09:36:25 +0000226
227 if (defined $stdout_filter) {
228 mysystem("$stdout_filter < $name.stdout.out > $tmp");
229 rename($tmp, "$name.stdout.out");
230 }
231
232 mysystem("$stderr_filter < $name.stderr.out > $tmp");
233 rename($tmp, "$name.stderr.out");
234
235 # If stdout expected empty, .exp file might be missing so diff with
236 # /dev/null
237 my $stdout_exp = ( -r "$name.stdout.exp"
238 ? "$name.stdout.exp"
239 : "/dev/null" );
240
njnd8ced862003-04-08 00:47:05 +0000241 my $stderr_exp = "$name.stderr.exp";
njn25e49d8e72002-09-23 09:36:25 +0000242 (-r $stderr_exp) or die "Could not read `$stderr_exp'\n";
243
244 mysystem("diff -C0 $stdout_exp $name.stdout.out > $name.stdout.diff");
245 mysystem("diff -C0 $stderr_exp $name.stderr.out > $name.stderr.diff");
246
247 for my $ext ("stdout", "stderr") {
248 if (-s "$name.$ext.diff") {
njndb3c4692002-10-04 10:03:34 +0000249 print "*** $name failed ($ext) ***\n";
250 push(@failures, sprintf("%-40s $ext", "$fullname"));
njn25e49d8e72002-09-23 09:36:25 +0000251 } else {
252 unlink("$name.$ext.out", "$name.$ext.diff");
253 }
254 }
njnd8ced862003-04-08 00:47:05 +0000255 $num_tests_done++;
njn25e49d8e72002-09-23 09:36:25 +0000256}
257
258#----------------------------------------------------------------------------
njn25cac76cb2002-09-23 11:21:57 +0000259# Test one directory (and any subdirs)
njn25e49d8e72002-09-23 09:36:25 +0000260#----------------------------------------------------------------------------
njndb3c4692002-10-04 10:03:34 +0000261sub test_one_dir($$); # forward declaration
njn25cac76cb2002-09-23 11:21:57 +0000262
njndb3c4692002-10-04 10:03:34 +0000263sub test_one_dir($$)
njn25e49d8e72002-09-23 09:36:25 +0000264{
njndb3c4692002-10-04 10:03:34 +0000265 my ($dir, $prev_dirs) = @_;
njn25e49d8e72002-09-23 09:36:25 +0000266 $dir =~ s/\/$//; # trim a trailing '/'
267
njnd8ced862003-04-08 00:47:05 +0000268 # ignore CVS/ and docs/ and doc/ dirs
269 if ($dir =~ /^(CVS|docs|doc)$/) { return; }
njn25cac76cb2002-09-23 11:21:57 +0000270
njn25e49d8e72002-09-23 09:36:25 +0000271 chdir($dir) or die "Could not change into $dir\n";
272
njn584eaac2002-10-04 15:30:48 +0000273 # Nb: Don't prepend a '/' to the base directory
274 my $full_dir = $prev_dirs . ($prev_dirs eq "" ? "" : "/") . $dir;
njndb3c4692002-10-04 10:03:34 +0000275 my $dashes = "-" x (50 - length $full_dir);
njn25cac76cb2002-09-23 11:21:57 +0000276
njndb3c4692002-10-04 10:03:34 +0000277 my @fs = glob "*";
278 my @vgtests = grep { $_ =~ /\.vgtest$/ } @fs;
279 my $found_tests = (0 != (grep { $_ =~ /\.vgtest$/ } @fs));
280
281 if ($found_tests) {
282 print "-- Running tests in $full_dir $dashes\n";
283 } else {
284 print "-- Found no tests in $full_dir $dashes\n";
285 }
njn25cac76cb2002-09-23 11:21:57 +0000286 foreach my $f (@fs) {
287 if (-d $f) {
njndb3c4692002-10-04 10:03:34 +0000288 test_one_dir($f, $full_dir);
njn25cac76cb2002-09-23 11:21:57 +0000289 } elsif ($f =~ /\.vgtest$/) {
njndb3c4692002-10-04 10:03:34 +0000290 do_one_test($full_dir, $f);
njn25cac76cb2002-09-23 11:21:57 +0000291 }
njn25e49d8e72002-09-23 09:36:25 +0000292 }
njndb3c4692002-10-04 10:03:34 +0000293 if ($found_tests) {
294 print "-- Finished tests in $full_dir $dashes\n";
295 }
296
njn25e49d8e72002-09-23 09:36:25 +0000297 chdir("..");
njn25e49d8e72002-09-23 09:36:25 +0000298}
299
300#----------------------------------------------------------------------------
301# Summarise results
302#----------------------------------------------------------------------------
303sub summarise_results
304{
njnd8ced862003-04-08 00:47:05 +0000305 my $x = ( $num_tests_done == 1 ? "test" : "tests" );
306 my $n_failures = @failures;
307
308 print "\n== $num_tests_done $x, $n_failures failed: ===============================\n";
309 if (0 == $n_failures) {
njn25e49d8e72002-09-23 09:36:25 +0000310 print " (none)\n";
311 } else {
312 foreach my $failure (@failures) {
313 print "$failure\n";
314 }
315 }
316}
317
318#----------------------------------------------------------------------------
319# main(), sort of
320#----------------------------------------------------------------------------
321
322# undefine $VALGRIND_OPTS
njnbf89f312002-10-30 15:23:05 +0000323# (Commented out 2002-Oct-30; was causing "Use of uninitialized value in undef
324# operator" warning --njn)
325# if ( exists $ENV{VALGRIND_OPTS} ) {
326# undef $ENV{VALGRIND_OPTS};
327# }
njn25e49d8e72002-09-23 09:36:25 +0000328
329my @fs = process_command_line();
330foreach my $f (@fs) {
331 if (-d $f) {
njn584eaac2002-10-04 15:30:48 +0000332 test_one_dir($f, "");
njn25e49d8e72002-09-23 09:36:25 +0000333 } else {
334 # Allow the .vgtest suffix to be given or omitted
335 if ($f =~ /.vgtest$/ && -r $f) {
336 # do nothing
337 } elsif (-r "$f.vgtest") {
338 $f = "$f.vgtest";
339 } else {
340 die "`$f' neither a directory nor a readable test file/name\n"
341 }
342 my $dir = `dirname $f`; chomp $dir;
343 my $file = `basename $f`; chomp $file;
344 chdir($dir) or die "Could not change into $dir\n";
345 do_one_test($dir, $file);
346 chdir($tests_dir);
347 }
348}
349summarise_results();
350
njnc2e7f482002-09-27 08:44:17 +0000351##--------------------------------------------------------------------##
352##--- end vg_regtest ---##
353##--------------------------------------------------------------------##