blob: 5d31f48a83b62a5244ef559b26959525c1a73337 [file] [log] [blame]
njnec0c27a2005-12-10 23:11:28 +00001#! @PERL@
2##--------------------------------------------------------------------##
3##--- Valgrind performance testing script vg_perf ---##
4##--------------------------------------------------------------------##
5
6# This file is part of Valgrind, a dynamic binary instrumentation
7# framework.
8#
9# Copyright (C) 2005 Nicholas Nethercote
10# njn@valgrind.org
11#
12# This program is free software; you can redistribute it and/or
13# modify it under the terms of the GNU General Public License as
14# published by the Free Software Foundation; either version 2 of the
15# License, or (at your option) any later version.
16#
17# This program is distributed in the hope that it will be useful, but
18# WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20# General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License
23# along with this program; if not, write to the Free Software
24# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
25# 02111-1307, USA.
26#
27# The GNU General Public License is contained in the file COPYING.
28
29#----------------------------------------------------------------------------
njn57781862005-12-14 02:58:23 +000030# usage: see usage message.
njnec0c27a2005-12-10 23:11:28 +000031#
njnec0c27a2005-12-10 23:11:28 +000032# You can specify individual files to test, or whole directories, or both.
33# Directories are traversed recursively, except for ones named, for example,
34# CVS/ or docs/.
35#
36# Each test is defined in a file <test>.vgperf, containing one or more of the
37# following lines, in any order:
38# - prog: <prog to run> (compulsory)
njnec0c27a2005-12-10 23:11:28 +000039# - args: <args for prog> (default: none)
40# - vgopts: <Valgrind options> (default: none)
41# - prereq: <prerequisite command> (default: none)
42# - cleanup: <post-test cleanup cmd to run> (default: none)
43#
44# The prerequisite command, if present, must return 0 otherwise the test is
45# skipped.
philippe215b2972012-07-24 19:47:46 +000046# Sometimes it is useful to run all the tests at a high sanity check
47# level or with arbitrary other flags. To make this simple, extra
48# options, applied to all tests run, are read from $EXTRA_REGTEST_OPTS,
49# and handed to valgrind prior to any other flags specified by the
50# .vgperf file. Note: the env var is the same as vg_regtest.
njnec0c27a2005-12-10 23:11:28 +000051#----------------------------------------------------------------------------
52
53use warnings;
54use strict;
55
56#----------------------------------------------------------------------------
57# Global vars
58#----------------------------------------------------------------------------
njnee45f1d2005-12-13 21:55:16 +000059my $usage = <<END
60usage: vg_perf [options] [files or dirs]
njnec0c27a2005-12-10 23:11:28 +000061
njnee45f1d2005-12-13 21:55:16 +000062 options for the user, with defaults in [ ], are:
63 -h --help show this message
njn3fbf17a2006-11-22 00:39:08 +000064 --reps=<n> number of repeats for each program [1]
njn768cd5d2006-11-22 00:52:00 +000065 --tools=<t1,t2,t3> tools to run [Nulgrind and Memcheck]
florian74c37d82013-01-24 21:32:32 +000066 --vg=<dir> top-level directory containing Valgrind to measure
67 [Valgrind in the current directory, i.e. --vg=.]
68 Can be specified multiple times.
philippe14ab1a32012-04-08 19:52:38 +000069 The "in-place" build is used.
70
71 --outer-valgrind: run these Valgrind(s) under the given outer valgrind.
72 These Valgrind(s) must be configured with --enable-inner.
73 --outer-tool: tool to use by the outer valgrind (default cachegrind).
74 --outer-args: use this as outer tool args.
njn768cd5d2006-11-22 00:52:00 +000075
76 Any tools named in --tools must be present in all directories specified
77 with --vg. (This is not checked.)
philippe215b2972012-07-24 19:47:46 +000078 Use EXTRA_REGTEST_OPTS to supply extra args for all tests
njnee45f1d2005-12-13 21:55:16 +000079END
80;
njnec0c27a2005-12-10 23:11:28 +000081
82# Test variables
83my $vgopts; # valgrind options
84my $prog; # test prog
85my $args; # test prog args
86my $prereq; # prerequisite test to satisfy before running test
87my $cleanup; # cleanup command to run
njnec0c27a2005-12-10 23:11:28 +000088
njn57781862005-12-14 02:58:23 +000089# Command line options
njn3fbf17a2006-11-22 00:39:08 +000090my $n_reps = 1; # Run each test $n_reps times and choose the best one.
91my @vgdirs; # Dirs of the various Valgrinds being measured.
njn768cd5d2006-11-22 00:52:00 +000092my @tools = ("none", "memcheck"); # tools being measured
njnec0c27a2005-12-10 23:11:28 +000093
philippe14ab1a32012-04-08 19:52:38 +000094# Outer valgrind to use, and args to use for it.
95# If this is set, --valgrind should be set to the installed inner valgrind,
96# and --valgrind-lib will be ignore
97my $outer_valgrind;
98my $outer_tool = "cachegrind";
99my $outer_args;
100
101
njnec0c27a2005-12-10 23:11:28 +0000102my $num_tests_done = 0;
103my $num_timings_done = 0;
104
105# Starting directory
106chomp(my $tests_dir = `pwd`);
107
njnec0c27a2005-12-10 23:11:28 +0000108#----------------------------------------------------------------------------
109# Process command line, setup
110#----------------------------------------------------------------------------
111
112# If $prog is a relative path, it prepends $dir to it. Useful for two reasons:
113#
114# 1. Can prepend "." onto programs to avoid trouble with users who don't have
115# "." in their path (by making $dir = ".")
116# 2. Can prepend the current dir to make the command absolute to avoid
117# subsequent trouble when we change directories.
118#
119# Also checks the program exists and is executable.
120sub validate_program ($$$$)
121{
122 my ($dir, $prog, $must_exist, $must_be_executable) = @_;
123
124 # If absolute path, leave it alone. If relative, make it
125 # absolute -- by prepending current dir -- so we can change
126 # dirs and still use it.
127 $prog = "$dir/$prog" if ($prog !~ /^\//);
128 if ($must_exist) {
129 (-f $prog) or die "vg_perf: '$prog' not found or not a file ($dir)\n";
130 }
131 if ($must_be_executable) {
132 (-x $prog) or die "vg_perf: '$prog' not executable ($dir)\n";
133 }
134
135 return $prog;
136}
137
njn57781862005-12-14 02:58:23 +0000138sub add_vgdir($)
139{
140 my ($vgdir) = @_;
141 if ($vgdir !~ /^\//) { $vgdir = "$tests_dir/$vgdir"; }
njn57781862005-12-14 02:58:23 +0000142 push(@vgdirs, $vgdir);
143}
144
njnec0c27a2005-12-10 23:11:28 +0000145sub process_command_line()
146{
njnec0c27a2005-12-10 23:11:28 +0000147 my @fs;
148
149 for my $arg (@ARGV) {
150 if ($arg =~ /^-/) {
njn1a95e052009-07-01 04:50:41 +0000151 if ($arg =~ /^--reps=(\d+)$/) {
njnee45f1d2005-12-13 21:55:16 +0000152 $n_reps = $1;
njn30e23ac2005-12-15 17:22:37 +0000153 if ($n_reps < 1) { die "bad --reps value: $n_reps\n"; }
njn57781862005-12-14 02:58:23 +0000154 } elsif ($arg =~ /^--vg=(.+)$/) {
155 # Make dir absolute if not already
156 add_vgdir($1);
njn3fbf17a2006-11-22 00:39:08 +0000157 } elsif ($arg =~ /^--tools=(.+)$/) {
158 @tools = split(/,/, $1);
philippe14ab1a32012-04-08 19:52:38 +0000159 } elsif ($arg =~ /^--outer-valgrind=(.*)$/) {
160 $outer_valgrind = $1;
161 } elsif ($arg =~ /^--outer-tool=(.*)$/) {
162 $outer_tool = $1;
163 } elsif ($arg =~ /^--outer-args=(.*)$/) {
164 $outer_args = $1;
njnec0c27a2005-12-10 23:11:28 +0000165 } else {
166 die $usage;
167 }
168 } else {
169 push(@fs, $arg);
170 }
171 }
njn57781862005-12-14 02:58:23 +0000172
173 # If no --vg options were specified, use the current tree.
174 if (0 == @vgdirs) {
175 add_vgdir($tests_dir);
176 }
njnec0c27a2005-12-10 23:11:28 +0000177
njnec0c27a2005-12-10 23:11:28 +0000178 (0 != @fs) or die "No test files or directories specified\n";
179
180 return @fs;
181}
182
183#----------------------------------------------------------------------------
184# Read a .vgperf file
185#----------------------------------------------------------------------------
186sub read_vgperf_file($)
187{
188 my ($f) = @_;
189
190 # Defaults.
191 ($vgopts, $prog, $args, $prereq, $cleanup)
192 = ("", undef, "", undef, undef, undef, undef);
193
194 open(INPUTFILE, "< $f") || die "File $f not openable\n";
195
196 while (my $line = <INPUTFILE>) {
197 if ($line =~ /^\s*#/ || $line =~ /^\s*$/) {
198 next;
199 } elsif ($line =~ /^\s*vgopts:\s*(.*)$/) {
200 $vgopts = $1;
201 } elsif ($line =~ /^\s*prog:\s*(.*)$/) {
njnc9582a22005-12-13 21:44:48 +0000202 $prog = validate_program(".", $1, 1, 1);
njnec0c27a2005-12-10 23:11:28 +0000203 } elsif ($line =~ /^\s*args:\s*(.*)$/) {
204 $args = $1;
205 } elsif ($line =~ /^\s*prereq:\s*(.*)$/) {
206 $prereq = $1;
207 } elsif ($line =~ /^\s*cleanup:\s*(.*)$/) {
208 $cleanup = $1;
209 } else {
210 die "Bad line in $f: $line\n";
211 }
212 }
213 close(INPUTFILE);
214
215 if (!defined $prog) {
216 $prog = ""; # allow no prog for testing error and --help cases
217 }
218 if (0 == @tools) {
219 die "vg_perf: missing 'tools' line in $f\n";
220 }
221}
222
223#----------------------------------------------------------------------------
224# Do one test
225#----------------------------------------------------------------------------
226# Since most of the program time is spent in system() calls, need this to
227# propagate a Ctrl-C enabling us to quit.
228sub mysystem($)
229{
njnecb47442005-12-13 16:54:58 +0000230 my ($cmd) = @_;
231 my $retval = system($cmd);
232 if ($retval == 2) {
233 exit 1;
234 } else {
235 return $retval;
236 }
njnec0c27a2005-12-10 23:11:28 +0000237}
238
sewardjb4860be2006-10-17 02:30:17 +0000239# Run program N times, return the best user time. Use the POSIX
240# -p flag on /usr/bin/time so as to get something parseable on AIX.
njnec0c27a2005-12-10 23:11:28 +0000241sub time_prog($$)
242{
243 my ($cmd, $n) = @_;
244 my $tmin = 999999;
245 for (my $i = 0; $i < $n; $i++) {
njnecb47442005-12-13 16:54:58 +0000246 mysystem("echo '$cmd' > perf.cmd");
247 my $retval = mysystem("$cmd > perf.stdout 2> perf.stderr");
248 (0 == $retval) or
njn57781862005-12-14 02:58:23 +0000249 die "\n*** Command returned non-zero ($retval)"
250 . "\n*** See perf.{cmd,stdout,stderr} to determine what went wrong.\n";
njnecb47442005-12-13 16:54:58 +0000251 my $out = `cat perf.stderr`;
njnae7f6822007-02-02 23:23:01 +0000252 ($out =~ /[Uu]ser +([\d\.]+)/) or
njn02383bc2005-12-13 20:23:38 +0000253 die "\n*** missing usertime in perf.stderr\n";
njnae7f6822007-02-02 23:23:01 +0000254 $tmin = $1 if ($1 < $tmin);
njnec0c27a2005-12-10 23:11:28 +0000255 }
florian7ac82602012-09-17 21:20:06 +0000256
257 # Successful run; cleanup
258 unlink("perf.cmd");
259 unlink("perf.stderr");
260 unlink("perf.stdout");
261
njn30e23ac2005-12-15 17:22:37 +0000262 # Avoid divisions by zero!
263 return (0 == $tmin ? 0.01 : $tmin);
njnec0c27a2005-12-10 23:11:28 +0000264}
265
266sub do_one_test($$)
267{
268 my ($dir, $vgperf) = @_;
269 $vgperf =~ /^(.*)\.vgperf/;
270 my $name = $1;
njn30e23ac2005-12-15 17:22:37 +0000271 my %first_tTool; # For doing percentage speedups when comparing
272 # multiple Valgrinds
njnec0c27a2005-12-10 23:11:28 +0000273
274 read_vgperf_file($vgperf);
275
276 if (defined $prereq) {
277 if (system("$prereq") != 0) {
278 printf("%-16s (skipping, prereq failed: $prereq)\n", "$name:");
279 return;
280 }
281 }
282
sewardjb4860be2006-10-17 02:30:17 +0000283 my $timecmd = "/usr/bin/time -p";
njnec0c27a2005-12-10 23:11:28 +0000284
285 # Do the native run(s).
njn57781862005-12-14 02:58:23 +0000286 printf("-- $name --\n") if (@vgdirs > 1);
njnec0c27a2005-12-10 23:11:28 +0000287 my $cmd = "$timecmd $prog $args";
njnee45f1d2005-12-13 21:55:16 +0000288 my $tNative = time_prog($cmd, $n_reps);
njnec0c27a2005-12-10 23:11:28 +0000289
philippe14ab1a32012-04-08 19:52:38 +0000290 if (defined $outer_valgrind) {
291 $outer_valgrind = validate_program($tests_dir, $outer_valgrind, 1, 1);
292 foreach my $vgdir (@vgdirs) {
293 validate_program($vgdir, "./coregrind/valgrind", 1, 1);
294 }
295 } else {
296 foreach my $vgdir (@vgdirs) {
297 validate_program($vgdir, "./coregrind/valgrind", 1, 1);
298 }
299 }
300
philippe215b2972012-07-24 19:47:46 +0000301 # Pull any extra options (for example, --sanity-level=4)
302 # from $EXTRA_REGTEST_OPTS.
303 my $maybe_extraopts = $ENV{"EXTRA_REGTEST_OPTS"};
304 my $extraopts = $maybe_extraopts ? $maybe_extraopts : "";
305
njn57781862005-12-14 02:58:23 +0000306 foreach my $vgdir (@vgdirs) {
307 # Benchmark name
308 printf("%-8s ", $name);
njnec0c27a2005-12-10 23:11:28 +0000309
njn57781862005-12-14 02:58:23 +0000310 # Print the Valgrind version if we are measuring more than one.
311 my $vgdirname = $vgdir;
312 chomp($vgdirname = `basename $vgdir`);
313 printf("%-10s:", $vgdirname);
314
315 # Native execution time
sewardj8e339a12006-10-14 14:04:42 +0000316 printf("%4.2fs", $tNative);
njnec0c27a2005-12-10 23:11:28 +0000317
njn57781862005-12-14 02:58:23 +0000318 foreach my $tool (@tools) {
njn341f98b2006-11-03 19:37:50 +0000319 # First two chars of toolname for abbreviation
320 my $tool_abbrev = $tool;
321 $tool_abbrev =~ s/(..).*/$1/;
njn341f98b2006-11-03 19:37:50 +0000322 printf(" %s:", $tool_abbrev);
philippe14ab1a32012-04-08 19:52:38 +0000323 my $run_outer_args = "";
324 if (not defined $outer_args) {
325 $run_outer_args =
326 " -v --command-line-only=yes"
327 . " --run-libc-freeres=no --sim-hints=enable-outer"
328 . " --smc-check=all-non-file"
329 . " --vgdb=no --trace-children=yes --read-var-info=no"
330 . " --suppressions=../tests/outer_inner.supp"
331 . " --memcheck:leak-check=full --memcheck:show-reachable=no"
332 . " --cachegrind:cache-sim=yes --cachegrind:branch-sim=yes"
333 . " --cachegrind:cachegrind-out-file=cachegrind.out.$vgdirname.$tool_abbrev.$name.%p"
334 . " --callgrind:cache-sim=yes --callgrind:branch-sim=yes"
335 . " --callgrind:dump-instr=yes --callgrind:collect-jumps=yes"
336 . " --callgrind:callgrind-out-file=callgrind.out.$vgdirname.$tool_abbrev.$name.%p"
337 . " ";
338 } else {
339 $run_outer_args = $outer_args;
340 }
341
342 my $vgsetup = "";
njn57781862005-12-14 02:58:23 +0000343 my $vgcmd = "$vgdir/coregrind/valgrind "
philippe215b2972012-07-24 19:47:46 +0000344 . "--command-line-only=yes --tool=$tool $extraopts -q "
njn276054a2006-11-03 19:30:33 +0000345 . "--memcheck:leak-check=no "
346 . "--trace-children=yes "
njn57781862005-12-14 02:58:23 +0000347 . "$vgopts ";
philippe14ab1a32012-04-08 19:52:38 +0000348 # Do the tool run(s).
349 if (defined $outer_valgrind ) {
350 # in an outer-inner setup, only set VALGRIND_LIB_INNER
351 $vgsetup = "VALGRIND_LIB_INNER=$vgdir/.in_place ";
352 $vgcmd = "$outer_valgrind "
353 . "--tool=" . $outer_tool . " "
354 . "$run_outer_args "
355 . "--log-file=" . "$outer_tool.outer.log.$vgdirname.$tool_abbrev.$name.%p "
356 . $vgcmd;
357 } else {
358 # Set both VALGRIND_LIB and VALGRIND_LIB_INNER
359 # in case this Valgrind was configured with --enable-inner. And
360 # also VALGRINDLIB, which was the old name for the variable, to
361 # allow comparison against old Valgrind versions (eg. 2.4.X).
362 $vgsetup = "VALGRINDLIB=$vgdir/.in_place "
363 . "VALGRIND_LIB=$vgdir/.in_place "
364 . "VALGRIND_LIB_INNER=$vgdir/.in_place ";
365 }
njn57781862005-12-14 02:58:23 +0000366 my $cmd = "$vgsetup $timecmd $vgcmd $prog $args";
367 my $tTool = time_prog($cmd, $n_reps);
njn30e23ac2005-12-15 17:22:37 +0000368 printf("%4.1fs (%4.1fx,", $tTool, $tTool/$tNative);
369
370 # If it's the first timing for this tool on this benchmark,
371 # record the time so we can get the percentage speedup of the
372 # subsequent Valgrinds. Otherwise, compute and print
373 # the speedup.
374 if (not defined $first_tTool{$tool}) {
375 $first_tTool{$tool} = $tTool;
njn715cc592006-03-27 00:39:43 +0000376 print(" -----)");
njn30e23ac2005-12-15 17:22:37 +0000377 } else {
378 my $speedup = 100 - (100 * $tTool / $first_tTool{$tool});
njn715cc592006-03-27 00:39:43 +0000379 printf("%5.1f%%)", $speedup);
njn30e23ac2005-12-15 17:22:37 +0000380 }
njn57781862005-12-14 02:58:23 +0000381
382 $num_timings_done++;
383
384 if (defined $cleanup) {
385 (system("$cleanup") == 0) or
386 print(" ($name cleanup operation failed: $cleanup)\n");
387 }
388 }
389 printf("\n");
njnec0c27a2005-12-10 23:11:28 +0000390 }
391
392 $num_tests_done++;
393}
394
395#----------------------------------------------------------------------------
396# Test one directory (and any subdirs)
397#----------------------------------------------------------------------------
398sub test_one_dir($$); # forward declaration
399
400sub test_one_dir($$)
401{
402 my ($dir, $prev_dirs) = @_;
403 $dir =~ s/\/$//; # trim a trailing '/'
404
njn356ffa32006-11-03 19:35:20 +0000405 chomp(my $initial_dir = `pwd`); # record where we started
406
njnec0c27a2005-12-10 23:11:28 +0000407 # Ignore dirs into which we should not recurse.
408 if ($dir =~ /^(BitKeeper|CVS|SCCS|docs|doc)$/) { return; }
409
410 chdir($dir) or die "Could not change into $dir\n";
411
412 # Nb: Don't prepend a '/' to the base directory
413 my $full_dir = $prev_dirs . ($prev_dirs eq "" ? "" : "/") . $dir;
414 my $dashes = "-" x (50 - length $full_dir);
415
416 my @fs = glob "*";
417 my $found_tests = (0 != (grep { $_ =~ /\.vgperf$/ } @fs));
418
419 if ($found_tests) {
420 print "-- Running tests in $full_dir $dashes\n";
421 }
422 foreach my $f (@fs) {
423 if (-d $f) {
424 test_one_dir($f, $full_dir);
425 } elsif ($f =~ /\.vgperf$/) {
426 do_one_test($full_dir, $f);
427 }
428 }
429 if ($found_tests) {
430 print "-- Finished tests in $full_dir $dashes\n";
431 }
432
njn356ffa32006-11-03 19:35:20 +0000433 chdir("$initial_dir");
njnec0c27a2005-12-10 23:11:28 +0000434}
435
436#----------------------------------------------------------------------------
437# Summarise results
438#----------------------------------------------------------------------------
439sub summarise_results
440{
441 printf("\n== %d programs, %d timings =================\n\n",
442 $num_tests_done, $num_timings_done);
443}
444
445#----------------------------------------------------------------------------
446# main()
447#----------------------------------------------------------------------------
philippe215b2972012-07-24 19:47:46 +0000448sub warn_about_EXTRA_REGTEST_OPTS()
449{
450 print "WARNING: \$EXTRA_REGTEST_OPTS is set. You probably don't want\n";
451 print "to run the perf tests with it set, unless you are doing some\n";
452 print "strange experiment, and/or you really know what you are doing.\n";
453 print "\n";
454}
njnec0c27a2005-12-10 23:11:28 +0000455
456# nuke VALGRIND_OPTS
457$ENV{"VALGRIND_OPTS"} = "";
458
philippe215b2972012-07-24 19:47:46 +0000459if ($ENV{"EXTRA_REGTEST_OPTS"}) {
460 print "\n";
461 warn_about_EXTRA_REGTEST_OPTS();
462}
463
njnec0c27a2005-12-10 23:11:28 +0000464my @fs = process_command_line();
465foreach my $f (@fs) {
466 if (-d $f) {
467 test_one_dir($f, "");
468 } else {
469 # Allow the .vgperf suffix to be given or omitted
470 if ($f =~ /.vgperf$/ && -r $f) {
471 # do nothing
472 } elsif (-r "$f.vgperf") {
473 $f = "$f.vgperf";
474 } else {
475 die "`$f' neither a directory nor a readable test file/name\n"
476 }
477 my $dir = `dirname $f`; chomp $dir;
478 my $file = `basename $f`; chomp $file;
479 chdir($dir) or die "Could not change into $dir\n";
480 do_one_test($dir, $file);
481 chdir($tests_dir);
482 }
483}
484summarise_results();
485
philippe215b2972012-07-24 19:47:46 +0000486if ($ENV{"EXTRA_REGTEST_OPTS"}) {
487 warn_about_EXTRA_REGTEST_OPTS();
488}
489
njnec0c27a2005-12-10 23:11:28 +0000490##--------------------------------------------------------------------##
491##--- end ---##
492##--------------------------------------------------------------------##