blob: 9be023e18e8a8b130011199f0b6c7f1794d2f589 [file] [log] [blame]
njnec0c27a2005-12-10 23:11:28 +00001#! @PERL@
2##--------------------------------------------------------------------##
3##--- Valgrind performance testing script vg_perf ---##
4##--------------------------------------------------------------------##
5
6# This file is part of Valgrind, a dynamic binary instrumentation
7# framework.
8#
9# Copyright (C) 2005 Nicholas Nethercote
10# njn@valgrind.org
11#
12# This program is free software; you can redistribute it and/or
13# modify it under the terms of the GNU General Public License as
14# published by the Free Software Foundation; either version 2 of the
15# License, or (at your option) any later version.
16#
17# This program is distributed in the hope that it will be useful, but
18# WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20# General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License
23# along with this program; if not, write to the Free Software
24# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
25# 02111-1307, USA.
26#
27# The GNU General Public License is contained in the file COPYING.
28
29#----------------------------------------------------------------------------
njn57781862005-12-14 02:58:23 +000030# usage: see usage message.
njnec0c27a2005-12-10 23:11:28 +000031#
njnec0c27a2005-12-10 23:11:28 +000032# You can specify individual files to test, or whole directories, or both.
33# Directories are traversed recursively, except for ones named, for example,
34# CVS/ or docs/.
35#
36# Each test is defined in a file <test>.vgperf, containing one or more of the
37# following lines, in any order:
38# - prog: <prog to run> (compulsory)
njnec0c27a2005-12-10 23:11:28 +000039# - args: <args for prog> (default: none)
40# - vgopts: <Valgrind options> (default: none)
41# - prereq: <prerequisite command> (default: none)
42# - cleanup: <post-test cleanup cmd to run> (default: none)
43#
44# The prerequisite command, if present, must return 0 otherwise the test is
45# skipped.
46#----------------------------------------------------------------------------
47
48use warnings;
49use strict;
50
51#----------------------------------------------------------------------------
52# Global vars
53#----------------------------------------------------------------------------
njnee45f1d2005-12-13 21:55:16 +000054my $usage = <<END
55usage: vg_perf [options] [files or dirs]
njnec0c27a2005-12-10 23:11:28 +000056
njnee45f1d2005-12-13 21:55:16 +000057 options for the user, with defaults in [ ], are:
58 -h --help show this message
njn3fbf17a2006-11-22 00:39:08 +000059 --reps=<n> number of repeats for each program [1]
njn768cd5d2006-11-22 00:52:00 +000060 --tools=<t1,t2,t3> tools to run [Nulgrind and Memcheck]
philippe14ab1a32012-04-08 19:52:38 +000061 --vg Valgrind(s) to measure [Valgrind in the current directory]
62 (can be specified multiple times).
63 The "in-place" build is used.
64
65 --outer-valgrind: run these Valgrind(s) under the given outer valgrind.
66 These Valgrind(s) must be configured with --enable-inner.
67 --outer-tool: tool to use by the outer valgrind (default cachegrind).
68 --outer-args: use this as outer tool args.
njn768cd5d2006-11-22 00:52:00 +000069
70 Any tools named in --tools must be present in all directories specified
71 with --vg. (This is not checked.)
njnee45f1d2005-12-13 21:55:16 +000072END
73;
njnec0c27a2005-12-10 23:11:28 +000074
75# Test variables
76my $vgopts; # valgrind options
77my $prog; # test prog
78my $args; # test prog args
79my $prereq; # prerequisite test to satisfy before running test
80my $cleanup; # cleanup command to run
njnec0c27a2005-12-10 23:11:28 +000081
njn57781862005-12-14 02:58:23 +000082# Command line options
njn3fbf17a2006-11-22 00:39:08 +000083my $n_reps = 1; # Run each test $n_reps times and choose the best one.
84my @vgdirs; # Dirs of the various Valgrinds being measured.
njn768cd5d2006-11-22 00:52:00 +000085my @tools = ("none", "memcheck"); # tools being measured
njnec0c27a2005-12-10 23:11:28 +000086
philippe14ab1a32012-04-08 19:52:38 +000087# Outer valgrind to use, and args to use for it.
88# If this is set, --valgrind should be set to the installed inner valgrind,
89# and --valgrind-lib will be ignore
90my $outer_valgrind;
91my $outer_tool = "cachegrind";
92my $outer_args;
93
94
njnec0c27a2005-12-10 23:11:28 +000095my $num_tests_done = 0;
96my $num_timings_done = 0;
97
98# Starting directory
99chomp(my $tests_dir = `pwd`);
100
njnec0c27a2005-12-10 23:11:28 +0000101#----------------------------------------------------------------------------
102# Process command line, setup
103#----------------------------------------------------------------------------
104
105# If $prog is a relative path, it prepends $dir to it. Useful for two reasons:
106#
107# 1. Can prepend "." onto programs to avoid trouble with users who don't have
108# "." in their path (by making $dir = ".")
109# 2. Can prepend the current dir to make the command absolute to avoid
110# subsequent trouble when we change directories.
111#
112# Also checks the program exists and is executable.
113sub validate_program ($$$$)
114{
115 my ($dir, $prog, $must_exist, $must_be_executable) = @_;
116
117 # If absolute path, leave it alone. If relative, make it
118 # absolute -- by prepending current dir -- so we can change
119 # dirs and still use it.
120 $prog = "$dir/$prog" if ($prog !~ /^\//);
121 if ($must_exist) {
122 (-f $prog) or die "vg_perf: '$prog' not found or not a file ($dir)\n";
123 }
124 if ($must_be_executable) {
125 (-x $prog) or die "vg_perf: '$prog' not executable ($dir)\n";
126 }
127
128 return $prog;
129}
130
njn57781862005-12-14 02:58:23 +0000131sub add_vgdir($)
132{
133 my ($vgdir) = @_;
134 if ($vgdir !~ /^\//) { $vgdir = "$tests_dir/$vgdir"; }
njn57781862005-12-14 02:58:23 +0000135 push(@vgdirs, $vgdir);
136}
137
njnec0c27a2005-12-10 23:11:28 +0000138sub process_command_line()
139{
njnec0c27a2005-12-10 23:11:28 +0000140 my @fs;
141
142 for my $arg (@ARGV) {
143 if ($arg =~ /^-/) {
njn1a95e052009-07-01 04:50:41 +0000144 if ($arg =~ /^--reps=(\d+)$/) {
njnee45f1d2005-12-13 21:55:16 +0000145 $n_reps = $1;
njn30e23ac2005-12-15 17:22:37 +0000146 if ($n_reps < 1) { die "bad --reps value: $n_reps\n"; }
njn57781862005-12-14 02:58:23 +0000147 } elsif ($arg =~ /^--vg=(.+)$/) {
148 # Make dir absolute if not already
149 add_vgdir($1);
njn3fbf17a2006-11-22 00:39:08 +0000150 } elsif ($arg =~ /^--tools=(.+)$/) {
151 @tools = split(/,/, $1);
philippe14ab1a32012-04-08 19:52:38 +0000152 } elsif ($arg =~ /^--outer-valgrind=(.*)$/) {
153 $outer_valgrind = $1;
154 } elsif ($arg =~ /^--outer-tool=(.*)$/) {
155 $outer_tool = $1;
156 } elsif ($arg =~ /^--outer-args=(.*)$/) {
157 $outer_args = $1;
njnec0c27a2005-12-10 23:11:28 +0000158 } else {
159 die $usage;
160 }
161 } else {
162 push(@fs, $arg);
163 }
164 }
njn57781862005-12-14 02:58:23 +0000165
166 # If no --vg options were specified, use the current tree.
167 if (0 == @vgdirs) {
168 add_vgdir($tests_dir);
169 }
njnec0c27a2005-12-10 23:11:28 +0000170
njnec0c27a2005-12-10 23:11:28 +0000171 (0 != @fs) or die "No test files or directories specified\n";
172
173 return @fs;
174}
175
176#----------------------------------------------------------------------------
177# Read a .vgperf file
178#----------------------------------------------------------------------------
179sub read_vgperf_file($)
180{
181 my ($f) = @_;
182
183 # Defaults.
184 ($vgopts, $prog, $args, $prereq, $cleanup)
185 = ("", undef, "", undef, undef, undef, undef);
186
187 open(INPUTFILE, "< $f") || die "File $f not openable\n";
188
189 while (my $line = <INPUTFILE>) {
190 if ($line =~ /^\s*#/ || $line =~ /^\s*$/) {
191 next;
192 } elsif ($line =~ /^\s*vgopts:\s*(.*)$/) {
193 $vgopts = $1;
194 } elsif ($line =~ /^\s*prog:\s*(.*)$/) {
njnc9582a22005-12-13 21:44:48 +0000195 $prog = validate_program(".", $1, 1, 1);
njnec0c27a2005-12-10 23:11:28 +0000196 } elsif ($line =~ /^\s*args:\s*(.*)$/) {
197 $args = $1;
198 } elsif ($line =~ /^\s*prereq:\s*(.*)$/) {
199 $prereq = $1;
200 } elsif ($line =~ /^\s*cleanup:\s*(.*)$/) {
201 $cleanup = $1;
202 } else {
203 die "Bad line in $f: $line\n";
204 }
205 }
206 close(INPUTFILE);
207
208 if (!defined $prog) {
209 $prog = ""; # allow no prog for testing error and --help cases
210 }
211 if (0 == @tools) {
212 die "vg_perf: missing 'tools' line in $f\n";
213 }
214}
215
216#----------------------------------------------------------------------------
217# Do one test
218#----------------------------------------------------------------------------
219# Since most of the program time is spent in system() calls, need this to
220# propagate a Ctrl-C enabling us to quit.
221sub mysystem($)
222{
njnecb47442005-12-13 16:54:58 +0000223 my ($cmd) = @_;
224 my $retval = system($cmd);
225 if ($retval == 2) {
226 exit 1;
227 } else {
228 return $retval;
229 }
njnec0c27a2005-12-10 23:11:28 +0000230}
231
sewardjb4860be2006-10-17 02:30:17 +0000232# Run program N times, return the best user time. Use the POSIX
233# -p flag on /usr/bin/time so as to get something parseable on AIX.
njnec0c27a2005-12-10 23:11:28 +0000234sub time_prog($$)
235{
236 my ($cmd, $n) = @_;
237 my $tmin = 999999;
238 for (my $i = 0; $i < $n; $i++) {
njnecb47442005-12-13 16:54:58 +0000239 mysystem("echo '$cmd' > perf.cmd");
240 my $retval = mysystem("$cmd > perf.stdout 2> perf.stderr");
241 (0 == $retval) or
njn57781862005-12-14 02:58:23 +0000242 die "\n*** Command returned non-zero ($retval)"
243 . "\n*** See perf.{cmd,stdout,stderr} to determine what went wrong.\n";
njnecb47442005-12-13 16:54:58 +0000244 my $out = `cat perf.stderr`;
njnae7f6822007-02-02 23:23:01 +0000245 ($out =~ /[Uu]ser +([\d\.]+)/) or
njn02383bc2005-12-13 20:23:38 +0000246 die "\n*** missing usertime in perf.stderr\n";
njnae7f6822007-02-02 23:23:01 +0000247 $tmin = $1 if ($1 < $tmin);
njnec0c27a2005-12-10 23:11:28 +0000248 }
njn30e23ac2005-12-15 17:22:37 +0000249 # Avoid divisions by zero!
250 return (0 == $tmin ? 0.01 : $tmin);
njnec0c27a2005-12-10 23:11:28 +0000251}
252
253sub do_one_test($$)
254{
255 my ($dir, $vgperf) = @_;
256 $vgperf =~ /^(.*)\.vgperf/;
257 my $name = $1;
njn30e23ac2005-12-15 17:22:37 +0000258 my %first_tTool; # For doing percentage speedups when comparing
259 # multiple Valgrinds
njnec0c27a2005-12-10 23:11:28 +0000260
261 read_vgperf_file($vgperf);
262
263 if (defined $prereq) {
264 if (system("$prereq") != 0) {
265 printf("%-16s (skipping, prereq failed: $prereq)\n", "$name:");
266 return;
267 }
268 }
269
sewardjb4860be2006-10-17 02:30:17 +0000270 my $timecmd = "/usr/bin/time -p";
njnec0c27a2005-12-10 23:11:28 +0000271
272 # Do the native run(s).
njn57781862005-12-14 02:58:23 +0000273 printf("-- $name --\n") if (@vgdirs > 1);
njnec0c27a2005-12-10 23:11:28 +0000274 my $cmd = "$timecmd $prog $args";
njnee45f1d2005-12-13 21:55:16 +0000275 my $tNative = time_prog($cmd, $n_reps);
njnec0c27a2005-12-10 23:11:28 +0000276
philippe14ab1a32012-04-08 19:52:38 +0000277 if (defined $outer_valgrind) {
278 $outer_valgrind = validate_program($tests_dir, $outer_valgrind, 1, 1);
279 foreach my $vgdir (@vgdirs) {
280 validate_program($vgdir, "./coregrind/valgrind", 1, 1);
281 }
282 } else {
283 foreach my $vgdir (@vgdirs) {
284 validate_program($vgdir, "./coregrind/valgrind", 1, 1);
285 }
286 }
287
njn57781862005-12-14 02:58:23 +0000288 foreach my $vgdir (@vgdirs) {
289 # Benchmark name
290 printf("%-8s ", $name);
njnec0c27a2005-12-10 23:11:28 +0000291
njn57781862005-12-14 02:58:23 +0000292 # Print the Valgrind version if we are measuring more than one.
293 my $vgdirname = $vgdir;
294 chomp($vgdirname = `basename $vgdir`);
295 printf("%-10s:", $vgdirname);
296
297 # Native execution time
sewardj8e339a12006-10-14 14:04:42 +0000298 printf("%4.2fs", $tNative);
njnec0c27a2005-12-10 23:11:28 +0000299
njn57781862005-12-14 02:58:23 +0000300 foreach my $tool (@tools) {
njn341f98b2006-11-03 19:37:50 +0000301 # First two chars of toolname for abbreviation
302 my $tool_abbrev = $tool;
303 $tool_abbrev =~ s/(..).*/$1/;
njn341f98b2006-11-03 19:37:50 +0000304 printf(" %s:", $tool_abbrev);
philippe14ab1a32012-04-08 19:52:38 +0000305 my $run_outer_args = "";
306 if (not defined $outer_args) {
307 $run_outer_args =
308 " -v --command-line-only=yes"
309 . " --run-libc-freeres=no --sim-hints=enable-outer"
310 . " --smc-check=all-non-file"
311 . " --vgdb=no --trace-children=yes --read-var-info=no"
312 . " --suppressions=../tests/outer_inner.supp"
313 . " --memcheck:leak-check=full --memcheck:show-reachable=no"
314 . " --cachegrind:cache-sim=yes --cachegrind:branch-sim=yes"
315 . " --cachegrind:cachegrind-out-file=cachegrind.out.$vgdirname.$tool_abbrev.$name.%p"
316 . " --callgrind:cache-sim=yes --callgrind:branch-sim=yes"
317 . " --callgrind:dump-instr=yes --callgrind:collect-jumps=yes"
318 . " --callgrind:callgrind-out-file=callgrind.out.$vgdirname.$tool_abbrev.$name.%p"
319 . " ";
320 } else {
321 $run_outer_args = $outer_args;
322 }
323
324 my $vgsetup = "";
njn57781862005-12-14 02:58:23 +0000325 my $vgcmd = "$vgdir/coregrind/valgrind "
326 . "--command-line-only=yes --tool=$tool -q "
njn276054a2006-11-03 19:30:33 +0000327 . "--memcheck:leak-check=no "
328 . "--trace-children=yes "
njn57781862005-12-14 02:58:23 +0000329 . "$vgopts ";
philippe14ab1a32012-04-08 19:52:38 +0000330 # Do the tool run(s).
331 if (defined $outer_valgrind ) {
332 # in an outer-inner setup, only set VALGRIND_LIB_INNER
333 $vgsetup = "VALGRIND_LIB_INNER=$vgdir/.in_place ";
334 $vgcmd = "$outer_valgrind "
335 . "--tool=" . $outer_tool . " "
336 . "$run_outer_args "
337 . "--log-file=" . "$outer_tool.outer.log.$vgdirname.$tool_abbrev.$name.%p "
338 . $vgcmd;
339 } else {
340 # Set both VALGRIND_LIB and VALGRIND_LIB_INNER
341 # in case this Valgrind was configured with --enable-inner. And
342 # also VALGRINDLIB, which was the old name for the variable, to
343 # allow comparison against old Valgrind versions (eg. 2.4.X).
344 $vgsetup = "VALGRINDLIB=$vgdir/.in_place "
345 . "VALGRIND_LIB=$vgdir/.in_place "
346 . "VALGRIND_LIB_INNER=$vgdir/.in_place ";
347 }
njn57781862005-12-14 02:58:23 +0000348 my $cmd = "$vgsetup $timecmd $vgcmd $prog $args";
349 my $tTool = time_prog($cmd, $n_reps);
njn30e23ac2005-12-15 17:22:37 +0000350 printf("%4.1fs (%4.1fx,", $tTool, $tTool/$tNative);
351
352 # If it's the first timing for this tool on this benchmark,
353 # record the time so we can get the percentage speedup of the
354 # subsequent Valgrinds. Otherwise, compute and print
355 # the speedup.
356 if (not defined $first_tTool{$tool}) {
357 $first_tTool{$tool} = $tTool;
njn715cc592006-03-27 00:39:43 +0000358 print(" -----)");
njn30e23ac2005-12-15 17:22:37 +0000359 } else {
360 my $speedup = 100 - (100 * $tTool / $first_tTool{$tool});
njn715cc592006-03-27 00:39:43 +0000361 printf("%5.1f%%)", $speedup);
njn30e23ac2005-12-15 17:22:37 +0000362 }
njn57781862005-12-14 02:58:23 +0000363
364 $num_timings_done++;
365
366 if (defined $cleanup) {
367 (system("$cleanup") == 0) or
368 print(" ($name cleanup operation failed: $cleanup)\n");
369 }
370 }
371 printf("\n");
njnec0c27a2005-12-10 23:11:28 +0000372 }
373
374 $num_tests_done++;
375}
376
377#----------------------------------------------------------------------------
378# Test one directory (and any subdirs)
379#----------------------------------------------------------------------------
380sub test_one_dir($$); # forward declaration
381
382sub test_one_dir($$)
383{
384 my ($dir, $prev_dirs) = @_;
385 $dir =~ s/\/$//; # trim a trailing '/'
386
njn356ffa32006-11-03 19:35:20 +0000387 chomp(my $initial_dir = `pwd`); # record where we started
388
njnec0c27a2005-12-10 23:11:28 +0000389 # Ignore dirs into which we should not recurse.
390 if ($dir =~ /^(BitKeeper|CVS|SCCS|docs|doc)$/) { return; }
391
392 chdir($dir) or die "Could not change into $dir\n";
393
394 # Nb: Don't prepend a '/' to the base directory
395 my $full_dir = $prev_dirs . ($prev_dirs eq "" ? "" : "/") . $dir;
396 my $dashes = "-" x (50 - length $full_dir);
397
398 my @fs = glob "*";
399 my $found_tests = (0 != (grep { $_ =~ /\.vgperf$/ } @fs));
400
401 if ($found_tests) {
402 print "-- Running tests in $full_dir $dashes\n";
403 }
404 foreach my $f (@fs) {
405 if (-d $f) {
406 test_one_dir($f, $full_dir);
407 } elsif ($f =~ /\.vgperf$/) {
408 do_one_test($full_dir, $f);
409 }
410 }
411 if ($found_tests) {
412 print "-- Finished tests in $full_dir $dashes\n";
413 }
414
njn356ffa32006-11-03 19:35:20 +0000415 chdir("$initial_dir");
njnec0c27a2005-12-10 23:11:28 +0000416}
417
418#----------------------------------------------------------------------------
419# Summarise results
420#----------------------------------------------------------------------------
421sub summarise_results
422{
423 printf("\n== %d programs, %d timings =================\n\n",
424 $num_tests_done, $num_timings_done);
425}
426
427#----------------------------------------------------------------------------
428# main()
429#----------------------------------------------------------------------------
430
431# nuke VALGRIND_OPTS
432$ENV{"VALGRIND_OPTS"} = "";
433
434my @fs = process_command_line();
435foreach my $f (@fs) {
436 if (-d $f) {
437 test_one_dir($f, "");
438 } else {
439 # Allow the .vgperf suffix to be given or omitted
440 if ($f =~ /.vgperf$/ && -r $f) {
441 # do nothing
442 } elsif (-r "$f.vgperf") {
443 $f = "$f.vgperf";
444 } else {
445 die "`$f' neither a directory nor a readable test file/name\n"
446 }
447 my $dir = `dirname $f`; chomp $dir;
448 my $file = `basename $f`; chomp $file;
449 chdir($dir) or die "Could not change into $dir\n";
450 do_one_test($dir, $file);
451 chdir($tests_dir);
452 }
453}
454summarise_results();
455
456##--------------------------------------------------------------------##
457##--- end ---##
458##--------------------------------------------------------------------##