njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 1 | #! @PERL@ |
| 2 | ##--------------------------------------------------------------------## |
| 3 | ##--- Valgrind performance testing script vg_perf ---## |
| 4 | ##--------------------------------------------------------------------## |
| 5 | |
| 6 | # This file is part of Valgrind, a dynamic binary instrumentation |
| 7 | # framework. |
| 8 | # |
| 9 | # Copyright (C) 2005 Nicholas Nethercote |
| 10 | # njn@valgrind.org |
| 11 | # |
| 12 | # This program is free software; you can redistribute it and/or |
| 13 | # modify it under the terms of the GNU General Public License as |
| 14 | # published by the Free Software Foundation; either version 2 of the |
| 15 | # License, or (at your option) any later version. |
| 16 | # |
| 17 | # This program is distributed in the hope that it will be useful, but |
| 18 | # WITHOUT ANY WARRANTY; without even the implied warranty of |
| 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 20 | # General Public License for more details. |
| 21 | # |
| 22 | # You should have received a copy of the GNU General Public License |
| 23 | # along with this program; if not, write to the Free Software |
| 24 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA |
| 25 | # 02111-1307, USA. |
| 26 | # |
| 27 | # The GNU General Public License is contained in the file COPYING. |
| 28 | |
| 29 | #---------------------------------------------------------------------------- |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 30 | # usage: see usage message. |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 31 | # |
| 32 | # The easiest way is to run all tests in valgrind/ with (assuming you installed |
| 33 | # in $PREFIX): |
| 34 | # |
| 35 | # perl perf/vg_perf --all |
| 36 | # |
| 37 | # You can specify individual files to test, or whole directories, or both. |
| 38 | # Directories are traversed recursively, except for ones named, for example, |
| 39 | # CVS/ or docs/. |
| 40 | # |
| 41 | # Each test is defined in a file <test>.vgperf, containing one or more of the |
| 42 | # following lines, in any order: |
| 43 | # - prog: <prog to run> (compulsory) |
| 44 | # - tools: <Valgrind tools> (compulsory) |
| 45 | # - args: <args for prog> (default: none) |
| 46 | # - vgopts: <Valgrind options> (default: none) |
| 47 | # - prereq: <prerequisite command> (default: none) |
| 48 | # - cleanup: <post-test cleanup cmd to run> (default: none) |
| 49 | # |
| 50 | # The prerequisite command, if present, must return 0 otherwise the test is |
| 51 | # skipped. |
| 52 | #---------------------------------------------------------------------------- |
| 53 | |
| 54 | use warnings; |
| 55 | use strict; |
| 56 | |
| 57 | #---------------------------------------------------------------------------- |
| 58 | # Global vars |
| 59 | #---------------------------------------------------------------------------- |
njn | ee45f1d | 2005-12-13 21:55:16 +0000 | [diff] [blame] | 60 | my $usage = <<END |
| 61 | usage: vg_perf [options] [files or dirs] |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 62 | |
njn | ee45f1d | 2005-12-13 21:55:16 +0000 | [diff] [blame] | 63 | options for the user, with defaults in [ ], are: |
| 64 | -h --help show this message |
| 65 | --all run all tests under this directory |
| 66 | --reps number of repeats for each program [3] |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 67 | --vg Valgrind(s) to measure (can be specified multiple |
| 68 | times). The "in-place" build is used. |
| 69 | [Valgrind in the current directory] |
njn | ee45f1d | 2005-12-13 21:55:16 +0000 | [diff] [blame] | 70 | END |
| 71 | ; |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 72 | |
| 73 | # Test variables |
| 74 | my $vgopts; # valgrind options |
| 75 | my $prog; # test prog |
| 76 | my $args; # test prog args |
| 77 | my $prereq; # prerequisite test to satisfy before running test |
| 78 | my $cleanup; # cleanup command to run |
| 79 | my @tools; # which tools are we measuring the program with |
| 80 | |
| 81 | # Abbreviations used in output |
| 82 | my %toolnames = ( |
| 83 | none => "nl", |
| 84 | memcheck => "mc", |
| 85 | cachegrind => "cg", |
| 86 | massif => "ms" |
| 87 | ); |
| 88 | |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 89 | # Command line options |
njn | fbfbd4f | 2005-12-23 23:08:04 +0000 | [diff] [blame] | 90 | my $n_reps = 1; # Run each program $n_reps times and choose the best one. |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 91 | my @vgdirs; # Dirs of the various Valgrinds being measured. |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 92 | |
| 93 | my $num_tests_done = 0; |
| 94 | my $num_timings_done = 0; |
| 95 | |
| 96 | # Starting directory |
| 97 | chomp(my $tests_dir = `pwd`); |
| 98 | |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 99 | #---------------------------------------------------------------------------- |
| 100 | # Process command line, setup |
| 101 | #---------------------------------------------------------------------------- |
| 102 | |
| 103 | # If $prog is a relative path, it prepends $dir to it. Useful for two reasons: |
| 104 | # |
| 105 | # 1. Can prepend "." onto programs to avoid trouble with users who don't have |
| 106 | # "." in their path (by making $dir = ".") |
| 107 | # 2. Can prepend the current dir to make the command absolute to avoid |
| 108 | # subsequent trouble when we change directories. |
| 109 | # |
| 110 | # Also checks the program exists and is executable. |
| 111 | sub validate_program ($$$$) |
| 112 | { |
| 113 | my ($dir, $prog, $must_exist, $must_be_executable) = @_; |
| 114 | |
| 115 | # If absolute path, leave it alone. If relative, make it |
| 116 | # absolute -- by prepending current dir -- so we can change |
| 117 | # dirs and still use it. |
| 118 | $prog = "$dir/$prog" if ($prog !~ /^\//); |
| 119 | if ($must_exist) { |
| 120 | (-f $prog) or die "vg_perf: '$prog' not found or not a file ($dir)\n"; |
| 121 | } |
| 122 | if ($must_be_executable) { |
| 123 | (-x $prog) or die "vg_perf: '$prog' not executable ($dir)\n"; |
| 124 | } |
| 125 | |
| 126 | return $prog; |
| 127 | } |
| 128 | |
| 129 | sub validate_tools($) |
| 130 | { |
| 131 | # XXX: should check they exist! |
| 132 | my ($toolnames) = @_; |
| 133 | my @t = split(/\s+/, $toolnames); |
| 134 | return @t; |
| 135 | } |
| 136 | |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 137 | sub add_vgdir($) |
| 138 | { |
| 139 | my ($vgdir) = @_; |
| 140 | if ($vgdir !~ /^\//) { $vgdir = "$tests_dir/$vgdir"; } |
| 141 | validate_program($vgdir, "./coregrind/valgrind", 1, 1); |
| 142 | push(@vgdirs, $vgdir); |
| 143 | } |
| 144 | |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 145 | sub process_command_line() |
| 146 | { |
| 147 | my $alldirs = 0; |
| 148 | my @fs; |
| 149 | |
| 150 | for my $arg (@ARGV) { |
| 151 | if ($arg =~ /^-/) { |
| 152 | if ($arg =~ /^--all$/) { |
| 153 | $alldirs = 1; |
njn | b2b4f46 | 2005-12-13 22:00:17 +0000 | [diff] [blame] | 154 | } elsif ($arg =~ /^--reps=(\d+)$/) { |
njn | ee45f1d | 2005-12-13 21:55:16 +0000 | [diff] [blame] | 155 | $n_reps = $1; |
njn | 30e23ac | 2005-12-15 17:22:37 +0000 | [diff] [blame] | 156 | if ($n_reps < 1) { die "bad --reps value: $n_reps\n"; } |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 157 | } elsif ($arg =~ /^--vg=(.+)$/) { |
| 158 | # Make dir absolute if not already |
| 159 | add_vgdir($1); |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 160 | } else { |
| 161 | die $usage; |
| 162 | } |
| 163 | } else { |
| 164 | push(@fs, $arg); |
| 165 | } |
| 166 | } |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 167 | |
| 168 | # If no --vg options were specified, use the current tree. |
| 169 | if (0 == @vgdirs) { |
| 170 | add_vgdir($tests_dir); |
| 171 | } |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 172 | |
| 173 | if ($alldirs) { |
| 174 | @fs = (); |
| 175 | foreach my $f (glob "*") { |
| 176 | push(@fs, $f) if (-d $f); |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | (0 != @fs) or die "No test files or directories specified\n"; |
| 181 | |
| 182 | return @fs; |
| 183 | } |
| 184 | |
| 185 | #---------------------------------------------------------------------------- |
| 186 | # Read a .vgperf file |
| 187 | #---------------------------------------------------------------------------- |
| 188 | sub read_vgperf_file($) |
| 189 | { |
| 190 | my ($f) = @_; |
| 191 | |
| 192 | # Defaults. |
| 193 | ($vgopts, $prog, $args, $prereq, $cleanup) |
| 194 | = ("", undef, "", undef, undef, undef, undef); |
| 195 | |
| 196 | open(INPUTFILE, "< $f") || die "File $f not openable\n"; |
| 197 | |
| 198 | while (my $line = <INPUTFILE>) { |
| 199 | if ($line =~ /^\s*#/ || $line =~ /^\s*$/) { |
| 200 | next; |
| 201 | } elsif ($line =~ /^\s*vgopts:\s*(.*)$/) { |
| 202 | $vgopts = $1; |
| 203 | } elsif ($line =~ /^\s*prog:\s*(.*)$/) { |
njn | c9582a2 | 2005-12-13 21:44:48 +0000 | [diff] [blame] | 204 | $prog = validate_program(".", $1, 1, 1); |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 205 | } elsif ($line =~ /^\s*tools:\s*(.*)$/) { |
| 206 | @tools = validate_tools($1); |
| 207 | } elsif ($line =~ /^\s*args:\s*(.*)$/) { |
| 208 | $args = $1; |
| 209 | } elsif ($line =~ /^\s*prereq:\s*(.*)$/) { |
| 210 | $prereq = $1; |
| 211 | } elsif ($line =~ /^\s*cleanup:\s*(.*)$/) { |
| 212 | $cleanup = $1; |
| 213 | } else { |
| 214 | die "Bad line in $f: $line\n"; |
| 215 | } |
| 216 | } |
| 217 | close(INPUTFILE); |
| 218 | |
| 219 | if (!defined $prog) { |
| 220 | $prog = ""; # allow no prog for testing error and --help cases |
| 221 | } |
| 222 | if (0 == @tools) { |
| 223 | die "vg_perf: missing 'tools' line in $f\n"; |
| 224 | } |
| 225 | } |
| 226 | |
| 227 | #---------------------------------------------------------------------------- |
| 228 | # Do one test |
| 229 | #---------------------------------------------------------------------------- |
| 230 | # Since most of the program time is spent in system() calls, need this to |
| 231 | # propagate a Ctrl-C enabling us to quit. |
| 232 | sub mysystem($) |
| 233 | { |
njn | ecb4744 | 2005-12-13 16:54:58 +0000 | [diff] [blame] | 234 | my ($cmd) = @_; |
| 235 | my $retval = system($cmd); |
| 236 | if ($retval == 2) { |
| 237 | exit 1; |
| 238 | } else { |
| 239 | return $retval; |
| 240 | } |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 241 | } |
| 242 | |
njn | 02383bc | 2005-12-13 20:23:38 +0000 | [diff] [blame] | 243 | # Run program N times, return the best user time. |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 244 | sub time_prog($$) |
| 245 | { |
| 246 | my ($cmd, $n) = @_; |
| 247 | my $tmin = 999999; |
| 248 | for (my $i = 0; $i < $n; $i++) { |
njn | ecb4744 | 2005-12-13 16:54:58 +0000 | [diff] [blame] | 249 | mysystem("echo '$cmd' > perf.cmd"); |
| 250 | my $retval = mysystem("$cmd > perf.stdout 2> perf.stderr"); |
| 251 | (0 == $retval) or |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 252 | die "\n*** Command returned non-zero ($retval)" |
| 253 | . "\n*** See perf.{cmd,stdout,stderr} to determine what went wrong.\n"; |
njn | ecb4744 | 2005-12-13 16:54:58 +0000 | [diff] [blame] | 254 | my $out = `cat perf.stderr`; |
njn | 02383bc | 2005-12-13 20:23:38 +0000 | [diff] [blame] | 255 | ($out =~ /usertime: ([\d\.]+)s/) or |
| 256 | die "\n*** missing usertime in perf.stderr\n"; |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 257 | $tmin = $1 if ($1 < $tmin); |
| 258 | } |
njn | 30e23ac | 2005-12-15 17:22:37 +0000 | [diff] [blame] | 259 | # Avoid divisions by zero! |
| 260 | return (0 == $tmin ? 0.01 : $tmin); |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 261 | } |
| 262 | |
| 263 | sub do_one_test($$) |
| 264 | { |
| 265 | my ($dir, $vgperf) = @_; |
| 266 | $vgperf =~ /^(.*)\.vgperf/; |
| 267 | my $name = $1; |
njn | 30e23ac | 2005-12-15 17:22:37 +0000 | [diff] [blame] | 268 | my %first_tTool; # For doing percentage speedups when comparing |
| 269 | # multiple Valgrinds |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 270 | |
| 271 | read_vgperf_file($vgperf); |
| 272 | |
| 273 | if (defined $prereq) { |
| 274 | if (system("$prereq") != 0) { |
| 275 | printf("%-16s (skipping, prereq failed: $prereq)\n", "$name:"); |
| 276 | return; |
| 277 | } |
| 278 | } |
| 279 | |
njn | 02383bc | 2005-12-13 20:23:38 +0000 | [diff] [blame] | 280 | my $timecmd = "/usr/bin/time -f 'usertime: %Us'"; |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 281 | |
| 282 | # Do the native run(s). |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 283 | printf("-- $name --\n") if (@vgdirs > 1); |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 284 | my $cmd = "$timecmd $prog $args"; |
njn | ee45f1d | 2005-12-13 21:55:16 +0000 | [diff] [blame] | 285 | my $tNative = time_prog($cmd, $n_reps); |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 286 | |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 287 | foreach my $vgdir (@vgdirs) { |
| 288 | # Benchmark name |
| 289 | printf("%-8s ", $name); |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 290 | |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 291 | # Print the Valgrind version if we are measuring more than one. |
| 292 | my $vgdirname = $vgdir; |
| 293 | chomp($vgdirname = `basename $vgdir`); |
| 294 | printf("%-10s:", $vgdirname); |
| 295 | |
| 296 | # Native execution time |
sewardj | 8e339a1 | 2006-10-14 14:04:42 +0000 | [diff] [blame^] | 297 | printf("%4.2fs", $tNative); |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 298 | |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 299 | foreach my $tool (@tools) { |
| 300 | (defined $toolnames{$tool}) or |
| 301 | die "unknown tool $tool, please add to %toolnames\n"; |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 302 | |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 303 | # Do the tool run(s). Set both VALGRIND_LIB and VALGRIND_LIB_INNER |
njn | 8c20d87 | 2006-04-06 22:59:35 +0000 | [diff] [blame] | 304 | # in case this Valgrind was configured with --enable-inner. And |
| 305 | # also VALGRINDLIB, which was the old name for the variable, to |
| 306 | # allow comparison against old Valgrind versions (eg. 2.4.X). |
njn | 715cc59 | 2006-03-27 00:39:43 +0000 | [diff] [blame] | 307 | printf(" %s:", $toolnames{$tool}); |
njn | 8c20d87 | 2006-04-06 22:59:35 +0000 | [diff] [blame] | 308 | my $vgsetup = "VALGRINDLIB=$vgdir/.in_place " |
| 309 | . "VALGRIND_LIB=$vgdir/.in_place " |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 310 | . "VALGRIND_LIB_INNER=$vgdir/.in_place "; |
| 311 | my $vgcmd = "$vgdir/coregrind/valgrind " |
| 312 | . "--command-line-only=yes --tool=$tool -q " |
| 313 | . "--memcheck:leak-check=no --addrcheck:leak-check=no " |
| 314 | . "$vgopts "; |
| 315 | my $cmd = "$vgsetup $timecmd $vgcmd $prog $args"; |
| 316 | my $tTool = time_prog($cmd, $n_reps); |
njn | 30e23ac | 2005-12-15 17:22:37 +0000 | [diff] [blame] | 317 | printf("%4.1fs (%4.1fx,", $tTool, $tTool/$tNative); |
| 318 | |
| 319 | # If it's the first timing for this tool on this benchmark, |
| 320 | # record the time so we can get the percentage speedup of the |
| 321 | # subsequent Valgrinds. Otherwise, compute and print |
| 322 | # the speedup. |
| 323 | if (not defined $first_tTool{$tool}) { |
| 324 | $first_tTool{$tool} = $tTool; |
njn | 715cc59 | 2006-03-27 00:39:43 +0000 | [diff] [blame] | 325 | print(" -----)"); |
njn | 30e23ac | 2005-12-15 17:22:37 +0000 | [diff] [blame] | 326 | } else { |
| 327 | my $speedup = 100 - (100 * $tTool / $first_tTool{$tool}); |
njn | 715cc59 | 2006-03-27 00:39:43 +0000 | [diff] [blame] | 328 | printf("%5.1f%%)", $speedup); |
njn | 30e23ac | 2005-12-15 17:22:37 +0000 | [diff] [blame] | 329 | } |
njn | 5778186 | 2005-12-14 02:58:23 +0000 | [diff] [blame] | 330 | |
| 331 | $num_timings_done++; |
| 332 | |
| 333 | if (defined $cleanup) { |
| 334 | (system("$cleanup") == 0) or |
| 335 | print(" ($name cleanup operation failed: $cleanup)\n"); |
| 336 | } |
| 337 | } |
| 338 | printf("\n"); |
njn | ec0c27a | 2005-12-10 23:11:28 +0000 | [diff] [blame] | 339 | } |
| 340 | |
| 341 | $num_tests_done++; |
| 342 | } |
| 343 | |
| 344 | #---------------------------------------------------------------------------- |
| 345 | # Test one directory (and any subdirs) |
| 346 | #---------------------------------------------------------------------------- |
| 347 | sub test_one_dir($$); # forward declaration |
| 348 | |
| 349 | sub test_one_dir($$) |
| 350 | { |
| 351 | my ($dir, $prev_dirs) = @_; |
| 352 | $dir =~ s/\/$//; # trim a trailing '/' |
| 353 | |
| 354 | # Ignore dirs into which we should not recurse. |
| 355 | if ($dir =~ /^(BitKeeper|CVS|SCCS|docs|doc)$/) { return; } |
| 356 | |
| 357 | chdir($dir) or die "Could not change into $dir\n"; |
| 358 | |
| 359 | # Nb: Don't prepend a '/' to the base directory |
| 360 | my $full_dir = $prev_dirs . ($prev_dirs eq "" ? "" : "/") . $dir; |
| 361 | my $dashes = "-" x (50 - length $full_dir); |
| 362 | |
| 363 | my @fs = glob "*"; |
| 364 | my $found_tests = (0 != (grep { $_ =~ /\.vgperf$/ } @fs)); |
| 365 | |
| 366 | if ($found_tests) { |
| 367 | print "-- Running tests in $full_dir $dashes\n"; |
| 368 | } |
| 369 | foreach my $f (@fs) { |
| 370 | if (-d $f) { |
| 371 | test_one_dir($f, $full_dir); |
| 372 | } elsif ($f =~ /\.vgperf$/) { |
| 373 | do_one_test($full_dir, $f); |
| 374 | } |
| 375 | } |
| 376 | if ($found_tests) { |
| 377 | print "-- Finished tests in $full_dir $dashes\n"; |
| 378 | } |
| 379 | |
| 380 | chdir(".."); |
| 381 | } |
| 382 | |
| 383 | #---------------------------------------------------------------------------- |
| 384 | # Summarise results |
| 385 | #---------------------------------------------------------------------------- |
| 386 | sub summarise_results |
| 387 | { |
| 388 | printf("\n== %d programs, %d timings =================\n\n", |
| 389 | $num_tests_done, $num_timings_done); |
| 390 | } |
| 391 | |
| 392 | #---------------------------------------------------------------------------- |
| 393 | # main() |
| 394 | #---------------------------------------------------------------------------- |
| 395 | |
| 396 | # nuke VALGRIND_OPTS |
| 397 | $ENV{"VALGRIND_OPTS"} = ""; |
| 398 | |
| 399 | my @fs = process_command_line(); |
| 400 | foreach my $f (@fs) { |
| 401 | if (-d $f) { |
| 402 | test_one_dir($f, ""); |
| 403 | } else { |
| 404 | # Allow the .vgperf suffix to be given or omitted |
| 405 | if ($f =~ /.vgperf$/ && -r $f) { |
| 406 | # do nothing |
| 407 | } elsif (-r "$f.vgperf") { |
| 408 | $f = "$f.vgperf"; |
| 409 | } else { |
| 410 | die "`$f' neither a directory nor a readable test file/name\n" |
| 411 | } |
| 412 | my $dir = `dirname $f`; chomp $dir; |
| 413 | my $file = `basename $f`; chomp $file; |
| 414 | chdir($dir) or die "Could not change into $dir\n"; |
| 415 | do_one_test($dir, $file); |
| 416 | chdir($tests_dir); |
| 417 | } |
| 418 | } |
| 419 | summarise_results(); |
| 420 | |
| 421 | ##--------------------------------------------------------------------## |
| 422 | ##--- end ---## |
| 423 | ##--------------------------------------------------------------------## |