yangguo@chromium.org | 5645471 | 2012-02-16 15:33:53 +0000 | [diff] [blame] | 1 | // Copyright 2012 the V8 project authors. All rights reserved. |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 2 | // Redistribution and use in source and binary forms, with or without |
| 3 | // modification, are permitted provided that the following conditions are |
| 4 | // met: |
| 5 | // |
| 6 | // * Redistributions of source code must retain the above copyright |
| 7 | // notice, this list of conditions and the following disclaimer. |
| 8 | // * Redistributions in binary form must reproduce the above |
| 9 | // copyright notice, this list of conditions and the following |
| 10 | // disclaimer in the documentation and/or other materials provided |
| 11 | // with the distribution. |
| 12 | // * Neither the name of Google Inc. nor the names of its |
| 13 | // contributors may be used to endorse or promote products derived |
| 14 | // from this software without specific prior written permission. |
| 15 | // |
| 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 27 | |
| 28 | |
| 29 | // Simple framework for running the benchmark suites and |
| 30 | // computing a score based on the timing measurements. |
| 31 | |
| 32 | |
| 33 | // A benchmark has a name (string) and a function that will be run to |
kasperl@chromium.org | b3284ad | 2009-05-18 06:12:45 +0000 | [diff] [blame] | 34 | // do the performance measurement. The optional setup and tearDown |
| 35 | // arguments are functions that will be invoked before and after |
| 36 | // running the benchmark, but the running time of these functions will |
| 37 | // not be accounted for in the benchmark score. |
| 38 | function Benchmark(name, run, setup, tearDown) { |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 39 | this.name = name; |
| 40 | this.run = run; |
kasperl@chromium.org | b3284ad | 2009-05-18 06:12:45 +0000 | [diff] [blame] | 41 | this.Setup = setup ? setup : function() { }; |
| 42 | this.TearDown = tearDown ? tearDown : function() { }; |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 43 | } |
| 44 | |
| 45 | |
| 46 | // Benchmark results hold the benchmark and the measured time used to |
| 47 | // run the benchmark. The benchmark score is computed later once a |
| 48 | // full benchmark suite has run to completion. |
| 49 | function BenchmarkResult(benchmark, time) { |
| 50 | this.benchmark = benchmark; |
| 51 | this.time = time; |
| 52 | } |
| 53 | |
| 54 | |
| 55 | // Automatically convert results to numbers. Used by the geometric |
| 56 | // mean computation. |
| 57 | BenchmarkResult.prototype.valueOf = function() { |
| 58 | return this.time; |
| 59 | } |
| 60 | |
| 61 | |
| 62 | // Suites of benchmarks consist of a name and the set of benchmarks in |
| 63 | // addition to the reference timing that the final score will be based |
| 64 | // on. This way, all scores are relative to a reference run and higher |
| 65 | // scores implies better performance. |
| 66 | function BenchmarkSuite(name, reference, benchmarks) { |
| 67 | this.name = name; |
| 68 | this.reference = reference; |
| 69 | this.benchmarks = benchmarks; |
| 70 | BenchmarkSuite.suites.push(this); |
| 71 | } |
| 72 | |
| 73 | |
| 74 | // Keep track of all declared benchmark suites. |
| 75 | BenchmarkSuite.suites = []; |
| 76 | |
| 77 | |
| 78 | // Scores are not comparable across versions. Bump the version if |
| 79 | // you're making changes that will affect that scores, e.g. if you add |
| 80 | // a new benchmark or change an existing one. |
yangguo@chromium.org | 5645471 | 2012-02-16 15:33:53 +0000 | [diff] [blame] | 81 | BenchmarkSuite.version = '7'; |
kasperl@chromium.org | 41044eb | 2008-10-06 08:24:46 +0000 | [diff] [blame] | 82 | |
| 83 | |
| 84 | // To make the benchmark results predictable, we replace Math.random |
| 85 | // with a 100% deterministic alternative. |
| 86 | Math.random = (function() { |
| 87 | var seed = 49734321; |
| 88 | return function() { |
| 89 | // Robert Jenkins' 32 bit integer hash function. |
| 90 | seed = ((seed + 0x7ed55d16) + (seed << 12)) & 0xffffffff; |
| 91 | seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff; |
| 92 | seed = ((seed + 0x165667b1) + (seed << 5)) & 0xffffffff; |
| 93 | seed = ((seed + 0xd3a2646c) ^ (seed << 9)) & 0xffffffff; |
kasperl@chromium.org | 41044eb | 2008-10-06 08:24:46 +0000 | [diff] [blame] | 94 | seed = ((seed + 0xfd7046c5) + (seed << 3)) & 0xffffffff; |
| 95 | seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff; |
| 96 | return (seed & 0xfffffff) / 0x10000000; |
| 97 | }; |
| 98 | })(); |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 99 | |
| 100 | |
| 101 | // Runs all registered benchmark suites and optionally yields between |
| 102 | // each individual benchmark to avoid running for too long in the |
| 103 | // context of browsers. Once done, the final score is reported to the |
| 104 | // runner. |
| 105 | BenchmarkSuite.RunSuites = function(runner) { |
| 106 | var continuation = null; |
| 107 | var suites = BenchmarkSuite.suites; |
| 108 | var length = suites.length; |
| 109 | BenchmarkSuite.scores = []; |
| 110 | var index = 0; |
| 111 | function RunStep() { |
| 112 | while (continuation || index < length) { |
| 113 | if (continuation) { |
| 114 | continuation = continuation(); |
| 115 | } else { |
| 116 | var suite = suites[index++]; |
| 117 | if (runner.NotifyStart) runner.NotifyStart(suite.name); |
| 118 | continuation = suite.RunStep(runner); |
| 119 | } |
| 120 | if (continuation && typeof window != 'undefined' && window.setTimeout) { |
kasperl@chromium.org | b3284ad | 2009-05-18 06:12:45 +0000 | [diff] [blame] | 121 | window.setTimeout(RunStep, 25); |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 122 | return; |
| 123 | } |
| 124 | } |
| 125 | if (runner.NotifyScore) { |
| 126 | var score = BenchmarkSuite.GeometricMean(BenchmarkSuite.scores); |
ager@chromium.org | 3291210 | 2009-01-16 10:38:43 +0000 | [diff] [blame] | 127 | var formatted = BenchmarkSuite.FormatScore(100 * score); |
| 128 | runner.NotifyScore(formatted); |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 129 | } |
| 130 | } |
| 131 | RunStep(); |
| 132 | } |
| 133 | |
| 134 | |
| 135 | // Counts the total number of registered benchmarks. Useful for |
| 136 | // showing progress as a percentage. |
| 137 | BenchmarkSuite.CountBenchmarks = function() { |
| 138 | var result = 0; |
| 139 | var suites = BenchmarkSuite.suites; |
| 140 | for (var i = 0; i < suites.length; i++) { |
| 141 | result += suites[i].benchmarks.length; |
| 142 | } |
| 143 | return result; |
| 144 | } |
| 145 | |
| 146 | |
| 147 | // Computes the geometric mean of a set of numbers. |
| 148 | BenchmarkSuite.GeometricMean = function(numbers) { |
| 149 | var log = 0; |
| 150 | for (var i = 0; i < numbers.length; i++) { |
| 151 | log += Math.log(numbers[i]); |
| 152 | } |
| 153 | return Math.pow(Math.E, log / numbers.length); |
| 154 | } |
| 155 | |
| 156 | |
ager@chromium.org | 3291210 | 2009-01-16 10:38:43 +0000 | [diff] [blame] | 157 | // Converts a score value to a string with at least three significant |
| 158 | // digits. |
| 159 | BenchmarkSuite.FormatScore = function(value) { |
| 160 | if (value > 100) { |
| 161 | return value.toFixed(0); |
| 162 | } else { |
| 163 | return value.toPrecision(3); |
| 164 | } |
| 165 | } |
| 166 | |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 167 | // Notifies the runner that we're done running a single benchmark in |
| 168 | // the benchmark suite. This can be useful to report progress. |
| 169 | BenchmarkSuite.prototype.NotifyStep = function(result) { |
| 170 | this.results.push(result); |
| 171 | if (this.runner.NotifyStep) this.runner.NotifyStep(result.benchmark.name); |
| 172 | } |
| 173 | |
| 174 | |
| 175 | // Notifies the runner that we're done with running a suite and that |
| 176 | // we have a result which can be reported to the user if needed. |
| 177 | BenchmarkSuite.prototype.NotifyResult = function() { |
| 178 | var mean = BenchmarkSuite.GeometricMean(this.results); |
| 179 | var score = this.reference / mean; |
| 180 | BenchmarkSuite.scores.push(score); |
| 181 | if (this.runner.NotifyResult) { |
ager@chromium.org | 3291210 | 2009-01-16 10:38:43 +0000 | [diff] [blame] | 182 | var formatted = BenchmarkSuite.FormatScore(100 * score); |
| 183 | this.runner.NotifyResult(this.name, formatted); |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 184 | } |
| 185 | } |
| 186 | |
| 187 | |
kasperl@chromium.org | 5a8ca6c | 2008-10-23 13:57:19 +0000 | [diff] [blame] | 188 | // Notifies the runner that running a benchmark resulted in an error. |
| 189 | BenchmarkSuite.prototype.NotifyError = function(error) { |
| 190 | if (this.runner.NotifyError) { |
| 191 | this.runner.NotifyError(this.name, error); |
| 192 | } |
| 193 | if (this.runner.NotifyStep) { |
| 194 | this.runner.NotifyStep(this.name); |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 199 | // Runs a single benchmark for at least a second and computes the |
| 200 | // average time it takes to run a single iteration. |
lrn@chromium.org | 32d961d | 2010-06-30 09:09:34 +0000 | [diff] [blame] | 201 | BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) { |
| 202 | function Measure(data) { |
| 203 | var elapsed = 0; |
| 204 | var start = new Date(); |
| 205 | for (var n = 0; elapsed < 1000; n++) { |
| 206 | benchmark.run(); |
| 207 | elapsed = new Date() - start; |
| 208 | } |
| 209 | if (data != null) { |
| 210 | data.runs += n; |
| 211 | data.elapsed += elapsed; |
| 212 | } |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 213 | } |
lrn@chromium.org | 32d961d | 2010-06-30 09:09:34 +0000 | [diff] [blame] | 214 | |
| 215 | if (data == null) { |
| 216 | // Measure the benchmark once for warm up and throw the result |
| 217 | // away. Return a fresh data object. |
| 218 | Measure(null); |
| 219 | return { runs: 0, elapsed: 0 }; |
| 220 | } else { |
| 221 | Measure(data); |
| 222 | // If we've run too few iterations, we continue for another second. |
| 223 | if (data.runs < 32) return data; |
| 224 | var usec = (data.elapsed * 1000) / data.runs; |
| 225 | this.NotifyStep(new BenchmarkResult(benchmark, usec)); |
| 226 | return null; |
| 227 | } |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 228 | } |
| 229 | |
| 230 | |
| 231 | // This function starts running a suite, but stops between each |
| 232 | // individual benchmark in the suite and returns a continuation |
| 233 | // function which can be invoked to run the next benchmark. Once the |
| 234 | // last benchmark has been executed, null is returned. |
| 235 | BenchmarkSuite.prototype.RunStep = function(runner) { |
| 236 | this.results = []; |
| 237 | this.runner = runner; |
| 238 | var length = this.benchmarks.length; |
| 239 | var index = 0; |
| 240 | var suite = this; |
lrn@chromium.org | 32d961d | 2010-06-30 09:09:34 +0000 | [diff] [blame] | 241 | var data; |
kasperl@chromium.org | b3284ad | 2009-05-18 06:12:45 +0000 | [diff] [blame] | 242 | |
| 243 | // Run the setup, the actual benchmark, and the tear down in three |
| 244 | // separate steps to allow the framework to yield between any of the |
| 245 | // steps. |
| 246 | |
| 247 | function RunNextSetup() { |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 248 | if (index < length) { |
kasperl@chromium.org | 5a8ca6c | 2008-10-23 13:57:19 +0000 | [diff] [blame] | 249 | try { |
kasperl@chromium.org | b3284ad | 2009-05-18 06:12:45 +0000 | [diff] [blame] | 250 | suite.benchmarks[index].Setup(); |
kasperl@chromium.org | 5a8ca6c | 2008-10-23 13:57:19 +0000 | [diff] [blame] | 251 | } catch (e) { |
| 252 | suite.NotifyError(e); |
| 253 | return null; |
| 254 | } |
kasperl@chromium.org | b3284ad | 2009-05-18 06:12:45 +0000 | [diff] [blame] | 255 | return RunNextBenchmark; |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 256 | } |
| 257 | suite.NotifyResult(); |
| 258 | return null; |
| 259 | } |
kasperl@chromium.org | b3284ad | 2009-05-18 06:12:45 +0000 | [diff] [blame] | 260 | |
| 261 | function RunNextBenchmark() { |
| 262 | try { |
lrn@chromium.org | 32d961d | 2010-06-30 09:09:34 +0000 | [diff] [blame] | 263 | data = suite.RunSingleBenchmark(suite.benchmarks[index], data); |
kasperl@chromium.org | b3284ad | 2009-05-18 06:12:45 +0000 | [diff] [blame] | 264 | } catch (e) { |
| 265 | suite.NotifyError(e); |
| 266 | return null; |
| 267 | } |
lrn@chromium.org | 32d961d | 2010-06-30 09:09:34 +0000 | [diff] [blame] | 268 | // If data is null, we're done with this benchmark. |
| 269 | return (data == null) ? RunNextTearDown : RunNextBenchmark(); |
kasperl@chromium.org | b3284ad | 2009-05-18 06:12:45 +0000 | [diff] [blame] | 270 | } |
| 271 | |
| 272 | function RunNextTearDown() { |
| 273 | try { |
| 274 | suite.benchmarks[index++].TearDown(); |
| 275 | } catch (e) { |
| 276 | suite.NotifyError(e); |
| 277 | return null; |
| 278 | } |
| 279 | return RunNextSetup; |
| 280 | } |
| 281 | |
| 282 | // Start out running the setup. |
| 283 | return RunNextSetup(); |
v8.team.kasperl | 727e995 | 2008-09-02 14:56:44 +0000 | [diff] [blame] | 284 | } |