Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright 2014 the V8 project authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | from collections import namedtuple |
| 7 | import coverage |
| 8 | import json |
| 9 | from mock import DEFAULT |
| 10 | from mock import MagicMock |
| 11 | import os |
| 12 | from os import path, sys |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 13 | import platform |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 14 | import shutil |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 15 | import subprocess |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 16 | import tempfile |
| 17 | import unittest |
| 18 | |
| 19 | # Requires python-coverage and python-mock. Native python coverage |
| 20 | # version >= 3.7.1 should be installed to get the best speed. |
| 21 | |
| 22 | TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf") |
| 23 | |
| 24 | V8_JSON = { |
| 25 | "path": ["."], |
| 26 | "binary": "d7", |
| 27 | "flags": ["--flag"], |
| 28 | "main": "run.js", |
| 29 | "run_count": 1, |
| 30 | "results_regexp": "^%s: (.+)$", |
| 31 | "tests": [ |
| 32 | {"name": "Richards"}, |
| 33 | {"name": "DeltaBlue"}, |
| 34 | ] |
| 35 | } |
| 36 | |
| 37 | V8_NESTED_SUITES_JSON = { |
| 38 | "path": ["."], |
| 39 | "flags": ["--flag"], |
| 40 | "run_count": 1, |
| 41 | "units": "score", |
| 42 | "tests": [ |
| 43 | {"name": "Richards", |
| 44 | "path": ["richards"], |
| 45 | "binary": "d7", |
| 46 | "main": "run.js", |
| 47 | "resources": ["file1.js", "file2.js"], |
| 48 | "run_count": 2, |
| 49 | "results_regexp": "^Richards: (.+)$"}, |
| 50 | {"name": "Sub", |
| 51 | "path": ["sub"], |
| 52 | "tests": [ |
| 53 | {"name": "Leaf", |
| 54 | "path": ["leaf"], |
| 55 | "run_count_x64": 3, |
| 56 | "units": "ms", |
| 57 | "main": "run.js", |
| 58 | "results_regexp": "^Simple: (.+) ms.$"}, |
| 59 | ] |
| 60 | }, |
| 61 | {"name": "DeltaBlue", |
| 62 | "path": ["delta_blue"], |
| 63 | "main": "run.js", |
| 64 | "flags": ["--flag2"], |
| 65 | "results_regexp": "^DeltaBlue: (.+)$"}, |
| 66 | {"name": "ShouldntRun", |
| 67 | "path": ["."], |
| 68 | "archs": ["arm"], |
| 69 | "main": "run.js"}, |
| 70 | ] |
| 71 | } |
| 72 | |
| 73 | V8_GENERIC_JSON = { |
| 74 | "path": ["."], |
| 75 | "binary": "cc", |
| 76 | "flags": ["--flag"], |
| 77 | "generic": True, |
| 78 | "run_count": 1, |
| 79 | "units": "ms", |
| 80 | } |
| 81 | |
| 82 | Output = namedtuple("Output", "stdout, stderr, timed_out") |
| 83 | |
| 84 | class PerfTest(unittest.TestCase): |
| 85 | @classmethod |
| 86 | def setUpClass(cls): |
| 87 | cls.base = path.dirname(path.dirname(path.abspath(__file__))) |
| 88 | sys.path.append(cls.base) |
| 89 | cls._cov = coverage.coverage( |
| 90 | include=([os.path.join(cls.base, "run_perf.py")])) |
| 91 | cls._cov.start() |
| 92 | import run_perf |
| 93 | from testrunner.local import commands |
| 94 | global commands |
| 95 | global run_perf |
| 96 | |
| 97 | @classmethod |
| 98 | def tearDownClass(cls): |
| 99 | cls._cov.stop() |
| 100 | print "" |
| 101 | print cls._cov.report() |
| 102 | |
| 103 | def setUp(self): |
| 104 | self.maxDiff = None |
| 105 | if path.exists(TEST_WORKSPACE): |
| 106 | shutil.rmtree(TEST_WORKSPACE) |
| 107 | os.makedirs(TEST_WORKSPACE) |
| 108 | |
| 109 | def tearDown(self): |
| 110 | if path.exists(TEST_WORKSPACE): |
| 111 | shutil.rmtree(TEST_WORKSPACE) |
| 112 | |
| 113 | def _WriteTestInput(self, json_content): |
| 114 | self._test_input = path.join(TEST_WORKSPACE, "test.json") |
| 115 | with open(self._test_input, "w") as f: |
| 116 | f.write(json.dumps(json_content)) |
| 117 | |
| 118 | def _MockCommand(self, *args, **kwargs): |
| 119 | # Fake output for each test run. |
| 120 | test_outputs = [Output(stdout=arg, |
| 121 | stderr=None, |
| 122 | timed_out=kwargs.get("timed_out", False)) |
| 123 | for arg in args[1]] |
| 124 | def execute(*args, **kwargs): |
| 125 | return test_outputs.pop() |
| 126 | commands.Execute = MagicMock(side_effect=execute) |
| 127 | |
| 128 | # Check that d8 is called from the correct cwd for each test run. |
| 129 | dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]] |
| 130 | def chdir(*args, **kwargs): |
| 131 | self.assertEquals(dirs.pop(), args[0]) |
| 132 | os.chdir = MagicMock(side_effect=chdir) |
| 133 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 134 | subprocess.check_call = MagicMock() |
| 135 | platform.system = MagicMock(return_value='Linux') |
| 136 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 137 | def _CallMain(self, *args): |
| 138 | self._test_output = path.join(TEST_WORKSPACE, "results.json") |
| 139 | all_args=[ |
| 140 | "--json-test-results", |
| 141 | self._test_output, |
| 142 | self._test_input, |
| 143 | ] |
| 144 | all_args += args |
| 145 | return run_perf.Main(all_args) |
| 146 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 147 | def _LoadResults(self, file_name=None): |
| 148 | with open(file_name or self._test_output) as f: |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 149 | return json.load(f) |
| 150 | |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 151 | def _VerifyResults(self, suite, units, traces, file_name=None): |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 152 | self.assertEquals([ |
| 153 | {"units": units, |
| 154 | "graphs": [suite, trace["name"]], |
| 155 | "results": trace["results"], |
| 156 | "stddev": trace["stddev"]} for trace in traces], |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 157 | self._LoadResults(file_name)["traces"]) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 158 | |
| 159 | def _VerifyErrors(self, errors): |
| 160 | self.assertEquals(errors, self._LoadResults()["errors"]) |
| 161 | |
| 162 | def _VerifyMock(self, binary, *args, **kwargs): |
| 163 | arg = [path.join(path.dirname(self.base), binary)] |
| 164 | arg += args |
| 165 | commands.Execute.assert_called_with( |
| 166 | arg, timeout=kwargs.get("timeout", 60)) |
| 167 | |
| 168 | def _VerifyMockMultiple(self, *args, **kwargs): |
| 169 | expected = [] |
| 170 | for arg in args: |
| 171 | a = [path.join(path.dirname(self.base), arg[0])] |
| 172 | a += arg[1:] |
| 173 | expected.append(((a,), {"timeout": kwargs.get("timeout", 60)})) |
| 174 | self.assertEquals(expected, commands.Execute.call_args_list) |
| 175 | |
| 176 | def testOneRun(self): |
| 177 | self._WriteTestInput(V8_JSON) |
| 178 | self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) |
| 179 | self.assertEquals(0, self._CallMain()) |
| 180 | self._VerifyResults("test", "score", [ |
| 181 | {"name": "Richards", "results": ["1.234"], "stddev": ""}, |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 182 | {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 183 | ]) |
| 184 | self._VerifyErrors([]) |
| 185 | self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
| 186 | |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 187 | def testOneRunWithTestFlags(self): |
| 188 | test_input = dict(V8_JSON) |
| 189 | test_input["test_flags"] = ["2", "test_name"] |
| 190 | self._WriteTestInput(test_input) |
| 191 | self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567"]) |
| 192 | self.assertEquals(0, self._CallMain()) |
| 193 | self._VerifyResults("test", "score", [ |
| 194 | {"name": "Richards", "results": ["1.234"], "stddev": ""}, |
| 195 | {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, |
| 196 | ]) |
| 197 | self._VerifyErrors([]) |
| 198 | self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js", |
| 199 | "--", "2", "test_name") |
| 200 | |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 201 | def testTwoRuns_Units_SuiteName(self): |
| 202 | test_input = dict(V8_JSON) |
| 203 | test_input["run_count"] = 2 |
| 204 | test_input["name"] = "v8" |
| 205 | test_input["units"] = "ms" |
| 206 | self._WriteTestInput(test_input) |
| 207 | self._MockCommand([".", "."], |
| 208 | ["Richards: 100\nDeltaBlue: 200\n", |
| 209 | "Richards: 50\nDeltaBlue: 300\n"]) |
| 210 | self.assertEquals(0, self._CallMain()) |
| 211 | self._VerifyResults("v8", "ms", [ |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 212 | {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""}, |
| 213 | {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""}, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 214 | ]) |
| 215 | self._VerifyErrors([]) |
| 216 | self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
| 217 | |
| 218 | def testTwoRuns_SubRegexp(self): |
| 219 | test_input = dict(V8_JSON) |
| 220 | test_input["run_count"] = 2 |
| 221 | del test_input["results_regexp"] |
| 222 | test_input["tests"][0]["results_regexp"] = "^Richards: (.+)$" |
| 223 | test_input["tests"][1]["results_regexp"] = "^DeltaBlue: (.+)$" |
| 224 | self._WriteTestInput(test_input) |
| 225 | self._MockCommand([".", "."], |
| 226 | ["Richards: 100\nDeltaBlue: 200\n", |
| 227 | "Richards: 50\nDeltaBlue: 300\n"]) |
| 228 | self.assertEquals(0, self._CallMain()) |
| 229 | self._VerifyResults("test", "score", [ |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 230 | {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""}, |
| 231 | {"name": "DeltaBlue", "results": ["300.0", "200.0"], "stddev": ""}, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 232 | ]) |
| 233 | self._VerifyErrors([]) |
| 234 | self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
| 235 | |
| 236 | def testNestedSuite(self): |
| 237 | self._WriteTestInput(V8_NESTED_SUITES_JSON) |
| 238 | self._MockCommand(["delta_blue", "sub/leaf", "richards"], |
| 239 | ["DeltaBlue: 200\n", |
| 240 | "Simple: 1 ms.\n", |
| 241 | "Simple: 2 ms.\n", |
| 242 | "Simple: 3 ms.\n", |
| 243 | "Richards: 100\n", |
| 244 | "Richards: 50\n"]) |
| 245 | self.assertEquals(0, self._CallMain()) |
| 246 | self.assertEquals([ |
| 247 | {"units": "score", |
| 248 | "graphs": ["test", "Richards"], |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 249 | "results": ["50.0", "100.0"], |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 250 | "stddev": ""}, |
| 251 | {"units": "ms", |
| 252 | "graphs": ["test", "Sub", "Leaf"], |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 253 | "results": ["3.0", "2.0", "1.0"], |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 254 | "stddev": ""}, |
| 255 | {"units": "score", |
| 256 | "graphs": ["test", "DeltaBlue"], |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 257 | "results": ["200.0"], |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 258 | "stddev": ""}, |
| 259 | ], self._LoadResults()["traces"]) |
| 260 | self._VerifyErrors([]) |
| 261 | self._VerifyMockMultiple( |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 262 | (path.join("out", "x64.release", "d7"), "--flag", "run.js"), |
| 263 | (path.join("out", "x64.release", "d7"), "--flag", "run.js"), |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 264 | (path.join("out", "x64.release", "d8"), "--flag", "run.js"), |
| 265 | (path.join("out", "x64.release", "d8"), "--flag", "run.js"), |
| 266 | (path.join("out", "x64.release", "d8"), "--flag", "run.js"), |
| 267 | (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js")) |
| 268 | |
| 269 | def testOneRunStdDevRegExp(self): |
| 270 | test_input = dict(V8_JSON) |
| 271 | test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" |
| 272 | self._WriteTestInput(test_input) |
| 273 | self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n" |
| 274 | "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"]) |
| 275 | self.assertEquals(0, self._CallMain()) |
| 276 | self._VerifyResults("test", "score", [ |
| 277 | {"name": "Richards", "results": ["1.234"], "stddev": "0.23"}, |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 278 | {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": "106"}, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 279 | ]) |
| 280 | self._VerifyErrors([]) |
| 281 | self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
| 282 | |
| 283 | def testTwoRunsStdDevRegExp(self): |
| 284 | test_input = dict(V8_JSON) |
| 285 | test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" |
| 286 | test_input["run_count"] = 2 |
| 287 | self._WriteTestInput(test_input) |
| 288 | self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n" |
| 289 | "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n", |
| 290 | "Richards: 2\nRichards-stddev: 0.5\n" |
| 291 | "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"]) |
| 292 | self.assertEquals(1, self._CallMain()) |
| 293 | self._VerifyResults("test", "score", [ |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 294 | {"name": "Richards", "results": ["2.0", "3.0"], "stddev": "0.7"}, |
| 295 | {"name": "DeltaBlue", "results": ["5.0", "6.0"], "stddev": "0.8"}, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 296 | ]) |
| 297 | self._VerifyErrors( |
| 298 | ["Test Richards should only run once since a stddev is provided " |
| 299 | "by the test.", |
| 300 | "Test DeltaBlue should only run once since a stddev is provided " |
| 301 | "by the test.", |
| 302 | "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test " |
| 303 | "DeltaBlue."]) |
| 304 | self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
| 305 | |
| 306 | def testBuildbot(self): |
| 307 | self._WriteTestInput(V8_JSON) |
| 308 | self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) |
| 309 | self.assertEquals(0, self._CallMain("--buildbot")) |
| 310 | self._VerifyResults("test", "score", [ |
| 311 | {"name": "Richards", "results": ["1.234"], "stddev": ""}, |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 312 | {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 313 | ]) |
| 314 | self._VerifyErrors([]) |
| 315 | self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") |
| 316 | |
| 317 | def testBuildbotWithTotal(self): |
| 318 | test_input = dict(V8_JSON) |
| 319 | test_input["total"] = True |
| 320 | self._WriteTestInput(test_input) |
| 321 | self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) |
| 322 | self.assertEquals(0, self._CallMain("--buildbot")) |
| 323 | self._VerifyResults("test", "score", [ |
| 324 | {"name": "Richards", "results": ["1.234"], "stddev": ""}, |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 325 | {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 326 | {"name": "Total", "results": ["3626.49109719"], "stddev": ""}, |
| 327 | ]) |
| 328 | self._VerifyErrors([]) |
| 329 | self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") |
| 330 | |
| 331 | def testBuildbotWithTotalAndErrors(self): |
| 332 | test_input = dict(V8_JSON) |
| 333 | test_input["total"] = True |
| 334 | self._WriteTestInput(test_input) |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 335 | self._MockCommand(["."], ["x\nRichards: bla\nDeltaBlue: 10657567\ny\n"]) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 336 | self.assertEquals(1, self._CallMain("--buildbot")) |
| 337 | self._VerifyResults("test", "score", [ |
| 338 | {"name": "Richards", "results": [], "stddev": ""}, |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 339 | {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 340 | ]) |
| 341 | self._VerifyErrors( |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 342 | ["Regexp \"^Richards: (.+)$\" " |
| 343 | "returned a non-numeric for test Richards.", |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 344 | "Not all traces have the same number of results."]) |
| 345 | self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") |
| 346 | |
| 347 | def testRegexpNoMatch(self): |
| 348 | self._WriteTestInput(V8_JSON) |
| 349 | self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) |
| 350 | self.assertEquals(1, self._CallMain()) |
| 351 | self._VerifyResults("test", "score", [ |
| 352 | {"name": "Richards", "results": [], "stddev": ""}, |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 353 | {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 354 | ]) |
| 355 | self._VerifyErrors( |
| 356 | ["Regexp \"^Richards: (.+)$\" didn't match for test Richards."]) |
| 357 | self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") |
| 358 | |
| 359 | def testOneRunGeneric(self): |
| 360 | test_input = dict(V8_GENERIC_JSON) |
| 361 | self._WriteTestInput(test_input) |
| 362 | self._MockCommand(["."], [ |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 363 | "RESULT Infra: Constant1= 11 count\n" |
| 364 | "RESULT Infra: Constant2= [10,5,10,15] count\n" |
| 365 | "RESULT Infra: Constant3= {12,1.2} count\n" |
| 366 | "RESULT Infra: Constant4= [10,5,error,15] count\n"]) |
| 367 | self.assertEquals(1, self._CallMain()) |
| 368 | self.assertEquals([ |
| 369 | {"units": "count", |
| 370 | "graphs": ["test", "Infra", "Constant1"], |
| 371 | "results": ["11.0"], |
| 372 | "stddev": ""}, |
| 373 | {"units": "count", |
| 374 | "graphs": ["test", "Infra", "Constant2"], |
| 375 | "results": ["10.0", "5.0", "10.0", "15.0"], |
| 376 | "stddev": ""}, |
| 377 | {"units": "count", |
| 378 | "graphs": ["test", "Infra", "Constant3"], |
| 379 | "results": ["12.0"], |
| 380 | "stddev": "1.2"}, |
| 381 | {"units": "count", |
| 382 | "graphs": ["test", "Infra", "Constant4"], |
| 383 | "results": [], |
| 384 | "stddev": ""}, |
| 385 | ], self._LoadResults()["traces"]) |
| 386 | self._VerifyErrors(["Found non-numeric in test/Infra/Constant4"]) |
Ben Murdoch | b8a8cc1 | 2014-11-26 15:28:44 +0000 | [diff] [blame] | 387 | self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "") |
| 388 | |
| 389 | def testOneRunTimingOut(self): |
| 390 | test_input = dict(V8_JSON) |
| 391 | test_input["timeout"] = 70 |
| 392 | self._WriteTestInput(test_input) |
| 393 | self._MockCommand(["."], [""], timed_out=True) |
| 394 | self.assertEquals(1, self._CallMain()) |
| 395 | self._VerifyResults("test", "score", [ |
| 396 | {"name": "Richards", "results": [], "stddev": ""}, |
| 397 | {"name": "DeltaBlue", "results": [], "stddev": ""}, |
| 398 | ]) |
| 399 | self._VerifyErrors([ |
| 400 | "Regexp \"^Richards: (.+)$\" didn't match for test Richards.", |
| 401 | "Regexp \"^DeltaBlue: (.+)$\" didn't match for test DeltaBlue.", |
| 402 | ]) |
| 403 | self._VerifyMock( |
| 404 | path.join("out", "x64.release", "d7"), "--flag", "run.js", timeout=70) |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 405 | |
| 406 | # Simple test that mocks out the android platform. Testing the platform would |
| 407 | # require lots of complicated mocks for the android tools. |
| 408 | def testAndroid(self): |
| 409 | self._WriteTestInput(V8_JSON) |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 410 | # FIXME(machenbach): This is not test-local! |
| 411 | platform = run_perf.AndroidPlatform |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 412 | platform.PreExecution = MagicMock(return_value=None) |
| 413 | platform.PostExecution = MagicMock(return_value=None) |
| 414 | platform.PreTests = MagicMock(return_value=None) |
| 415 | platform.Run = MagicMock( |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 416 | return_value=("Richards: 1.234\nDeltaBlue: 10657567\n", None)) |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 417 | run_perf.AndroidPlatform = MagicMock(return_value=platform) |
| 418 | self.assertEquals( |
| 419 | 0, self._CallMain("--android-build-tools", "/some/dir", |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 420 | "--arch", "arm")) |
Emily Bernier | d0a1eb7 | 2015-03-24 16:35:39 -0400 | [diff] [blame] | 421 | self._VerifyResults("test", "score", [ |
| 422 | {"name": "Richards", "results": ["1.234"], "stddev": ""}, |
| 423 | {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, |
| 424 | ]) |
Ben Murdoch | 4a90d5f | 2016-03-22 12:00:34 +0000 | [diff] [blame] | 425 | |
| 426 | def testTwoRuns_Trybot(self): |
| 427 | test_input = dict(V8_JSON) |
| 428 | test_input["run_count"] = 2 |
| 429 | self._WriteTestInput(test_input) |
| 430 | self._MockCommand([".", ".", ".", "."], |
| 431 | ["Richards: 100\nDeltaBlue: 200\n", |
| 432 | "Richards: 200\nDeltaBlue: 20\n", |
| 433 | "Richards: 50\nDeltaBlue: 200\n", |
| 434 | "Richards: 100\nDeltaBlue: 20\n"]) |
| 435 | test_output_no_patch = path.join(TEST_WORKSPACE, "results_no_patch.json") |
| 436 | self.assertEquals(0, self._CallMain( |
| 437 | "--outdir-no-patch", "out-no-patch", |
| 438 | "--json-test-results-no-patch", test_output_no_patch, |
| 439 | )) |
| 440 | self._VerifyResults("test", "score", [ |
| 441 | {"name": "Richards", "results": ["100.0", "200.0"], "stddev": ""}, |
| 442 | {"name": "DeltaBlue", "results": ["20.0", "20.0"], "stddev": ""}, |
| 443 | ]) |
| 444 | self._VerifyResults("test", "score", [ |
| 445 | {"name": "Richards", "results": ["50.0", "100.0"], "stddev": ""}, |
| 446 | {"name": "DeltaBlue", "results": ["200.0", "200.0"], "stddev": ""}, |
| 447 | ], test_output_no_patch) |
| 448 | self._VerifyErrors([]) |
| 449 | self._VerifyMockMultiple( |
| 450 | (path.join("out", "x64.release", "d7"), "--flag", "run.js"), |
| 451 | (path.join("out-no-patch", "x64.release", "d7"), "--flag", "run.js"), |
| 452 | (path.join("out", "x64.release", "d7"), "--flag", "run.js"), |
| 453 | (path.join("out-no-patch", "x64.release", "d7"), "--flag", "run.js"), |
| 454 | ) |
| 455 | |
| 456 | def testWrongBinaryWithProf(self): |
| 457 | test_input = dict(V8_JSON) |
| 458 | self._WriteTestInput(test_input) |
| 459 | self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) |
| 460 | self.assertEquals(0, self._CallMain("--extra-flags=--prof")) |
| 461 | self._VerifyResults("test", "score", [ |
| 462 | {"name": "Richards", "results": ["1.234"], "stddev": ""}, |
| 463 | {"name": "DeltaBlue", "results": ["10657567.0"], "stddev": ""}, |
| 464 | ]) |
| 465 | self._VerifyErrors([]) |
| 466 | self._VerifyMock(path.join("out", "x64.release", "d7"), |
| 467 | "--flag", "--prof", "run.js") |
| 468 | |
| 469 | def testUnzip(self): |
| 470 | def Gen(): |
| 471 | for i in [1, 2, 3]: |
| 472 | yield i, i + 1 |
| 473 | l, r = run_perf.Unzip(Gen()) |
| 474 | self.assertEquals([1, 2, 3], list(l())) |
| 475 | self.assertEquals([2, 3, 4], list(r())) |