Mike Frysinger | c7f1593 | 2013-03-20 13:43:35 -0400 | [diff] [blame] | 1 | #!/usr/bin/python |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 2 | # |
| 3 | # Copyright 2010 Google Inc. All Rights Reserved. |
| 4 | |
| 5 | """Script to run ChromeOS benchmarks |
| 6 | |
| 7 | Inputs: |
| 8 | chromeos_root |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 9 | board |
| 10 | [chromeos/cpu/<benchname>|chromeos/browser/[pagecycler|sunspider]|chromeos/startup] |
| 11 | hostname/IP of Chromeos machine |
| 12 | |
| 13 | chromeos/cpu/<benchname> |
| 14 | - Read run script rules from bench.mk perflab-bin, copy benchmark to host, run |
| 15 | and return results. |
| 16 | |
| 17 | chromeos/startup |
| 18 | - Re-image host with image in perflab-bin |
| 19 | - Call run_tests to run startup test, gather results. |
| 20 | - Restore host back to what it was. |
| 21 | |
| 22 | chromeos/browser/* |
| 23 | - Call build_chromebrowser to build image with new browser |
| 24 | - Copy image to perflab-bin |
| 25 | |
| 26 | """ |
| 27 | |
| 28 | __author__ = "bjanakiraman@google.com (Bhaskar Janakiraman)" |
| 29 | |
| 30 | import optparse |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 31 | import os |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 32 | import re |
| 33 | import sys |
kbaclawski | 20082a0 | 2013-02-16 02:12:57 +0000 | [diff] [blame] | 34 | |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 35 | import image_chromeos |
kbaclawski | 20082a0 | 2013-02-16 02:12:57 +0000 | [diff] [blame] | 36 | import run_tests |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 37 | from utils import command_executer |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 38 | from utils import logger |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 39 | |
| 40 | |
| 41 | KNOWN_BENCHMARKS = [ |
| 42 | "chromeos/startup", |
| 43 | "chromeos/browser/pagecycler", |
| 44 | "chromeos/browser/sunspider", |
bjanakiraman | e7a379a | 2013-02-15 10:24:30 +0000 | [diff] [blame] | 45 | "chromeos/browser/v8bench", |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 46 | "chromeos/cpu/bikjmp"] |
| 47 | |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 48 | name_map = { |
| 49 | "pagecycler" : "Page", |
| 50 | "sunspider" : "SunSpider", |
bjanakiraman | e7a379a | 2013-02-15 10:24:30 +0000 | [diff] [blame] | 51 | "v8bench" : "V8Bench", |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 52 | "startup" : "BootPerfServer"} |
| 53 | |
| 54 | |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 55 | # Run command template |
| 56 | |
| 57 | |
| 58 | # Common initializations |
| 59 | cmd_executer = command_executer.GetCommandExecuter() |
| 60 | |
| 61 | |
| 62 | def Usage(parser, message): |
| 63 | print "ERROR: " + message |
| 64 | parser.print_help() |
| 65 | sys.exit(0) |
| 66 | |
| 67 | |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 68 | def RunBrowserBenchmark(chromeos_root, board, bench, workdir, machine): |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 69 | """Run browser benchmarks. |
| 70 | |
| 71 | Args: |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 72 | chromeos_root: ChromeOS src dir |
| 73 | board: Board being tested |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 74 | bench: Name of benchmark (chromeos/browser/*) |
| 75 | workdir: Directory containing benchmark directory |
| 76 | machine: name of chromeos machine |
| 77 | """ |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 78 | benchname = re.split('/', bench)[2] |
| 79 | benchdir = '%s/%s' % (workdir, benchname) |
| 80 | benchname = name_map[benchname] |
| 81 | retval = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) |
| 82 | return retval |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 83 | |
| 84 | |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 85 | def RunStartupBenchmark(chromeos_root, board, bench, workdir, machine): |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 86 | """Run browser benchmarks. |
| 87 | |
| 88 | Args: |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 89 | chromeos_root: ChromeOS src dir |
| 90 | board: Board being tested |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 91 | bench: Name of benchmark (chromeos/browser/*) |
| 92 | workdir: Directory containing benchmark directory |
| 93 | machine: name of chromeos machine |
| 94 | """ |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 95 | benchname = 'startup' |
| 96 | benchdir = '%s/%s' % (workdir, benchname) |
| 97 | benchname = name_map[benchname] |
| 98 | retval = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) |
| 99 | return retval |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 100 | |
| 101 | |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 102 | def RunCpuBenchmark(chromeos_root, bench, workdir, machine): |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 103 | """Run CPU benchmark. |
| 104 | |
| 105 | Args: |
| 106 | bench: Name of benchmark |
| 107 | workdir: directory containing benchmark directory |
| 108 | machine: name of chromeos machine |
| 109 | |
| 110 | Returns: |
| 111 | status: 0 on success |
| 112 | """ |
| 113 | |
| 114 | benchname = re.split('/', bench)[2] |
| 115 | benchdir = '%s/%s' % (workdir, benchname) |
| 116 | |
| 117 | # Delete any existing run directories on machine. |
| 118 | # Since this has exclusive access to the machine, |
| 119 | # we do not worry about duplicates. |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 120 | args = 'rm -rf /tmp/%s' % benchname |
| 121 | retval = cmd_executer.CrosRunCommand(args, chromeos_root=chromeos_root, |
| 122 | machine=machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 123 | if retval: |
| 124 | return retval |
| 125 | |
| 126 | # Copy benchmark directory. |
asharif | 29775b2 | 2013-02-15 05:15:38 +0000 | [diff] [blame] | 127 | retval = cmd_executer.CopyFiles(benchdir, "/tmp/" + benchname, |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 128 | chromeos_root=chromeos_root, |
| 129 | dest_machine=machine, |
| 130 | dest_cros=True) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 131 | if retval: |
| 132 | return retval |
| 133 | |
| 134 | # Parse bench.mk to extract run flags. |
| 135 | |
| 136 | benchmk_file = open('%s/bench.mk' % benchdir, 'r') |
| 137 | for line in benchmk_file: |
| 138 | line.rstrip() |
| 139 | if re.match('^run_cmd', line): |
bjanakiraman | 4e8bdf2 | 2013-02-15 04:57:09 +0000 | [diff] [blame] | 140 | line = re.sub('^run_cmd.*\${PERFLAB_PATH}', './out', line) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 141 | line = re.sub('\${PERFLAB_INPUT}', './data', line) |
| 142 | run_cmd = line |
| 143 | break |
| 144 | |
| 145 | # Execute on remote machine |
| 146 | # Capture output and process it. |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 147 | sshargs = "'cd /tmp/%s;" % benchname |
| 148 | sshargs += "time -p %s'" % run_cmd |
| 149 | cmd_executer.CrosRunCommand(sshargs, chromeos_root=chromeos_root, |
| 150 | machine=machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 151 | |
| 152 | return retval |
| 153 | |
| 154 | |
| 155 | def Main(argv): |
| 156 | """Build ChromeOS.""" |
| 157 | # Common initializations |
| 158 | |
| 159 | parser = optparse.OptionParser() |
| 160 | parser.add_option("-c", "--chromeos_root", dest="chromeos_root", |
| 161 | help="Target directory for ChromeOS installation.") |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 162 | parser.add_option("-m", "--machine", dest="machine", |
| 163 | help="The chromeos host machine.") |
| 164 | parser.add_option("--workdir", dest="workdir", default="./perflab-bin", |
| 165 | help="Work directory for perflab outputs.") |
| 166 | parser.add_option("--board", dest="board", |
| 167 | help="ChromeOS target board, e.g. x86-generic") |
| 168 | |
| 169 | (options, args) = parser.parse_args(argv[1:]) |
| 170 | |
| 171 | # validate args |
| 172 | for arg in args: |
| 173 | if arg not in KNOWN_BENCHMARKS: |
kbaclawski | 6999ada | 2013-02-15 19:57:09 +0000 | [diff] [blame] | 174 | logger.GetLogger().LogFatal("Bad benchmark %s specified" % arg) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 175 | |
| 176 | if options.chromeos_root is None: |
| 177 | Usage(parser, "--chromeos_root must be set") |
| 178 | |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 179 | if options.board is None: |
| 180 | Usage(parser, "--board must be set") |
| 181 | |
| 182 | if options.machine is None: |
| 183 | Usage(parser, "--machine must be set") |
| 184 | |
| 185 | found_err = 0 |
| 186 | retval = 0 |
| 187 | for arg in args: |
| 188 | # CPU benchmarks |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 189 | comps = re.split('/', arg) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 190 | if re.match('chromeos/cpu', arg): |
bjanakiraman | 1c19a7f | 2013-02-15 06:31:19 +0000 | [diff] [blame] | 191 | benchname = comps[2] |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 192 | print "RUNNING %s" % benchname |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 193 | retval = RunCpuBenchmark(options.chromeos_root, |
| 194 | arg, options.workdir, options.machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 195 | if not found_err: |
| 196 | found_err = retval |
| 197 | elif re.match('chromeos/startup', arg): |
bjanakiraman | 1c19a7f | 2013-02-15 06:31:19 +0000 | [diff] [blame] | 198 | benchname = comps[1] |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 199 | image_args = [os.path.dirname(os.path.abspath(__file__)) + |
| 200 | "/image_chromeos.py", |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 201 | "--chromeos_root=" + options.chromeos_root, |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 202 | "--remote=" + options.machine, |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 203 | "--image=" + options.workdir + "/" + |
| 204 | benchname + "/chromiumos_image.bin" |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 205 | ] |
| 206 | logger.GetLogger().LogOutput("Reimaging machine %s" % options.machine) |
asharif | 8697d4e | 2013-02-15 09:18:09 +0000 | [diff] [blame] | 207 | image_chromeos.Main(image_args) |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 208 | |
| 209 | logger.GetLogger().LogOutput("Running %s" % arg) |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 210 | retval = RunStartupBenchmark(options.chromeos_root, |
| 211 | options.board, |
| 212 | arg, options.workdir, options.machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 213 | if not found_err: |
| 214 | found_err = retval |
| 215 | elif re.match('chromeos/browser', arg): |
bjanakiraman | 1c19a7f | 2013-02-15 06:31:19 +0000 | [diff] [blame] | 216 | benchname = comps[2] |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 217 | image_args = [os.path.dirname(os.path.abspath(__file__)) + |
| 218 | "/image_chromeos.py", |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 219 | "--chromeos_root=" + options.chromeos_root, |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 220 | "--remote=" + options.machine, |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 221 | "--image=" + options.workdir + "/" + |
| 222 | benchname + "/chromiumos_image.bin" |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 223 | ] |
| 224 | logger.GetLogger().LogOutput("Reimaging machine %s" % options.machine) |
asharif | 8697d4e | 2013-02-15 09:18:09 +0000 | [diff] [blame] | 225 | image_chromeos.Main(image_args) |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 226 | |
| 227 | logger.GetLogger().LogOutput("Running %s" % arg) |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 228 | retval = RunBrowserBenchmark(options.chromeos_root, |
| 229 | options.board, |
| 230 | arg, options.workdir, options.machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 231 | if not found_err: |
| 232 | found_err = retval |
| 233 | |
| 234 | return found_err |
| 235 | |
| 236 | if __name__ == "__main__": |
asharif | 2198c51 | 2013-02-15 09:21:35 +0000 | [diff] [blame] | 237 | retval = Main(sys.argv) |
| 238 | sys.exit(retval) |