bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 1 | #!/usr/bin/python2.6 |
| 2 | # |
| 3 | # Copyright 2010 Google Inc. All Rights Reserved. |
| 4 | |
| 5 | """Script to run ChromeOS benchmarks |
| 6 | |
| 7 | Inputs: |
| 8 | chromeos_root |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 9 | board |
| 10 | [chromeos/cpu/<benchname>|chromeos/browser/[pagecycler|sunspider]|chromeos/startup] |
| 11 | hostname/IP of Chromeos machine |
| 12 | |
| 13 | chromeos/cpu/<benchname> |
| 14 | - Read run script rules from bench.mk perflab-bin, copy benchmark to host, run |
| 15 | and return results. |
| 16 | |
| 17 | chromeos/startup |
| 18 | - Re-image host with image in perflab-bin |
| 19 | - Call run_tests to run startup test, gather results. |
| 20 | - Restore host back to what it was. |
| 21 | |
| 22 | chromeos/browser/* |
| 23 | - Call build_chromebrowser to build image with new browser |
| 24 | - Copy image to perflab-bin |
| 25 | |
| 26 | """ |
| 27 | |
| 28 | __author__ = "bjanakiraman@google.com (Bhaskar Janakiraman)" |
| 29 | |
| 30 | import optparse |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 31 | import os |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 32 | import re |
| 33 | import sys |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 34 | import run_tests |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 35 | import image_chromeos |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 36 | from utils import command_executer |
| 37 | from utils import utils |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 38 | from utils import logger |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 39 | |
| 40 | |
| 41 | KNOWN_BENCHMARKS = [ |
| 42 | "chromeos/startup", |
| 43 | "chromeos/browser/pagecycler", |
| 44 | "chromeos/browser/sunspider", |
| 45 | "chromeos/cpu/bikjmp"] |
| 46 | |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 47 | name_map = { |
| 48 | "pagecycler" : "Page", |
| 49 | "sunspider" : "SunSpider", |
| 50 | "startup" : "BootPerfServer"} |
| 51 | |
| 52 | |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 53 | # Run command template |
| 54 | |
| 55 | |
| 56 | # Common initializations |
| 57 | cmd_executer = command_executer.GetCommandExecuter() |
| 58 | |
| 59 | |
| 60 | def Usage(parser, message): |
| 61 | print "ERROR: " + message |
| 62 | parser.print_help() |
| 63 | sys.exit(0) |
| 64 | |
| 65 | |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 66 | def RunBrowserBenchmark(chromeos_root, board, bench, workdir, machine): |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 67 | """Run browser benchmarks. |
| 68 | |
| 69 | Args: |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 70 | chromeos_root: ChromeOS src dir |
| 71 | board: Board being tested |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 72 | bench: Name of benchmark (chromeos/browser/*) |
| 73 | workdir: Directory containing benchmark directory |
| 74 | machine: name of chromeos machine |
| 75 | """ |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 76 | benchname = re.split('/', bench)[2] |
| 77 | benchdir = '%s/%s' % (workdir, benchname) |
| 78 | benchname = name_map[benchname] |
| 79 | retval = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) |
| 80 | return retval |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 81 | |
| 82 | |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 83 | def RunStartupBenchmark(chromeos_root, board, bench, workdir, machine): |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 84 | """Run browser benchmarks. |
| 85 | |
| 86 | Args: |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 87 | chromeos_root: ChromeOS src dir |
| 88 | board: Board being tested |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 89 | bench: Name of benchmark (chromeos/browser/*) |
| 90 | workdir: Directory containing benchmark directory |
| 91 | machine: name of chromeos machine |
| 92 | """ |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 93 | benchname = 'startup' |
| 94 | benchdir = '%s/%s' % (workdir, benchname) |
| 95 | benchname = name_map[benchname] |
| 96 | retval = run_tests.RunRemoteTests(chromeos_root, machine, board, benchname) |
| 97 | return retval |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 98 | |
| 99 | |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 100 | def RunCpuBenchmark(chromeos_root, bench, workdir, machine): |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 101 | """Run CPU benchmark. |
| 102 | |
| 103 | Args: |
| 104 | bench: Name of benchmark |
| 105 | workdir: directory containing benchmark directory |
| 106 | machine: name of chromeos machine |
| 107 | |
| 108 | Returns: |
| 109 | status: 0 on success |
| 110 | """ |
| 111 | |
| 112 | benchname = re.split('/', bench)[2] |
| 113 | benchdir = '%s/%s' % (workdir, benchname) |
| 114 | |
| 115 | # Delete any existing run directories on machine. |
| 116 | # Since this has exclusive access to the machine, |
| 117 | # we do not worry about duplicates. |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 118 | args = 'rm -rf /tmp/%s' % benchname |
| 119 | retval = cmd_executer.CrosRunCommand(args, chromeos_root=chromeos_root, |
| 120 | machine=machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 121 | if retval: |
| 122 | return retval |
| 123 | |
| 124 | # Copy benchmark directory. |
asharif | 29775b2 | 2013-02-15 05:15:38 +0000 | [diff] [blame] | 125 | retval = cmd_executer.CopyFiles(benchdir, "/tmp/" + benchname, |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 126 | chromeos_root=chromeos_root, |
| 127 | dest_machine=machine, |
| 128 | dest_cros=True) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 129 | if retval: |
| 130 | return retval |
| 131 | |
| 132 | # Parse bench.mk to extract run flags. |
| 133 | |
| 134 | benchmk_file = open('%s/bench.mk' % benchdir, 'r') |
| 135 | for line in benchmk_file: |
| 136 | line.rstrip() |
| 137 | if re.match('^run_cmd', line): |
bjanakiraman | 4e8bdf2 | 2013-02-15 04:57:09 +0000 | [diff] [blame] | 138 | line = re.sub('^run_cmd.*\${PERFLAB_PATH}', './out', line) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 139 | line = re.sub('\${PERFLAB_INPUT}', './data', line) |
| 140 | run_cmd = line |
| 141 | break |
| 142 | |
| 143 | # Execute on remote machine |
| 144 | # Capture output and process it. |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 145 | sshargs = "'cd /tmp/%s;" % benchname |
| 146 | sshargs += "time -p %s'" % run_cmd |
| 147 | cmd_executer.CrosRunCommand(sshargs, chromeos_root=chromeos_root, |
| 148 | machine=machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 149 | |
| 150 | return retval |
| 151 | |
| 152 | |
| 153 | def Main(argv): |
| 154 | """Build ChromeOS.""" |
| 155 | # Common initializations |
| 156 | |
| 157 | parser = optparse.OptionParser() |
| 158 | parser.add_option("-c", "--chromeos_root", dest="chromeos_root", |
| 159 | help="Target directory for ChromeOS installation.") |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 160 | parser.add_option("-m", "--machine", dest="machine", |
| 161 | help="The chromeos host machine.") |
| 162 | parser.add_option("--workdir", dest="workdir", default="./perflab-bin", |
| 163 | help="Work directory for perflab outputs.") |
| 164 | parser.add_option("--board", dest="board", |
| 165 | help="ChromeOS target board, e.g. x86-generic") |
| 166 | |
| 167 | (options, args) = parser.parse_args(argv[1:]) |
| 168 | |
| 169 | # validate args |
| 170 | for arg in args: |
| 171 | if arg not in KNOWN_BENCHMARKS: |
| 172 | utils.AssertExit(False, "Bad benchmark %s specified" % arg) |
| 173 | |
| 174 | |
| 175 | if options.chromeos_root is None: |
| 176 | Usage(parser, "--chromeos_root must be set") |
| 177 | |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 178 | if options.board is None: |
| 179 | Usage(parser, "--board must be set") |
| 180 | |
| 181 | if options.machine is None: |
| 182 | Usage(parser, "--machine must be set") |
| 183 | |
| 184 | found_err = 0 |
| 185 | retval = 0 |
| 186 | for arg in args: |
| 187 | # CPU benchmarks |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 188 | comps = re.split('/', arg) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 189 | if re.match('chromeos/cpu', arg): |
bjanakiraman | 1c19a7f | 2013-02-15 06:31:19 +0000 | [diff] [blame] | 190 | benchname = comps[2] |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 191 | print "RUNNING %s" % benchname |
asharif | 4d2b716 | 2013-02-15 05:14:57 +0000 | [diff] [blame] | 192 | retval = RunCpuBenchmark(options.chromeos_root, |
| 193 | arg, options.workdir, options.machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 194 | if not found_err: |
| 195 | found_err = retval |
| 196 | elif re.match('chromeos/startup', arg): |
bjanakiraman | 1c19a7f | 2013-02-15 06:31:19 +0000 | [diff] [blame] | 197 | benchname = comps[1] |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 198 | image_args = [os.path.dirname(os.path.abspath(__file__)) + |
| 199 | "/image_chromeos.py", |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 200 | "--chromeos_root=" + options.chromeos_root, |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 201 | "--remote=" + options.machine, |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 202 | "--image=" + options.workdir + "/" + |
| 203 | benchname + "/chromiumos_image.bin" |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 204 | ] |
| 205 | logger.GetLogger().LogOutput("Reimaging machine %s" % options.machine) |
asharif | 0e0e268 | 2013-02-15 05:15:29 +0000 | [diff] [blame] | 206 | image_chromeos.ImageChromeOS(image_args) |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 207 | |
| 208 | logger.GetLogger().LogOutput("Running %s" % arg) |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 209 | retval = RunStartupBenchmark(options.chromeos_root, |
| 210 | options.board, |
| 211 | arg, options.workdir, options.machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 212 | if not found_err: |
| 213 | found_err = retval |
| 214 | elif re.match('chromeos/browser', arg): |
bjanakiraman | 1c19a7f | 2013-02-15 06:31:19 +0000 | [diff] [blame] | 215 | benchname = comps[2] |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 216 | image_args = [os.path.dirname(os.path.abspath(__file__)) + |
| 217 | "/image_chromeos.py", |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 218 | "--chromeos_root=" + options.chromeos_root, |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 219 | "--remote=" + options.machine, |
asharif | 253e88b | 2013-02-15 05:15:34 +0000 | [diff] [blame] | 220 | "--image=" + options.workdir + "/" + |
| 221 | benchname + "/chromiumos_image.bin" |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 222 | ] |
| 223 | logger.GetLogger().LogOutput("Reimaging machine %s" % options.machine) |
asharif | 0e0e268 | 2013-02-15 05:15:29 +0000 | [diff] [blame] | 224 | image_chromeos.ImageChromeOS(image_args) |
asharif | 8de2c73 | 2013-02-15 05:15:25 +0000 | [diff] [blame] | 225 | |
| 226 | logger.GetLogger().LogOutput("Running %s" % arg) |
bjanakiraman | 81a30d0 | 2013-02-15 04:57:20 +0000 | [diff] [blame] | 227 | retval = RunBrowserBenchmark(options.chromeos_root, |
| 228 | options.board, |
| 229 | arg, options.workdir, options.machine) |
bjanakiraman | 229d626 | 2013-02-15 04:56:46 +0000 | [diff] [blame] | 230 | if not found_err: |
| 231 | found_err = retval |
| 232 | |
| 233 | return found_err |
| 234 | |
| 235 | if __name__ == "__main__": |
| 236 | Main(sys.argv) |