Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 1 | #!/usr/bin/pyton |
| 2 | |
| 3 | # Copyright 2017 Google Inc. |
| 4 | # |
| 5 | # Use of this source code is governed by a BSD-style license that can be |
| 6 | # found in the LICENSE file. |
| 7 | |
| 8 | import os |
| 9 | import sys |
| 10 | import subprocess |
| 11 | import multiprocessing |
| 12 | |
| 13 | from argparse import ArgumentParser |
| 14 | |
| 15 | |
| 16 | README = """ |
| 17 | Simply run |
| 18 | \033[36m |
| 19 | python {0} TEST_GIT_BRANCH |
| 20 | \033[0m |
| 21 | to see if TEST_GIT_BRANCH has performance regressions against master in 8888. |
| 22 | |
| 23 | To compare a specific config with svg and skp resources included, add --config |
| 24 | and --extraarg option. For exampe, |
| 25 | \033[36m |
| 26 | python {0} TEST_GIT_BRANCH --config gl \\ |
| 27 | --extraarg "--svgs ~/Desktop/bots/svgs --skps ~/Desktop/bots/skps" |
| 28 | \033[0m |
| 29 | For more options, please see |
| 30 | |
| 31 | python {0} --help |
| 32 | """.format(__file__) |
| 33 | |
| 34 | |
| 35 | CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 36 | AB_SCRIPT = "ab.py" |
| 37 | |
| 38 | |
| 39 | def parse_args(): |
| 40 | if len(sys.argv) <= 1 or sys.argv[1] == '-h' or sys.argv[1] == '--help': |
| 41 | print README |
| 42 | |
| 43 | parser = ArgumentParser( |
| 44 | description='Noiselessly (hence calm) becnhmark a git branch against ' + |
| 45 | 'another baseline branch (e.g., master) using multiple ' + |
| 46 | ' nanobench runs.' |
| 47 | ) |
| 48 | |
| 49 | default_threads = max(1, multiprocessing.cpu_count() / 2); |
| 50 | default_skiadir = os.path.normpath(CURRENT_DIR + "/../../") |
| 51 | |
| 52 | config_help = ( |
| 53 | 'nanobench config; we currently support only one config ' |
| 54 | 'at a time (default: %(default)s)') |
| 55 | reps_help = ( |
| 56 | 'initial repititions of the nanobench run; this may be ' |
| 57 | 'overridden when we have many threads (default: %(default)s)') |
| 58 | extraarg_help = ( |
| 59 | 'nanobench args (example: --svgs ~/Desktop/bots/svgs --skps ' |
| 60 | '~/Desktop/bots/skps)') |
| 61 | baseline_help = ( |
| 62 | 'baseline branch to compare against (default: %(default)s)') |
| 63 | basearg_help = ( |
| 64 | 'nanobench arg for the baseline branch; if not given, we use ' |
| 65 | ' the same arg for both the test branch and the baseline branch') |
| 66 | threads_help = ( |
| 67 | 'number of threads to be used (default: %(default)s); ' |
| 68 | 'for GPU config, this will always be 1') |
| 69 | no_compile_help = ( |
| 70 | 'whether NOT to compile nanobench and copy it to WRITEDIR ' |
| 71 | '(i.e., reuse previous nanobench compiled)') |
| 72 | skip_base_help = ( |
| 73 | 'whether NOT to run nanobench on baseline branch ' |
| 74 | '(i.e., reuse previous baseline measurements)') |
| 75 | noinit_help = ( |
| 76 | 'whether to skip initial nanobench runs (default: %(default)s)') |
Yuqian Li | c81aaaa | 2017-10-16 12:24:43 -0400 | [diff] [blame] | 77 | branch_help = ( |
| 78 | "the test branch to benchmark; if it's 'modified', we'll benchmark the " |
| 79 | "current modified code against 'git stash'.") |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 80 | |
| 81 | definitions = [ |
| 82 | # argname, type, default value, help |
| 83 | ['--config', str, '8888', config_help], |
| 84 | ['--skiadir', str, default_skiadir, 'default: %(default)s'], |
| 85 | ['--ninjadir', str, 'out/Release', 'default: %(default)s'], |
| 86 | ['--writedir', str, '/var/tmp', 'default: %(default)s'], |
| 87 | ['--extraarg', str, '', extraarg_help], |
| 88 | ['--baseline', str, 'master', baseline_help], |
| 89 | ['--basearg', str, '', basearg_help], |
| 90 | ['--reps', int, 2, reps_help], |
Yuqian Li | 228da62 | 2017-10-26 15:38:30 -0400 | [diff] [blame] | 91 | ['--threads', int, default_threads, threads_help], |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 92 | ] |
| 93 | |
| 94 | for d in definitions: |
| 95 | parser.add_argument(d[0], type=d[1], default=d[2], help=d[3]) |
| 96 | |
Yuqian Li | c81aaaa | 2017-10-16 12:24:43 -0400 | [diff] [blame] | 97 | parser.add_argument('branch', type=str, help=branch_help) |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 98 | parser.add_argument('--no-compile', dest='no_compile', action="store_true", |
| 99 | help=no_compile_help) |
| 100 | parser.add_argument('--skip-base', dest='skipbase', action="store_true", |
| 101 | help=skip_base_help) |
| 102 | parser.add_argument('--noinit', dest='noinit', action="store_true", |
| 103 | help=noinit_help) |
Yuqian Li | 84366d2 | 2017-10-17 16:26:32 -0400 | [diff] [blame] | 104 | parser.add_argument('--concise', dest='concise', action="store_true", |
| 105 | help="If set, no verbose thread info will be printed.") |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 106 | parser.set_defaults(no_compile=False); |
| 107 | parser.set_defaults(skipbase=False); |
| 108 | parser.set_defaults(noinit=False); |
Yuqian Li | 84366d2 | 2017-10-17 16:26:32 -0400 | [diff] [blame] | 109 | parser.set_defaults(concise=False); |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 110 | |
Yuqian Li | 228da62 | 2017-10-26 15:38:30 -0400 | [diff] [blame] | 111 | # Additional args for bots |
| 112 | BHELP = "bot specific options" |
| 113 | parser.add_argument('--githash', type=str, help=BHELP) |
| 114 | parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP) |
| 115 | |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 116 | args = parser.parse_args() |
| 117 | if not args.basearg: |
| 118 | args.basearg = args.extraarg |
| 119 | |
| 120 | return args |
| 121 | |
| 122 | |
| 123 | def nano_path(args, branch): |
| 124 | return args.writedir + '/nanobench_' + branch |
| 125 | |
| 126 | |
| 127 | def compile_branch(args, branch): |
| 128 | print "Compiling branch %s" % args.branch |
| 129 | |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 130 | commands = [ |
| 131 | ['git', 'checkout', branch], |
| 132 | ['ninja', '-C', args.ninjadir, 'nanobench'], |
| 133 | ['cp', args.ninjadir + '/nanobench', nano_path(args, branch)] |
| 134 | ] |
| 135 | for command in commands: |
| 136 | subprocess.check_call(command, cwd=args.skiadir) |
| 137 | |
| 138 | |
Yuqian Li | c81aaaa | 2017-10-16 12:24:43 -0400 | [diff] [blame] | 139 | def compile_modified(args): |
| 140 | print "Compiling modified code" |
| 141 | subprocess.check_call( |
| 142 | ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) |
| 143 | subprocess.check_call( |
| 144 | ['cp', args.ninjadir + '/nanobench', nano_path(args, args.branch)], |
| 145 | cwd=args.skiadir) |
| 146 | |
| 147 | print "Compiling stashed code" |
Yuqian Li | 5be3a8e | 2017-10-16 16:01:19 -0400 | [diff] [blame] | 148 | stash_output = subprocess.check_output(['git', 'stash'], cwd=args.skiadir) |
| 149 | if 'No local changes to save' in stash_output: |
| 150 | subprocess.check_call(['git', 'reset', 'HEAD^', '--soft']) |
| 151 | subprocess.check_call(['git', 'stash']) |
| 152 | |
Yuqian Li | 632156d | 2018-01-02 11:56:52 -0500 | [diff] [blame] | 153 | subprocess.check_call(['gclient', 'sync'], cwd=args.skiadir) |
Yuqian Li | c81aaaa | 2017-10-16 12:24:43 -0400 | [diff] [blame] | 154 | subprocess.check_call( |
| 155 | ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) |
| 156 | subprocess.check_call( |
| 157 | ['cp', args.ninjadir + '/nanobench', nano_path(args, args.baseline)], |
| 158 | cwd=args.skiadir) |
| 159 | subprocess.check_call(['git', 'stash', 'pop'], cwd=args.skiadir) |
| 160 | |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 161 | def compile_nanobench(args): |
Yuqian Li | c81aaaa | 2017-10-16 12:24:43 -0400 | [diff] [blame] | 162 | if args.branch == 'modified': |
| 163 | compile_modified(args) |
| 164 | else: |
| 165 | compile_branch(args, args.branch) |
| 166 | compile_branch(args, args.baseline) |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 167 | |
| 168 | |
| 169 | def main(): |
| 170 | args = parse_args() |
| 171 | |
| 172 | # copy in case that it will be gone after git branch switching |
| 173 | orig_ab_name = CURRENT_DIR + "/" + AB_SCRIPT |
| 174 | temp_ab_name = args.writedir + "/" + AB_SCRIPT |
| 175 | subprocess.check_call(['cp', orig_ab_name, temp_ab_name]) |
| 176 | |
| 177 | if not args.no_compile: |
| 178 | compile_nanobench(args) |
| 179 | |
| 180 | command = [ |
| 181 | 'python', |
| 182 | temp_ab_name, |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 183 | args.writedir, |
| 184 | args.branch + ("_A" if args.branch == args.baseline else ""), |
| 185 | args.baseline + ("_B" if args.branch == args.baseline else ""), |
| 186 | nano_path(args, args.branch), |
| 187 | nano_path(args, args.baseline), |
| 188 | args.extraarg, |
| 189 | args.basearg, |
| 190 | str(args.reps), |
| 191 | "true" if args.skipbase else "false", |
| 192 | args.config, |
| 193 | str(args.threads if args.config in ["8888", "565"] else 1), |
| 194 | "true" if args.noinit else "false" |
| 195 | ] |
| 196 | |
Yuqian Li | 228da62 | 2017-10-26 15:38:30 -0400 | [diff] [blame] | 197 | if args.githash: |
| 198 | command += ['--githash', args.githash] |
| 199 | if args.keys: |
| 200 | command += (['--keys'] + args.keys) |
| 201 | |
Yuqian Li | 84366d2 | 2017-10-17 16:26:32 -0400 | [diff] [blame] | 202 | if args.concise: |
| 203 | command.append("--concise") |
| 204 | |
Yuqian Li | 980379d | 2017-09-29 11:20:01 -0400 | [diff] [blame] | 205 | p = subprocess.Popen(command, cwd=args.skiadir) |
| 206 | try: |
| 207 | p.wait() |
| 208 | except KeyboardInterrupt: |
| 209 | try: |
| 210 | p.terminate() |
| 211 | except OSError as e: |
| 212 | print e |
| 213 | |
| 214 | |
| 215 | if __name__ == "__main__": |
| 216 | main() |