ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python2.7 |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 2 | # |
ncteisen | 0ff7384 | 2017-06-08 08:15:03 -0700 | [diff] [blame] | 3 | # Copyright 2017 gRPC authors. |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 4 | # |
ncteisen | 0ff7384 | 2017-06-08 08:15:03 -0700 | [diff] [blame] | 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 8 | # |
ncteisen | 0ff7384 | 2017-06-08 08:15:03 -0700 | [diff] [blame] | 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 16 | """ Runs the entire bm_*.py pipeline, and possible comments on the PR """ |
| 17 | |
| 18 | import bm_constants |
| 19 | import bm_build |
| 20 | import bm_run |
| 21 | import bm_diff |
| 22 | |
| 23 | import sys |
| 24 | import os |
ncteisen | 460af54 | 2017-09-28 16:12:15 -0700 | [diff] [blame] | 25 | import random |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 26 | import argparse |
| 27 | import multiprocessing |
| 28 | import subprocess |
| 29 | |
ncteisen | 738be24 | 2017-05-09 17:36:11 -0700 | [diff] [blame] | 30 | sys.path.append( |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 31 | os.path.join( |
| 32 | os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 33 | import comment_on_pr |
| 34 | |
ncteisen | 460af54 | 2017-09-28 16:12:15 -0700 | [diff] [blame] | 35 | sys.path.append( |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 36 | os.path.join( |
| 37 | os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests', |
| 38 | 'python_utils')) |
ncteisen | 460af54 | 2017-09-28 16:12:15 -0700 | [diff] [blame] | 39 | import jobset |
| 40 | |
ncteisen | 738be24 | 2017-05-09 17:36:11 -0700 | [diff] [blame] | 41 | |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 42 | def _args(): |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 43 | argp = argparse.ArgumentParser( |
| 44 | description='Perform diff on microbenchmarks') |
| 45 | argp.add_argument( |
| 46 | '-t', |
| 47 | '--track', |
| 48 | choices=sorted(bm_constants._INTERESTING), |
| 49 | nargs='+', |
| 50 | default=sorted(bm_constants._INTERESTING), |
| 51 | help='Which metrics to track') |
| 52 | argp.add_argument( |
| 53 | '-b', |
| 54 | '--benchmarks', |
| 55 | nargs='+', |
| 56 | choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
| 57 | default=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
| 58 | help='Which benchmarks to run') |
| 59 | argp.add_argument( |
| 60 | '-d', |
| 61 | '--diff_base', |
| 62 | type=str, |
| 63 | help='Commit or branch to compare the current one to') |
| 64 | argp.add_argument( |
| 65 | '-o', |
| 66 | '--old', |
| 67 | default='old', |
| 68 | type=str, |
| 69 | help='Name of baseline run to compare to. Ususally just called "old"') |
| 70 | argp.add_argument( |
| 71 | '-r', |
| 72 | '--regex', |
| 73 | type=str, |
| 74 | default="", |
| 75 | help='Regex to filter benchmarks run') |
| 76 | argp.add_argument( |
| 77 | '-l', |
| 78 | '--loops', |
| 79 | type=int, |
| 80 | default=10, |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 81 | help= |
| 82 | 'Number of times to loops the benchmarks. More loops cuts down on noise' |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 83 | ) |
| 84 | argp.add_argument( |
| 85 | '-j', |
| 86 | '--jobs', |
| 87 | type=int, |
| 88 | default=multiprocessing.cpu_count(), |
| 89 | help='Number of CPUs to use') |
| 90 | argp.add_argument( |
| 91 | '--pr_comment_name', |
| 92 | type=str, |
| 93 | default="microbenchmarks", |
| 94 | help='Name that Jenkins will use to commen on the PR') |
| 95 | argp.add_argument('--counters', dest='counters', action='store_true') |
| 96 | argp.add_argument('--no-counters', dest='counters', action='store_false') |
| 97 | argp.set_defaults(counters=True) |
| 98 | args = argp.parse_args() |
| 99 | assert args.diff_base or args.old, "One of diff_base or old must be set!" |
| 100 | if args.loops < 3: |
| 101 | print "WARNING: This run will likely be noisy. Increase loops." |
| 102 | return args |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 103 | |
| 104 | |
| 105 | def eintr_be_gone(fn): |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 106 | """Run fn until it doesn't stop because of EINTR""" |
ncteisen | 738be24 | 2017-05-09 17:36:11 -0700 | [diff] [blame] | 107 | |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 108 | def inner(*args): |
| 109 | while True: |
| 110 | try: |
| 111 | return fn(*args) |
| 112 | except IOError, e: |
| 113 | if e.errno != errno.EINTR: |
| 114 | raise |
ncteisen | 738be24 | 2017-05-09 17:36:11 -0700 | [diff] [blame] | 115 | |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 116 | return inner |
ncteisen | 738be24 | 2017-05-09 17:36:11 -0700 | [diff] [blame] | 117 | |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 118 | |
| 119 | def main(args): |
| 120 | |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 121 | bm_build.build('new', args.benchmarks, args.jobs, args.counters) |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 122 | |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 123 | old = args.old |
| 124 | if args.diff_base: |
| 125 | old = 'old' |
| 126 | where_am_i = subprocess.check_output( |
| 127 | ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() |
| 128 | subprocess.check_call(['git', 'checkout', args.diff_base]) |
| 129 | try: |
| 130 | bm_build.build(old, args.benchmarks, args.jobs, args.counters) |
| 131 | finally: |
| 132 | subprocess.check_call(['git', 'checkout', where_am_i]) |
| 133 | subprocess.check_call(['git', 'submodule', 'update']) |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 134 | |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 135 | jobs_list = [] |
| 136 | jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, |
| 137 | args.regex, args.counters) |
| 138 | jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, |
| 139 | args.regex, args.counters) |
ncteisen | 460af54 | 2017-09-28 16:12:15 -0700 | [diff] [blame] | 140 | |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 141 | # shuffle all jobs to eliminate noise from GCE CPU drift |
| 142 | random.shuffle(jobs_list, random.SystemRandom().random) |
| 143 | jobset.run(jobs_list, maxjobs=args.jobs) |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 144 | |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 145 | diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex, |
| 146 | args.track, old, 'new', args.counters) |
| 147 | if diff: |
| 148 | text = '[%s] Performance differences noted:\n%s' % ( |
| 149 | args.pr_comment_name, diff) |
| 150 | else: |
| 151 | text = '[%s] No significant performance differences' % args.pr_comment_name |
| 152 | if note: |
| 153 | text = note + '\n\n' + text |
| 154 | print('%s' % text) |
| 155 | comment_on_pr.comment_on_pr('```\n%s\n```' % text) |
ncteisen | 738be24 | 2017-05-09 17:36:11 -0700 | [diff] [blame] | 156 | |
ncteisen | dc76c66 | 2017-05-05 17:13:07 -0700 | [diff] [blame] | 157 | |
| 158 | if __name__ == '__main__': |
ncteisen | 173c477 | 2017-12-11 16:52:44 -0800 | [diff] [blame] | 159 | args = _args() |
| 160 | main(args) |