| #!/usr/bin/env python2.7 |
| # Copyright 2017, Google Inc. |
| # All rights reserved. |
| # |
| # Redistribution and use in source and binary forms, with or without |
| # modification, are permitted provided that the following conditions are |
| # met: |
| # |
| # * Redistributions of source code must retain the above copyright |
| # notice, this list of conditions and the following disclaimer. |
| # * Redistributions in binary form must reproduce the above |
| # copyright notice, this list of conditions and the following disclaimer |
| # in the documentation and/or other materials provided with the |
| # distribution. |
| # * Neither the name of Google Inc. nor the names of its |
| # contributors may be used to endorse or promote products derived from |
| # this software without specific prior written permission. |
| # |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| """ Runs the entire bm_*.py pipeline, and possible comments on the PR """ |
| |
| import bm_constants |
| import bm_build |
| import bm_run |
| import bm_diff |
| |
| import sys |
| import os |
| import argparse |
| import multiprocessing |
| import subprocess |
| |
| sys.path.append( |
| os.path.join( |
| os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) |
| import comment_on_pr |
| |
| |
| def _args(): |
| argp = argparse.ArgumentParser( |
| description='Perform diff on microbenchmarks') |
| argp.add_argument( |
| '-t', |
| '--track', |
| choices=sorted(bm_constants._INTERESTING), |
| nargs='+', |
| default=sorted(bm_constants._INTERESTING), |
| help='Which metrics to track') |
| argp.add_argument( |
| '-b', |
| '--benchmarks', |
| nargs='+', |
| choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
| default=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
| help='Which benchmarks to run') |
| argp.add_argument( |
| '-d', |
| '--diff_base', |
| type=str, |
| help='Commit or branch to compare the current one to') |
| argp.add_argument( |
| '-o', |
| '--old', |
| default='old', |
| type=str, |
| help='Name of baseline run to compare to. Ususally just called "old"') |
| argp.add_argument( |
| '-r', |
| '--repetitions', |
| type=int, |
| default=1, |
| help='Number of repetitions to pass to the benchmarks') |
| argp.add_argument( |
| '-l', |
| '--loops', |
| type=int, |
| default=20, |
| help='Number of times to loops the benchmarks. More loops cuts down on noise' |
| ) |
| argp.add_argument( |
| '-j', |
| '--jobs', |
| type=int, |
| default=multiprocessing.cpu_count(), |
| help='Number of CPUs to use') |
| args = argp.parse_args() |
| assert args.diff_base or args.old, "One of diff_base or old must be set!" |
| if args.loops < 3: |
| print "WARNING: This run will likely be noisy. Increase loops." |
| return args |
| |
| |
| def eintr_be_gone(fn): |
| """Run fn until it doesn't stop because of EINTR""" |
| |
| def inner(*args): |
| while True: |
| try: |
| return fn(*args) |
| except IOError, e: |
| if e.errno != errno.EINTR: |
| raise |
| |
| return inner |
| |
| |
| def main(args): |
| |
| bm_build.build('new', args.benchmarks, args.jobs) |
| |
| old = args.old |
| if args.diff_base: |
| old = 'old' |
| where_am_i = subprocess.check_output( |
| ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() |
| subprocess.check_call(['git', 'checkout', args.diff_base]) |
| try: |
| bm_build.build('old', args.benchmarks, args.jobs) |
| finally: |
| subprocess.check_call(['git', 'checkout', where_am_i]) |
| subprocess.check_call(['git', 'submodule', 'update']) |
| |
| bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions) |
| bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions) |
| |
| diff, note = bm_diff.diff(args.benchmarks, args.loops, args.track, old, |
| 'new') |
| if diff: |
| text = 'Performance differences noted:\n' + diff |
| else: |
| text = 'No significant performance differences' |
| print('%s\n%s' % (note, text)) |
| comment_on_pr.comment_on_pr('```\n%s\n\n%s\n```' % (note, text)) |
| |
| |
| if __name__ == '__main__': |
| args = _args() |
| main(args) |