Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 1 | #!/usr/bin/env python2.7 |
| 2 | # Copyright 2017, Google Inc. |
| 3 | # All rights reserved. |
| 4 | # |
| 5 | # Redistribution and use in source and binary forms, with or without |
| 6 | # modification, are permitted provided that the following conditions are |
| 7 | # met: |
| 8 | # |
| 9 | # * Redistributions of source code must retain the above copyright |
| 10 | # notice, this list of conditions and the following disclaimer. |
| 11 | # * Redistributions in binary form must reproduce the above |
| 12 | # copyright notice, this list of conditions and the following disclaimer |
| 13 | # in the documentation and/or other materials provided with the |
| 14 | # distribution. |
| 15 | # * Neither the name of Google Inc. nor the names of its |
| 16 | # contributors may be used to endorse or promote products derived from |
| 17 | # this software without specific prior written permission. |
| 18 | # |
| 19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 | |
Craig Tiller | 891e816 | 2017-02-15 23:30:27 -0800 | [diff] [blame] | 31 | import cgi |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 32 | import multiprocessing |
| 33 | import os |
| 34 | import subprocess |
| 35 | import sys |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 36 | import argparse |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 37 | |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 38 | import python_utils.jobset as jobset |
| 39 | import python_utils.start_port_server as start_port_server |
| 40 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 41 | flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph') |
| 42 | |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 43 | os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..')) |
| 44 | if not os.path.exists('reports'): |
| 45 | os.makedirs('reports') |
| 46 | |
Craig Tiller | cba864b | 2017-02-17 10:27:56 -0800 | [diff] [blame] | 47 | start_port_server.start_port_server() |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 48 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 49 | def fnize(s): |
| 50 | out = '' |
| 51 | for c in s: |
| 52 | if c in '<>, /': |
| 53 | if len(out) and out[-1] == '_': continue |
| 54 | out += '_' |
| 55 | else: |
| 56 | out += c |
| 57 | return out |
| 58 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 59 | # index html |
| 60 | index_html = """ |
| 61 | <html> |
| 62 | <head> |
| 63 | <title>Microbenchmark Results</title> |
| 64 | </head> |
| 65 | <body> |
| 66 | """ |
| 67 | |
| 68 | def heading(name): |
| 69 | global index_html |
| 70 | index_html += "<h1>%s</h1>\n" % name |
| 71 | |
| 72 | def link(txt, tgt): |
| 73 | global index_html |
Craig Tiller | 891e816 | 2017-02-15 23:30:27 -0800 | [diff] [blame] | 74 | index_html += "<p><a href=\"%s\">%s</a></p>\n" % ( |
| 75 | cgi.escape(tgt, quote=True), cgi.escape(txt)) |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 76 | |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 77 | def text(txt): |
| 78 | global index_html |
Craig Tiller | 891e816 | 2017-02-15 23:30:27 -0800 | [diff] [blame] | 79 | index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt) |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 80 | |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 81 | def collect_latency(bm_name, args): |
| 82 | """generate latency profiles""" |
| 83 | benchmarks = [] |
| 84 | profile_analysis = [] |
| 85 | cleanup = [] |
| 86 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 87 | heading('Latency Profiles: %s' % bm_name) |
| 88 | subprocess.check_call( |
| 89 | ['make', bm_name, |
| 90 | 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()]) |
| 91 | for line in subprocess.check_output(['bins/basicprof/%s' % bm_name, |
| 92 | '--benchmark_list_tests']).splitlines(): |
Craig Tiller | 3940179 | 2017-02-02 12:22:07 -0800 | [diff] [blame] | 93 | link(line, '%s.txt' % fnize(line)) |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 94 | benchmarks.append( |
Craig Tiller | ece502f | 2017-02-17 16:20:50 -0800 | [diff] [blame] | 95 | jobset.JobSpec(['bins/basicprof/%s' % bm_name, |
| 96 | '--benchmark_filter=^%s$' % line, |
| 97 | '--benchmark_min_time=0.05'], |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 98 | environ={'LATENCY_TRACE': '%s.trace' % fnize(line)})) |
| 99 | profile_analysis.append( |
| 100 | jobset.JobSpec([sys.executable, |
| 101 | 'tools/profiling/latency_profile/profile_analyzer.py', |
| 102 | '--source', '%s.trace' % fnize(line), '--fmt', 'simple', |
| 103 | '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None)) |
Craig Tiller | 715e43b | 2017-02-07 11:13:16 -0800 | [diff] [blame] | 104 | cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)])) |
Craig Tiller | 360c0d5 | 2017-02-08 13:36:44 -0800 | [diff] [blame] | 105 | # periodically flush out the list of jobs: profile_analysis jobs at least |
| 106 | # consume upwards of five gigabytes of ram in some cases, and so analysing |
| 107 | # hundreds of them at once is impractical -- but we want at least some |
| 108 | # concurrency or the work takes too long |
Craig Tiller | ece502f | 2017-02-17 16:20:50 -0800 | [diff] [blame] | 109 | if len(benchmarks) >= min(16, multiprocessing.cpu_count()): |
Craig Tiller | 360c0d5 | 2017-02-08 13:36:44 -0800 | [diff] [blame] | 110 | # run up to half the cpu count: each benchmark can use up to two cores |
| 111 | # (one for the microbenchmark, one for the data flush) |
Craig Tiller | cba864b | 2017-02-17 10:27:56 -0800 | [diff] [blame] | 112 | jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2)) |
Craig Tiller | 6911d08 | 2017-02-07 10:30:44 -0800 | [diff] [blame] | 113 | jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) |
| 114 | jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) |
| 115 | benchmarks = [] |
| 116 | profile_analysis = [] |
| 117 | cleanup = [] |
Craig Tiller | 360c0d5 | 2017-02-08 13:36:44 -0800 | [diff] [blame] | 118 | # run the remaining benchmarks that weren't flushed |
Craig Tiller | 6911d08 | 2017-02-07 10:30:44 -0800 | [diff] [blame] | 119 | if len(benchmarks): |
Craig Tiller | cba864b | 2017-02-17 10:27:56 -0800 | [diff] [blame] | 120 | jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2)) |
Craig Tiller | 6911d08 | 2017-02-07 10:30:44 -0800 | [diff] [blame] | 121 | jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) |
| 122 | jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 123 | |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 124 | def collect_perf(bm_name, args): |
| 125 | """generate flamegraphs""" |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 126 | heading('Flamegraphs: %s' % bm_name) |
| 127 | subprocess.check_call( |
| 128 | ['make', bm_name, |
| 129 | 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()]) |
Craig Tiller | 6ad0072 | 2017-02-15 09:14:24 -0800 | [diff] [blame] | 130 | benchmarks = [] |
| 131 | profile_analysis = [] |
| 132 | cleanup = [] |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 133 | for line in subprocess.check_output(['bins/mutrace/%s' % bm_name, |
| 134 | '--benchmark_list_tests']).splitlines(): |
Craig Tiller | 5a8c586 | 2017-02-15 07:59:05 -0800 | [diff] [blame] | 135 | link(line, '%s.svg' % fnize(line)) |
Craig Tiller | 6ad0072 | 2017-02-15 09:14:24 -0800 | [diff] [blame] | 136 | benchmarks.append( |
| 137 | jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line), |
Craig Tiller | c2c0c6f | 2017-02-15 11:27:37 -0800 | [diff] [blame] | 138 | '-g', '-F', '997', |
Craig Tiller | 6ad0072 | 2017-02-15 09:14:24 -0800 | [diff] [blame] | 139 | 'bins/mutrace/%s' % bm_name, |
| 140 | '--benchmark_filter=^%s$' % line, |
| 141 | '--benchmark_min_time=10'])) |
| 142 | profile_analysis.append( |
| 143 | jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'], |
| 144 | environ = { |
| 145 | 'PERF_BASE_NAME': fnize(line), |
| 146 | 'OUTPUT_DIR': 'reports', |
| 147 | 'OUTPUT_FILENAME': fnize(line), |
| 148 | })) |
| 149 | cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)])) |
| 150 | cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)])) |
| 151 | # periodically flush out the list of jobs: temporary space required for this |
| 152 | # processing is large |
| 153 | if len(benchmarks) >= 20: |
| 154 | # run up to half the cpu count: each benchmark can use up to two cores |
| 155 | # (one for the microbenchmark, one for the data flush) |
Craig Tiller | cba864b | 2017-02-17 10:27:56 -0800 | [diff] [blame] | 156 | jobset.run(benchmarks, maxjobs=1) |
Craig Tiller | 6ad0072 | 2017-02-15 09:14:24 -0800 | [diff] [blame] | 157 | jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) |
| 158 | jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) |
| 159 | benchmarks = [] |
| 160 | profile_analysis = [] |
| 161 | cleanup = [] |
| 162 | # run the remaining benchmarks that weren't flushed |
| 163 | if len(benchmarks): |
Craig Tiller | cba864b | 2017-02-17 10:27:56 -0800 | [diff] [blame] | 164 | jobset.run(benchmarks, maxjobs=1) |
Craig Tiller | 6ad0072 | 2017-02-15 09:14:24 -0800 | [diff] [blame] | 165 | jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) |
| 166 | jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 167 | |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 168 | def run_summary(bm_name, cfg, base_json_name): |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 169 | subprocess.check_call( |
| 170 | ['make', bm_name, |
Craig Tiller | 541b87e | 2017-03-01 08:42:52 -0800 | [diff] [blame] | 171 | 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()]) |
| 172 | cmd = ['bins/%s/%s' % (cfg, bm_name), |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 173 | '--benchmark_out=%s.%s.json' % (base_json_name, cfg), |
Craig Tiller | d9bc210 | 2017-02-15 08:24:55 -0800 | [diff] [blame] | 174 | '--benchmark_out_format=json'] |
| 175 | if args.summary_time is not None: |
| 176 | cmd += ['--benchmark_min_time=%d' % args.summary_time] |
Craig Tiller | 541b87e | 2017-03-01 08:42:52 -0800 | [diff] [blame] | 177 | return subprocess.check_output(cmd) |
| 178 | |
| 179 | def collect_summary(bm_name, args): |
| 180 | heading('Summary: %s [no counters]' % bm_name) |
Craig Tiller | 0a3d5f9 | 2017-03-01 17:08:39 -0800 | [diff] [blame] | 181 | text(run_summary(bm_name, 'opt', 'out')) |
Craig Tiller | 541b87e | 2017-03-01 08:42:52 -0800 | [diff] [blame] | 182 | heading('Summary: %s [with counters]' % bm_name) |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 183 | text(run_summary(bm_name, 'counters', 'out')) |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 184 | if args.bigquery_upload: |
Craig Tiller | d12731e | 2017-02-13 12:41:27 -0800 | [diff] [blame] | 185 | with open('out.csv', 'w') as f: |
Craig Tiller | 541b87e | 2017-03-01 08:42:52 -0800 | [diff] [blame] | 186 | f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.counters.json', 'out.opt.json'])) |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 187 | subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', 'out.csv']) |
| 188 | |
| 189 | collectors = { |
| 190 | 'latency': collect_latency, |
| 191 | 'perf': collect_perf, |
| 192 | 'summary': collect_summary, |
| 193 | } |
| 194 | |
| 195 | argp = argparse.ArgumentParser(description='Collect data from microbenchmarks') |
| 196 | argp.add_argument('-c', '--collect', |
| 197 | choices=sorted(collectors.keys()), |
Craig Tiller | 5ef448d | 2017-03-01 14:12:47 -0800 | [diff] [blame] | 198 | nargs='*', |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 199 | default=sorted(collectors.keys()), |
| 200 | help='Which collectors should be run against each benchmark') |
| 201 | argp.add_argument('-b', '--benchmarks', |
Craig Tiller | 0dc57ad | 2017-03-03 13:15:13 -0800 | [diff] [blame] | 202 | default=['bm_fullstack_unary_ping_pong', |
Craig Tiller | 0d8b11f | 2017-03-03 11:10:09 -0800 | [diff] [blame] | 203 | 'bm_fullstack_streaming_ping_pong', |
| 204 | 'bm_fullstack_streaming_pump', |
Craig Tiller | 523d54b | 2017-02-23 08:52:38 -0800 | [diff] [blame] | 205 | 'bm_closure', |
| 206 | 'bm_cq', |
| 207 | 'bm_call_create', |
| 208 | 'bm_error', |
Craig Tiller | 510f38a | 2017-02-24 17:00:19 -0800 | [diff] [blame] | 209 | 'bm_chttp2_hpack', |
Craig Tiller | 0dc57ad | 2017-03-03 13:15:13 -0800 | [diff] [blame] | 210 | 'bm_metadata', |
| 211 | 'bm_fullstack_trickle', |
| 212 | ], |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 213 | nargs='+', |
| 214 | type=str, |
| 215 | help='Which microbenchmarks should be run') |
Craig Tiller | d753f45 | 2017-03-01 14:00:35 -0800 | [diff] [blame] | 216 | argp.add_argument('--diff_perf', |
| 217 | default=None, |
| 218 | type=str, |
| 219 | help='Diff microbenchmarks against this git revision') |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 220 | argp.add_argument('--bigquery_upload', |
| 221 | default=False, |
| 222 | action='store_const', |
| 223 | const=True, |
| 224 | help='Upload results from summary collection to bigquery') |
Craig Tiller | d9bc210 | 2017-02-15 08:24:55 -0800 | [diff] [blame] | 225 | argp.add_argument('--summary_time', |
| 226 | default=None, |
| 227 | type=int, |
| 228 | help='Minimum time to run benchmarks for the summary collection') |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 229 | args = argp.parse_args() |
| 230 | |
| 231 | for bm_name in args.benchmarks: |
| 232 | for collect in args.collect: |
| 233 | collectors[collect](bm_name, args) |
Craig Tiller | d753f45 | 2017-03-01 14:00:35 -0800 | [diff] [blame] | 234 | if args.diff_perf: |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 235 | for bm_name in args.benchmarks: |
Craig Tiller | 0a3d5f9 | 2017-03-01 17:08:39 -0800 | [diff] [blame] | 236 | run_summary(bm_name, 'opt', '%s.new' % bm_name) |
Craig Tiller | 6d91f2d | 2017-03-01 14:17:37 -0800 | [diff] [blame] | 237 | where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() |
Craig Tiller | 5ef448d | 2017-03-01 14:12:47 -0800 | [diff] [blame] | 238 | subprocess.check_call(['git', 'checkout', args.diff_perf]) |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 239 | comparables = [] |
Craig Tiller | dce26f4 | 2017-03-01 14:15:46 -0800 | [diff] [blame] | 240 | subprocess.check_call(['make', 'clean']) |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 241 | try: |
| 242 | for bm_name in args.benchmarks: |
| 243 | try: |
Craig Tiller | 0a3d5f9 | 2017-03-01 17:08:39 -0800 | [diff] [blame] | 244 | run_summary(bm_name, 'opt', '%s.old' % bm_name) |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 245 | comparables.append(bm_name) |
| 246 | except subprocess.CalledProcessError, e: |
| 247 | pass |
| 248 | finally: |
Craig Tiller | 5ef448d | 2017-03-01 14:12:47 -0800 | [diff] [blame] | 249 | subprocess.check_call(['git', 'checkout', where_am_i]) |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 250 | for bm_name in comparables: |
Craig Tiller | 5ef448d | 2017-03-01 14:12:47 -0800 | [diff] [blame] | 251 | subprocess.check_call(['third_party/benchmark/tools/compare_bench.py', |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 252 | '%s.new.opt.json' % bm_name, |
| 253 | '%s.old.opt.json' % bm_name]) |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 254 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 255 | index_html += "</body>\n</html>\n" |
| 256 | with open('reports/index.html', 'w') as f: |
Craig Tiller | 360c0d5 | 2017-02-08 13:36:44 -0800 | [diff] [blame] | 257 | f.write(index_html) |