Siddharth Shukla | 8e64d90 | 2017-03-12 19:50:18 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 2 | # Copyright 2017 gRPC authors. |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 3 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 7 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 9 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 15 | |
Craig Tiller | 891e816 | 2017-02-15 23:30:27 -0800 | [diff] [blame] | 16 | import cgi |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 17 | import multiprocessing |
| 18 | import os |
| 19 | import subprocess |
| 20 | import sys |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 21 | import argparse |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 22 | |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 23 | import python_utils.jobset as jobset |
| 24 | import python_utils.start_port_server as start_port_server |
| 25 | |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 26 | sys.path.append( |
| 27 | os.path.join( |
| 28 | os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks', |
| 29 | 'bm_diff')) |
ncteisen | 64637b7 | 2017-05-09 14:23:16 -0700 | [diff] [blame] | 30 | import bm_constants |
Matt Kwong | d0ee10d | 2017-03-10 22:37:52 -0800 | [diff] [blame] | 31 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 32 | flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph') |
| 33 | |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 34 | os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..')) |
| 35 | if not os.path.exists('reports'): |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 36 | os.makedirs('reports') |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 37 | |
Craig Tiller | cba864b | 2017-02-17 10:27:56 -0800 | [diff] [blame] | 38 | start_port_server.start_port_server() |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 39 | |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 40 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 41 | def fnize(s): |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 42 | out = '' |
| 43 | for c in s: |
| 44 | if c in '<>, /': |
| 45 | if len(out) and out[-1] == '_': continue |
| 46 | out += '_' |
| 47 | else: |
| 48 | out += c |
| 49 | return out |
| 50 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 51 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 52 | # index html |
| 53 | index_html = """ |
| 54 | <html> |
| 55 | <head> |
| 56 | <title>Microbenchmark Results</title> |
| 57 | </head> |
| 58 | <body> |
| 59 | """ |
| 60 | |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 61 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 62 | def heading(name): |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 63 | global index_html |
| 64 | index_html += "<h1>%s</h1>\n" % name |
| 65 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 66 | |
| 67 | def link(txt, tgt): |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 68 | global index_html |
| 69 | index_html += "<p><a href=\"%s\">%s</a></p>\n" % ( |
| 70 | cgi.escape(tgt, quote=True), cgi.escape(txt)) |
| 71 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 72 | |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 73 | def text(txt): |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 74 | global index_html |
| 75 | index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt) |
| 76 | |
Craig Tiller | 7dc4ea6 | 2017-02-02 16:08:05 -0800 | [diff] [blame] | 77 | |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 78 | def collect_latency(bm_name, args): |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 79 | """generate latency profiles""" |
| 80 | benchmarks = [] |
| 81 | profile_analysis = [] |
| 82 | cleanup = [] |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 83 | |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 84 | heading('Latency Profiles: %s' % bm_name) |
| 85 | subprocess.check_call([ |
| 86 | 'make', bm_name, 'CONFIG=basicprof', '-j', |
| 87 | '%d' % multiprocessing.cpu_count() |
| 88 | ]) |
| 89 | for line in subprocess.check_output( |
| 90 | ['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines(): |
| 91 | link(line, '%s.txt' % fnize(line)) |
| 92 | benchmarks.append( |
| 93 | jobset.JobSpec( |
| 94 | [ |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 95 | 'bins/basicprof/%s' % bm_name, |
| 96 | '--benchmark_filter=^%s$' % line, |
| 97 | '--benchmark_min_time=0.05' |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 98 | ], |
| 99 | environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}, |
| 100 | shortname='profile-%s' % fnize(line))) |
| 101 | profile_analysis.append( |
| 102 | jobset.JobSpec( |
| 103 | [ |
| 104 | sys.executable, |
| 105 | 'tools/profiling/latency_profile/profile_analyzer.py', |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 106 | '--source', |
| 107 | '%s.trace' % fnize(line), '--fmt', 'simple', '--out', |
| 108 | 'reports/%s.txt' % fnize(line) |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 109 | ], |
| 110 | timeout_seconds=20 * 60, |
| 111 | shortname='analyze-%s' % fnize(line))) |
| 112 | cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)])) |
| 113 | # periodically flush out the list of jobs: profile_analysis jobs at least |
| 114 | # consume upwards of five gigabytes of ram in some cases, and so analysing |
| 115 | # hundreds of them at once is impractical -- but we want at least some |
| 116 | # concurrency or the work takes too long |
| 117 | if len(benchmarks) >= min(16, multiprocessing.cpu_count()): |
| 118 | # run up to half the cpu count: each benchmark can use up to two cores |
| 119 | # (one for the microbenchmark, one for the data flush) |
| 120 | jobset.run( |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 121 | benchmarks, maxjobs=max(1, |
| 122 | multiprocessing.cpu_count() / 2)) |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 123 | jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) |
| 124 | jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) |
| 125 | benchmarks = [] |
| 126 | profile_analysis = [] |
| 127 | cleanup = [] |
| 128 | # run the remaining benchmarks that weren't flushed |
| 129 | if len(benchmarks): |
| 130 | jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2)) |
| 131 | jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) |
| 132 | jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) |
| 133 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 134 | |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 135 | def collect_perf(bm_name, args): |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 136 | """generate flamegraphs""" |
| 137 | heading('Flamegraphs: %s' % bm_name) |
| 138 | subprocess.check_call([ |
| 139 | 'make', bm_name, 'CONFIG=mutrace', '-j', |
| 140 | '%d' % multiprocessing.cpu_count() |
| 141 | ]) |
| 142 | benchmarks = [] |
| 143 | profile_analysis = [] |
| 144 | cleanup = [] |
| 145 | for line in subprocess.check_output( |
| 146 | ['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines(): |
| 147 | link(line, '%s.svg' % fnize(line)) |
| 148 | benchmarks.append( |
| 149 | jobset.JobSpec( |
| 150 | [ |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 151 | 'perf', 'record', '-o', |
| 152 | '%s-perf.data' % fnize(line), '-g', '-F', '997', |
| 153 | 'bins/mutrace/%s' % bm_name, |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 154 | '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10' |
| 155 | ], |
| 156 | shortname='perf-%s' % fnize(line))) |
| 157 | profile_analysis.append( |
| 158 | jobset.JobSpec( |
| 159 | [ |
| 160 | 'tools/run_tests/performance/process_local_perf_flamegraphs.sh' |
| 161 | ], |
| 162 | environ={ |
| 163 | 'PERF_BASE_NAME': fnize(line), |
| 164 | 'OUTPUT_DIR': 'reports', |
| 165 | 'OUTPUT_FILENAME': fnize(line), |
| 166 | }, |
| 167 | shortname='flame-%s' % fnize(line))) |
| 168 | cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)])) |
| 169 | cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)])) |
| 170 | # periodically flush out the list of jobs: temporary space required for this |
| 171 | # processing is large |
| 172 | if len(benchmarks) >= 20: |
| 173 | # run up to half the cpu count: each benchmark can use up to two cores |
| 174 | # (one for the microbenchmark, one for the data flush) |
| 175 | jobset.run(benchmarks, maxjobs=1) |
| 176 | jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) |
| 177 | jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) |
| 178 | benchmarks = [] |
| 179 | profile_analysis = [] |
| 180 | cleanup = [] |
| 181 | # run the remaining benchmarks that weren't flushed |
| 182 | if len(benchmarks): |
| 183 | jobset.run(benchmarks, maxjobs=1) |
| 184 | jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) |
| 185 | jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) |
| 186 | |
Craig Tiller | f7af2a9 | 2017-01-31 15:08:31 -0800 | [diff] [blame] | 187 | |
Craig Tiller | ff84b36 | 2017-03-01 14:11:15 -0800 | [diff] [blame] | 188 | def run_summary(bm_name, cfg, base_json_name): |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 189 | subprocess.check_call([ |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 190 | 'make', bm_name, |
| 191 | 'CONFIG=%s' % cfg, '-j', |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 192 | '%d' % multiprocessing.cpu_count() |
| 193 | ]) |
| 194 | cmd = [ |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 195 | 'bins/%s/%s' % (cfg, bm_name), |
| 196 | '--benchmark_out=%s.%s.json' % (base_json_name, cfg), |
| 197 | '--benchmark_out_format=json' |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 198 | ] |
| 199 | if args.summary_time is not None: |
| 200 | cmd += ['--benchmark_min_time=%d' % args.summary_time] |
| 201 | return subprocess.check_output(cmd) |
| 202 | |
Craig Tiller | 541b87e | 2017-03-01 08:42:52 -0800 | [diff] [blame] | 203 | |
| 204 | def collect_summary(bm_name, args): |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 205 | heading('Summary: %s [no counters]' % bm_name) |
| 206 | text(run_summary(bm_name, 'opt', bm_name)) |
| 207 | heading('Summary: %s [with counters]' % bm_name) |
| 208 | text(run_summary(bm_name, 'counters', bm_name)) |
| 209 | if args.bigquery_upload: |
| 210 | with open('%s.csv' % bm_name, 'w') as f: |
| 211 | f.write( |
| 212 | subprocess.check_output([ |
| 213 | 'tools/profiling/microbenchmarks/bm2bq.py', |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 214 | '%s.counters.json' % bm_name, |
| 215 | '%s.opt.json' % bm_name |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 216 | ])) |
| 217 | subprocess.check_call([ |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 218 | 'bq', 'load', 'microbenchmarks.microbenchmarks', |
| 219 | '%s.csv' % bm_name |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 220 | ]) |
| 221 | |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 222 | |
| 223 | collectors = { |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 224 | 'latency': collect_latency, |
| 225 | 'perf': collect_perf, |
| 226 | 'summary': collect_summary, |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 227 | } |
| 228 | |
| 229 | argp = argparse.ArgumentParser(description='Collect data from microbenchmarks') |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 230 | argp.add_argument( |
| 231 | '-c', |
| 232 | '--collect', |
| 233 | choices=sorted(collectors.keys()), |
| 234 | nargs='*', |
| 235 | default=sorted(collectors.keys()), |
| 236 | help='Which collectors should be run against each benchmark') |
| 237 | argp.add_argument( |
| 238 | '-b', |
| 239 | '--benchmarks', |
| 240 | choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
| 241 | default=bm_constants._AVAILABLE_BENCHMARK_TESTS, |
| 242 | nargs='+', |
| 243 | type=str, |
| 244 | help='Which microbenchmarks should be run') |
| 245 | argp.add_argument( |
| 246 | '--bigquery_upload', |
| 247 | default=False, |
| 248 | action='store_const', |
| 249 | const=True, |
| 250 | help='Upload results from summary collection to bigquery') |
| 251 | argp.add_argument( |
| 252 | '--summary_time', |
| 253 | default=None, |
| 254 | type=int, |
| 255 | help='Minimum time to run benchmarks for the summary collection') |
Craig Tiller | aa64ddf | 2017-02-08 14:20:08 -0800 | [diff] [blame] | 256 | args = argp.parse_args() |
| 257 | |
Craig Tiller | b432885 | 2017-03-08 13:08:40 -0800 | [diff] [blame] | 258 | try: |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 259 | for collect in args.collect: |
| 260 | for bm_name in args.benchmarks: |
| 261 | collectors[collect](bm_name, args) |
Craig Tiller | b432885 | 2017-03-08 13:08:40 -0800 | [diff] [blame] | 262 | finally: |
ncteisen | 888093c | 2017-12-11 18:00:40 -0800 | [diff] [blame] | 263 | if not os.path.exists('reports'): |
| 264 | os.makedirs('reports') |
| 265 | index_html += "</body>\n</html>\n" |
| 266 | with open('reports/index.html', 'w') as f: |
| 267 | f.write(index_html) |