Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / grpc-grpc / 888093c6ed0d24eed699173b5fb35235fe7a6069 / . / tools / run_tests / run_microbenchmark.py
blob: 561217ceb1fa0f8ecb800eaef172eb0192804462 [file] [log] [blame]
Siddharth Shukla8e64d902017-03-12 19:50:18 +0100[diff] [blame]1#!/usr/bin/env python
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]2# Copyright 2017 gRPC authors.
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]3#
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]7#
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]8# http://www.apache.org/licenses/LICENSE-2.0
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]9#
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]15
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]16import cgi
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]17import multiprocessing
18import os
19import subprocess
20import sys
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]21import argparse
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]22
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]23import python_utils.jobset as jobset
24import python_utils.start_port_server as start_port_server
25
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]26sys.path.append(
27 os.path.join(
28 os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks',
29 'bm_diff'))
ncteisen64637b72017-05-09 14:23:16 -0700[diff] [blame]30import bm_constants
Matt Kwongd0ee10d2017-03-10 22:37:52 -0800[diff] [blame]31
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]32flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')
33
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]34os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
35if not os.path.exists('reports'):
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]36 os.makedirs('reports')
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]37
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]38start_port_server.start_port_server()
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]39
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]40
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]41def fnize(s):
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]42 out = ''
43 for c in s:
44 if c in '<>, /':
45 if len(out) and out[-1] == '_': continue
46 out += '_'
47 else:
48 out += c
49 return out
50
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]51
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]52# index html
53index_html = """
54<html>
55<head>
56<title>Microbenchmark Results</title>
57</head>
58<body>
59"""
60
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]61
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]62def heading(name):
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]63 global index_html
64 index_html += "<h1>%s</h1>\n" % name
65
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]66
67def link(txt, tgt):
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]68 global index_html
69 index_html += "<p><a href=\"%s\">%s</a></p>\n" % (
70 cgi.escape(tgt, quote=True), cgi.escape(txt))
71
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]72
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]73def text(txt):
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]74 global index_html
75 index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)
76
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]77
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]78def collect_latency(bm_name, args):
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]79 """generate latency profiles"""
80 benchmarks = []
81 profile_analysis = []
82 cleanup = []
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]83
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]84 heading('Latency Profiles: %s' % bm_name)
85 subprocess.check_call([
86 'make', bm_name, 'CONFIG=basicprof', '-j',
87 '%d' % multiprocessing.cpu_count()
88 ])
89 for line in subprocess.check_output(
90 ['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines():
91 link(line, '%s.txt' % fnize(line))
92 benchmarks.append(
93 jobset.JobSpec(
94 [
95 'bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' %
96 line, '--benchmark_min_time=0.05'
97 ],
98 environ={'LATENCY_TRACE': '%s.trace' % fnize(line)},
99 shortname='profile-%s' % fnize(line)))
100 profile_analysis.append(
101 jobset.JobSpec(
102 [
103 sys.executable,
104 'tools/profiling/latency_profile/profile_analyzer.py',
105 '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
106 '--out', 'reports/%s.txt' % fnize(line)
107 ],
108 timeout_seconds=20 * 60,
109 shortname='analyze-%s' % fnize(line)))
110 cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
111 # periodically flush out the list of jobs: profile_analysis jobs at least
112 # consume upwards of five gigabytes of ram in some cases, and so analysing
113 # hundreds of them at once is impractical -- but we want at least some
114 # concurrency or the work takes too long
115 if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
116 # run up to half the cpu count: each benchmark can use up to two cores
117 # (one for the microbenchmark, one for the data flush)
118 jobset.run(
119 benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2))
120 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
121 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
122 benchmarks = []
123 profile_analysis = []
124 cleanup = []
125 # run the remaining benchmarks that weren't flushed
126 if len(benchmarks):
127 jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2))
128 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
129 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
130
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]131
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]132def collect_perf(bm_name, args):
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]133 """generate flamegraphs"""
134 heading('Flamegraphs: %s' % bm_name)
135 subprocess.check_call([
136 'make', bm_name, 'CONFIG=mutrace', '-j',
137 '%d' % multiprocessing.cpu_count()
138 ])
139 benchmarks = []
140 profile_analysis = []
141 cleanup = []
142 for line in subprocess.check_output(
143 ['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines():
144 link(line, '%s.svg' % fnize(line))
145 benchmarks.append(
146 jobset.JobSpec(
147 [
148 'perf', 'record', '-o', '%s-perf.data' % fnize(
149 line), '-g', '-F', '997', 'bins/mutrace/%s' % bm_name,
150 '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10'
151 ],
152 shortname='perf-%s' % fnize(line)))
153 profile_analysis.append(
154 jobset.JobSpec(
155 [
156 'tools/run_tests/performance/process_local_perf_flamegraphs.sh'
157 ],
158 environ={
159 'PERF_BASE_NAME': fnize(line),
160 'OUTPUT_DIR': 'reports',
161 'OUTPUT_FILENAME': fnize(line),
162 },
163 shortname='flame-%s' % fnize(line)))
164 cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
165 cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
166 # periodically flush out the list of jobs: temporary space required for this
167 # processing is large
168 if len(benchmarks) >= 20:
169 # run up to half the cpu count: each benchmark can use up to two cores
170 # (one for the microbenchmark, one for the data flush)
171 jobset.run(benchmarks, maxjobs=1)
172 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
173 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
174 benchmarks = []
175 profile_analysis = []
176 cleanup = []
177 # run the remaining benchmarks that weren't flushed
178 if len(benchmarks):
179 jobset.run(benchmarks, maxjobs=1)
180 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
181 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
182
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]183
Craig Tillerff84b362017-03-01 14:11:15 -0800[diff] [blame]184def run_summary(bm_name, cfg, base_json_name):
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]185 subprocess.check_call([
186 'make', bm_name, 'CONFIG=%s' % cfg, '-j',
187 '%d' % multiprocessing.cpu_count()
188 ])
189 cmd = [
190 'bins/%s/%s' % (cfg, bm_name), '--benchmark_out=%s.%s.json' %
191 (base_json_name, cfg), '--benchmark_out_format=json'
192 ]
193 if args.summary_time is not None:
194 cmd += ['--benchmark_min_time=%d' % args.summary_time]
195 return subprocess.check_output(cmd)
196
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]197
198def collect_summary(bm_name, args):
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]199 heading('Summary: %s [no counters]' % bm_name)
200 text(run_summary(bm_name, 'opt', bm_name))
201 heading('Summary: %s [with counters]' % bm_name)
202 text(run_summary(bm_name, 'counters', bm_name))
203 if args.bigquery_upload:
204 with open('%s.csv' % bm_name, 'w') as f:
205 f.write(
206 subprocess.check_output([
207 'tools/profiling/microbenchmarks/bm2bq.py',
208 '%s.counters.json' % bm_name, '%s.opt.json' % bm_name
209 ]))
210 subprocess.check_call([
211 'bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name
212 ])
213
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]214
215collectors = {
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]216 'latency': collect_latency,
217 'perf': collect_perf,
218 'summary': collect_summary,
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]219}
220
221argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]222argp.add_argument(
223 '-c',
224 '--collect',
225 choices=sorted(collectors.keys()),
226 nargs='*',
227 default=sorted(collectors.keys()),
228 help='Which collectors should be run against each benchmark')
229argp.add_argument(
230 '-b',
231 '--benchmarks',
232 choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
233 default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
234 nargs='+',
235 type=str,
236 help='Which microbenchmarks should be run')
237argp.add_argument(
238 '--bigquery_upload',
239 default=False,
240 action='store_const',
241 const=True,
242 help='Upload results from summary collection to bigquery')
243argp.add_argument(
244 '--summary_time',
245 default=None,
246 type=int,
247 help='Minimum time to run benchmarks for the summary collection')
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]248args = argp.parse_args()
249
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]250try:
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]251 for collect in args.collect:
252 for bm_name in args.benchmarks:
253 collectors[collect](bm_name, args)
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]254finally:
ncteisen888093c2017-12-11 18:00:40 -0800[diff] [blame^]255 if not os.path.exists('reports'):
256 os.makedirs('reports')
257 index_html += "</body>\n</html>\n"
258 with open('reports/index.html', 'w') as f:
259 f.write(index_html)