Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / grpc-grpc / 15494b5d3e2b4ed8d8dc2243f32d06c41d668e9d / . / tools / run_tests / run_microbenchmark.py
blob: 312abd59c48120f2c243a9128c515848215beb58 [file] [log] [blame]
Siddharth Shukla8e64d902017-03-12 19:50:18 +0100[diff] [blame]1#!/usr/bin/env python
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]2# Copyright 2017 gRPC authors.
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]3#
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]7#
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]8# http://www.apache.org/licenses/LICENSE-2.0
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]9#
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]15
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]16import cgi
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]17import multiprocessing
18import os
19import subprocess
20import sys
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]21import argparse
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]22
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]23import python_utils.jobset as jobset
24import python_utils.start_port_server as start_port_server
25
ncteisen64637b72017-05-09 14:23:16 -0700[diff] [blame]26sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks', 'bm_diff'))
27import bm_constants
Matt Kwongd0ee10d2017-03-10 22:37:52 -0800[diff] [blame]28
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]29flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')
30
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]31os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
32if not os.path.exists('reports'):
33 os.makedirs('reports')
34
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]35start_port_server.start_port_server()
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]36
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]37def fnize(s):
38 out = ''
39 for c in s:
40 if c in '<>, /':
41 if len(out) and out[-1] == '_': continue
42 out += '_'
43 else:
44 out += c
45 return out
46
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]47# index html
48index_html = """
49<html>
50<head>
51<title>Microbenchmark Results</title>
52</head>
53<body>
54"""
55
56def heading(name):
57 global index_html
58 index_html += "<h1>%s</h1>\n" % name
59
60def link(txt, tgt):
61 global index_html
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]62 index_html += "<p><a href=\"%s\">%s</a></p>\n" % (
63 cgi.escape(tgt, quote=True), cgi.escape(txt))
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]64
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]65def text(txt):
66 global index_html
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]67 index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]68
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]69def collect_latency(bm_name, args):
70 """generate latency profiles"""
71 benchmarks = []
72 profile_analysis = []
73 cleanup = []
74
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]75 heading('Latency Profiles: %s' % bm_name)
76 subprocess.check_call(
77 ['make', bm_name,
78 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])
79 for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,
80 '--benchmark_list_tests']).splitlines():
Craig Tiller39401792017-02-02 12:22:07 -0800[diff] [blame]81 link(line, '%s.txt' % fnize(line))
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]82 benchmarks.append(
Craig Tillerece502f2017-02-17 16:20:50 -0800[diff] [blame]83 jobset.JobSpec(['bins/basicprof/%s' % bm_name,
84 '--benchmark_filter=^%s$' % line,
85 '--benchmark_min_time=0.05'],
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]86 environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}))
87 profile_analysis.append(
88 jobset.JobSpec([sys.executable,
89 'tools/profiling/latency_profile/profile_analyzer.py',
90 '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
91 '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None))
Craig Tiller715e43b2017-02-07 11:13:16 -0800[diff] [blame]92 cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
Craig Tiller360c0d52017-02-08 13:36:44 -0800[diff] [blame]93 # periodically flush out the list of jobs: profile_analysis jobs at least
94 # consume upwards of five gigabytes of ram in some cases, and so analysing
95 # hundreds of them at once is impractical -- but we want at least some
96 # concurrency or the work takes too long
Craig Tillerece502f2017-02-17 16:20:50 -0800[diff] [blame]97 if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
Craig Tiller360c0d52017-02-08 13:36:44 -0800[diff] [blame]98 # run up to half the cpu count: each benchmark can use up to two cores
99 # (one for the microbenchmark, one for the data flush)
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]100 jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
Craig Tiller6911d082017-02-07 10:30:44 -0800[diff] [blame]101 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
102 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
103 benchmarks = []
104 profile_analysis = []
105 cleanup = []
Craig Tiller360c0d52017-02-08 13:36:44 -0800[diff] [blame]106 # run the remaining benchmarks that weren't flushed
Craig Tiller6911d082017-02-07 10:30:44 -0800[diff] [blame]107 if len(benchmarks):
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]108 jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
Craig Tiller6911d082017-02-07 10:30:44 -0800[diff] [blame]109 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
110 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]111
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]112def collect_perf(bm_name, args):
113 """generate flamegraphs"""
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]114 heading('Flamegraphs: %s' % bm_name)
115 subprocess.check_call(
116 ['make', bm_name,
117 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]118 benchmarks = []
119 profile_analysis = []
120 cleanup = []
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]121 for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,
122 '--benchmark_list_tests']).splitlines():
Craig Tiller5a8c5862017-02-15 07:59:05 -0800[diff] [blame]123 link(line, '%s.svg' % fnize(line))
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]124 benchmarks.append(
125 jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),
Craig Tillerc2c0c6f2017-02-15 11:27:37 -0800[diff] [blame]126 '-g', '-F', '997',
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]127 'bins/mutrace/%s' % bm_name,
128 '--benchmark_filter=^%s$' % line,
129 '--benchmark_min_time=10']))
130 profile_analysis.append(
131 jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],
132 environ = {
133 'PERF_BASE_NAME': fnize(line),
134 'OUTPUT_DIR': 'reports',
135 'OUTPUT_FILENAME': fnize(line),
136 }))
137 cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
138 cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
139 # periodically flush out the list of jobs: temporary space required for this
140 # processing is large
141 if len(benchmarks) >= 20:
142 # run up to half the cpu count: each benchmark can use up to two cores
143 # (one for the microbenchmark, one for the data flush)
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]144 jobset.run(benchmarks, maxjobs=1)
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]145 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
146 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
147 benchmarks = []
148 profile_analysis = []
149 cleanup = []
150 # run the remaining benchmarks that weren't flushed
151 if len(benchmarks):
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]152 jobset.run(benchmarks, maxjobs=1)
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]153 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
154 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]155
Craig Tillerff84b362017-03-01 14:11:15 -0800[diff] [blame]156def run_summary(bm_name, cfg, base_json_name):
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]157 subprocess.check_call(
158 ['make', bm_name,
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]159 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])
160 cmd = ['bins/%s/%s' % (cfg, bm_name),
Craig Tillerff84b362017-03-01 14:11:15 -0800[diff] [blame]161 '--benchmark_out=%s.%s.json' % (base_json_name, cfg),
Craig Tillerd9bc2102017-02-15 08:24:55 -0800[diff] [blame]162 '--benchmark_out_format=json']
163 if args.summary_time is not None:
164 cmd += ['--benchmark_min_time=%d' % args.summary_time]
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]165 return subprocess.check_output(cmd)
166
167def collect_summary(bm_name, args):
168 heading('Summary: %s [no counters]' % bm_name)
Craig Tiller26995eb2017-03-08 13:10:28 -0800[diff] [blame]169 text(run_summary(bm_name, 'opt', bm_name))
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]170 heading('Summary: %s [with counters]' % bm_name)
Craig Tiller26995eb2017-03-08 13:10:28 -0800[diff] [blame]171 text(run_summary(bm_name, 'counters', bm_name))
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]172 if args.bigquery_upload:
Craig Tiller26995eb2017-03-08 13:10:28 -0800[diff] [blame]173 with open('%s.csv' % bm_name, 'w') as f:
174 f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py',
175 '%s.counters.json' % bm_name,
176 '%s.opt.json' % bm_name]))
177 subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name])
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]178
179collectors = {
180 'latency': collect_latency,
181 'perf': collect_perf,
182 'summary': collect_summary,
183}
184
185argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
186argp.add_argument('-c', '--collect',
187 choices=sorted(collectors.keys()),
Craig Tiller5ef448d2017-03-01 14:12:47 -0800[diff] [blame]188 nargs='*',
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]189 default=sorted(collectors.keys()),
190 help='Which collectors should be run against each benchmark')
191argp.add_argument('-b', '--benchmarks',
ncteisen64637b72017-05-09 14:23:16 -0700[diff] [blame]192 choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
193 default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]194 nargs='+',
195 type=str,
196 help='Which microbenchmarks should be run')
197argp.add_argument('--bigquery_upload',
198 default=False,
199 action='store_const',
200 const=True,
201 help='Upload results from summary collection to bigquery')
Craig Tillerd9bc2102017-02-15 08:24:55 -0800[diff] [blame]202argp.add_argument('--summary_time',
203 default=None,
204 type=int,
205 help='Minimum time to run benchmarks for the summary collection')
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]206args = argp.parse_args()
207
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]208try:
Craig Tiller47f37f32017-03-10 10:44:56 -0800[diff] [blame]209 for collect in args.collect:
210 for bm_name in args.benchmarks:
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]211 collectors[collect](bm_name, args)
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]212finally:
Matt Kwongaff1c052017-03-09 15:08:01 -0800[diff] [blame]213 if not os.path.exists('reports'):
214 os.makedirs('reports')
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]215 index_html += "</body>\n</html>\n"
216 with open('reports/index.html', 'w') as f:
217 f.write(index_html)