Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / grpc-grpc / 9f9f0f82f31d7dd3cfe58d1ba5a0c181c601e5c0 / . / tools / run_tests / run_microbenchmark.py
blob: c136af58cb87740dc01aee9ddd20a6731a804680 [file] [log] [blame]
Siddharth Shukla8e64d902017-03-12 19:50:18 +0100[diff] [blame]1#!/usr/bin/env python
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]2# Copyright 2017 gRPC authors.
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]3#
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]7#
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]8# http://www.apache.org/licenses/LICENSE-2.0
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]9#
Jan Tattermusch7897ae92017-06-07 22:57:36 +0200[diff] [blame]10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]15
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]16import cgi
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]17import multiprocessing
18import os
19import subprocess
20import sys
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]21import argparse
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]22
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]23import python_utils.jobset as jobset
24import python_utils.start_port_server as start_port_server
25
ncteisen64637b72017-05-09 14:23:16 -0700[diff] [blame]26sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks', 'bm_diff'))
27import bm_constants
Matt Kwongd0ee10d2017-03-10 22:37:52 -0800[diff] [blame]28
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]29flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')
30
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]31os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
32if not os.path.exists('reports'):
33 os.makedirs('reports')
34
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]35start_port_server.start_port_server()
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]36
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]37def fnize(s):
38 out = ''
39 for c in s:
40 if c in '<>, /':
41 if len(out) and out[-1] == '_': continue
42 out += '_'
43 else:
44 out += c
45 return out
46
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]47# index html
48index_html = """
49<html>
50<head>
51<title>Microbenchmark Results</title>
52</head>
53<body>
54"""
55
56def heading(name):
57 global index_html
58 index_html += "<h1>%s</h1>\n" % name
59
60def link(txt, tgt):
61 global index_html
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]62 index_html += "<p><a href=\"%s\">%s</a></p>\n" % (
63 cgi.escape(tgt, quote=True), cgi.escape(txt))
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]64
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]65def text(txt):
66 global index_html
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]67 index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]68
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]69def collect_latency(bm_name, args):
70 """generate latency profiles"""
71 benchmarks = []
72 profile_analysis = []
73 cleanup = []
74
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]75 heading('Latency Profiles: %s' % bm_name)
76 subprocess.check_call(
77 ['make', bm_name,
78 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])
79 for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,
80 '--benchmark_list_tests']).splitlines():
Craig Tiller39401792017-02-02 12:22:07 -0800[diff] [blame]81 link(line, '%s.txt' % fnize(line))
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]82 benchmarks.append(
Craig Tillerece502f2017-02-17 16:20:50 -0800[diff] [blame]83 jobset.JobSpec(['bins/basicprof/%s' % bm_name,
84 '--benchmark_filter=^%s$' % line,
85 '--benchmark_min_time=0.05'],
Noah Eisen89563ca2017-08-28 17:37:43 -0700[diff] [blame]86 environ={'LATENCY_TRACE': '%s.trace' % fnize(line)},
87 shortname='profile-%s' % fnize(line)))
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]88 profile_analysis.append(
89 jobset.JobSpec([sys.executable,
90 'tools/profiling/latency_profile/profile_analyzer.py',
91 '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
Noah Eisen89563ca2017-08-28 17:37:43 -0700[diff] [blame]92 '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=20*60,
93 shortname='analyze-%s' % fnize(line)))
Craig Tiller715e43b2017-02-07 11:13:16 -0800[diff] [blame]94 cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
Craig Tiller360c0d52017-02-08 13:36:44 -0800[diff] [blame]95 # periodically flush out the list of jobs: profile_analysis jobs at least
96 # consume upwards of five gigabytes of ram in some cases, and so analysing
97 # hundreds of them at once is impractical -- but we want at least some
98 # concurrency or the work takes too long
Craig Tillerece502f2017-02-17 16:20:50 -0800[diff] [blame]99 if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
Craig Tiller360c0d52017-02-08 13:36:44 -0800[diff] [blame]100 # run up to half the cpu count: each benchmark can use up to two cores
101 # (one for the microbenchmark, one for the data flush)
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]102 jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
Craig Tiller6911d082017-02-07 10:30:44 -0800[diff] [blame]103 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
104 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
105 benchmarks = []
106 profile_analysis = []
107 cleanup = []
Craig Tiller360c0d52017-02-08 13:36:44 -0800[diff] [blame]108 # run the remaining benchmarks that weren't flushed
Craig Tiller6911d082017-02-07 10:30:44 -0800[diff] [blame]109 if len(benchmarks):
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]110 jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
Craig Tiller6911d082017-02-07 10:30:44 -0800[diff] [blame]111 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
112 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]113
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]114def collect_perf(bm_name, args):
115 """generate flamegraphs"""
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]116 heading('Flamegraphs: %s' % bm_name)
117 subprocess.check_call(
118 ['make', bm_name,
119 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]120 benchmarks = []
121 profile_analysis = []
122 cleanup = []
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]123 for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,
124 '--benchmark_list_tests']).splitlines():
Craig Tiller5a8c5862017-02-15 07:59:05 -0800[diff] [blame]125 link(line, '%s.svg' % fnize(line))
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]126 benchmarks.append(
127 jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),
Craig Tillerc2c0c6f2017-02-15 11:27:37 -0800[diff] [blame]128 '-g', '-F', '997',
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]129 'bins/mutrace/%s' % bm_name,
130 '--benchmark_filter=^%s$' % line,
Noah Eisen89563ca2017-08-28 17:37:43 -0700[diff] [blame]131 '--benchmark_min_time=10'],
132 shortname='perf-%s' % fnize(line)))
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]133 profile_analysis.append(
134 jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],
135 environ = {
136 'PERF_BASE_NAME': fnize(line),
137 'OUTPUT_DIR': 'reports',
138 'OUTPUT_FILENAME': fnize(line),
Noah Eisen89563ca2017-08-28 17:37:43 -0700[diff] [blame]139 },
140 shortname='flame-%s' % fnize(line)))
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]141 cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
142 cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
143 # periodically flush out the list of jobs: temporary space required for this
144 # processing is large
145 if len(benchmarks) >= 20:
146 # run up to half the cpu count: each benchmark can use up to two cores
147 # (one for the microbenchmark, one for the data flush)
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]148 jobset.run(benchmarks, maxjobs=1)
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]149 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
150 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
151 benchmarks = []
152 profile_analysis = []
153 cleanup = []
154 # run the remaining benchmarks that weren't flushed
155 if len(benchmarks):
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]156 jobset.run(benchmarks, maxjobs=1)
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]157 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
158 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]159
Craig Tillerff84b362017-03-01 14:11:15 -0800[diff] [blame]160def run_summary(bm_name, cfg, base_json_name):
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]161 subprocess.check_call(
162 ['make', bm_name,
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]163 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])
164 cmd = ['bins/%s/%s' % (cfg, bm_name),
Craig Tillerff84b362017-03-01 14:11:15 -0800[diff] [blame]165 '--benchmark_out=%s.%s.json' % (base_json_name, cfg),
Craig Tillerd9bc2102017-02-15 08:24:55 -0800[diff] [blame]166 '--benchmark_out_format=json']
167 if args.summary_time is not None:
168 cmd += ['--benchmark_min_time=%d' % args.summary_time]
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]169 return subprocess.check_output(cmd)
170
171def collect_summary(bm_name, args):
172 heading('Summary: %s [no counters]' % bm_name)
Craig Tiller26995eb2017-03-08 13:10:28 -0800[diff] [blame]173 text(run_summary(bm_name, 'opt', bm_name))
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]174 heading('Summary: %s [with counters]' % bm_name)
Craig Tiller26995eb2017-03-08 13:10:28 -0800[diff] [blame]175 text(run_summary(bm_name, 'counters', bm_name))
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]176 if args.bigquery_upload:
Craig Tiller26995eb2017-03-08 13:10:28 -0800[diff] [blame]177 with open('%s.csv' % bm_name, 'w') as f:
178 f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py',
179 '%s.counters.json' % bm_name,
180 '%s.opt.json' % bm_name]))
181 subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name])
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]182
183collectors = {
184 'latency': collect_latency,
185 'perf': collect_perf,
186 'summary': collect_summary,
187}
188
189argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
190argp.add_argument('-c', '--collect',
191 choices=sorted(collectors.keys()),
Craig Tiller5ef448d2017-03-01 14:12:47 -0800[diff] [blame]192 nargs='*',
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]193 default=sorted(collectors.keys()),
194 help='Which collectors should be run against each benchmark')
195argp.add_argument('-b', '--benchmarks',
ncteisen64637b72017-05-09 14:23:16 -0700[diff] [blame]196 choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
197 default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]198 nargs='+',
199 type=str,
200 help='Which microbenchmarks should be run')
201argp.add_argument('--bigquery_upload',
202 default=False,
203 action='store_const',
204 const=True,
205 help='Upload results from summary collection to bigquery')
Craig Tillerd9bc2102017-02-15 08:24:55 -0800[diff] [blame]206argp.add_argument('--summary_time',
207 default=None,
208 type=int,
209 help='Minimum time to run benchmarks for the summary collection')
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]210args = argp.parse_args()
211
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]212try:
Craig Tiller47f37f32017-03-10 10:44:56 -0800[diff] [blame]213 for collect in args.collect:
214 for bm_name in args.benchmarks:
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]215 collectors[collect](bm_name, args)
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]216finally:
Matt Kwongaff1c052017-03-09 15:08:01 -0800[diff] [blame]217 if not os.path.exists('reports'):
218 os.makedirs('reports')
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]219 index_html += "</body>\n</html>\n"
220 with open('reports/index.html', 'w') as f:
221 f.write(index_html)