Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / grpc-grpc / 64637b7c8c040bfad7043b62b462612eb0b88a22 / . / tools / run_tests / run_microbenchmark.py
blob: dadebb1b54b564baa20b5cad0ebb3a79ae36d9dc [file] [log] [blame]
Siddharth Shukla8e64d902017-03-12 19:50:18 +0100[diff] [blame]1#!/usr/bin/env python
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]2# Copyright 2017, Google Inc.
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9# * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11# * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15# * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]31import cgi
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]32import multiprocessing
33import os
34import subprocess
35import sys
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]36import argparse
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]37
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]38import python_utils.jobset as jobset
39import python_utils.start_port_server as start_port_server
40
ncteisen64637b72017-05-09 14:23:16 -0700[diff] [blame^]41sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks', 'bm_diff'))
42import bm_constants
Matt Kwongd0ee10d2017-03-10 22:37:52 -0800[diff] [blame]43
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]44flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')
45
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]46os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
47if not os.path.exists('reports'):
48 os.makedirs('reports')
49
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]50start_port_server.start_port_server()
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]51
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]52def fnize(s):
53 out = ''
54 for c in s:
55 if c in '<>, /':
56 if len(out) and out[-1] == '_': continue
57 out += '_'
58 else:
59 out += c
60 return out
61
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]62# index html
63index_html = """
64<html>
65<head>
66<title>Microbenchmark Results</title>
67</head>
68<body>
69"""
70
71def heading(name):
72 global index_html
73 index_html += "<h1>%s</h1>\n" % name
74
75def link(txt, tgt):
76 global index_html
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]77 index_html += "<p><a href=\"%s\">%s</a></p>\n" % (
78 cgi.escape(tgt, quote=True), cgi.escape(txt))
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]79
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]80def text(txt):
81 global index_html
Craig Tiller891e8162017-02-15 23:30:27 -0800[diff] [blame]82 index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]83
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]84def collect_latency(bm_name, args):
85 """generate latency profiles"""
86 benchmarks = []
87 profile_analysis = []
88 cleanup = []
89
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]90 heading('Latency Profiles: %s' % bm_name)
91 subprocess.check_call(
92 ['make', bm_name,
93 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])
94 for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,
95 '--benchmark_list_tests']).splitlines():
Craig Tiller39401792017-02-02 12:22:07 -0800[diff] [blame]96 link(line, '%s.txt' % fnize(line))
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]97 benchmarks.append(
Craig Tillerece502f2017-02-17 16:20:50 -0800[diff] [blame]98 jobset.JobSpec(['bins/basicprof/%s' % bm_name,
99 '--benchmark_filter=^%s$' % line,
100 '--benchmark_min_time=0.05'],
Craig Tiller7dc4ea62017-02-02 16:08:05 -0800[diff] [blame]101 environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}))
102 profile_analysis.append(
103 jobset.JobSpec([sys.executable,
104 'tools/profiling/latency_profile/profile_analyzer.py',
105 '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
106 '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None))
Craig Tiller715e43b2017-02-07 11:13:16 -0800[diff] [blame]107 cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
Craig Tiller360c0d52017-02-08 13:36:44 -0800[diff] [blame]108 # periodically flush out the list of jobs: profile_analysis jobs at least
109 # consume upwards of five gigabytes of ram in some cases, and so analysing
110 # hundreds of them at once is impractical -- but we want at least some
111 # concurrency or the work takes too long
Craig Tillerece502f2017-02-17 16:20:50 -0800[diff] [blame]112 if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
Craig Tiller360c0d52017-02-08 13:36:44 -0800[diff] [blame]113 # run up to half the cpu count: each benchmark can use up to two cores
114 # (one for the microbenchmark, one for the data flush)
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]115 jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
Craig Tiller6911d082017-02-07 10:30:44 -0800[diff] [blame]116 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
117 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
118 benchmarks = []
119 profile_analysis = []
120 cleanup = []
Craig Tiller360c0d52017-02-08 13:36:44 -0800[diff] [blame]121 # run the remaining benchmarks that weren't flushed
Craig Tiller6911d082017-02-07 10:30:44 -0800[diff] [blame]122 if len(benchmarks):
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]123 jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
Craig Tiller6911d082017-02-07 10:30:44 -0800[diff] [blame]124 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
125 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]126
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]127def collect_perf(bm_name, args):
128 """generate flamegraphs"""
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]129 heading('Flamegraphs: %s' % bm_name)
130 subprocess.check_call(
131 ['make', bm_name,
132 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]133 benchmarks = []
134 profile_analysis = []
135 cleanup = []
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]136 for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,
137 '--benchmark_list_tests']).splitlines():
Craig Tiller5a8c5862017-02-15 07:59:05 -0800[diff] [blame]138 link(line, '%s.svg' % fnize(line))
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]139 benchmarks.append(
140 jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),
Craig Tillerc2c0c6f2017-02-15 11:27:37 -0800[diff] [blame]141 '-g', '-F', '997',
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]142 'bins/mutrace/%s' % bm_name,
143 '--benchmark_filter=^%s$' % line,
144 '--benchmark_min_time=10']))
145 profile_analysis.append(
146 jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],
147 environ = {
148 'PERF_BASE_NAME': fnize(line),
149 'OUTPUT_DIR': 'reports',
150 'OUTPUT_FILENAME': fnize(line),
151 }))
152 cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
153 cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
154 # periodically flush out the list of jobs: temporary space required for this
155 # processing is large
156 if len(benchmarks) >= 20:
157 # run up to half the cpu count: each benchmark can use up to two cores
158 # (one for the microbenchmark, one for the data flush)
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]159 jobset.run(benchmarks, maxjobs=1)
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]160 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
161 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
162 benchmarks = []
163 profile_analysis = []
164 cleanup = []
165 # run the remaining benchmarks that weren't flushed
166 if len(benchmarks):
Craig Tillercba864b2017-02-17 10:27:56 -0800[diff] [blame]167 jobset.run(benchmarks, maxjobs=1)
Craig Tiller6ad00722017-02-15 09:14:24 -0800[diff] [blame]168 jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
169 jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
Craig Tillerf7af2a92017-01-31 15:08:31 -0800[diff] [blame]170
Craig Tillerff84b362017-03-01 14:11:15 -0800[diff] [blame]171def run_summary(bm_name, cfg, base_json_name):
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]172 subprocess.check_call(
173 ['make', bm_name,
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]174 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])
175 cmd = ['bins/%s/%s' % (cfg, bm_name),
Craig Tillerff84b362017-03-01 14:11:15 -0800[diff] [blame]176 '--benchmark_out=%s.%s.json' % (base_json_name, cfg),
Craig Tillerd9bc2102017-02-15 08:24:55 -0800[diff] [blame]177 '--benchmark_out_format=json']
178 if args.summary_time is not None:
179 cmd += ['--benchmark_min_time=%d' % args.summary_time]
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]180 return subprocess.check_output(cmd)
181
182def collect_summary(bm_name, args):
183 heading('Summary: %s [no counters]' % bm_name)
Craig Tiller26995eb2017-03-08 13:10:28 -0800[diff] [blame]184 text(run_summary(bm_name, 'opt', bm_name))
Craig Tiller541b87e2017-03-01 08:42:52 -0800[diff] [blame]185 heading('Summary: %s [with counters]' % bm_name)
Craig Tiller26995eb2017-03-08 13:10:28 -0800[diff] [blame]186 text(run_summary(bm_name, 'counters', bm_name))
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]187 if args.bigquery_upload:
Craig Tiller26995eb2017-03-08 13:10:28 -0800[diff] [blame]188 with open('%s.csv' % bm_name, 'w') as f:
189 f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py',
190 '%s.counters.json' % bm_name,
191 '%s.opt.json' % bm_name]))
192 subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name])
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]193
194collectors = {
195 'latency': collect_latency,
196 'perf': collect_perf,
197 'summary': collect_summary,
198}
199
200argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
201argp.add_argument('-c', '--collect',
202 choices=sorted(collectors.keys()),
Craig Tiller5ef448d2017-03-01 14:12:47 -0800[diff] [blame]203 nargs='*',
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]204 default=sorted(collectors.keys()),
205 help='Which collectors should be run against each benchmark')
206argp.add_argument('-b', '--benchmarks',
ncteisen64637b72017-05-09 14:23:16 -0700[diff] [blame^]207 choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
208 default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]209 nargs='+',
210 type=str,
211 help='Which microbenchmarks should be run')
212argp.add_argument('--bigquery_upload',
213 default=False,
214 action='store_const',
215 const=True,
216 help='Upload results from summary collection to bigquery')
Craig Tillerd9bc2102017-02-15 08:24:55 -0800[diff] [blame]217argp.add_argument('--summary_time',
218 default=None,
219 type=int,
220 help='Minimum time to run benchmarks for the summary collection')
Craig Tilleraa64ddf2017-02-08 14:20:08 -0800[diff] [blame]221args = argp.parse_args()
222
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]223try:
Craig Tiller47f37f32017-03-10 10:44:56 -0800[diff] [blame]224 for collect in args.collect:
225 for bm_name in args.benchmarks:
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]226 collectors[collect](bm_name, args)
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]227finally:
Matt Kwongaff1c052017-03-09 15:08:01 -0800[diff] [blame]228 if not os.path.exists('reports'):
229 os.makedirs('reports')
Craig Tillerb4328852017-03-08 13:08:40 -0800[diff] [blame]230 index_html += "</body>\n</html>\n"
231 with open('reports/index.html', 'w') as f:
232 f.write(index_html)