Blame - tools/run_tests/run_microbenchmark.py - platform/external/grpc-grpc

2017-01-31 15:08:31 -0800

[diff] [blame]

#

# Redistribution and use in source and binary forms, with or without

6

# modification, are permitted provided that the following conditions are

7

# met:

8

#

9

# * Redistributions of source code must retain the above copyright

10

# notice, this list of conditions and the following disclaimer.

11

# * Redistributions in binary form must reproduce the above

12

# copyright notice, this list of conditions and the following disclaimer

13

# in the documentation and/or other materials provided with the

14

# distribution.

15

# * Neither the name of Google Inc. nor the names of its

16

# contributors may be used to endorse or promote products derived from

17

# this software without specific prior written permission.

18

#

19

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

20

# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

21

# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

22

# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

23

# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

24

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

25

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

26

# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

27

# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

28

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

29

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

30

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

31

import cgi

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

32

import multiprocessing

33

import os

34

import subprocess

35

import sys

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

36

import argparse

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

37

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

38

import python_utils.jobset as jobset

39

import python_utils.start_port_server as start_port_server

40

Matt Kwong

d0ee10d

2017-03-10 22:37:52 -0800

[diff] [blame]

41

_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong',

42

'bm_fullstack_streaming_ping_pong',

43

'bm_fullstack_streaming_pump',

'bm_closure',

'bm_cq',

'bm_call_create',

'bm_error',

'bm_chttp2_hpack',

Matt Kwong

063adf3

2017-03-22 12:48:34 -0700

[diff] [blame]

49

'bm_chttp2_transport',

50

'bm_pollset',

Matt Kwong

d0ee10d

2017-03-10 22:37:52 -0800

[diff] [blame]

51

'bm_metadata',

52

'bm_fullstack_trickle']

53

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

54

flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')

55

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

56

os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))

57

if not os.path.exists('reports'):

58

os.makedirs('reports')

59

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

60

start_port_server.start_port_server()

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

61

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

def fnize(s):

out = ''

for c in s:

if c in '<>, /':

if len(out) and out[-1] == '_': continue

out += '_'

else:

out += c

return out

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

# index html

index_html = """

<html>

<head>

<title>Microbenchmark Results</title>

</head>

<body>

"""

def heading(name):

global index_html

index_html += "<h1>%s</h1>\n" % name

84

85

def link(txt, tgt):

86

global index_html

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

87

index_html += "<p><a href=\"%s\">%s</a></p>\n" % (

88

cgi.escape(tgt, quote=True), cgi.escape(txt))

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

89

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

90

def text(txt):

91

global index_html

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

92

index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

93

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

94

def collect_latency(bm_name, args):

95

"""generate latency profiles"""

96

benchmarks = []

97

profile_analysis = []

98

cleanup = []

99

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

100

heading('Latency Profiles: %s' % bm_name)

101

subprocess.check_call(

102

['make', bm_name,

103

'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])

104

for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,

105

'--benchmark_list_tests']).splitlines():

Craig Tiller

3940179

2017-02-02 12:22:07 -0800

[diff] [blame]

106

link(line, '%s.txt' % fnize(line))

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

107

benchmarks.append(

Craig Tiller

ece502f

2017-02-17 16:20:50 -0800

[diff] [blame]

108

jobset.JobSpec(['bins/basicprof/%s' % bm_name,

109

'--benchmark_filter=^%s$' % line,

110

'--benchmark_min_time=0.05'],

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

111

environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}))

112

profile_analysis.append(

113

jobset.JobSpec([sys.executable,

114

'tools/profiling/latency_profile/profile_analyzer.py',

115

'--source', '%s.trace' % fnize(line), '--fmt', 'simple',

116

'--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None))

Craig Tiller

715e43b

2017-02-07 11:13:16 -0800

[diff] [blame]

117

cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))

Craig Tiller

360c0d5

2017-02-08 13:36:44 -0800

[diff] [blame]

118

# periodically flush out the list of jobs: profile_analysis jobs at least

119

# consume upwards of five gigabytes of ram in some cases, and so analysing

120

# hundreds of them at once is impractical -- but we want at least some

121

# concurrency or the work takes too long

Craig Tiller

ece502f

2017-02-17 16:20:50 -0800

[diff] [blame]

122

if len(benchmarks) >= min(16, multiprocessing.cpu_count()):

Craig Tiller

360c0d5

2017-02-08 13:36:44 -0800

[diff] [blame]

123

# run up to half the cpu count: each benchmark can use up to two cores

124

# (one for the microbenchmark, one for the data flush)

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

125

jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

126

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

127

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

128

benchmarks = []

129

profile_analysis = []

130

cleanup = []

Craig Tiller

360c0d5

2017-02-08 13:36:44 -0800

[diff] [blame]

131

# run the remaining benchmarks that weren't flushed

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

132

if len(benchmarks):

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

133

jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

134

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

135

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

136

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

137

def collect_perf(bm_name, args):

138

"""generate flamegraphs"""

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

139

heading('Flamegraphs: %s' % bm_name)

140

subprocess.check_call(

141

['make', bm_name,

142

'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

143

benchmarks = []

144

profile_analysis = []

145

cleanup = []

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

146

for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,

147

'--benchmark_list_tests']).splitlines():

Craig Tiller

5a8c586

2017-02-15 07:59:05 -0800

[diff] [blame]

148

link(line, '%s.svg' % fnize(line))

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

149

benchmarks.append(

150

jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),

Craig Tiller

c2c0c6f

2017-02-15 11:27:37 -0800

[diff] [blame]

151

'-g', '-F', '997',

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

152

'bins/mutrace/%s' % bm_name,

153

'--benchmark_filter=^%s$' % line,

154

'--benchmark_min_time=10']))

155

profile_analysis.append(

156

jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],

157

environ = {

158

'PERF_BASE_NAME': fnize(line),

159

'OUTPUT_DIR': 'reports',

160

'OUTPUT_FILENAME': fnize(line),

161

}))

162

cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))

163

cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))

164

# periodically flush out the list of jobs: temporary space required for this

165

# processing is large

166

if len(benchmarks) >= 20:

167

# run up to half the cpu count: each benchmark can use up to two cores

168

# (one for the microbenchmark, one for the data flush)

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

169

jobset.run(benchmarks, maxjobs=1)

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

170

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

171

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

172

benchmarks = []

173

profile_analysis = []

174

cleanup = []

175

# run the remaining benchmarks that weren't flushed

176

if len(benchmarks):

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

177

jobset.run(benchmarks, maxjobs=1)

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

178

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

179

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

180

Craig Tiller

ff84b36

2017-03-01 14:11:15 -0800

[diff] [blame]

181

def run_summary(bm_name, cfg, base_json_name):

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

182

subprocess.check_call(

183

['make', bm_name,

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

184

'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])

185

cmd = ['bins/%s/%s' % (cfg, bm_name),

Craig Tiller

ff84b36

2017-03-01 14:11:15 -0800

[diff] [blame]

186

'--benchmark_out=%s.%s.json' % (base_json_name, cfg),

Craig Tiller

d9bc210

2017-02-15 08:24:55 -0800

[diff] [blame]

187

'--benchmark_out_format=json']

188

if args.summary_time is not None:

189

cmd += ['--benchmark_min_time=%d' % args.summary_time]

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

190

return subprocess.check_output(cmd)

191

192

def collect_summary(bm_name, args):

193

heading('Summary: %s [no counters]' % bm_name)

Craig Tiller

26995eb

2017-03-08 13:10:28 -0800

[diff] [blame]

194

text(run_summary(bm_name, 'opt', bm_name))

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

195

heading('Summary: %s [with counters]' % bm_name)

Craig Tiller

26995eb

2017-03-08 13:10:28 -0800

[diff] [blame]

196

text(run_summary(bm_name, 'counters', bm_name))

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

197

if args.bigquery_upload:

Craig Tiller

26995eb

2017-03-08 13:10:28 -0800

[diff] [blame]

198

with open('%s.csv' % bm_name, 'w') as f:

199

f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py',

200

'%s.counters.json' % bm_name,

201

'%s.opt.json' % bm_name]))

202

subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name])

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

203

204

collectors = {

205

'latency': collect_latency,

206

'perf': collect_perf,

207

'summary': collect_summary,

208

}

209

210

argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')

211

argp.add_argument('-c', '--collect',

212

choices=sorted(collectors.keys()),

Craig Tiller

5ef448d

2017-03-01 14:12:47 -0800

[diff] [blame]

213

nargs='*',

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

214

default=sorted(collectors.keys()),

215

help='Which collectors should be run against each benchmark')

216

argp.add_argument('-b', '--benchmarks',

Matt Kwong

d0ee10d

2017-03-10 22:37:52 -0800

[diff] [blame]

217

choices=_AVAILABLE_BENCHMARK_TESTS,

218

default=_AVAILABLE_BENCHMARK_TESTS,

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

219

nargs='+',

220

type=str,

221

help='Which microbenchmarks should be run')

222

argp.add_argument('--bigquery_upload',

223

default=False,

224

action='store_const',

225

const=True,

226

help='Upload results from summary collection to bigquery')

Craig Tiller

d9bc210

2017-02-15 08:24:55 -0800

[diff] [blame]

227

argp.add_argument('--summary_time',

228

default=None,

229

type=int,

230

help='Minimum time to run benchmarks for the summary collection')

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

231

args = argp.parse_args()

232

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

233

try:

Craig Tiller

47f37f3

2017-03-10 10:44:56 -0800

[diff] [blame]

234

for collect in args.collect:

235

for bm_name in args.benchmarks:

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

236

collectors[collect](bm_name, args)

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

237

finally:

Matt Kwong

aff1c05

2017-03-09 15:08:01 -0800

[diff] [blame]

238

if not os.path.exists('reports'):

239

os.makedirs('reports')

Craig Tiller