Blame - tools/run_tests/run_microbenchmark.py - platform/external/grpc-grpc

2017-01-31 15:08:31 -0800

[diff] [blame]

#

# Redistribution and use in source and binary forms, with or without

6

# modification, are permitted provided that the following conditions are

7

# met:

8

#

9

# * Redistributions of source code must retain the above copyright

10

# notice, this list of conditions and the following disclaimer.

11

# * Redistributions in binary form must reproduce the above

12

# copyright notice, this list of conditions and the following disclaimer

13

# in the documentation and/or other materials provided with the

14

# distribution.

15

# * Neither the name of Google Inc. nor the names of its

16

# contributors may be used to endorse or promote products derived from

17

# this software without specific prior written permission.

18

#

19

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

20

# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

21

# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

22

# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

23

# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

24

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

25

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

26

# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

27

# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

28

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

29

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

30

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

31

import cgi

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

32

import multiprocessing

33

import os

34

import subprocess

35

import sys

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

36

import argparse

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

37

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

38

import python_utils.jobset as jobset

39

import python_utils.start_port_server as start_port_server

40

Matt Kwong

2017-03-10 22:37:52 -0800

[diff] [blame^]

41

_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong',

42

'bm_fullstack_streaming_ping_pong',

43

'bm_fullstack_streaming_pump',

'bm_closure',

'bm_cq',

'bm_call_create',

'bm_error',

'bm_chttp2_hpack',

'bm_metadata',

'bm_fullstack_trickle']

51

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

52

flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')

53

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

54

os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))

55

if not os.path.exists('reports'):

56

os.makedirs('reports')

57

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

58

start_port_server.start_port_server()

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

59

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

def fnize(s):

out = ''

for c in s:

if c in '<>, /':

if len(out) and out[-1] == '_': continue

out += '_'

else:

out += c

return out

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

# index html

index_html = """

<html>

<head>

<title>Microbenchmark Results</title>

</head>

<body>

"""

def heading(name):

global index_html

index_html += "<h1>%s</h1>\n" % name

82

83

def link(txt, tgt):

84

global index_html

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

85

index_html += "<p><a href=\"%s\">%s</a></p>\n" % (

86

cgi.escape(tgt, quote=True), cgi.escape(txt))

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

87

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

88

def text(txt):

89

global index_html

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

90

index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

91

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

92

def collect_latency(bm_name, args):

93

"""generate latency profiles"""

94

benchmarks = []

95

profile_analysis = []

96

cleanup = []

97

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

98

heading('Latency Profiles: %s' % bm_name)

99

subprocess.check_call(

100

['make', bm_name,

101

'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])

102

for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,

103

'--benchmark_list_tests']).splitlines():

Craig Tiller

3940179

2017-02-02 12:22:07 -0800

[diff] [blame]

104

link(line, '%s.txt' % fnize(line))

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

105

benchmarks.append(

Craig Tiller

ece502f

2017-02-17 16:20:50 -0800

[diff] [blame]

106

jobset.JobSpec(['bins/basicprof/%s' % bm_name,

107

'--benchmark_filter=^%s$' % line,

108

'--benchmark_min_time=0.05'],

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

109

environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}))

110

profile_analysis.append(

111

jobset.JobSpec([sys.executable,

112

'tools/profiling/latency_profile/profile_analyzer.py',

113

'--source', '%s.trace' % fnize(line), '--fmt', 'simple',

114

'--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None))

Craig Tiller

715e43b

2017-02-07 11:13:16 -0800

[diff] [blame]

115

cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))

Craig Tiller

360c0d5

2017-02-08 13:36:44 -0800

[diff] [blame]

116

# periodically flush out the list of jobs: profile_analysis jobs at least

117

# consume upwards of five gigabytes of ram in some cases, and so analysing

118

# hundreds of them at once is impractical -- but we want at least some

119

# concurrency or the work takes too long

Craig Tiller

ece502f

2017-02-17 16:20:50 -0800

[diff] [blame]

120

if len(benchmarks) >= min(16, multiprocessing.cpu_count()):

Craig Tiller

360c0d5

2017-02-08 13:36:44 -0800

[diff] [blame]

121

# run up to half the cpu count: each benchmark can use up to two cores

122

# (one for the microbenchmark, one for the data flush)

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

123

jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

124

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

125

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

126

benchmarks = []

127

profile_analysis = []

128

cleanup = []

Craig Tiller

360c0d5

2017-02-08 13:36:44 -0800

[diff] [blame]

129

# run the remaining benchmarks that weren't flushed

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

130

if len(benchmarks):

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

131

jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

132

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

133

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

134

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

135

def collect_perf(bm_name, args):

136

"""generate flamegraphs"""

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

137

heading('Flamegraphs: %s' % bm_name)

138

subprocess.check_call(

139

['make', bm_name,

140

'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

141

benchmarks = []

142

profile_analysis = []

143

cleanup = []

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

144

for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,

145

'--benchmark_list_tests']).splitlines():

Craig Tiller

5a8c586

2017-02-15 07:59:05 -0800

[diff] [blame]

146

link(line, '%s.svg' % fnize(line))

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

147

benchmarks.append(

148

jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),

Craig Tiller

c2c0c6f

2017-02-15 11:27:37 -0800

[diff] [blame]

149

'-g', '-F', '997',

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

150

'bins/mutrace/%s' % bm_name,

151

'--benchmark_filter=^%s$' % line,

152

'--benchmark_min_time=10']))

153

profile_analysis.append(

154

jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],

155

environ = {

156

'PERF_BASE_NAME': fnize(line),

157

'OUTPUT_DIR': 'reports',

158

'OUTPUT_FILENAME': fnize(line),

159

}))

160

cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))

161

cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))

162

# periodically flush out the list of jobs: temporary space required for this

163

# processing is large

164

if len(benchmarks) >= 20:

165

# run up to half the cpu count: each benchmark can use up to two cores

166

# (one for the microbenchmark, one for the data flush)

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

167

jobset.run(benchmarks, maxjobs=1)

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

168

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

169

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

170

benchmarks = []

171

profile_analysis = []

172

cleanup = []

173

# run the remaining benchmarks that weren't flushed

174

if len(benchmarks):

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

175

jobset.run(benchmarks, maxjobs=1)

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

176

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

177

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

178

Craig Tiller

ff84b36

2017-03-01 14:11:15 -0800

[diff] [blame]

179

def run_summary(bm_name, cfg, base_json_name):

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

180

subprocess.check_call(

181

['make', bm_name,

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

182

'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])

183

cmd = ['bins/%s/%s' % (cfg, bm_name),

Craig Tiller

ff84b36

2017-03-01 14:11:15 -0800

[diff] [blame]

184

'--benchmark_out=%s.%s.json' % (base_json_name, cfg),

Craig Tiller

d9bc210

2017-02-15 08:24:55 -0800

[diff] [blame]

185

'--benchmark_out_format=json']

186

if args.summary_time is not None:

187

cmd += ['--benchmark_min_time=%d' % args.summary_time]

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

188

return subprocess.check_output(cmd)

189

190

def collect_summary(bm_name, args):

191

heading('Summary: %s [no counters]' % bm_name)

Craig Tiller

26995eb

2017-03-08 13:10:28 -0800

[diff] [blame]

192

text(run_summary(bm_name, 'opt', bm_name))

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

193

heading('Summary: %s [with counters]' % bm_name)

Craig Tiller

26995eb

2017-03-08 13:10:28 -0800

[diff] [blame]

194

text(run_summary(bm_name, 'counters', bm_name))

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

195

if args.bigquery_upload:

Craig Tiller

26995eb

2017-03-08 13:10:28 -0800

[diff] [blame]

196

with open('%s.csv' % bm_name, 'w') as f:

197

f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py',

198

'%s.counters.json' % bm_name,

199

'%s.opt.json' % bm_name]))

200

subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name])

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

201

202

collectors = {

203

'latency': collect_latency,

204

'perf': collect_perf,

205

'summary': collect_summary,

206

}

207

208

argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')

209

argp.add_argument('-c', '--collect',

210

choices=sorted(collectors.keys()),

Craig Tiller

5ef448d

2017-03-01 14:12:47 -0800

[diff] [blame]

211

nargs='*',

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

212

default=sorted(collectors.keys()),

213

help='Which collectors should be run against each benchmark')

214

argp.add_argument('-b', '--benchmarks',

Matt Kwong

2017-03-10 22:37:52 -0800

[diff] [blame^]

215

choices=_AVAILABLE_BENCHMARK_TESTS,

216

default=_AVAILABLE_BENCHMARK_TESTS,

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

217

nargs='+',

218

type=str,

219

help='Which microbenchmarks should be run')

Craig Tiller

d753f45

2017-03-01 14:00:35 -0800

[diff] [blame]

220

argp.add_argument('--diff_perf',

221

default=None,

222

type=str,

223

help='Diff microbenchmarks against this git revision')

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

224

argp.add_argument('--bigquery_upload',

225

default=False,

226

action='store_const',

227

const=True,

228

help='Upload results from summary collection to bigquery')

Craig Tiller

d9bc210

2017-02-15 08:24:55 -0800

[diff] [blame]

229

argp.add_argument('--summary_time',

230

default=None,

231

type=int,

232

help='Minimum time to run benchmarks for the summary collection')

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

233

args = argp.parse_args()

234

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

235

try:

Craig Tiller

47f37f3

2017-03-10 10:44:56 -0800

[diff] [blame]

236

for collect in args.collect:

237

for bm_name in args.benchmarks:

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

238

collectors[collect](bm_name, args)

239

if args.diff_perf:

Matt Kwong

2017-03-10 22:37:52 -0800

[diff] [blame^]

240

git_comment = 'Performance differences between this PR and %s\\n' % args.diff_perf

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

241

if 'summary' not in args.collect:

242

for bm_name in args.benchmarks:

243

run_summary(bm_name, 'opt', bm_name)

244

run_summary(bm_name, 'counters', bm_name)

245

where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()

Matt Kwong

2017-03-10 22:37:52 -0800

[diff] [blame^]

246

# todo(mattkwong): uncomment this before merging

247

# subprocess.check_call(['git', 'checkout', args.diff_perf])

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

248

comparables = []

249

subprocess.check_call(['make', 'clean'])

250

try:

251

for bm_name in args.benchmarks:

252

try:

253

run_summary(bm_name, 'opt', '%s.old' % bm_name)

254

run_summary(bm_name, 'counters', '%s.old' % bm_name)

255

comparables.append(bm_name)

256

except subprocess.CalledProcessError, e:

257

pass

258

finally:

259

subprocess.check_call(['git', 'checkout', where_am_i])

260

for bm_name in comparables:

261

diff = subprocess.check_output(['tools/profiling/microbenchmarks/bm_diff.py',

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

262

'%s.counters.json' % bm_name,

Craig Tiller

5adc93e

2017-03-08 13:17:53 -0800

[diff] [blame]

263

'%s.opt.json' % bm_name,

264

'%s.old.counters.json' % bm_name,

265

'%s.old.opt.json' % bm_name]).strip()

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

266

if diff:

267

heading('Performance diff: %s' % bm_name)

268

text(diff)

Matt Kwong

aff1c05

2017-03-09 15:08:01 -0800

[diff] [blame]

269

git_comment += '```\\nPerformance diff: %s\\n%s\\n```\\n' % (bm_name, diff.replace('\n', '\\n'))

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

270

finally:

Matt Kwong

aff1c05

2017-03-09 15:08:01 -0800

[diff] [blame]

271

if args.diff_perf:

272

subprocess.call(['tools/jenkins/comment_on_pr.sh "%s"' % git_comment.replace('`', '\`')],

273

stdout=subprocess.PIPE,

274

shell=True)

275

if not os.path.exists('reports'):

276

os.makedirs('reports')

Craig Tiller