Blame - tools/run_tests/run_microbenchmark.py - platform/external/grpc-grpc

2017-01-31 15:08:31 -0800

[diff] [blame]

1

#!/usr/bin/env python2.7

#

# Redistribution and use in source and binary forms, with or without

6

# modification, are permitted provided that the following conditions are

7

# met:

8

#

9

# * Redistributions of source code must retain the above copyright

10

# notice, this list of conditions and the following disclaimer.

11

# * Redistributions in binary form must reproduce the above

12

# copyright notice, this list of conditions and the following disclaimer

13

# in the documentation and/or other materials provided with the

14

# distribution.

15

# * Neither the name of Google Inc. nor the names of its

16

# contributors may be used to endorse or promote products derived from

17

# this software without specific prior written permission.

18

#

19

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

20

# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

21

# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

22

# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

23

# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

24

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

25

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

26

# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

27

# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

28

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

29

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

30

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

31

import cgi

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

32

import multiprocessing

33

import os

34

import subprocess

35

import sys

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

36

import argparse

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

37

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

38

import python_utils.jobset as jobset

39

import python_utils.start_port_server as start_port_server

40

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

41

flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')

42

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

43

os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))

44

if not os.path.exists('reports'):

45

os.makedirs('reports')

46

47

port_server_port = 32766

48

start_port_server.start_port_server(port_server_port)

49

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

def fnize(s):

out = ''

for c in s:

if c in '<>, /':

if len(out) and out[-1] == '_': continue

out += '_'

else:

out += c

return out

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

# index html

index_html = """

<html>

<head>

<title>Microbenchmark Results</title>

</head>

<body>

"""

def heading(name):

global index_html

index_html += "<h1>%s</h1>\n" % name

72

73

def link(txt, tgt):

74

global index_html

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

75

index_html += "<p><a href=\"%s\">%s</a></p>\n" % (

76

cgi.escape(tgt, quote=True), cgi.escape(txt))

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

77

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

78

def text(txt):

79

global index_html

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

80

index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

81

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

82

def collect_latency(bm_name, args):

83

"""generate latency profiles"""

84

benchmarks = []

85

profile_analysis = []

86

cleanup = []

87

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

88

heading('Latency Profiles: %s' % bm_name)

89

subprocess.check_call(

90

['make', bm_name,

91

'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])

92

for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,

93

'--benchmark_list_tests']).splitlines():

Craig Tiller

3940179

2017-02-02 12:22:07 -0800

[diff] [blame]

94

link(line, '%s.txt' % fnize(line))

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

95

benchmarks.append(

Craig Tiller

ece502f

2017-02-17 16:20:50 -0800

[diff] [blame]

96

jobset.JobSpec(['bins/basicprof/%s' % bm_name,

97

'--benchmark_filter=^%s$' % line,

98

'--benchmark_min_time=0.05'],

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

99

environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}))

100

profile_analysis.append(

101

jobset.JobSpec([sys.executable,

102

'tools/profiling/latency_profile/profile_analyzer.py',

103

'--source', '%s.trace' % fnize(line), '--fmt', 'simple',

104

'--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None))

Craig Tiller

715e43b

2017-02-07 11:13:16 -0800

[diff] [blame]

105

cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))

Craig Tiller

2017-02-08 13:36:44 -0800

[diff] [blame]

106

# periodically flush out the list of jobs: profile_analysis jobs at least

107

# consume upwards of five gigabytes of ram in some cases, and so analysing

108

# hundreds of them at once is impractical -- but we want at least some

109

# concurrency or the work takes too long

Craig Tiller

ece502f

2017-02-17 16:20:50 -0800

[diff] [blame]

110

if len(benchmarks) >= min(16, multiprocessing.cpu_count()):

Craig Tiller

2017-02-08 13:36:44 -0800

[diff] [blame]

111

# run up to half the cpu count: each benchmark can use up to two cores

112

# (one for the microbenchmark, one for the data flush)

Craig Tiller

2ef0d54

2017-02-08 13:53:18 -0800

[diff] [blame]

113

jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2),

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

114

add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})

115

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

116

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

117

benchmarks = []

118

profile_analysis = []

119

cleanup = []

Craig Tiller

2017-02-08 13:36:44 -0800

[diff] [blame]

120

# run the remaining benchmarks that weren't flushed

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

121

if len(benchmarks):

Craig Tiller

2ef0d54

2017-02-08 13:53:18 -0800

[diff] [blame]

122

jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2),

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

123

add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})

124

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

125

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

126

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

127

def collect_perf(bm_name, args):

128

"""generate flamegraphs"""

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

129

heading('Flamegraphs: %s' % bm_name)

130

subprocess.check_call(

131

['make', bm_name,

132

'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

133

benchmarks = []

134

profile_analysis = []

135

cleanup = []

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

136

for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,

137

'--benchmark_list_tests']).splitlines():

Craig Tiller

5a8c586

2017-02-15 07:59:05 -0800

[diff] [blame]

138

link(line, '%s.svg' % fnize(line))

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

139

benchmarks.append(

140

jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),

Craig Tiller

c2c0c6f

2017-02-15 11:27:37 -0800

[diff] [blame]

141

'-g', '-F', '997',

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

142

'bins/mutrace/%s' % bm_name,

143

'--benchmark_filter=^%s$' % line,

144

'--benchmark_min_time=10']))

145

profile_analysis.append(

146

jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],

147

environ = {

148

'PERF_BASE_NAME': fnize(line),

149

'OUTPUT_DIR': 'reports',

150

'OUTPUT_FILENAME': fnize(line),

151

}))

152

cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))

153

cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))

154

# periodically flush out the list of jobs: temporary space required for this

155

# processing is large

156

if len(benchmarks) >= 20:

157

# run up to half the cpu count: each benchmark can use up to two cores

158

# (one for the microbenchmark, one for the data flush)

Craig Tiller

ab95194

2017-02-15 11:26:54 -0800

[diff] [blame]

159

jobset.run(benchmarks, maxjobs=1,

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

160

add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})

161

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

162

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

163

benchmarks = []

164

profile_analysis = []

165

cleanup = []

166

# run the remaining benchmarks that weren't flushed

167

if len(benchmarks):

Craig Tiller

ab95194

2017-02-15 11:26:54 -0800

[diff] [blame]

168

jobset.run(benchmarks, maxjobs=1,

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

169

add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})

170

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

171

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

172

Craig Tiller

2017-03-01 14:11:15 -0800

[diff] [blame^]

173

def run_summary(bm_name, cfg, base_json_name):

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

174

subprocess.check_call(

175

['make', bm_name,

Craig Tiller

2017-03-01 08:42:52 -0800

[diff] [blame]

176

'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])

177

cmd = ['bins/%s/%s' % (cfg, bm_name),

Craig Tiller

2017-03-01 14:11:15 -0800

[diff] [blame^]

178

'--benchmark_out=%s.%s.json' % (base_json_name, cfg),

Craig Tiller

d9bc210

2017-02-15 08:24:55 -0800

[diff] [blame]

179

'--benchmark_out_format=json']

180

if args.summary_time is not None:

181

cmd += ['--benchmark_min_time=%d' % args.summary_time]

Craig Tiller

2017-03-01 08:42:52 -0800

[diff] [blame]

182

return subprocess.check_output(cmd)

183

184

def collect_summary(bm_name, args):

185

heading('Summary: %s [no counters]' % bm_name)

Craig Tiller

2017-03-01 14:11:15 -0800

[diff] [blame^]

186

text(run_summary(bm_name, 'opt', 'out'))

Craig Tiller

2017-03-01 08:42:52 -0800

[diff] [blame]

187

heading('Summary: %s [with counters]' % bm_name)

Craig Tiller

2017-03-01 14:11:15 -0800

[diff] [blame^]

188

text(run_summary(bm_name, 'counters', 'out'))

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

189

if args.bigquery_upload:

Craig Tiller

d12731e

2017-02-13 12:41:27 -0800

[diff] [blame]

190

with open('out.csv', 'w') as f:

Craig Tiller

2017-03-01 08:42:52 -0800

[diff] [blame]

191

f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py', 'out.counters.json', 'out.opt.json']))

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

192

subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', 'out.csv'])

193

194

collectors = {

195

'latency': collect_latency,

196

'perf': collect_perf,

197

'summary': collect_summary,

198

}

199

200

argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')

201

argp.add_argument('-c', '--collect',

202

choices=sorted(collectors.keys()),

203

nargs='+',

204

default=sorted(collectors.keys()),

205

help='Which collectors should be run against each benchmark')

206

argp.add_argument('-b', '--benchmarks',

Craig Tiller

523d54b

2017-02-23 08:52:38 -0800

[diff] [blame]

207

default=['bm_fullstack',

'bm_closure',

'bm_cq',

'bm_call_create',

'bm_error',

Craig Tiller

510f38a

2017-02-24 17:00:19 -0800

[diff] [blame]

212

'bm_chttp2_hpack',

213

'bm_metadata'],

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

214

nargs='+',

215

type=str,

216

help='Which microbenchmarks should be run')

Craig Tiller

d753f45

2017-03-01 14:00:35 -0800

[diff] [blame]

217

argp.add_argument('--diff_perf',

218

default=None,

219

type=str,

220

help='Diff microbenchmarks against this git revision')

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

221

argp.add_argument('--bigquery_upload',

222

default=False,

223

action='store_const',

224

const=True,

225

help='Upload results from summary collection to bigquery')

Craig Tiller

d9bc210

2017-02-15 08:24:55 -0800

[diff] [blame]

226

argp.add_argument('--summary_time',

227

default=None,

228

type=int,

229

help='Minimum time to run benchmarks for the summary collection')

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

230

args = argp.parse_args()

231

232

for bm_name in args.benchmarks:

233

for collect in args.collect:

234

collectors[collect](bm_name, args)

Craig Tiller

d753f45

2017-03-01 14:00:35 -0800

[diff] [blame]

235

if args.diff_perf:

Craig Tiller

2017-03-01 14:11:15 -0800

[diff] [blame^]

236

for bm_name in args.benchmarks:

237

run_summary(bm_name, 'opt', '%s.new' % bm_name)

238

where_am_i = submodule.check_call(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])

239

submodule.check_call(['git', 'checkout', args.diff_perf])

240

comparables = []

241

try:

242

for bm_name in args.benchmarks:

243

try:

244

run_summary(bm_name, 'opt', '%s.old' % bm_name)

245

comparables.append(bm_name)

246

except subprocess.CalledProcessError, e:

247

pass

248

finally:

249

submodule.check_call(['git', 'checkout', where_am_i])

250

for bm_name in comparables:

251

submodule.check_call(['third_party/benchmark/tools/compare_bench.py',

252

'%s.new.opt.json' % bm_name,

253

'%s.old.opt.json' % bm_name])

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

254

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

255

index_html += "</body>\n</html>\n"

256

with open('reports/index.html', 'w') as f:

Craig Tiller