Blame - tools/run_tests/run_microbenchmark.py - platform/external/grpc-grpc

2017-06-07 22:57:36 +0200

[diff] [blame^]

2

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

3

#

Jan Tattermusch

2017-06-07 22:57:36 +0200

[diff] [blame^]

4

# Licensed under the Apache License, Version 2.0 (the "License");

5

# you may not use this file except in compliance with the License.

6

# You may obtain a copy of the License at

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

7

#

Jan Tattermusch

2017-06-07 22:57:36 +0200

[diff] [blame^]

8

# http://www.apache.org/licenses/LICENSE-2.0

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

9

#

Jan Tattermusch

2017-06-07 22:57:36 +0200

[diff] [blame^]

10

# Unless required by applicable law or agreed to in writing, software

11

# distributed under the License is distributed on an "AS IS" BASIS,

12

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

13

# See the License for the specific language governing permissions and

14

# limitations under the License.

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

15

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

16

import cgi

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

17

import multiprocessing

18

import os

19

import subprocess

20

import sys

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

21

import argparse

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

22

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

23

import python_utils.jobset as jobset

24

import python_utils.start_port_server as start_port_server

25

Matt Kwong

d0ee10d

2017-03-10 22:37:52 -0800

[diff] [blame]

26

_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong',

27

'bm_fullstack_streaming_ping_pong',

28

'bm_fullstack_streaming_pump',

'bm_closure',

'bm_cq',

'bm_call_create',

'bm_error',

'bm_chttp2_hpack',

Matt Kwong

063adf3

2017-03-22 12:48:34 -0700

[diff] [blame]

34

'bm_chttp2_transport',

35

'bm_pollset',

Matt Kwong

d0ee10d

2017-03-10 22:37:52 -0800

[diff] [blame]

36

'bm_metadata',

37

'bm_fullstack_trickle']

38

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

39

flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')

40

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

41

os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))

42

if not os.path.exists('reports'):

43

os.makedirs('reports')

44

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

45

start_port_server.start_port_server()

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

46

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

def fnize(s):

out = ''

for c in s:

if c in '<>, /':

if len(out) and out[-1] == '_': continue

out += '_'

else:

out += c

return out

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

# index html

index_html = """

<html>

<head>

<title>Microbenchmark Results</title>

</head>

<body>

"""

def heading(name):

global index_html

index_html += "<h1>%s</h1>\n" % name

69

70

def link(txt, tgt):

71

global index_html

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

72

index_html += "<p><a href=\"%s\">%s</a></p>\n" % (

73

cgi.escape(tgt, quote=True), cgi.escape(txt))

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

74

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

75

def text(txt):

76

global index_html

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

77

index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

78

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

79

def collect_latency(bm_name, args):

80

"""generate latency profiles"""

81

benchmarks = []

82

profile_analysis = []

83

cleanup = []

84

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

85

heading('Latency Profiles: %s' % bm_name)

86

subprocess.check_call(

87

['make', bm_name,

88

'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])

89

for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,

90

'--benchmark_list_tests']).splitlines():

Craig Tiller

3940179

2017-02-02 12:22:07 -0800

[diff] [blame]

91

link(line, '%s.txt' % fnize(line))

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

92

benchmarks.append(

Craig Tiller

ece502f

2017-02-17 16:20:50 -0800

[diff] [blame]

93

jobset.JobSpec(['bins/basicprof/%s' % bm_name,

94

'--benchmark_filter=^%s$' % line,

95

'--benchmark_min_time=0.05'],

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

96

environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}))

97

profile_analysis.append(

98

jobset.JobSpec([sys.executable,

99

'tools/profiling/latency_profile/profile_analyzer.py',

100

'--source', '%s.trace' % fnize(line), '--fmt', 'simple',

101

'--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None))

Craig Tiller

715e43b

2017-02-07 11:13:16 -0800

[diff] [blame]

102

cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))

Craig Tiller

360c0d5

2017-02-08 13:36:44 -0800

[diff] [blame]

103

# periodically flush out the list of jobs: profile_analysis jobs at least

104

# consume upwards of five gigabytes of ram in some cases, and so analysing

105

# hundreds of them at once is impractical -- but we want at least some

106

# concurrency or the work takes too long

Craig Tiller

ece502f

2017-02-17 16:20:50 -0800

[diff] [blame]

107

if len(benchmarks) >= min(16, multiprocessing.cpu_count()):

Craig Tiller

360c0d5

2017-02-08 13:36:44 -0800

[diff] [blame]

108

# run up to half the cpu count: each benchmark can use up to two cores

109

# (one for the microbenchmark, one for the data flush)

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

110

jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

111

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

112

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

113

benchmarks = []

114

profile_analysis = []

115

cleanup = []

Craig Tiller

360c0d5

2017-02-08 13:36:44 -0800

[diff] [blame]

116

# run the remaining benchmarks that weren't flushed

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

117

if len(benchmarks):

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

118

jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))

Craig Tiller

6911d08

2017-02-07 10:30:44 -0800

[diff] [blame]

119

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

120

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

121

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

122

def collect_perf(bm_name, args):

123

"""generate flamegraphs"""

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

124

heading('Flamegraphs: %s' % bm_name)

125

subprocess.check_call(

126

['make', bm_name,

127

'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

128

benchmarks = []

129

profile_analysis = []

130

cleanup = []

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

131

for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,

132

'--benchmark_list_tests']).splitlines():

Craig Tiller

5a8c586

2017-02-15 07:59:05 -0800

[diff] [blame]

133

link(line, '%s.svg' % fnize(line))

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

134

benchmarks.append(

135

jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),

Craig Tiller

c2c0c6f

2017-02-15 11:27:37 -0800

[diff] [blame]

136

'-g', '-F', '997',

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

137

'bins/mutrace/%s' % bm_name,

138

'--benchmark_filter=^%s$' % line,

139

'--benchmark_min_time=10']))

140

profile_analysis.append(

141

jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],

142

environ = {

143

'PERF_BASE_NAME': fnize(line),

144

'OUTPUT_DIR': 'reports',

145

'OUTPUT_FILENAME': fnize(line),

146

}))

147

cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))

148

cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))

149

# periodically flush out the list of jobs: temporary space required for this

150

# processing is large

151

if len(benchmarks) >= 20:

152

# run up to half the cpu count: each benchmark can use up to two cores

153

# (one for the microbenchmark, one for the data flush)

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

154

jobset.run(benchmarks, maxjobs=1)

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

155

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

156

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

157

benchmarks = []

158

profile_analysis = []

159

cleanup = []

160

# run the remaining benchmarks that weren't flushed

161

if len(benchmarks):

Craig Tiller

2017-02-17 10:27:56 -0800

[diff] [blame]

162

jobset.run(benchmarks, maxjobs=1)

Craig Tiller

2017-02-15 09:14:24 -0800

[diff] [blame]

163

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

164

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

165

Craig Tiller

ff84b36

2017-03-01 14:11:15 -0800

[diff] [blame]

166

def run_summary(bm_name, cfg, base_json_name):

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

167

subprocess.check_call(

168

['make', bm_name,

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

169

'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])

170

cmd = ['bins/%s/%s' % (cfg, bm_name),

Craig Tiller

ff84b36

2017-03-01 14:11:15 -0800

[diff] [blame]

171

'--benchmark_out=%s.%s.json' % (base_json_name, cfg),

Craig Tiller

d9bc210

2017-02-15 08:24:55 -0800

[diff] [blame]

172

'--benchmark_out_format=json']

173

if args.summary_time is not None:

174

cmd += ['--benchmark_min_time=%d' % args.summary_time]

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

175

return subprocess.check_output(cmd)

176

177

def collect_summary(bm_name, args):

178

heading('Summary: %s [no counters]' % bm_name)

Craig Tiller

26995eb

2017-03-08 13:10:28 -0800

[diff] [blame]

179

text(run_summary(bm_name, 'opt', bm_name))

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

180

heading('Summary: %s [with counters]' % bm_name)

Craig Tiller

26995eb

2017-03-08 13:10:28 -0800

[diff] [blame]

181

text(run_summary(bm_name, 'counters', bm_name))

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

182

if args.bigquery_upload:

Craig Tiller

26995eb

2017-03-08 13:10:28 -0800

[diff] [blame]

183

with open('%s.csv' % bm_name, 'w') as f:

184

f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py',

185

'%s.counters.json' % bm_name,

186

'%s.opt.json' % bm_name]))

187

subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name])

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

188

189

collectors = {

190

'latency': collect_latency,

191

'perf': collect_perf,

192

'summary': collect_summary,

193

}

194

195

argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')

196

argp.add_argument('-c', '--collect',

197

choices=sorted(collectors.keys()),

Craig Tiller

5ef448d

2017-03-01 14:12:47 -0800

[diff] [blame]

198

nargs='*',

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

199

default=sorted(collectors.keys()),

200

help='Which collectors should be run against each benchmark')

201

argp.add_argument('-b', '--benchmarks',

Matt Kwong

d0ee10d

2017-03-10 22:37:52 -0800

[diff] [blame]

202

choices=_AVAILABLE_BENCHMARK_TESTS,

203

default=_AVAILABLE_BENCHMARK_TESTS,

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

204

nargs='+',

205

type=str,

206

help='Which microbenchmarks should be run')

207

argp.add_argument('--bigquery_upload',

208

default=False,

209

action='store_const',

210

const=True,

211

help='Upload results from summary collection to bigquery')

Craig Tiller

d9bc210

2017-02-15 08:24:55 -0800

[diff] [blame]

212

argp.add_argument('--summary_time',

213

default=None,

214

type=int,

215

help='Minimum time to run benchmarks for the summary collection')

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

216

args = argp.parse_args()

217

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

218

try:

Craig Tiller

47f37f3

2017-03-10 10:44:56 -0800

[diff] [blame]

219

for collect in args.collect:

220

for bm_name in args.benchmarks:

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

221

collectors[collect](bm_name, args)

Craig Tiller

2017-03-08 13:08:40 -0800

[diff] [blame]

222

finally:

Matt Kwong

aff1c05

2017-03-09 15:08:01 -0800

[diff] [blame]

223

if not os.path.exists('reports'):

224

os.makedirs('reports')

Craig Tiller