Blame - tools/run_tests/run_microbenchmark.py - platform/external/grpc-grpc

2017-06-07 22:57:36 +0200

[diff] [blame]

2

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

3

#

Jan Tattermusch

2017-06-07 22:57:36 +0200

[diff] [blame]

4

# Licensed under the Apache License, Version 2.0 (the "License");

5

# you may not use this file except in compliance with the License.

6

# You may obtain a copy of the License at

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

7

#

Jan Tattermusch

2017-06-07 22:57:36 +0200

[diff] [blame]

8

# http://www.apache.org/licenses/LICENSE-2.0

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

9

#

Jan Tattermusch

2017-06-07 22:57:36 +0200

[diff] [blame]

10

# Unless required by applicable law or agreed to in writing, software

11

# distributed under the License is distributed on an "AS IS" BASIS,

12

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

13

# See the License for the specific language governing permissions and

14

# limitations under the License.

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

15

Craig Tiller

891e816

2017-02-15 23:30:27 -0800

[diff] [blame]

16

import cgi

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

17

import multiprocessing

18

import os

19

import subprocess

20

import sys

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

21

import argparse

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

22

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

23

import python_utils.jobset as jobset

24

import python_utils.start_port_server as start_port_server

25

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

26

sys.path.append(

27

os.path.join(

28

os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks',

29

'bm_diff'))

ncteisen

64637b7

2017-05-09 14:23:16 -0700

[diff] [blame]

30

import bm_constants

Matt Kwong

d0ee10d

2017-03-10 22:37:52 -0800

[diff] [blame]

31

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

32

flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')

33

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

34

os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))

35

if not os.path.exists('reports'):

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

36

os.makedirs('reports')

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

37

Craig Tiller

cba864b

2017-02-17 10:27:56 -0800

[diff] [blame]

38

start_port_server.start_port_server()

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

39

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

40

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

41

def fnize(s):

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

out = ''

for c in s:

if c in '<>, /':

if len(out) and out[-1] == '_': continue

out += '_'

else:

out += c

return out

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

51

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

# index html

index_html = """

<html>

<head>

<title>Microbenchmark Results</title>

</head>

<body>

"""

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

61

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

62

def heading(name):

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

63

global index_html

64

index_html += "<h1>%s</h1>\n" % name

65

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

66

67

def link(txt, tgt):

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

68

global index_html

69

index_html += "<p><a href=\"%s\">%s</a></p>\n" % (

70

cgi.escape(tgt, quote=True), cgi.escape(txt))

71

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

72

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

73

def text(txt):

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

74

global index_html

75

index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)

76

Craig Tiller

2017-02-02 16:08:05 -0800

[diff] [blame]

77

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

78

def collect_latency(bm_name, args):

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

79

"""generate latency profiles"""

80

benchmarks = []

81

profile_analysis = []

82

cleanup = []

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

83

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

84

heading('Latency Profiles: %s' % bm_name)

85

subprocess.check_call([

86

'make', bm_name, 'CONFIG=basicprof', '-j',

87

'%d' % multiprocessing.cpu_count()

88

])

89

for line in subprocess.check_output(

90

['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines():

91

link(line, '%s.txt' % fnize(line))

92

benchmarks.append(

93

jobset.JobSpec(

94

[

Mehrdad Afshari

2018-01-02 14:40:00 -0800

[diff] [blame]

95

'bins/basicprof/%s' % bm_name,

96

'--benchmark_filter=^%s$' % line,

97

'--benchmark_min_time=0.05'

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

98

],

99

environ={'LATENCY_TRACE': '%s.trace' % fnize(line)},

100

shortname='profile-%s' % fnize(line)))

101

profile_analysis.append(

jobset.JobSpec(

[

sys.executable,

'tools/profiling/latency_profile/profile_analyzer.py',

Mehrdad Afshari

2018-01-02 14:40:00 -0800

[diff] [blame]

106

'--source',

107

'%s.trace' % fnize(line), '--fmt', 'simple', '--out',

108

'reports/%s.txt' % fnize(line)

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

109

],

110

timeout_seconds=20 * 60,

111

shortname='analyze-%s' % fnize(line)))

112

cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))

113

# periodically flush out the list of jobs: profile_analysis jobs at least

114

# consume upwards of five gigabytes of ram in some cases, and so analysing

115

# hundreds of them at once is impractical -- but we want at least some

116

# concurrency or the work takes too long

117

if len(benchmarks) >= min(16, multiprocessing.cpu_count()):

118

# run up to half the cpu count: each benchmark can use up to two cores

119

# (one for the microbenchmark, one for the data flush)

120

jobset.run(

Mehrdad Afshari

2018-01-02 14:40:00 -0800

[diff] [blame]

121

benchmarks, maxjobs=max(1,

122

multiprocessing.cpu_count() / 2))

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

123

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

124

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

125

benchmarks = []

126

profile_analysis = []

127

cleanup = []

128

# run the remaining benchmarks that weren't flushed

129

if len(benchmarks):

130

jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2))

131

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

132

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

133

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

134

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

135

def collect_perf(bm_name, args):

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

136

"""generate flamegraphs"""

137

heading('Flamegraphs: %s' % bm_name)

138

subprocess.check_call([

139

'make', bm_name, 'CONFIG=mutrace', '-j',

140

'%d' % multiprocessing.cpu_count()

141

])

142

benchmarks = []

143

profile_analysis = []

144

cleanup = []

145

for line in subprocess.check_output(

146

['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines():

147

link(line, '%s.svg' % fnize(line))

148

benchmarks.append(

149

jobset.JobSpec(

150

[

Mehrdad Afshari

2018-01-02 14:40:00 -0800

[diff] [blame]

151

'perf', 'record', '-o',

152

'%s-perf.data' % fnize(line), '-g', '-F', '997',

153

'bins/mutrace/%s' % bm_name,

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

154

'--benchmark_filter=^%s$' % line, '--benchmark_min_time=10'

155

],

156

shortname='perf-%s' % fnize(line)))

157

profile_analysis.append(

158

jobset.JobSpec(

159

[

160

'tools/run_tests/performance/process_local_perf_flamegraphs.sh'

161

],

162

environ={

163

'PERF_BASE_NAME': fnize(line),

164

'OUTPUT_DIR': 'reports',

165

'OUTPUT_FILENAME': fnize(line),

166

},

167

shortname='flame-%s' % fnize(line)))

168

cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))

169

cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))

170

# periodically flush out the list of jobs: temporary space required for this

171

# processing is large

172

if len(benchmarks) >= 20:

173

# run up to half the cpu count: each benchmark can use up to two cores

174

# (one for the microbenchmark, one for the data flush)

175

jobset.run(benchmarks, maxjobs=1)

176

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

177

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

178

benchmarks = []

179

profile_analysis = []

180

cleanup = []

181

# run the remaining benchmarks that weren't flushed

182

if len(benchmarks):

183

jobset.run(benchmarks, maxjobs=1)

184

jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())

185

jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())

186

Craig Tiller

2017-01-31 15:08:31 -0800

[diff] [blame]

187

Craig Tiller

ff84b36

2017-03-01 14:11:15 -0800

[diff] [blame]

188

def run_summary(bm_name, cfg, base_json_name):

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

189

subprocess.check_call([

Mehrdad Afshari

2018-01-02 14:40:00 -0800

[diff] [blame]

190

'make', bm_name,

191

'CONFIG=%s' % cfg, '-j',

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

192

'%d' % multiprocessing.cpu_count()

193

])

194

cmd = [

Mehrdad Afshari

2018-01-02 14:40:00 -0800

[diff] [blame]

195

'bins/%s/%s' % (cfg, bm_name),

196

'--benchmark_out=%s.%s.json' % (base_json_name, cfg),

197

'--benchmark_out_format=json'

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

198

]

199

if args.summary_time is not None:

200

cmd += ['--benchmark_min_time=%d' % args.summary_time]

201

return subprocess.check_output(cmd)

202

Craig Tiller

541b87e

2017-03-01 08:42:52 -0800

[diff] [blame]

203

204

def collect_summary(bm_name, args):

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

205

heading('Summary: %s [no counters]' % bm_name)

206

text(run_summary(bm_name, 'opt', bm_name))

207

heading('Summary: %s [with counters]' % bm_name)

208

text(run_summary(bm_name, 'counters', bm_name))

209

if args.bigquery_upload:

210

with open('%s.csv' % bm_name, 'w') as f:

211

f.write(

212

subprocess.check_output([

213

'tools/profiling/microbenchmarks/bm2bq.py',

Mehrdad Afshari

2018-01-02 14:40:00 -0800

[diff] [blame]

214

'%s.counters.json' % bm_name,

215

'%s.opt.json' % bm_name

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

216

]))

217

subprocess.check_call([

Mehrdad Afshari

2018-01-02 14:40:00 -0800

[diff] [blame]

218

'bq', 'load', 'microbenchmarks.microbenchmarks',

219

'%s.csv' % bm_name

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

220

])

221

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

222

223

collectors = {

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

224

'latency': collect_latency,

225

'perf': collect_perf,

226

'summary': collect_summary,

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

227

}

228

229

argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

argp.add_argument(

'-c',

'--collect',

choices=sorted(collectors.keys()),

234

nargs='*',

235

default=sorted(collectors.keys()),

236

help='Which collectors should be run against each benchmark')

argp.add_argument(

'-b',

'--benchmarks',

choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,

241

default=bm_constants._AVAILABLE_BENCHMARK_TESTS,

242

nargs='+',

243

type=str,

244

help='Which microbenchmarks should be run')

argp.add_argument(

'--bigquery_upload',

default=False,

action='store_const',

249

const=True,

250

help='Upload results from summary collection to bigquery')

argp.add_argument(

'--summary_time',

default=None,

type=int,

help='Minimum time to run benchmarks for the summary collection')

Craig Tiller

2017-02-08 14:20:08 -0800

[diff] [blame]

256

args = argp.parse_args()

257

Craig Tiller

b432885

2017-03-08 13:08:40 -0800

[diff] [blame]

258

try:

ncteisen

2017-12-11 18:00:40 -0800

[diff] [blame]

259

for collect in args.collect:

260

for bm_name in args.benchmarks:

261

collectors[collect](bm_name, args)

Craig Tiller

b432885

2017-03-08 13:08:40 -0800

[diff] [blame]

262

finally:

ncteisen