Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / grpc-grpc / 9c79e8de96c168c40691bebf8db18c6acd3e813b / . / tools / run_tests / run_tests_matrix.py
blob: 2344e2f9fd8e3d817d56af7cb2415ab83dc5809c [file] [log] [blame]
Jan Tattermusch9c79e8d2016-09-19 14:33:18 +0200[diff] [blame^]1#!/usr/bin/env python2.7
2# Copyright 2015, Google Inc.
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9# * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11# * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15# * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Run test matrix."""
32
33import argparse
34import jobset
35import os
36import report_utils
37import sys
38
39_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
40os.chdir(_ROOT)
41
42# TODO(jtattermusch): this is not going to be enough for sanitizers.
43_RUNTESTS_TIMEOUT = 30*60
44
45
46def _docker_jobspec(name, runtests_args=[]):
47 """Run a single instance of run_tests.py in a docker container"""
48 # TODO: fix copying report files from inside docker....
49 test_job = jobset.JobSpec(
50 cmdline=['python', 'tools/run_tests/run_tests.py',
51 '--use_docker',
52 '-t',
53 '-j', '3',
54 '-x', 'report_%s.xml' % name] + runtests_args,
55 shortname='run_tests_%s' % name,
56 timeout_seconds=_RUNTESTS_TIMEOUT)
57 return test_job
58
59
60def _workspace_jobspec(name, runtests_args=[], workspace_name=None):
61 """Run a single instance of run_tests.py in a separate workspace"""
62 env = {'WORKSPACE_NAME': workspace_name}
63 test_job = jobset.JobSpec(
64 cmdline=['tools/run_tests/run_tests_in_workspace.sh',
65 '-t',
66 '-j', '3',
67 '-x', '../report_%s.xml' % name] + runtests_args,
68 environ=env,
69 shortname='run_tests_%s' % name,
70 timeout_seconds=_RUNTESTS_TIMEOUT)
71 return test_job
72
73
74def _generate_jobs(languages, configs, platforms,
75 arch=None, compiler=None,
76 labels=[]):
77 result = []
78 for language in languages:
79 for platform in platforms:
80 for config in configs:
81 name = '%s_%s_%s' % (language, platform, config)
82 runtests_args = ['-l', language,
83 '-c', config]
84 if arch or compiler:
85 name += '_%s_%s' % (arch, compiler)
86 runtests_args += ['--arch', arch,
87 '--compiler', compiler]
88
89 if platform == 'linux':
90 job = _docker_jobspec(name=name, runtests_args=runtests_args)
91 else:
92 job = _workspace_jobspec(name=name, runtests_args=runtests_args)
93
94 job.labels = [platform, config, language] + labels
95 result.append(job)
96 return result
97
98
99def _create_test_jobs():
100 test_jobs = []
101 # supported on linux only
102 test_jobs += _generate_jobs(languages=['sanity', 'php7'],
103 configs=['dbg', 'opt'],
104 platforms=['linux'],
105 labels=['basictests'])
106
107 # supported on all platforms.
108 test_jobs += _generate_jobs(languages=['c', 'csharp', 'node', 'python'],
109 configs=['dbg', 'opt'],
110 platforms=['linux', 'macos', 'windows'],
111 labels=['basictests'])
112
113 # supported on linux and mac.
114 test_jobs += _generate_jobs(languages=['c++', 'ruby', 'php'],
115 configs=['dbg', 'opt'],
116 platforms=['linux', 'macos'],
117 labels=['basictests'])
118
119 # supported on mac only.
120 test_jobs += _generate_jobs(languages=['objc'],
121 configs=['dbg', 'opt'],
122 platforms=['macos'],
123 labels=['basictests'])
124
125 # sanitizers
126 test_jobs += _generate_jobs(languages=['c'],
127 configs=['msan', 'asan', 'tsan'],
128 platforms=['linux'],
129 labels=['sanitizers'])
130 test_jobs += _generate_jobs(languages=['c++'],
131 configs=['asan', 'tsan'],
132 platforms=['linux'],
133 labels=['sanitizers'])
134 return test_jobs
135
136
137def _create_portability_test_jobs():
138 test_jobs = []
139 # portability C x86
140 test_jobs += _generate_jobs(languages=['c'],
141 configs=['dbg'],
142 platforms=['linux'],
143 arch='x86',
144 compiler='default',
145 labels=['portability'])
146
147 # portability C and C++ on x64
148 for compiler in ['gcc4.4', 'gcc4.6', 'gcc5.3',
149 'clang3.5', 'clang3.6', 'clang3.7']:
150 test_jobs += _generate_jobs(languages=['c', 'c++'],
151 configs=['dbg'],
152 platforms=['linux'],
153 arch='x64',
154 compiler=compiler,
155 labels=['portability'])
156
157 # portability C on Windows
158 for arch in ['x86', 'x64']:
159 for compiler in ['vs2013', 'vs2015']:
160 test_jobs += _generate_jobs(languages=['c'],
161 configs=['dbg'],
162 platforms=['windows'],
163 arch=arch,
164 compiler=compiler,
165 labels=['portability'])
166
167 test_jobs += _generate_jobs(languages=['python'],
168 configs=['dbg'],
169 platforms=['linux'],
170 arch='default',
171 compiler='python3.4',
172 labels=['portability'])
173
174 test_jobs += _generate_jobs(languages=['csharp'],
175 configs=['dbg'],
176 platforms=['linux'],
177 arch='default',
178 compiler='coreclr',
179 labels=['portability'])
180
181 for compiler in ['node5', 'node6', 'node0.12']:
182 test_jobs += _generate_jobs(languages=['node'],
183 configs=['dbg'],
184 platforms=['linux'],
185 arch='default',
186 compiler=compiler,
187 labels=['portability'])
188 return test_jobs
189
190
191all_jobs = _create_test_jobs() + _create_portability_test_jobs()
192
193all_labels = set()
194for job in all_jobs:
195 for label in job.labels:
196 all_labels.add(label)
197
198argp = argparse.ArgumentParser(description='Run a matrix of run_tests.py tests.')
199argp.add_argument('-f', '--filter',
200 choices=sorted(all_labels),
201 nargs='+',
202 default=[],
203 help='Filter targets to run by label with AND semantics.')
204args = argp.parse_args()
205
206jobs = []
207for job in all_jobs:
208 if not args.filter or all(filter in job.labels for filter in args.filter):
209 jobs.append(job)
210
211if not jobs:
212 jobset.message('FAILED', 'No test suites match given criteria.',
213 do_newline=True)
214 sys.exit(1)
215
216print('IMPORTANT: The changes you are testing need to be locally committed')
217print('because only the committed changes in the current branch will be')
218print('copied to the docker environment or into subworkspaces.')
219
220print
221print 'Will run these tests:'
222for job in jobs:
223 print ' %s' % job.shortname
224print
225
226jobset.message('START', 'Running test matrix.', do_newline=True)
227num_failures, resultset = jobset.run(jobs,
228 newline_on_success=True,
229 travis=True,
230 maxjobs=2)
231report_utils.render_junit_xml_report(resultset, 'report.xml')
232
233if num_failures == 0:
234 jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.',
235 do_newline=True)
236else:
237 jobset.message('FAILED', 'Some run_tests.py instance have failed.',
238 do_newline=True)
239 sys.exit(1)