simplify run_tests.py
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 41cf335..7f156f7 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -118,6 +118,11 @@
not (travis and tgt['flaky'])]
+def _check_compiler(compiler, supported_compilers):
+ if compiler not in supported_compilers:
+ raise Exception('Compiler %s not supported.' % compiler)
+
+
class CLanguage(object):
def __init__(self, make_target, test_lang):
@@ -125,38 +130,50 @@
self.platform = platform_string()
self.test_lang = test_lang
- def test_specs(self, config, args):
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ if self.platform == 'windows':
+ _check_compiler(self.args.compiler, ['default',
+ 'vs2010',
+ 'vs2013',
+ 'vs2015'])
+ else:
+ _check_compiler(self.args.compiler, ['default'])
+
+ def test_specs(self):
out = []
- binaries = get_c_tests(args.travis, self.test_lang)
+ binaries = get_c_tests(self.args.travis, self.test_lang)
for target in binaries:
- if config.build_config in target['exclude_configs']:
+ if self.config.build_config in target['exclude_configs']:
continue
if self.platform == 'windows':
binary = 'vsprojects/%s%s/%s.exe' % (
- 'x64/' if args.arch == 'x64' else '',
- _WINDOWS_CONFIG[config.build_config],
+ 'x64/' if self.args.arch == 'x64' else '',
+ _WINDOWS_CONFIG[self.config.build_config],
target['name'])
else:
- binary = 'bins/%s/%s' % (config.build_config, target['name'])
+ binary = 'bins/%s/%s' % (self.config.build_config, target['name'])
if os.path.isfile(binary):
cmdline = [binary] + target['args']
- out.append(config.job_spec(cmdline, [binary],
- shortname=' '.join(cmdline),
- cpu_cost=target['cpu_cost'],
- environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
- os.path.abspath(os.path.dirname(
- sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')}))
- elif args.regex == '.*' or platform_string() == 'windows':
+ out.append(self.config.job_spec(cmdline, [binary],
+ shortname=' '.join(cmdline),
+ cpu_cost=target['cpu_cost'],
+ environ={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
+ os.path.abspath(os.path.dirname(
+ sys.argv[0]) + '/../../src/core/tsi/test_creds/ca.pem')}))
+ elif self.args.regex == '.*' or self.platform == 'windows':
print '\nWARNING: binary not found, skipping', binary
return sorted(out)
- def make_targets(self, test_regex):
- if platform_string() != 'windows' and test_regex != '.*':
+ def make_targets(self):
+ test_regex = self.args.regex
+ if self.platform != 'windows' and self.args.regex != '.*':
# use the regex to minimize the number of things to build
return [os.path.basename(target['name'])
for target in get_c_tests(False, self.test_lang)
if re.search(test_regex, '/' + target['name'])]
- if platform_string() == 'windows':
+ if self.platform == 'windows':
# don't build tools on windows just yet
return ['buildtests_%s' % self.make_target]
return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target]
@@ -182,11 +199,8 @@
def makefile_name(self):
return 'Makefile'
- def supports_multi_config(self):
- return True
-
- def dockerfile_dir(self, config, arch):
- return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(arch)
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/cxx_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return self.make_target
@@ -198,13 +212,18 @@
self.platform = platform_string()
self.node_version = '0.12'
- def test_specs(self, config, args):
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ _check_compiler(self.args.compiler, ['default'])
+
+ def test_specs(self):
if self.platform == 'windows':
- return [config.job_spec(['tools\\run_tests\\run_node.bat'], None)]
+ return [self.config.job_spec(['tools\\run_tests\\run_node.bat'], None)]
else:
- return [config.job_spec(['tools/run_tests/run_node.sh', self.node_version],
- None,
- environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+ return [self.config.job_spec(['tools/run_tests/run_node.sh', self.node_version],
+ None,
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self):
if self.platform == 'windows':
@@ -212,7 +231,7 @@
else:
return [['tools/run_tests/pre_build_node.sh', self.node_version]]
- def make_targets(self, test_regex):
+ def make_targets(self):
return []
def make_options(self):
@@ -230,11 +249,8 @@
def makefile_name(self):
return 'Makefile'
- def supports_multi_config(self):
- return False
-
- def dockerfile_dir(self, config, arch):
- return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(arch)
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/node_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'node'
@@ -242,14 +258,19 @@
class PhpLanguage(object):
- def test_specs(self, config, args):
- return [config.job_spec(['src/php/bin/run_tests.sh'], None,
- environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ _check_compiler(self.args.compiler, ['default'])
+
+ def test_specs(self):
+ return [self.config.job_spec(['src/php/bin/run_tests.sh'], None,
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self):
return []
- def make_targets(self, test_regex):
+ def make_targets(self):
return ['static_c', 'shared_c']
def make_options(self):
@@ -264,11 +285,8 @@
def makefile_name(self):
return 'Makefile'
- def supports_multi_config(self):
- return False
-
- def dockerfile_dir(self, config, arch):
- return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(arch)
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/php_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'php'
@@ -280,10 +298,15 @@
self._build_python_versions = ['2.7']
self._has_python_versions = []
- def test_specs(self, config, args):
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ _check_compiler(self.args.compiler, ['default'])
+
+ def test_specs(self):
environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
environment['PYVER'] = '2.7'
- return [config.job_spec(
+ return [self.config.job_spec(
['tools/run_tests/run_python.sh'],
None,
environ=environment,
@@ -294,7 +317,7 @@
def pre_build_steps(self):
return []
- def make_targets(self, test_regex):
+ def make_targets(self):
return ['static_c', 'grpc_python_plugin', 'shared_c']
def make_options(self):
@@ -320,11 +343,8 @@
def makefile_name(self):
return 'Makefile'
- def supports_multi_config(self):
- return False
-
- def dockerfile_dir(self, config, arch):
- return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(arch)
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'python'
@@ -332,10 +352,15 @@
class RubyLanguage(object):
- def test_specs(self, config, args):
- return [config.job_spec(['tools/run_tests/run_ruby.sh'], None,
- timeout_seconds=10*60,
- environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ _check_compiler(self.args.compiler, ['default'])
+
+ def test_specs(self):
+ return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], None,
+ timeout_seconds=10*60,
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self):
return [['tools/run_tests/pre_build_ruby.sh']]
@@ -355,27 +380,30 @@
def makefile_name(self):
return 'Makefile'
- def supports_multi_config(self):
- return False
-
- def dockerfile_dir(self, config, arch):
- return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(arch)
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/ruby_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'ruby'
class CSharpLanguage(object):
+
def __init__(self):
self.platform = platform_string()
- def test_specs(self, config, args):
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ _check_compiler(self.args.compiler, ['default'])
+
+ def test_specs(self):
with open('src/csharp/tests.json') as f:
tests_json = json.load(f)
assemblies = tests_json['assemblies']
tests = tests_json['tests']
- msbuild_config = _WINDOWS_CONFIG[config.build_config]
+ msbuild_config = _WINDOWS_CONFIG[self.config.build_config]
assembly_files = ['%s/bin/%s/%s.dll' % (a, msbuild_config, a)
for a in assemblies]
@@ -387,13 +415,13 @@
else:
script_name = 'tools/run_tests/run_csharp.sh'
- if config.build_config == 'gcov':
+ if self.config.build_config == 'gcov':
# On Windows, we only collect C# code coverage.
# On Linux, we only collect coverage for native extension.
# For code coverage all tests need to run as one suite.
- return [config.job_spec([script_name] + extra_args, None,
- shortname='csharp.coverage',
- environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+ return [self.config.job_spec([script_name] + extra_args, None,
+ shortname='csharp.coverage',
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
else:
specs = []
for test in tests:
@@ -402,9 +430,9 @@
# use different output directory for each test to prevent
# TestResult.xml clash between parallel test runs.
cmdline += ['-work=test-result/%s' % uuid.uuid4()]
- specs.append(config.job_spec(cmdline, None,
- shortname='csharp.%s' % test,
- environ=_FORCE_ENVIRON_FOR_WRAPPERS))
+ specs.append(self.config.job_spec(cmdline, None,
+ shortname='csharp.%s' % test,
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS))
return specs
def pre_build_steps(self):
@@ -413,7 +441,7 @@
else:
return [['tools/run_tests/pre_build_csharp.sh']]
- def make_targets(self, test_regex):
+ def make_targets(self):
# For Windows, this target doesn't really build anything,
# everything is build by buildall script later.
if self.platform == 'windows':
@@ -440,11 +468,8 @@
def makefile_name(self):
return 'Makefile'
- def supports_multi_config(self):
- return False
-
- def dockerfile_dir(self, config, arch):
- return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(arch)
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/csharp_jessie_%s' % _docker_arch_suffix(self.args.arch)
def __str__(self):
return 'csharp'
@@ -452,14 +477,19 @@
class ObjCLanguage(object):
- def test_specs(self, config, args):
- return [config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
- environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ _check_compiler(self.args.compiler, ['default'])
+
+ def test_specs(self):
+ return [self.config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
def pre_build_steps(self):
return []
- def make_targets(self, test_regex):
+ def make_targets(self):
return ['grpc_objective_c_plugin', 'interop_server']
def make_options(self):
@@ -474,10 +504,7 @@
def makefile_name(self):
return 'Makefile'
- def supports_multi_config(self):
- return False
-
- def dockerfile_dir(self, config, arch):
+ def dockerfile_dir(self):
return None
def __str__(self):
@@ -486,18 +513,23 @@
class Sanity(object):
- def test_specs(self, config, args):
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ _check_compiler(self.args.compiler, ['default'])
+
+ def test_specs(self):
import yaml
with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f:
- return [config.job_spec(cmd['script'].split(), None,
- timeout_seconds=None, environ={'TEST': 'true'},
- cpu_cost=cmd.get('cpu_cost', 1))
+ return [self.config.job_spec(cmd['script'].split(), None,
+ timeout_seconds=None, environ={'TEST': 'true'},
+ cpu_cost=cmd.get('cpu_cost', 1))
for cmd in yaml.load(f)]
def pre_build_steps(self):
return []
- def make_targets(self, test_regex):
+ def make_targets(self):
return ['run_dep_checks']
def make_options(self):
@@ -512,10 +544,7 @@
def makefile_name(self):
return 'Makefile'
- def supports_multi_config(self):
- return False
-
- def dockerfile_dir(self, config, arch):
+ def dockerfile_dir(self):
return 'tools/dockerfile/test/sanity'
def __str__(self):
@@ -527,7 +556,6 @@
_CONFIGS = dict((cfg['config'], Config(**cfg)) for cfg in ast.literal_eval(f.read()))
-_DEFAULT = ['opt']
_LANGUAGES = {
'c++': CLanguage('cxx', 'c++'),
'c': CLanguage('c', 'c'),
@@ -540,6 +568,7 @@
'sanity': Sanity()
}
+
_WINDOWS_CONFIG = {
'dbg': 'Debug',
'opt': 'Release',
@@ -648,9 +677,8 @@
# parse command line
argp = argparse.ArgumentParser(description='Run grpc tests.')
argp.add_argument('-c', '--config',
- choices=['all'] + sorted(_CONFIGS.keys()),
- nargs='+',
- default=_DEFAULT)
+ choices=sorted(_CONFIGS.keys()),
+ default='opt')
argp.add_argument('-n', '--runs_per_test', default=1, type=runs_per_test_type,
help='A positive integer or "inf". If "inf", all tests will run in an '
'infinite loop. Especially useful in combination with "-f"')
@@ -696,7 +724,7 @@
argp.add_argument('--compiler',
choices=['default', 'vs2010', 'vs2013', 'vs2015'],
default='default',
- help='Selects compiler to use. For some platforms "default" is the only supported choice.')
+ help='Selects compiler to use. Allowed values depend on the platform and language.')
argp.add_argument('--build_only',
default=False,
action='store_const',
@@ -742,11 +770,8 @@
# grab config
-run_configs = set(_CONFIGS[cfg]
- for cfg in itertools.chain.from_iterable(
- _CONFIGS.iterkeys() if x == 'all' else [x]
- for x in args.config))
-build_configs = set(cfg.build_config for cfg in run_configs)
+run_config = _CONFIGS[args.config]
+build_config = run_config.build_config
if args.travis:
_FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'}
@@ -762,12 +787,8 @@
lang_list.remove(bad)
languages = set(_LANGUAGES[l] for l in lang_list)
-
-if len(build_configs) > 1:
- for language in languages:
- if not language.supports_multi_config():
- print language, 'does not support multiple build configurations'
- sys.exit(1)
+for l in languages:
+ l.configure(run_config, args)
language_make_options=[]
if any(language.make_options() for language in languages):
@@ -777,8 +798,8 @@
else:
language_make_options = next(iter(languages)).make_options()
-if len(languages) != 1 or len(build_configs) != 1:
- print 'Multi-language and multi-config testing is not supported.'
+if len(languages) != 1:
+ print 'Multi-language testing is not supported.'
sys.exit(1)
if args.use_docker:
@@ -809,10 +830,6 @@
env=env)
sys.exit(0)
-if platform_string() != 'windows' and args.compiler != 'default':
- print 'Compiler %s not supported on current platform.' % args.compiler
- sys.exit(1)
-
_check_arch_option(args.arch)
def make_jobspec(cfg, targets, makefile='Makefile'):
@@ -852,7 +869,7 @@
for l in languages:
makefile = l.makefile_name()
make_targets[makefile] = make_targets.get(makefile, set()).union(
- set(l.make_targets(args.regex)))
+ set(l.make_targets()))
def build_step_environ(cfg):
environ = {'CONFIG': cfg}
@@ -862,22 +879,19 @@
return environ
build_steps = list(set(
- jobset.JobSpec(cmdline, environ=build_step_environ(cfg), flake_retries=5)
- for cfg in build_configs
+ jobset.JobSpec(cmdline, environ=build_step_environ(build_config), flake_retries=5)
for l in languages
for cmdline in l.pre_build_steps()))
if make_targets:
- make_commands = itertools.chain.from_iterable(make_jobspec(cfg, list(targets), makefile) for cfg in build_configs for (makefile, targets) in make_targets.iteritems())
+ make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems())
build_steps.extend(set(make_commands))
build_steps.extend(set(
- jobset.JobSpec(cmdline, environ=build_step_environ(cfg), timeout_seconds=None)
- for cfg in build_configs
+ jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None)
for l in languages
for cmdline in l.build_steps()))
post_tests_steps = list(set(
- jobset.JobSpec(cmdline, environ=build_step_environ(cfg))
- for cfg in build_configs
+ jobset.JobSpec(cmdline, environ=build_step_environ(build_config))
for l in languages
for cmdline in l.post_tests_steps()))
runs_per_test = args.runs_per_test
@@ -1068,9 +1082,8 @@
infinite_runs = runs_per_test == 0
one_run = set(
spec
- for config in run_configs
for language in languages
- for spec in language.test_specs(config, args)
+ for spec in language.test_specs()
if re.search(args.regex, spec.shortname))
# When running on travis, we want out test runs to be as similar as possible
# for reproducibility purposes.