yapf run_tests
diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py
index 7c58d8e..49be8f1 100755
--- a/tools/run_tests/run_tests_matrix.py
+++ b/tools/run_tests/run_tests_matrix.py
@@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
"""Run test matrix."""
from __future__ import print_function
@@ -29,14 +28,14 @@
_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
os.chdir(_ROOT)
-_DEFAULT_RUNTESTS_TIMEOUT = 1*60*60
+_DEFAULT_RUNTESTS_TIMEOUT = 1 * 60 * 60
# Set the timeout high to allow enough time for sanitizers and pre-building
# clang docker.
-_CPP_RUNTESTS_TIMEOUT = 4*60*60
+_CPP_RUNTESTS_TIMEOUT = 4 * 60 * 60
# C++ TSAN takes longer than other sanitizers
-_CPP_TSAN_RUNTESTS_TIMEOUT = 8*60*60
+_CPP_TSAN_RUNTESTS_TIMEOUT = 8 * 60 * 60
# Number of jobs assigned to each run_tests.py instance
_DEFAULT_INNER_JOBS = 2
@@ -46,448 +45,517 @@
def _report_filename(name):
- """Generates report file name"""
- return 'report_%s_%s' % (name, _REPORT_SUFFIX)
+ """Generates report file name"""
+ return 'report_%s_%s' % (name, _REPORT_SUFFIX)
def _report_filename_internal_ci(name):
- """Generates report file name that leads to better presentation by internal CI"""
- return '%s/%s' % (name, _REPORT_SUFFIX)
+ """Generates report file name that leads to better presentation by internal CI"""
+ return '%s/%s' % (name, _REPORT_SUFFIX)
-def _docker_jobspec(name, runtests_args=[], runtests_envs={},
+def _docker_jobspec(name,
+ runtests_args=[],
+ runtests_envs={},
inner_jobs=_DEFAULT_INNER_JOBS,
timeout_seconds=None):
- """Run a single instance of run_tests.py in a docker container"""
- if not timeout_seconds:
- timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
- test_job = jobset.JobSpec(
- cmdline=['python', 'tools/run_tests/run_tests.py',
- '--use_docker',
- '-t',
- '-j', str(inner_jobs),
- '-x', _report_filename(name),
- '--report_suite_name', '%s' % name] + runtests_args,
- environ=runtests_envs,
- shortname='run_tests_%s' % name,
- timeout_seconds=timeout_seconds)
- return test_job
+ """Run a single instance of run_tests.py in a docker container"""
+ if not timeout_seconds:
+ timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
+ test_job = jobset.JobSpec(
+ cmdline=[
+ 'python', 'tools/run_tests/run_tests.py', '--use_docker', '-t',
+ '-j', str(inner_jobs), '-x', _report_filename(name),
+ '--report_suite_name', '%s' % name
+ ] + runtests_args,
+ environ=runtests_envs,
+ shortname='run_tests_%s' % name,
+ timeout_seconds=timeout_seconds)
+ return test_job
-def _workspace_jobspec(name, runtests_args=[], workspace_name=None,
- runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS,
+def _workspace_jobspec(name,
+ runtests_args=[],
+ workspace_name=None,
+ runtests_envs={},
+ inner_jobs=_DEFAULT_INNER_JOBS,
timeout_seconds=None):
- """Run a single instance of run_tests.py in a separate workspace"""
- if not workspace_name:
- workspace_name = 'workspace_%s' % name
- if not timeout_seconds:
- timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
- env = {'WORKSPACE_NAME': workspace_name}
- env.update(runtests_envs)
- test_job = jobset.JobSpec(
- cmdline=['bash',
- 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
- '-t',
- '-j', str(inner_jobs),
- '-x', '../%s' % _report_filename(name),
- '--report_suite_name', '%s' % name] + runtests_args,
- environ=env,
- shortname='run_tests_%s' % name,
- timeout_seconds=timeout_seconds)
- return test_job
+ """Run a single instance of run_tests.py in a separate workspace"""
+ if not workspace_name:
+ workspace_name = 'workspace_%s' % name
+ if not timeout_seconds:
+ timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
+ env = {'WORKSPACE_NAME': workspace_name}
+ env.update(runtests_envs)
+ test_job = jobset.JobSpec(
+ cmdline=[
+ 'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
+ '-t', '-j', str(inner_jobs), '-x', '../%s' % _report_filename(name),
+ '--report_suite_name', '%s' % name
+ ] + runtests_args,
+ environ=env,
+ shortname='run_tests_%s' % name,
+ timeout_seconds=timeout_seconds)
+ return test_job
-def _generate_jobs(languages, configs, platforms, iomgr_platform = 'native',
- arch=None, compiler=None,
- labels=[], extra_args=[], extra_envs={},
- inner_jobs=_DEFAULT_INNER_JOBS,
- timeout_seconds=None):
- result = []
- for language in languages:
- for platform in platforms:
- for config in configs:
- name = '%s_%s_%s_%s' % (language, platform, config, iomgr_platform)
- runtests_args = ['-l', language,
- '-c', config,
- '--iomgr_platform', iomgr_platform]
- if arch or compiler:
- name += '_%s_%s' % (arch, compiler)
- runtests_args += ['--arch', arch,
- '--compiler', compiler]
- if '--build_only' in extra_args:
- name += '_buildonly'
- for extra_env in extra_envs:
- name += '_%s_%s' % (extra_env, extra_envs[extra_env])
+def _generate_jobs(languages,
+ configs,
+ platforms,
+ iomgr_platform='native',
+ arch=None,
+ compiler=None,
+ labels=[],
+ extra_args=[],
+ extra_envs={},
+ inner_jobs=_DEFAULT_INNER_JOBS,
+ timeout_seconds=None):
+ result = []
+ for language in languages:
+ for platform in platforms:
+ for config in configs:
+ name = '%s_%s_%s_%s' % (language, platform, config,
+ iomgr_platform)
+ runtests_args = [
+ '-l', language, '-c', config, '--iomgr_platform',
+ iomgr_platform
+ ]
+ if arch or compiler:
+ name += '_%s_%s' % (arch, compiler)
+ runtests_args += ['--arch', arch, '--compiler', compiler]
+ if '--build_only' in extra_args:
+ name += '_buildonly'
+ for extra_env in extra_envs:
+ name += '_%s_%s' % (extra_env, extra_envs[extra_env])
- runtests_args += extra_args
- if platform == 'linux':
- job = _docker_jobspec(name=name, runtests_args=runtests_args,
- runtests_envs=extra_envs, inner_jobs=inner_jobs,
- timeout_seconds=timeout_seconds)
- else:
- job = _workspace_jobspec(name=name, runtests_args=runtests_args,
- runtests_envs=extra_envs, inner_jobs=inner_jobs,
- timeout_seconds=timeout_seconds)
+ runtests_args += extra_args
+ if platform == 'linux':
+ job = _docker_jobspec(
+ name=name,
+ runtests_args=runtests_args,
+ runtests_envs=extra_envs,
+ inner_jobs=inner_jobs,
+ timeout_seconds=timeout_seconds)
+ else:
+ job = _workspace_jobspec(
+ name=name,
+ runtests_args=runtests_args,
+ runtests_envs=extra_envs,
+ inner_jobs=inner_jobs,
+ timeout_seconds=timeout_seconds)
- job.labels = [platform, config, language, iomgr_platform] + labels
- result.append(job)
- return result
+ job.labels = [platform, config, language, iomgr_platform
+ ] + labels
+ result.append(job)
+ return result
def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
- test_jobs = []
- # supported on linux only
- test_jobs += _generate_jobs(languages=['sanity', 'php7'],
- configs=['dbg', 'opt'],
- platforms=['linux'],
- labels=['basictests', 'multilang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs)
+ test_jobs = []
+ # supported on linux only
+ test_jobs += _generate_jobs(
+ languages=['sanity', 'php7'],
+ configs=['dbg', 'opt'],
+ platforms=['linux'],
+ labels=['basictests', 'multilang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs)
- # supported on all platforms.
- test_jobs += _generate_jobs(languages=['c'],
- configs=['dbg', 'opt'],
- platforms=['linux', 'macos', 'windows'],
- labels=['basictests', 'corelang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs,
- timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+ # supported on all platforms.
+ test_jobs += _generate_jobs(
+ languages=['c'],
+ configs=['dbg', 'opt'],
+ platforms=['linux', 'macos', 'windows'],
+ labels=['basictests', 'corelang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs,
+ timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
- test_jobs += _generate_jobs(languages=['csharp', 'python'],
- configs=['dbg', 'opt'],
- platforms=['linux', 'macos', 'windows'],
- labels=['basictests', 'multilang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs)
+ test_jobs += _generate_jobs(
+ languages=['csharp', 'python'],
+ configs=['dbg', 'opt'],
+ platforms=['linux', 'macos', 'windows'],
+ labels=['basictests', 'multilang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs)
- # supported on linux and mac.
- test_jobs += _generate_jobs(languages=['c++'],
- configs=['dbg', 'opt'],
- platforms=['linux', 'macos'],
- labels=['basictests', 'corelang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs,
- timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+ # supported on linux and mac.
+ test_jobs += _generate_jobs(
+ languages=['c++'],
+ configs=['dbg', 'opt'],
+ platforms=['linux', 'macos'],
+ labels=['basictests', 'corelang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs,
+ timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
- test_jobs += _generate_jobs(languages=['grpc-node', 'ruby', 'php'],
- configs=['dbg', 'opt'],
- platforms=['linux', 'macos'],
- labels=['basictests', 'multilang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs)
+ test_jobs += _generate_jobs(
+ languages=['grpc-node', 'ruby', 'php'],
+ configs=['dbg', 'opt'],
+ platforms=['linux', 'macos'],
+ labels=['basictests', 'multilang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs)
- # supported on mac only.
- test_jobs += _generate_jobs(languages=['objc'],
- configs=['dbg', 'opt'],
- platforms=['macos'],
- labels=['basictests', 'multilang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs)
+ # supported on mac only.
+ test_jobs += _generate_jobs(
+ languages=['objc'],
+ configs=['dbg', 'opt'],
+ platforms=['macos'],
+ labels=['basictests', 'multilang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs)
- # sanitizers
- test_jobs += _generate_jobs(languages=['c'],
- configs=['msan', 'asan', 'tsan', 'ubsan'],
- platforms=['linux'],
- labels=['sanitizers', 'corelang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs,
- timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
- test_jobs += _generate_jobs(languages=['c++'],
- configs=['asan'],
- platforms=['linux'],
- labels=['sanitizers', 'corelang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs,
- timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
- test_jobs += _generate_jobs(languages=['c++'],
- configs=['tsan'],
- platforms=['linux'],
- labels=['sanitizers', 'corelang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs,
- timeout_seconds=_CPP_TSAN_RUNTESTS_TIMEOUT)
+ # sanitizers
+ test_jobs += _generate_jobs(
+ languages=['c'],
+ configs=['msan', 'asan', 'tsan', 'ubsan'],
+ platforms=['linux'],
+ labels=['sanitizers', 'corelang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs,
+ timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+ test_jobs += _generate_jobs(
+ languages=['c++'],
+ configs=['asan'],
+ platforms=['linux'],
+ labels=['sanitizers', 'corelang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs,
+ timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+ test_jobs += _generate_jobs(
+ languages=['c++'],
+ configs=['tsan'],
+ platforms=['linux'],
+ labels=['sanitizers', 'corelang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs,
+ timeout_seconds=_CPP_TSAN_RUNTESTS_TIMEOUT)
- return test_jobs
+ return test_jobs
-def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
- test_jobs = []
- # portability C x86
- test_jobs += _generate_jobs(languages=['c'],
- configs=['dbg'],
- platforms=['linux'],
- arch='x86',
- compiler='default',
- labels=['portability', 'corelang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs)
+def _create_portability_test_jobs(extra_args=[],
+ inner_jobs=_DEFAULT_INNER_JOBS):
+ test_jobs = []
+ # portability C x86
+ test_jobs += _generate_jobs(
+ languages=['c'],
+ configs=['dbg'],
+ platforms=['linux'],
+ arch='x86',
+ compiler='default',
+ labels=['portability', 'corelang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs)
- # portability C and C++ on x64
- for compiler in ['gcc4.8', 'gcc5.3', 'gcc_musl',
- 'clang3.5', 'clang3.6', 'clang3.7']:
- test_jobs += _generate_jobs(languages=['c', 'c++'],
- configs=['dbg'],
- platforms=['linux'],
- arch='x64',
- compiler=compiler,
- labels=['portability', 'corelang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs,
- timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+ # portability C and C++ on x64
+ for compiler in [
+ 'gcc4.8', 'gcc5.3', 'gcc_musl', 'clang3.5', 'clang3.6', 'clang3.7'
+ ]:
+ test_jobs += _generate_jobs(
+ languages=['c', 'c++'],
+ configs=['dbg'],
+ platforms=['linux'],
+ arch='x64',
+ compiler=compiler,
+ labels=['portability', 'corelang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs,
+ timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
- # portability C on Windows 64-bit (x86 is the default)
- test_jobs += _generate_jobs(languages=['c'],
- configs=['dbg'],
- platforms=['windows'],
- arch='x64',
- compiler='default',
- labels=['portability', 'corelang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs)
+ # portability C on Windows 64-bit (x86 is the default)
+ test_jobs += _generate_jobs(
+ languages=['c'],
+ configs=['dbg'],
+ platforms=['windows'],
+ arch='x64',
+ compiler='default',
+ labels=['portability', 'corelang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs)
- # portability C++ on Windows
- # TODO(jtattermusch): some of the tests are failing, so we force --build_only
- test_jobs += _generate_jobs(languages=['c++'],
- configs=['dbg'],
- platforms=['windows'],
- arch='default',
- compiler='default',
- labels=['portability', 'corelang'],
- extra_args=extra_args + ['--build_only'],
- inner_jobs=inner_jobs)
+ # portability C++ on Windows
+ # TODO(jtattermusch): some of the tests are failing, so we force --build_only
+ test_jobs += _generate_jobs(
+ languages=['c++'],
+ configs=['dbg'],
+ platforms=['windows'],
+ arch='default',
+ compiler='default',
+ labels=['portability', 'corelang'],
+ extra_args=extra_args + ['--build_only'],
+ inner_jobs=inner_jobs)
- # portability C and C++ on Windows using VS2017 (build only)
- # TODO(jtattermusch): some of the tests are failing, so we force --build_only
- test_jobs += _generate_jobs(languages=['c', 'c++'],
- configs=['dbg'],
- platforms=['windows'],
- arch='x64',
- compiler='cmake_vs2017',
- labels=['portability', 'corelang'],
- extra_args=extra_args + ['--build_only'],
- inner_jobs=inner_jobs)
+ # portability C and C++ on Windows using VS2017 (build only)
+ # TODO(jtattermusch): some of the tests are failing, so we force --build_only
+ test_jobs += _generate_jobs(
+ languages=['c', 'c++'],
+ configs=['dbg'],
+ platforms=['windows'],
+ arch='x64',
+ compiler='cmake_vs2017',
+ labels=['portability', 'corelang'],
+ extra_args=extra_args + ['--build_only'],
+ inner_jobs=inner_jobs)
- # C and C++ with the c-ares DNS resolver on Linux
- test_jobs += _generate_jobs(languages=['c', 'c++'],
- configs=['dbg'], platforms=['linux'],
- labels=['portability', 'corelang'],
- extra_args=extra_args,
- extra_envs={'GRPC_DNS_RESOLVER': 'ares'},
- timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+ # C and C++ with the c-ares DNS resolver on Linux
+ test_jobs += _generate_jobs(
+ languages=['c', 'c++'],
+ configs=['dbg'],
+ platforms=['linux'],
+ labels=['portability', 'corelang'],
+ extra_args=extra_args,
+ extra_envs={'GRPC_DNS_RESOLVER': 'ares'},
+ timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
- # TODO(zyc): Turn on this test after adding c-ares support on windows.
- # C with the c-ares DNS resolver on Windows
- # test_jobs += _generate_jobs(languages=['c'],
- # configs=['dbg'], platforms=['windows'],
- # labels=['portability', 'corelang'],
- # extra_args=extra_args,
- # extra_envs={'GRPC_DNS_RESOLVER': 'ares'})
+ # TODO(zyc): Turn on this test after adding c-ares support on windows.
+ # C with the c-ares DNS resolver on Windows
+ # test_jobs += _generate_jobs(languages=['c'],
+ # configs=['dbg'], platforms=['windows'],
+ # labels=['portability', 'corelang'],
+ # extra_args=extra_args,
+ # extra_envs={'GRPC_DNS_RESOLVER': 'ares'})
- # C and C++ build with cmake on Linux
- # TODO(jtattermusch): some of the tests are failing, so we force --build_only
- # to make sure it's buildable at least.
- test_jobs += _generate_jobs(languages=['c', 'c++'],
- configs=['dbg'],
- platforms=['linux'],
- arch='default',
- compiler='cmake',
- labels=['portability', 'corelang'],
- extra_args=extra_args + ['--build_only'],
- inner_jobs=inner_jobs)
+ # C and C++ build with cmake on Linux
+ # TODO(jtattermusch): some of the tests are failing, so we force --build_only
+ # to make sure it's buildable at least.
+ test_jobs += _generate_jobs(
+ languages=['c', 'c++'],
+ configs=['dbg'],
+ platforms=['linux'],
+ arch='default',
+ compiler='cmake',
+ labels=['portability', 'corelang'],
+ extra_args=extra_args + ['--build_only'],
+ inner_jobs=inner_jobs)
- test_jobs += _generate_jobs(languages=['python'],
- configs=['dbg'],
- platforms=['linux'],
- arch='default',
- compiler='python_alpine',
- labels=['portability', 'multilang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs)
+ test_jobs += _generate_jobs(
+ languages=['python'],
+ configs=['dbg'],
+ platforms=['linux'],
+ arch='default',
+ compiler='python_alpine',
+ labels=['portability', 'multilang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs)
- test_jobs += _generate_jobs(languages=['csharp'],
- configs=['dbg'],
- platforms=['linux'],
- arch='default',
- compiler='coreclr',
- labels=['portability', 'multilang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs)
+ test_jobs += _generate_jobs(
+ languages=['csharp'],
+ configs=['dbg'],
+ platforms=['linux'],
+ arch='default',
+ compiler='coreclr',
+ labels=['portability', 'multilang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs)
- test_jobs += _generate_jobs(languages=['c'],
- configs=['dbg'],
- platforms=['linux'],
- iomgr_platform='uv',
- labels=['portability', 'corelang'],
- extra_args=extra_args,
- inner_jobs=inner_jobs,
- timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
+ test_jobs += _generate_jobs(
+ languages=['c'],
+ configs=['dbg'],
+ platforms=['linux'],
+ iomgr_platform='uv',
+ labels=['portability', 'corelang'],
+ extra_args=extra_args,
+ inner_jobs=inner_jobs,
+ timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
- return test_jobs
+ return test_jobs
def _allowed_labels():
- """Returns a list of existing job labels."""
- all_labels = set()
- for job in _create_test_jobs() + _create_portability_test_jobs():
- for label in job.labels:
- all_labels.add(label)
- return sorted(all_labels)
+ """Returns a list of existing job labels."""
+ all_labels = set()
+ for job in _create_test_jobs() + _create_portability_test_jobs():
+ for label in job.labels:
+ all_labels.add(label)
+ return sorted(all_labels)
def _runs_per_test_type(arg_str):
- """Auxiliary function to parse the "runs_per_test" flag."""
- try:
- n = int(arg_str)
- if n <= 0: raise ValueError
- return n
- except:
- msg = '\'{}\' is not a positive integer'.format(arg_str)
- raise argparse.ArgumentTypeError(msg)
+ """Auxiliary function to parse the "runs_per_test" flag."""
+ try:
+ n = int(arg_str)
+ if n <= 0: raise ValueError
+ return n
+ except:
+ msg = '\'{}\' is not a positive integer'.format(arg_str)
+ raise argparse.ArgumentTypeError(msg)
if __name__ == "__main__":
- argp = argparse.ArgumentParser(description='Run a matrix of run_tests.py tests.')
- argp.add_argument('-j', '--jobs',
- default=multiprocessing.cpu_count()/_DEFAULT_INNER_JOBS,
- type=int,
- help='Number of concurrent run_tests.py instances.')
- argp.add_argument('-f', '--filter',
- choices=_allowed_labels(),
- nargs='+',
- default=[],
- help='Filter targets to run by label with AND semantics.')
- argp.add_argument('--exclude',
- choices=_allowed_labels(),
- nargs='+',
- default=[],
- help='Exclude targets with any of given labels.')
- argp.add_argument('--build_only',
- default=False,
- action='store_const',
- const=True,
- help='Pass --build_only flag to run_tests.py instances.')
- argp.add_argument('--force_default_poller', default=False, action='store_const', const=True,
- help='Pass --force_default_poller to run_tests.py instances.')
- argp.add_argument('--dry_run',
- default=False,
- action='store_const',
- const=True,
- help='Only print what would be run.')
- argp.add_argument('--filter_pr_tests',
- default=False,
- action='store_const',
- const=True,
- help='Filters out tests irrelevant to pull request changes.')
- argp.add_argument('--base_branch',
- default='origin/master',
- type=str,
- help='Branch that pull request is requesting to merge into')
- argp.add_argument('--inner_jobs',
- default=_DEFAULT_INNER_JOBS,
- type=int,
- help='Number of jobs in each run_tests.py instance')
- argp.add_argument('-n', '--runs_per_test', default=1, type=_runs_per_test_type,
- help='How many times to run each tests. >1 runs implies ' +
- 'omitting passing test from the output & reports.')
- argp.add_argument('--max_time', default=-1, type=int,
- help='Maximum amount of time to run tests for' +
- '(other tests will be skipped)')
- argp.add_argument('--internal_ci',
- default=False,
- action='store_const',
- const=True,
- help='Put reports into subdirectories to improve presentation of '
- 'results by Internal CI.')
- argp.add_argument('--bq_result_table',
- default='',
- type=str,
- nargs='?',
- help='Upload test results to a specified BQ table.')
- args = argp.parse_args()
+ argp = argparse.ArgumentParser(
+ description='Run a matrix of run_tests.py tests.')
+ argp.add_argument(
+ '-j',
+ '--jobs',
+ default=multiprocessing.cpu_count() / _DEFAULT_INNER_JOBS,
+ type=int,
+ help='Number of concurrent run_tests.py instances.')
+ argp.add_argument(
+ '-f',
+ '--filter',
+ choices=_allowed_labels(),
+ nargs='+',
+ default=[],
+ help='Filter targets to run by label with AND semantics.')
+ argp.add_argument(
+ '--exclude',
+ choices=_allowed_labels(),
+ nargs='+',
+ default=[],
+ help='Exclude targets with any of given labels.')
+ argp.add_argument(
+ '--build_only',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Pass --build_only flag to run_tests.py instances.')
+ argp.add_argument(
+ '--force_default_poller',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Pass --force_default_poller to run_tests.py instances.')
+ argp.add_argument(
+ '--dry_run',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Only print what would be run.')
+ argp.add_argument(
+ '--filter_pr_tests',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Filters out tests irrelevant to pull request changes.')
+ argp.add_argument(
+ '--base_branch',
+ default='origin/master',
+ type=str,
+ help='Branch that pull request is requesting to merge into')
+ argp.add_argument(
+ '--inner_jobs',
+ default=_DEFAULT_INNER_JOBS,
+ type=int,
+ help='Number of jobs in each run_tests.py instance')
+ argp.add_argument(
+ '-n',
+ '--runs_per_test',
+ default=1,
+ type=_runs_per_test_type,
+ help='How many times to run each tests. >1 runs implies ' +
+ 'omitting passing test from the output & reports.')
+ argp.add_argument(
+ '--max_time',
+ default=-1,
+ type=int,
+ help='Maximum amount of time to run tests for' +
+ '(other tests will be skipped)')
+ argp.add_argument(
+ '--internal_ci',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Put reports into subdirectories to improve presentation of '
+ 'results by Internal CI.')
+ argp.add_argument(
+ '--bq_result_table',
+ default='',
+ type=str,
+ nargs='?',
+ help='Upload test results to a specified BQ table.')
+ args = argp.parse_args()
- if args.internal_ci:
- _report_filename = _report_filename_internal_ci # override the function
+ if args.internal_ci:
+ _report_filename = _report_filename_internal_ci # override the function
- extra_args = []
- if args.build_only:
- extra_args.append('--build_only')
- if args.force_default_poller:
- extra_args.append('--force_default_poller')
- if args.runs_per_test > 1:
- extra_args.append('-n')
- extra_args.append('%s' % args.runs_per_test)
- extra_args.append('--quiet_success')
- if args.max_time > 0:
- extra_args.extend(('--max_time', '%d' % args.max_time))
- if args.bq_result_table:
- extra_args.append('--bq_result_table')
- extra_args.append('%s' % args.bq_result_table)
- extra_args.append('--measure_cpu_costs')
- extra_args.append('--disable_auto_set_flakes')
+ extra_args = []
+ if args.build_only:
+ extra_args.append('--build_only')
+ if args.force_default_poller:
+ extra_args.append('--force_default_poller')
+ if args.runs_per_test > 1:
+ extra_args.append('-n')
+ extra_args.append('%s' % args.runs_per_test)
+ extra_args.append('--quiet_success')
+ if args.max_time > 0:
+ extra_args.extend(('--max_time', '%d' % args.max_time))
+ if args.bq_result_table:
+ extra_args.append('--bq_result_table')
+ extra_args.append('%s' % args.bq_result_table)
+ extra_args.append('--measure_cpu_costs')
+ extra_args.append('--disable_auto_set_flakes')
- all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \
- _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)
+ all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \
+ _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)
- jobs = []
- for job in all_jobs:
- if not args.filter or all(filter in job.labels for filter in args.filter):
- if not any(exclude_label in job.labels for exclude_label in args.exclude):
- jobs.append(job)
+ jobs = []
+ for job in all_jobs:
+ if not args.filter or all(filter in job.labels
+ for filter in args.filter):
+ if not any(exclude_label in job.labels
+ for exclude_label in args.exclude):
+ jobs.append(job)
- if not jobs:
- jobset.message('FAILED', 'No test suites match given criteria.',
- do_newline=True)
- sys.exit(1)
+ if not jobs:
+ jobset.message(
+ 'FAILED', 'No test suites match given criteria.', do_newline=True)
+ sys.exit(1)
- print('IMPORTANT: The changes you are testing need to be locally committed')
- print('because only the committed changes in the current branch will be')
- print('copied to the docker environment or into subworkspaces.')
+ print('IMPORTANT: The changes you are testing need to be locally committed')
+ print('because only the committed changes in the current branch will be')
+ print('copied to the docker environment or into subworkspaces.')
- skipped_jobs = []
+ skipped_jobs = []
- if args.filter_pr_tests:
- print('Looking for irrelevant tests to skip...')
- relevant_jobs = filter_tests(jobs, args.base_branch)
- if len(relevant_jobs) == len(jobs):
- print('No tests will be skipped.')
- else:
- print('These tests will be skipped:')
- skipped_jobs = list(set(jobs) - set(relevant_jobs))
- # Sort by shortnames to make printing of skipped tests consistent
- skipped_jobs.sort(key=lambda job: job.shortname)
- for job in list(skipped_jobs):
- print(' %s' % job.shortname)
- jobs = relevant_jobs
+ if args.filter_pr_tests:
+ print('Looking for irrelevant tests to skip...')
+ relevant_jobs = filter_tests(jobs, args.base_branch)
+ if len(relevant_jobs) == len(jobs):
+ print('No tests will be skipped.')
+ else:
+ print('These tests will be skipped:')
+ skipped_jobs = list(set(jobs) - set(relevant_jobs))
+ # Sort by shortnames to make printing of skipped tests consistent
+ skipped_jobs.sort(key=lambda job: job.shortname)
+ for job in list(skipped_jobs):
+ print(' %s' % job.shortname)
+ jobs = relevant_jobs
- print('Will run these tests:')
- for job in jobs:
+ print('Will run these tests:')
+ for job in jobs:
+ if args.dry_run:
+ print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline)))
+ else:
+ print(' %s' % job.shortname)
+ print
+
if args.dry_run:
- print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline)))
+ print('--dry_run was used, exiting')
+ sys.exit(1)
+
+ jobset.message('START', 'Running test matrix.', do_newline=True)
+ num_failures, resultset = jobset.run(
+ jobs, newline_on_success=True, travis=True, maxjobs=args.jobs)
+ # Merge skipped tests into results to show skipped tests on report.xml
+ if skipped_jobs:
+ ignored_num_skipped_failures, skipped_results = jobset.run(
+ skipped_jobs, skip_jobs=True)
+ resultset.update(skipped_results)
+ report_utils.render_junit_xml_report(
+ resultset,
+ _report_filename('aggregate_tests'),
+ suite_name='aggregate_tests')
+
+ if num_failures == 0:
+ jobset.message(
+ 'SUCCESS',
+ 'All run_tests.py instance finished successfully.',
+ do_newline=True)
else:
- print(' %s' % job.shortname)
- print
-
- if args.dry_run:
- print('--dry_run was used, exiting')
- sys.exit(1)
-
- jobset.message('START', 'Running test matrix.', do_newline=True)
- num_failures, resultset = jobset.run(jobs,
- newline_on_success=True,
- travis=True,
- maxjobs=args.jobs)
- # Merge skipped tests into results to show skipped tests on report.xml
- if skipped_jobs:
- ignored_num_skipped_failures, skipped_results = jobset.run(
- skipped_jobs, skip_jobs=True)
- resultset.update(skipped_results)
- report_utils.render_junit_xml_report(resultset, _report_filename('aggregate_tests'),
- suite_name='aggregate_tests')
-
- if num_failures == 0:
- jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.',
- do_newline=True)
- else:
- jobset.message('FAILED', 'Some run_tests.py instance have failed.',
- do_newline=True)
- sys.exit(1)
+ jobset.message(
+ 'FAILED',
+ 'Some run_tests.py instance have failed.',
+ do_newline=True)
+ sys.exit(1)