Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python2.7 |
| 2 | # Copyright 2017 gRPC authors. |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 15 | """Run tests using docker images in Google Container Registry per matrix.""" |
| 16 | |
| 17 | from __future__ import print_function |
| 18 | |
| 19 | import argparse |
| 20 | import atexit |
| 21 | import json |
| 22 | import multiprocessing |
| 23 | import os |
| 24 | import re |
| 25 | import subprocess |
| 26 | import sys |
| 27 | import uuid |
| 28 | |
| 29 | # Langauage Runtime Matrix |
| 30 | import client_matrix |
| 31 | |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 32 | python_util_dir = os.path.abspath( |
| 33 | os.path.join(os.path.dirname(__file__), '../run_tests/python_utils')) |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 34 | sys.path.append(python_util_dir) |
| 35 | import dockerjob |
| 36 | import jobset |
| 37 | import report_utils |
Adele Zhou | 10ab806 | 2017-10-11 11:30:56 -0700 | [diff] [blame] | 38 | import upload_test_results |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 39 | |
| 40 | _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys() |
| 41 | # All gRPC release tags, flattened, deduped and sorted. |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 42 | _RELEASES = sorted( |
| 43 | list( |
| 44 | set( |
| 45 | client_matrix.get_release_tag_name(info) |
| 46 | for lang in client_matrix.LANG_RELEASE_MATRIX.values() |
| 47 | for info in lang))) |
Adele Zhou | cda43a3 | 2018-05-18 11:31:19 -0700 | [diff] [blame] | 48 | _TEST_TIMEOUT = 60 |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 49 | |
| 50 | argp = argparse.ArgumentParser(description='Run interop tests.') |
| 51 | argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 52 | argp.add_argument( |
| 53 | '--gcr_path', |
| 54 | default='gcr.io/grpc-testing', |
| 55 | help='Path of docker images in Google Container Registry') |
| 56 | argp.add_argument( |
| 57 | '--release', |
| 58 | default='all', |
| 59 | choices=['all', 'master'] + _RELEASES, |
| 60 | help='Release tags to test. When testing all ' |
| 61 | 'releases defined in client_matrix.py, use "all".') |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 62 | argp.add_argument( |
| 63 | '-l', |
| 64 | '--language', |
| 65 | choices=['all'] + sorted(_LANGUAGES), |
| 66 | nargs='+', |
| 67 | default=['all'], |
| 68 | help='Languages to test') |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 69 | argp.add_argument( |
| 70 | '--keep', |
| 71 | action='store_true', |
| 72 | help='keep the created local images after finishing the tests.') |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 73 | argp.add_argument( |
| 74 | '--report_file', default='report.xml', help='The result file to create.') |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 75 | argp.add_argument( |
| 76 | '--allow_flakes', |
| 77 | default=False, |
| 78 | action='store_const', |
| 79 | const=True, |
| 80 | help=('Allow flaky tests to show as passing (re-runs failed ' |
| 81 | 'tests up to five times)')) |
| 82 | argp.add_argument( |
| 83 | '--bq_result_table', |
| 84 | default='', |
| 85 | type=str, |
| 86 | nargs='?', |
| 87 | help='Upload test results to a specified BQ table.') |
Adele Zhou | b1c9b5f | 2018-03-06 16:45:11 -0800 | [diff] [blame] | 88 | argp.add_argument( |
| 89 | '--server_host', |
| 90 | default='74.125.206.210', |
| 91 | type=str, |
| 92 | nargs='?', |
| 93 | help='The gateway to backend services.') |
Adele Zhou | e757adc | 2017-10-09 17:40:03 -0700 | [diff] [blame] | 94 | |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 95 | args = argp.parse_args() |
| 96 | |
Adele Zhou | 92b5a00 | 2017-11-07 11:25:49 -0800 | [diff] [blame] | 97 | print(str(args)) |
| 98 | |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 99 | |
| 100 | def find_all_images_for_lang(lang): |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 101 | """Find docker images for a language across releases and runtimes. |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 102 | |
| 103 | Returns dictionary of list of (<tag>, <image-full-path>) keyed by runtime. |
| 104 | """ |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 105 | # Find all defined releases. |
| 106 | if args.release == 'all': |
| 107 | releases = ['master'] + client_matrix.get_release_tags(lang) |
| 108 | else: |
| 109 | # Look for a particular release. |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 110 | if args.release not in ['master' |
| 111 | ] + client_matrix.get_release_tags(lang): |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 112 | jobset.message( |
| 113 | 'SKIPPED', |
| 114 | '%s for %s is not defined' % (args.release, lang), |
| 115 | do_newline=True) |
| 116 | return {} |
| 117 | releases = [args.release] |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 118 | |
Jan Tattermusch | b6a2eeb | 2018-05-30 17:13:32 +0200 | [diff] [blame] | 119 | # TODO(jtattermusch): why do we need to query the existing images/tags? |
| 120 | # From LANG_RUNTIME_MATRIX and LANG_RELEASE_MATRIX it should be obvious |
| 121 | # which tags we want to test - and it should be an error if they are |
| 122 | # missing. |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 123 | # Images tuples keyed by runtime. |
| 124 | images = {} |
| 125 | for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]: |
| 126 | image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime) |
| 127 | output = subprocess.check_output([ |
| 128 | 'gcloud', 'beta', 'container', 'images', 'list-tags', |
| 129 | '--format=json', image_path |
| 130 | ]) |
| 131 | docker_image_list = json.loads(output) |
| 132 | # All images should have a single tag or no tag. |
| 133 | # TODO(adelez): Remove tagless images. |
| 134 | tags = [i['tags'][0] for i in docker_image_list if i['tags']] |
| 135 | jobset.message( |
| 136 | 'START', |
| 137 | 'Found images for %s: %s' % (image_path, tags), |
| 138 | do_newline=True) |
| 139 | skipped = len(docker_image_list) - len(tags) |
| 140 | jobset.message( |
| 141 | 'SKIPPED', |
| 142 | 'Skipped images (no-tag/unknown-tag): %d' % skipped, |
| 143 | do_newline=True) |
| 144 | # Filter tags based on the releases. |
Mehrdad Afshari | 87cd994 | 2018-01-02 14:40:00 -0800 | [diff] [blame] | 145 | images[runtime] = [(tag, '%s:%s' % (image_path, tag)) |
| 146 | for tag in tags |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 147 | if tag in releases] |
| 148 | return images |
| 149 | |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 150 | |
| 151 | # caches test cases (list of JobSpec) loaded from file. Keyed by lang and runtime. |
Adele Zhou | 317641d | 2017-11-03 17:52:14 -0700 | [diff] [blame] | 152 | def find_test_cases(lang, runtime, release, suite_name): |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 153 | """Returns the list of test cases from testcase files per lang/release.""" |
Adele Zhou | 46fb165 | 2018-03-19 15:33:46 -0700 | [diff] [blame] | 154 | testcase_dir = os.path.join(os.path.dirname(__file__), 'testcases') |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 155 | filename_prefix = lang |
| 156 | if lang == 'csharp': |
| 157 | filename_prefix = runtime |
Adele Zhou | 46fb165 | 2018-03-19 15:33:46 -0700 | [diff] [blame] | 158 | # Check to see if we need to use a particular version of test cases. |
| 159 | lang_version = '%s_%s' % (filename_prefix, release) |
| 160 | if lang_version in client_matrix.TESTCASES_VERSION_MATRIX: |
| 161 | testcases = os.path.join( |
| 162 | testcase_dir, client_matrix.TESTCASES_VERSION_MATRIX[lang_version]) |
| 163 | else: |
| 164 | testcases = os.path.join(testcase_dir, '%s__master' % filename_prefix) |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 165 | |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 166 | job_spec_list = [] |
| 167 | try: |
| 168 | with open(testcases) as f: |
| 169 | # Only line start with 'docker run' are test cases. |
| 170 | for line in f.readlines(): |
| 171 | if line.startswith('docker run'): |
| 172 | m = re.search('--test_case=(.*)"', line) |
| 173 | shortname = m.group(1) if m else 'unknown_test' |
| 174 | m = re.search( |
| 175 | '--server_host_override=(.*).sandbox.googleapis.com', |
Adele Zhou | 091c3eb | 2017-10-11 16:51:06 -0700 | [diff] [blame] | 176 | line) |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 177 | server = m.group(1) if m else 'unknown_server' |
Adele Zhou | b1c9b5f | 2018-03-06 16:45:11 -0800 | [diff] [blame] | 178 | |
| 179 | # If server_host arg is not None, replace the original |
| 180 | # server_host with the one provided or append to the end of |
| 181 | # the command if server_host does not appear originally. |
| 182 | if args.server_host: |
| 183 | if line.find('--server_host=') > -1: |
| 184 | line = re.sub('--server_host=[^ ]*', |
| 185 | '--server_host=%s' % args.server_host, |
| 186 | line) |
| 187 | else: |
| 188 | line = '%s --server_host=%s"' % (line[:-1], |
| 189 | args.server_host) |
| 190 | print(line) |
| 191 | |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 192 | spec = jobset.JobSpec( |
| 193 | cmdline=line, |
| 194 | shortname='%s:%s:%s:%s' % (suite_name, lang, server, |
| 195 | shortname), |
| 196 | timeout_seconds=_TEST_TIMEOUT, |
| 197 | shell=True, |
| 198 | flake_retries=5 if args.allow_flakes else 0) |
| 199 | job_spec_list.append(spec) |
| 200 | jobset.message( |
| 201 | 'START', |
| 202 | 'Loaded %s tests from %s' % (len(job_spec_list), testcases), |
| 203 | do_newline=True) |
| 204 | except IOError as err: |
| 205 | jobset.message('FAILED', err, do_newline=True) |
| 206 | return job_spec_list |
| 207 | |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 208 | |
Yong Ni | 5f32c51 | 2017-07-05 11:43:29 -0700 | [diff] [blame] | 209 | _xml_report_tree = report_utils.new_junit_xml_tree() |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 210 | |
| 211 | |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 212 | def run_tests_for_lang(lang, runtime, images): |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 213 | """Find and run all test cases for a language. |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 214 | |
| 215 | images is a list of (<release-tag>, <image-full-path>) tuple. |
| 216 | """ |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 217 | total_num_failures = 0 |
Jan Tattermusch | b2f56a0 | 2018-06-11 11:36:21 +0200 | [diff] [blame] | 218 | for image_tuple in images: |
| 219 | release, image = image_tuple |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 220 | jobset.message('START', 'Testing %s' % image, do_newline=True) |
Jan Tattermusch | b2f56a0 | 2018-06-11 11:36:21 +0200 | [diff] [blame] | 221 | # Download the docker image before running each test case. |
| 222 | subprocess.check_call(['gcloud', 'docker', '--', 'pull', image]) |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 223 | suite_name = '%s__%s_%s' % (lang, runtime, release) |
| 224 | job_spec_list = find_test_cases(lang, runtime, release, suite_name) |
Adele Zhou | 317641d | 2017-11-03 17:52:14 -0700 | [diff] [blame] | 225 | |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 226 | if not job_spec_list: |
| 227 | jobset.message( |
| 228 | 'FAILED', 'No test cases were found.', do_newline=True) |
| 229 | return 1 |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 230 | |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 231 | num_failures, resultset = jobset.run( |
| 232 | job_spec_list, |
| 233 | newline_on_success=True, |
| 234 | add_env={'docker_image': image}, |
| 235 | maxjobs=args.jobs) |
| 236 | if args.bq_result_table and resultset: |
| 237 | upload_test_results.upload_interop_results_to_bq( |
| 238 | resultset, args.bq_result_table, args) |
| 239 | if num_failures: |
| 240 | jobset.message('FAILED', 'Some tests failed', do_newline=True) |
| 241 | total_num_failures += num_failures |
| 242 | else: |
| 243 | jobset.message('SUCCESS', 'All tests passed', do_newline=True) |
Adele Zhou | 92b5a00 | 2017-11-07 11:25:49 -0800 | [diff] [blame] | 244 | |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 245 | report_utils.append_junit_xml_results(_xml_report_tree, resultset, |
| 246 | 'grpc_interop_matrix', suite_name, |
| 247 | str(uuid.uuid4())) |
| 248 | |
| 249 | if not args.keep: |
| 250 | cleanup(image) |
| 251 | |
| 252 | return total_num_failures |
Adele Zhou | e6b9c78 | 2017-10-04 15:53:03 -0700 | [diff] [blame] | 253 | |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 254 | |
Adele Zhou | 92b5a00 | 2017-11-07 11:25:49 -0800 | [diff] [blame] | 255 | def cleanup(image): |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 256 | jobset.message('START', 'Cleanup docker image %s' % image, do_newline=True) |
| 257 | dockerjob.remove_image(image, skip_nonexistent=True) |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 258 | |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 259 | |
| 260 | languages = args.language if args.language != ['all'] else _LANGUAGES |
Adele Zhou | e6b9c78 | 2017-10-04 15:53:03 -0700 | [diff] [blame] | 261 | total_num_failures = 0 |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 262 | for lang in languages: |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 263 | docker_images = find_all_images_for_lang(lang) |
| 264 | for runtime in sorted(docker_images.keys()): |
| 265 | total_num_failures += run_tests_for_lang(lang, runtime, |
| 266 | docker_images[runtime]) |
Yong Ni | b7ea4ab | 2017-06-26 15:24:40 -0700 | [diff] [blame] | 267 | |
Yong Ni | 5f32c51 | 2017-07-05 11:43:29 -0700 | [diff] [blame] | 268 | report_utils.create_xml_report_file(_xml_report_tree, args.report_file) |
Adele Zhou | e6b9c78 | 2017-10-04 15:53:03 -0700 | [diff] [blame] | 269 | |
| 270 | if total_num_failures: |
ncteisen | e4bef08 | 2017-12-11 16:51:34 -0800 | [diff] [blame] | 271 | sys.exit(1) |
Adele Zhou | e6b9c78 | 2017-10-04 15:53:03 -0700 | [diff] [blame] | 272 | sys.exit(0) |