blob: 29d5a9aaa008fa41ef46eb17ee2d41be8ee099da [file] [log] [blame]
Siddharth Shukla8e64d902017-03-12 19:50:18 +01001#!/usr/bin/env python
Craig Tiller6169d5f2016-03-31 07:46:18 -07002# Copyright 2015, Google Inc.
Jan Tattermusch320bd612015-09-15 12:44:35 -07003# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9# * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11# * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15# * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Run interop (cross-language) tests in parallel."""
32
siddharthshukla0589e532016-07-07 16:08:01 +020033from __future__ import print_function
34
Jan Tattermusch320bd612015-09-15 12:44:35 -070035import argparse
Carl Mastrangeloe7f8e8e2015-12-08 17:22:44 -080036import atexit
Jan Tattermusch320bd612015-09-15 12:44:35 -070037import itertools
Carl Mastrangeloe7f8e8e2015-12-08 17:22:44 -080038import json
Jan Tattermusch210a0ea2015-10-02 15:05:36 -070039import multiprocessing
Jan Tattermusch8266c672015-09-17 09:18:03 -070040import os
Carl Mastrangeloe7f8e8e2015-12-08 17:22:44 -080041import re
Carl Mastrangeloe7f8e8e2015-12-08 17:22:44 -080042import subprocess
Jan Tattermusch8266c672015-09-17 09:18:03 -070043import sys
Jan Tattermusch91ad0182015-10-01 09:22:03 -070044import tempfile
Jan Tattermusch8266c672015-09-17 09:18:03 -070045import time
Jan Tattermusch91ad0182015-10-01 09:22:03 -070046import uuid
Siddharth Shuklad194f592017-03-11 19:12:43 +010047import six
Jan Tattermusch320bd612015-09-15 12:44:35 -070048
Jan Tattermusch5c79a312016-12-20 11:02:50 +010049import python_utils.dockerjob as dockerjob
50import python_utils.jobset as jobset
51import python_utils.report_utils as report_utils
52
Carl Mastrangeloe7f8e8e2015-12-08 17:22:44 -080053# Docker doesn't clean up after itself, so we do it on exit.
54atexit.register(lambda: subprocess.call(['stty', 'echo']))
55
Jan Tattermusch91ad0182015-10-01 09:22:03 -070056ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
57os.chdir(ROOT)
58
59_DEFAULT_SERVER_PORT=8080
Jan Tattermuschf49936a2015-09-16 15:44:26 -070060
Jan Tattermusch9fc079f2016-06-21 19:53:31 -070061_SKIP_CLIENT_COMPRESSION = ['client_compressed_unary',
62 'client_compressed_streaming']
63
64_SKIP_SERVER_COMPRESSION = ['server_compressed_unary',
65 'server_compressed_streaming']
66
67_SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -080068
Noah Eisen794afb32016-11-11 14:42:52 -080069_SKIP_ADVANCED = ['status_code_and_message',
70 'custom_metadata',
71 'unimplemented_method',
72 'unimplemented_service']
Jan Tattermusch8266c672015-09-17 09:18:03 -070073
Jan Tattermusch19f703d2016-03-03 15:33:29 -080074_TEST_TIMEOUT = 3*60
75
Jan Tattermuschf49936a2015-09-16 15:44:26 -070076class CXXLanguage:
77
78 def __init__(self):
Jan Tattermuschf49936a2015-09-16 15:44:26 -070079 self.client_cwd = None
Jan Tattermusch91ad0182015-10-01 09:22:03 -070080 self.server_cwd = None
Eric Gribkoffcaf7ea52017-03-01 08:15:25 -080081 self.http2_cwd = None
Jan Tattermusch0a14f622015-10-09 14:34:29 -070082 self.safename = 'cxx'
Jan Tattermuschf49936a2015-09-16 15:44:26 -070083
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -070084 def client_cmd(self, args):
85 return ['bins/opt/interop_client'] + args
Jan Tattermusch8266c672015-09-17 09:18:03 -070086
Eric Gribkoffcaf7ea52017-03-01 08:15:25 -080087 def client_cmd_http2interop(self, args):
88 return ['bins/opt/http2_client'] + args
89
Jan Tattermuschf49936a2015-09-16 15:44:26 -070090 def cloud_to_prod_env(self):
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -070091 return {}
Jan Tattermuschf49936a2015-09-16 15:44:26 -070092
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -070093 def server_cmd(self, args):
Adele Zhou57b02fc2017-01-17 14:18:41 -080094 return ['bins/opt/interop_server'] + args
Jan Tattermusch91ad0182015-10-01 09:22:03 -070095
Masood Malekghassemi18cc8422015-10-09 17:55:45 -070096 def global_env(self):
97 return {}
98
Jan Tattermusch289b7b92015-10-21 18:09:59 -070099 def unimplemented_test_cases(self):
Noah Eisen88477fd2016-10-18 12:25:25 -0700100 return []
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800101
102 def unimplemented_test_cases_server(self):
Noah Eisen88477fd2016-10-18 12:25:25 -0700103 return []
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700104
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700105 def __str__(self):
106 return 'c++'
107
108
109class CSharpLanguage:
110
111 def __init__(self):
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700112 self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug'
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700113 self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug'
Jan Tattermusch0a14f622015-10-09 14:34:29 -0700114 self.safename = str(self)
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700115
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700116 def client_cmd(self, args):
117 return ['mono', 'Grpc.IntegrationTesting.Client.exe'] + args
Jan Tattermusch8266c672015-09-17 09:18:03 -0700118
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700119 def cloud_to_prod_env(self):
Jan Tattermuscha6b2c4c2015-12-10 16:41:11 -0800120 return {}
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700121
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700122 def server_cmd(self, args):
Adele Zhou57b02fc2017-01-17 14:18:41 -0800123 return ['mono', 'Grpc.IntegrationTesting.Server.exe'] + args
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700124
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700125 def global_env(self):
126 return {}
127
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700128 def unimplemented_test_cases(self):
Jan Tattermusch9fc079f2016-06-21 19:53:31 -0700129 return _SKIP_SERVER_COMPRESSION
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800130
131 def unimplemented_test_cases_server(self):
132 return _SKIP_COMPRESSION
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700133
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700134 def __str__(self):
135 return 'csharp'
136
137
Jan Tattermuscha98a2ad2016-09-09 12:42:32 +0200138class CSharpCoreCLRLanguage:
139
140 def __init__(self):
141 self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug/netcoreapp1.0'
142 self.server_cwd = 'src/csharp/Grpc.IntegrationTesting.Server/bin/Debug/netcoreapp1.0'
143 self.safename = str(self)
144
145 def client_cmd(self, args):
146 return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Client.dll'] + args
147
148 def cloud_to_prod_env(self):
149 return {}
150
151 def server_cmd(self, args):
Adele Zhou57b02fc2017-01-17 14:18:41 -0800152 return ['dotnet', 'exec', 'Grpc.IntegrationTesting.Server.dll'] + args
Jan Tattermuscha98a2ad2016-09-09 12:42:32 +0200153
154 def global_env(self):
155 return {}
156
157 def unimplemented_test_cases(self):
158 return _SKIP_SERVER_COMPRESSION
159
160 def unimplemented_test_cases_server(self):
161 return _SKIP_COMPRESSION
162
163 def __str__(self):
164 return 'csharpcoreclr'
165
166
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700167class JavaLanguage:
168
169 def __init__(self):
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700170 self.client_cwd = '../grpc-java'
171 self.server_cwd = '../grpc-java'
Noah Eisende99fe82017-02-27 17:26:20 -0800172 self.http2_cwd = '../grpc-java'
Jan Tattermusch0a14f622015-10-09 14:34:29 -0700173 self.safename = str(self)
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700174
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700175 def client_cmd(self, args):
176 return ['./run-test-client.sh'] + args
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700177
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800178 def client_cmd_http2interop(self, args):
Eric Gribkoff4e511402017-01-20 11:07:59 -0800179 return ['./interop-testing/build/install/grpc-interop-testing/bin/http2-client'] + args
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800180
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700181 def cloud_to_prod_env(self):
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700182 return {}
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700183
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700184 def server_cmd(self, args):
Adele Zhou57b02fc2017-01-17 14:18:41 -0800185 return ['./run-test-server.sh'] + args
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700186
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700187 def global_env(self):
188 return {}
189
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700190 def unimplemented_test_cases(self):
Eric Gribkoffced87022017-01-06 09:16:29 -0800191 return _SKIP_COMPRESSION
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800192
193 def unimplemented_test_cases_server(self):
Eric Gribkoffced87022017-01-06 09:16:29 -0800194 return _SKIP_COMPRESSION
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700195
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700196 def __str__(self):
197 return 'java'
198
199
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700200class GoLanguage:
201
202 def __init__(self):
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700203 # TODO: this relies on running inside docker
204 self.client_cwd = '/go/src/google.golang.org/grpc/interop/client'
205 self.server_cwd = '/go/src/google.golang.org/grpc/interop/server'
Noah Eisende99fe82017-02-27 17:26:20 -0800206 self.http2_cwd = '/go/src/google.golang.org/grpc/interop/http2'
Jan Tattermusch0a14f622015-10-09 14:34:29 -0700207 self.safename = str(self)
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700208
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700209 def client_cmd(self, args):
210 return ['go', 'run', 'client.go'] + args
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700211
Noah Eisende99fe82017-02-27 17:26:20 -0800212 def client_cmd_http2interop(self, args):
213 return ['go', 'run', 'negative_http2_client.go'] + args
214
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700215 def cloud_to_prod_env(self):
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700216 return {}
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700217
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700218 def server_cmd(self, args):
Adele Zhou57b02fc2017-01-17 14:18:41 -0800219 return ['go', 'run', 'server.go'] + args
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700220
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700221 def global_env(self):
222 return {}
223
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700224 def unimplemented_test_cases(self):
Noah Eisen794afb32016-11-11 14:42:52 -0800225 return _SKIP_COMPRESSION
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800226
227 def unimplemented_test_cases_server(self):
Noah Eisen794afb32016-11-11 14:42:52 -0800228 return _SKIP_COMPRESSION
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700229
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700230 def __str__(self):
231 return 'go'
232
Makarand Dharmapurikar078e4212017-01-10 15:09:21 -0800233class Http2Server:
234 """Represents the HTTP/2 Interop Test server
235
236 This pretends to be a language in order to be built and run, but really it
237 isn't.
238 """
239 def __init__(self):
240 self.server_cwd = None
241 self.safename = str(self)
242
243 def server_cmd(self, args):
244 return ['python test/http2_test/http2_test_server.py']
245
246 def cloud_to_prod_env(self):
247 return {}
248
249 def global_env(self):
250 return {}
251
252 def unimplemented_test_cases(self):
253 return _TEST_CASES
254
255 def unimplemented_test_cases_server(self):
256 return _TEST_CASES
257
258 def __str__(self):
259 return 'http2'
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700260
Carl Mastrangelode449102015-10-28 11:05:49 -0700261class Http2Client:
262 """Represents the HTTP/2 Interop Test
263
264 This pretends to be a language in order to be built and run, but really it
265 isn't.
266 """
267 def __init__(self):
268 self.client_cwd = None
269 self.safename = str(self)
270
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800271 def client_cmd(self, args):
272 return ['tools/http2_interop/http2_interop.test', '-test.v'] + args
Carl Mastrangelode449102015-10-28 11:05:49 -0700273
274 def cloud_to_prod_env(self):
275 return {}
276
277 def global_env(self):
278 return {}
279
280 def unimplemented_test_cases(self):
281 return _TEST_CASES
282
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800283 def unimplemented_test_cases_server(self):
284 return []
285
Carl Mastrangelode449102015-10-28 11:05:49 -0700286 def __str__(self):
287 return 'http2'
288
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700289class NodeLanguage:
290
291 def __init__(self):
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700292 self.client_cwd = None
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700293 self.server_cwd = None
Jan Tattermusch0a14f622015-10-09 14:34:29 -0700294 self.safename = str(self)
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700295
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700296 def client_cmd(self, args):
Jan Tattermusch6d4dba92016-11-18 10:36:35 +0100297 return ['tools/run_tests/interop/with_nvm.sh',
298 'node', 'src/node/interop/interop_client.js'] + args
Jan Tattermusch8266c672015-09-17 09:18:03 -0700299
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700300 def cloud_to_prod_env(self):
Jan Tattermuscha6b2c4c2015-12-10 16:41:11 -0800301 return {}
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700302
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700303 def server_cmd(self, args):
Jan Tattermusch6d4dba92016-11-18 10:36:35 +0100304 return ['tools/run_tests/interop/with_nvm.sh',
Adele Zhou57b02fc2017-01-17 14:18:41 -0800305 'node', 'src/node/interop/interop_server.js'] + args
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700306
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700307 def global_env(self):
308 return {}
309
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700310 def unimplemented_test_cases(self):
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800311 return _SKIP_COMPRESSION
312
313 def unimplemented_test_cases_server(self):
314 return _SKIP_COMPRESSION
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700315
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700316 def __str__(self):
317 return 'node'
318
Jan Tattermuschf88f3e22015-09-16 17:50:45 -0700319
320class PHPLanguage:
321
322 def __init__(self):
Jan Tattermuschf88f3e22015-09-16 17:50:45 -0700323 self.client_cwd = None
Jan Tattermusch0a14f622015-10-09 14:34:29 -0700324 self.safename = str(self)
Jan Tattermuschf88f3e22015-09-16 17:50:45 -0700325
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700326 def client_cmd(self, args):
327 return ['src/php/bin/interop_client.sh'] + args
Jan Tattermusch8266c672015-09-17 09:18:03 -0700328
Jan Tattermuschf88f3e22015-09-16 17:50:45 -0700329 def cloud_to_prod_env(self):
Jan Tattermuscha6b2c4c2015-12-10 16:41:11 -0800330 return {}
Jan Tattermuschf88f3e22015-09-16 17:50:45 -0700331
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700332 def global_env(self):
Stanley Cheung22b6bed2015-10-15 16:54:52 -0700333 return {}
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700334
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700335 def unimplemented_test_cases(self):
Stanley Cheung83d8e372016-06-17 11:53:33 -0700336 return _SKIP_COMPRESSION
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800337
338 def unimplemented_test_cases_server(self):
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700339 return []
340
Jan Tattermuschf88f3e22015-09-16 17:50:45 -0700341 def __str__(self):
342 return 'php'
343
344
Stanley Cheung2e2cdff2016-07-23 19:07:36 -0700345class PHP7Language:
346
347 def __init__(self):
348 self.client_cwd = None
349 self.safename = str(self)
350
351 def client_cmd(self, args):
352 return ['src/php/bin/interop_client.sh'] + args
353
354 def cloud_to_prod_env(self):
355 return {}
356
357 def global_env(self):
358 return {}
359
360 def unimplemented_test_cases(self):
361 return _SKIP_COMPRESSION
362
363 def unimplemented_test_cases_server(self):
364 return []
365
366 def __str__(self):
367 return 'php7'
368
369
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700370class RubyLanguage:
371
372 def __init__(self):
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700373 self.client_cwd = None
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700374 self.server_cwd = None
Jan Tattermusch0a14f622015-10-09 14:34:29 -0700375 self.safename = str(self)
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700376
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700377 def client_cmd(self, args):
Jan Tattermusch34e8b912016-11-18 10:44:28 +0100378 return ['tools/run_tests/interop/with_rvm.sh',
379 'ruby', 'src/ruby/pb/test/client.rb'] + args
Jan Tattermusch8266c672015-09-17 09:18:03 -0700380
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700381 def cloud_to_prod_env(self):
Jan Tattermuscha6b2c4c2015-12-10 16:41:11 -0800382 return {}
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700383
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700384 def server_cmd(self, args):
Jan Tattermusch34e8b912016-11-18 10:44:28 +0100385 return ['tools/run_tests/interop/with_rvm.sh',
Adele Zhou57b02fc2017-01-17 14:18:41 -0800386 'ruby', 'src/ruby/pb/test/server.rb'] + args
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700387
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700388 def global_env(self):
389 return {}
390
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700391 def unimplemented_test_cases(self):
ncteisen9d1cc8c2017-01-10 13:57:25 -0800392 return _SKIP_SERVER_COMPRESSION
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800393
394 def unimplemented_test_cases_server(self):
ncteisen9d1cc8c2017-01-10 13:57:25 -0800395 return _SKIP_COMPRESSION
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700396
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700397 def __str__(self):
398 return 'ruby'
399
400
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700401class PythonLanguage:
402
403 def __init__(self):
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700404 self.client_cwd = None
405 self.server_cwd = None
Noah Eisende99fe82017-02-27 17:26:20 -0800406 self.http2_cwd = None
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700407 self.safename = str(self)
408
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700409 def client_cmd(self, args):
410 return [
Masood Malekghassemi3b5b2062016-06-02 20:27:20 -0700411 'py27/bin/python',
412 'src/python/grpcio_tests/setup.py',
413 'run_interop',
414 '--client',
415 '--args="{}"'.format(' '.join(args))
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700416 ]
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700417
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800418 def client_cmd_http2interop(self, args):
419 return [ 'py27/bin/python',
ncteisen459c93f2017-01-20 15:42:07 -0800420 'src/python/grpcio_tests/tests/http2/negative_http2_client.py',
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800421 ] + args
422
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700423 def cloud_to_prod_env(self):
Jan Tattermuscha6b2c4c2015-12-10 16:41:11 -0800424 return {}
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700425
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700426 def server_cmd(self, args):
427 return [
Masood Malekghassemi3b5b2062016-06-02 20:27:20 -0700428 'py27/bin/python',
429 'src/python/grpcio_tests/setup.py',
430 'run_interop',
431 '--server',
Adele Zhou57b02fc2017-01-17 14:18:41 -0800432 '--args="{}"'.format(' '.join(args))
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700433 ]
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700434
435 def global_env(self):
Ken Payson707c9e22016-04-20 09:42:19 -0700436 return {'LD_LIBRARY_PATH': '{}/libs/opt'.format(DOCKER_WORKDIR_ROOT),
437 'PYTHONPATH': '{}/src/python/gens'.format(DOCKER_WORKDIR_ROOT)}
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700438
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700439 def unimplemented_test_cases(self):
Noah Eisen9785c8f2016-10-25 12:04:24 -0700440 return _SKIP_COMPRESSION
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800441
442 def unimplemented_test_cases_server(self):
Noah Eisen9785c8f2016-10-25 12:04:24 -0700443 return _SKIP_COMPRESSION
Jan Tattermusch289b7b92015-10-21 18:09:59 -0700444
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700445 def __str__(self):
446 return 'python'
447
448
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700449_LANGUAGES = {
450 'c++' : CXXLanguage(),
451 'csharp' : CSharpLanguage(),
Jan Tattermuscha98a2ad2016-09-09 12:42:32 +0200452 'csharpcoreclr' : CSharpCoreCLRLanguage(),
Jan Tattermuschcc1bde72015-10-05 12:47:45 -0700453 'go' : GoLanguage(),
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700454 'java' : JavaLanguage(),
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700455 'node' : NodeLanguage(),
Jan Tattermuschf88f3e22015-09-16 17:50:45 -0700456 'php' : PHPLanguage(),
Stanley Cheung2e2cdff2016-07-23 19:07:36 -0700457 'php7' : PHP7Language(),
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700458 'ruby' : RubyLanguage(),
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700459 'python' : PythonLanguage(),
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700460}
Jan Tattermusch320bd612015-09-15 12:44:35 -0700461
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700462# languages supported as cloud_to_cloud servers
Jan Tattermuscha98a2ad2016-09-09 12:42:32 +0200463_SERVERS = ['c++', 'node', 'csharp', 'csharpcoreclr', 'java', 'go', 'ruby', 'python']
Jan Tattermusch8266c672015-09-17 09:18:03 -0700464
Jan Tattermusch320bd612015-09-15 12:44:35 -0700465_TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong',
Jan Tattermusch1f1c5c52015-10-09 14:36:28 -0700466 'empty_stream', 'client_streaming', 'server_streaming',
Jan Tattermusch13bf36a2015-10-14 17:01:00 -0700467 'cancel_after_begin', 'cancel_after_first_response',
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -0800468 'timeout_on_sleeping_server', 'custom_metadata',
469 'status_code_and_message', 'unimplemented_method',
David Garcia Quintasff32a862016-06-21 17:07:20 -0700470 'client_compressed_unary', 'server_compressed_unary',
Noah Eisen7fc1d4e2016-10-27 18:17:03 -0700471 'client_compressed_streaming', 'server_compressed_streaming',
472 'unimplemented_service']
Jan Tattermusch320bd612015-09-15 12:44:35 -0700473
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700474_AUTH_TEST_CASES = ['compute_engine_creds', 'jwt_token_creds',
475 'oauth2_auth_token', 'per_rpc_creds']
476
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800477_HTTP2_TEST_CASES = ['tls', 'framing']
478
479_HTTP2_BADSERVER_TEST_CASES = ['rst_after_header', 'rst_after_data', 'rst_during_data',
480 'goaway', 'ping', 'max_streams']
Jan Tattermusch8266c672015-09-17 09:18:03 -0700481
Eric Gribkoffcaf7ea52017-03-01 08:15:25 -0800482_LANGUAGES_FOR_HTTP2_BADSERVER_TESTS = ['java', 'go', 'python', 'c++']
Adele Zhou429e5602017-02-06 14:57:56 -0800483
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700484DOCKER_WORKDIR_ROOT = '/var/local/git/grpc'
485
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700486def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None):
487 """Wraps given cmdline array to create 'docker run' cmdline from it."""
488 docker_cmdline = ['docker', 'run', '-i', '--rm=true']
489
490 # turn environ into -e docker args
491 if environ:
siddharthshukla0589e532016-07-07 16:08:01 +0200492 for k,v in environ.items():
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700493 docker_cmdline += ['-e', '%s=%s' % (k,v)]
494
495 # set working directory
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700496 workdir = DOCKER_WORKDIR_ROOT
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700497 if cwd:
498 workdir = os.path.join(workdir, cwd)
499 docker_cmdline += ['-w', workdir]
500
501 docker_cmdline += docker_args + [image] + cmdline
502 return docker_cmdline
503
504
Jan Tattermusch7134a312017-03-07 19:00:25 +0100505def manual_cmdline(docker_cmdline):
506 """Returns docker cmdline adjusted for manual invocation."""
507 print_cmdline = []
508 for item in docker_cmdline:
509 if item.startswith('--name='):
510 continue
511 # add quotes when necessary
512 if any(character.isspace() for character in item):
513 item = "\"%s\"" % item
514 print_cmdline.append(item)
515 return ' '.join(print_cmdline)
516
517
518def write_cmdlog_maybe(cmdlog, filename):
519 """Returns docker cmdline adjusted for manual invocation."""
520 if cmdlog:
521 with open(filename, 'w') as logfile:
522 logfile.write('#!/bin/bash\n')
523 logfile.writelines("%s\n" % line for line in cmdlog)
524 print('Command log written to file %s' % filename)
525
526
Jan Tattermusch03b76122016-11-17 15:28:57 +0100527def bash_cmdline(cmdline):
528 """Creates bash -c cmdline from args list."""
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700529 # Use login shell:
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700530 # * makes error messages clearer if executables are missing
Jan Tattermusch03b76122016-11-17 15:28:57 +0100531 return ['bash', '-c', ' '.join(cmdline)]
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700532
533
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800534def auth_options(language, test_case):
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700535 """Returns (cmdline, env) tuple with cloud_to_prod_auth test options."""
536
537 language = str(language)
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800538 cmdargs = []
539 env = {}
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700540
541 # TODO(jtattermusch): this file path only works inside docker
542 key_filepath = '/root/service_account/stubbyCloudTestingTest-ee3fce360ac5.json'
543 oauth_scope_arg = '--oauth_scope=https://www.googleapis.com/auth/xapi.zoo'
544 key_file_arg = '--service_account_key_file=%s' % key_filepath
545 default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com'
546
547 if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']:
Jan Tattermuscha98a2ad2016-09-09 12:42:32 +0200548 if language in ['csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', 'ruby']:
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700549 env['GOOGLE_APPLICATION_CREDENTIALS'] = key_filepath
550 else:
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800551 cmdargs += [key_file_arg]
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700552
553 if test_case in ['per_rpc_creds', 'oauth2_auth_token']:
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800554 cmdargs += [oauth_scope_arg]
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700555
Jan Tattermusch64d7c242015-10-08 08:02:27 -0700556 if test_case == 'oauth2_auth_token' and language == 'c++':
557 # C++ oauth2 test uses GCE creds and thus needs to know the default account
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800558 cmdargs += [default_account_arg]
Jan Tattermusch64d7c242015-10-08 08:02:27 -0700559
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700560 if test_case == 'compute_engine_creds':
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800561 cmdargs += [oauth_scope_arg, default_account_arg]
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700562
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800563 return (cmdargs, env)
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700564
565
Jan Tattermusche2686282015-10-08 16:27:07 -0700566def _job_kill_handler(job):
567 if job._spec.container_name:
568 dockerjob.docker_kill(job._spec.container_name)
Jan Tattermusch39e3cb32015-10-22 18:21:08 -0700569 # When the job times out and we decide to kill it,
570 # we need to wait a before restarting the job
571 # to prevent "container name already in use" error.
572 # TODO(jtattermusch): figure out a cleaner way to to this.
573 time.sleep(2)
Jan Tattermusche2686282015-10-08 16:27:07 -0700574
575
murgatroid99090a2512016-02-19 11:32:31 -0800576def cloud_to_prod_jobspec(language, test_case, server_host_name,
Jan Tattermusch7134a312017-03-07 19:00:25 +0100577 server_host_detail, docker_image=None, auth=False,
578 manual_cmd_log=None):
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700579 """Creates jobspec for cloud-to-prod interop test"""
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800580 container_name = None
581 cmdargs = [
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800582 '--server_host=%s' % server_host_detail[0],
583 '--server_host_override=%s' % server_host_detail[1],
Jan Tattermuschc8b94412015-10-21 17:57:28 -0700584 '--server_port=443',
585 '--use_tls=true',
586 '--test_case=%s' % test_case]
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700587 environ = dict(language.cloud_to_prod_env(), **language.global_env())
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700588 if auth:
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800589 auth_cmdargs, auth_env = auth_options(language, test_case)
590 cmdargs += auth_cmdargs
591 environ.update(auth_env)
Jan Tattermusch03b76122016-11-17 15:28:57 +0100592 cmdline = bash_cmdline(language.client_cmd(cmdargs))
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800593 cwd = language.client_cwd
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700594
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700595 if docker_image:
murgatroid99090a2512016-02-19 11:32:31 -0800596 container_name = dockerjob.random_name('interop_client_%s' %
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800597 language.safename)
Jan Tattermusche2686282015-10-08 16:27:07 -0700598 cmdline = docker_run_cmdline(cmdline,
599 image=docker_image,
600 cwd=cwd,
601 environ=environ,
602 docker_args=['--net=host',
Jan Tattermusch7134a312017-03-07 19:00:25 +0100603 '--name=%s' % container_name])
604 if manual_cmd_log is not None:
605 manual_cmd_log.append(manual_cmdline(cmdline))
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700606 cwd = None
607 environ = None
608
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700609 suite_name='cloud_to_prod_auth' if auth else 'cloud_to_prod'
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700610 test_job = jobset.JobSpec(
611 cmdline=cmdline,
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700612 cwd=cwd,
613 environ=environ,
murgatroid99090a2512016-02-19 11:32:31 -0800614 shortname='%s:%s:%s:%s' % (suite_name, server_host_name, language,
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800615 test_case),
Jan Tattermusch19f703d2016-03-03 15:33:29 -0800616 timeout_seconds=_TEST_TIMEOUT,
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700617 flake_retries=5 if args.allow_flakes else 0,
Jan Tattermusche2686282015-10-08 16:27:07 -0700618 timeout_retries=2 if args.allow_flakes else 0,
619 kill_handler=_job_kill_handler)
murgatroid9901467562016-06-27 15:48:08 -0700620 if docker_image:
621 test_job.container_name = container_name
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700622 return test_job
623
Jan Tattermusch8266c672015-09-17 09:18:03 -0700624
625def cloud_to_cloud_jobspec(language, test_case, server_name, server_host,
Jan Tattermusch7134a312017-03-07 19:00:25 +0100626 server_port, docker_image=None, insecure=False,
627 manual_cmd_log=None):
Jan Tattermusch8266c672015-09-17 09:18:03 -0700628 """Creates jobspec for cloud-to-cloud interop test"""
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800629 interop_only_options = [
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700630 '--server_host_override=foo.test.google.fr',
Adele Zhou57b02fc2017-01-17 14:18:41 -0800631 '--use_tls=%s' % ('false' if insecure else 'true'),
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700632 '--use_test_ca=true',
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800633 ]
634 common_options = [
Masood Malekghassemi7566c9a2015-10-21 20:29:23 -0700635 '--test_case=%s' % test_case,
636 '--server_host=%s' % server_host,
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800637 ]
Makarand Dharmapurikar078e4212017-01-10 15:09:21 -0800638 if test_case in _HTTP2_BADSERVER_TEST_CASES:
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800639 # We are running the http2_badserver_interop test. Adjust command line accordingly.
640 offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case)
641 client_options = common_options + ['--server_port=%s' %
642 (int(server_port)+offset)]
643 cmdline = bash_cmdline(language.client_cmd_http2interop(client_options))
Noah Eisende99fe82017-02-27 17:26:20 -0800644 cwd = language.http2_cwd
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800645 else:
646 client_options = interop_only_options + common_options + ['--server_port=%s' % server_port]
647 cmdline = bash_cmdline(language.client_cmd(client_options))
Noah Eisende99fe82017-02-27 17:26:20 -0800648 cwd = language.client_cwd
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800649
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700650 environ = language.global_env()
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700651 if docker_image:
Jan Tattermusch0a14f622015-10-09 14:34:29 -0700652 container_name = dockerjob.random_name('interop_client_%s' % language.safename)
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700653 cmdline = docker_run_cmdline(cmdline,
654 image=docker_image,
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700655 environ=environ,
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700656 cwd=cwd,
Jan Tattermusche2686282015-10-08 16:27:07 -0700657 docker_args=['--net=host',
Jan Tattermusch7134a312017-03-07 19:00:25 +0100658 '--name=%s' % container_name])
659 if manual_cmd_log is not None:
660 manual_cmd_log.append(manual_cmdline(cmdline))
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700661 cwd = None
Jan Tattermusche2686282015-10-08 16:27:07 -0700662
Jan Tattermusch8266c672015-09-17 09:18:03 -0700663 test_job = jobset.JobSpec(
664 cmdline=cmdline,
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700665 cwd=cwd,
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700666 environ=environ,
Adele Zhoue4c35612015-10-16 15:34:23 -0700667 shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name,
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800668 test_case),
Jan Tattermusch19f703d2016-03-03 15:33:29 -0800669 timeout_seconds=_TEST_TIMEOUT,
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700670 flake_retries=5 if args.allow_flakes else 0,
Jan Tattermusche2686282015-10-08 16:27:07 -0700671 timeout_retries=2 if args.allow_flakes else 0,
672 kill_handler=_job_kill_handler)
murgatroid9901467562016-06-27 15:48:08 -0700673 if docker_image:
674 test_job.container_name = container_name
Jan Tattermusch8266c672015-09-17 09:18:03 -0700675 return test_job
676
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700677
Jan Tattermusch7134a312017-03-07 19:00:25 +0100678def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None):
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700679 """Create jobspec for running a server"""
Jan Tattermusch0a14f622015-10-09 14:34:29 -0700680 container_name = dockerjob.random_name('interop_server_%s' % language.safename)
Jan Tattermusch03b76122016-11-17 15:28:57 +0100681 cmdline = bash_cmdline(
Adele Zhou57b02fc2017-01-17 14:18:41 -0800682 language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT,
683 '--use_tls=%s' % ('false' if insecure else 'true')]))
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700684 environ = language.global_env()
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800685 if language.safename == 'http2':
686 # we are running the http2 interop server. Open next N ports beginning
687 # with the server port. These ports are used for http2 interop test
688 # (one test case per port). We also attach the docker container running
689 # the server to local network, so we don't have to mess with port mapping
690 port_args = [
691 '-p', str(_DEFAULT_SERVER_PORT+0),
692 '-p', str(_DEFAULT_SERVER_PORT+1),
693 '-p', str(_DEFAULT_SERVER_PORT+2),
694 '-p', str(_DEFAULT_SERVER_PORT+3),
695 '-p', str(_DEFAULT_SERVER_PORT+4),
696 '-p', str(_DEFAULT_SERVER_PORT+5),
697 '-p', str(_DEFAULT_SERVER_PORT+6),
698 '--net=host',
699 ]
700 else:
701 port_args = ['-p', str(_DEFAULT_SERVER_PORT)]
702
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700703 docker_cmdline = docker_run_cmdline(cmdline,
704 image=docker_image,
705 cwd=language.server_cwd,
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700706 environ=environ,
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800707 docker_args=port_args +
Jan Tattermusch7134a312017-03-07 19:00:25 +0100708 ['--name=%s' % container_name])
709 if manual_cmd_log is not None:
710 manual_cmd_log.append(manual_cmdline(docker_cmdline))
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700711 server_job = jobset.JobSpec(
712 cmdline=docker_cmdline,
Masood Malekghassemi18cc8422015-10-09 17:55:45 -0700713 environ=environ,
Adele Zhoue4c35612015-10-16 15:34:23 -0700714 shortname='interop_server_%s' % language,
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700715 timeout_seconds=30*60)
Jan Tattermusche2686282015-10-08 16:27:07 -0700716 server_job.container_name = container_name
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700717 return server_job
718
719
720def build_interop_image_jobspec(language, tag=None):
721 """Creates jobspec for building interop docker image for a language"""
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700722 if not tag:
Jan Tattermusch0a14f622015-10-09 14:34:29 -0700723 tag = 'grpc_interop_%s:%s' % (language.safename, uuid.uuid4())
Stanley Cheung22b6bed2015-10-15 16:54:52 -0700724 env = {'INTEROP_IMAGE': tag,
725 'BASE_NAME': 'grpc_interop_%s' % language.safename}
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700726 if not args.travis:
Stanley Cheung22b6bed2015-10-15 16:54:52 -0700727 env['TTY_FLAG'] = '-t'
728 # This env variable is used to get around the github rate limit
729 # error when running the PHP `composer install` command
Stanley Cheungd82efd82016-02-10 10:36:20 -0800730 host_file = '%s/.composer/auth.json' % os.environ['HOME']
731 if language.safename == 'php' and os.path.exists(host_file):
Stanley Cheung22b6bed2015-10-15 16:54:52 -0700732 env['BUILD_INTEROP_DOCKER_EXTRA_ARGS'] = \
Stanley Cheungd82efd82016-02-10 10:36:20 -0800733 '-v %s:/root/.composer/auth.json:ro' % host_file
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700734 build_job = jobset.JobSpec(
Jan Tattermusch9835d4b2016-04-29 15:05:05 -0700735 cmdline=['tools/run_tests/dockerize/build_interop_image.sh'],
Stanley Cheung22b6bed2015-10-15 16:54:52 -0700736 environ=env,
Adele Zhoue4c35612015-10-16 15:34:23 -0700737 shortname='build_docker_%s' % (language),
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700738 timeout_seconds=30*60)
739 build_job.tag = tag
740 return build_job
741
742
Carl Mastrangeloe7f8e8e2015-12-08 17:22:44 -0800743def aggregate_http2_results(stdout):
744 match = re.search(r'\{"cases[^\]]*\]\}', stdout)
745 if not match:
746 return None
murgatroid99c3910ca2016-01-06 13:14:23 -0800747
Carl Mastrangeloe7f8e8e2015-12-08 17:22:44 -0800748 results = json.loads(match.group(0))
749 skipped = 0
750 passed = 0
751 failed = 0
752 failed_cases = []
753 for case in results['cases']:
754 if case.get('skipped', False):
755 skipped += 1
756 else:
757 if case.get('passed', False):
758 passed += 1
759 else:
760 failed += 1
761 failed_cases.append(case.get('name', "NONAME"))
762 return {
763 'passed': passed,
764 'failed': failed,
765 'skipped': skipped,
766 'failed_cases': ', '.join(failed_cases),
767 'percent': 1.0 * passed / (passed + failed)
768 }
769
murgatroid99090a2512016-02-19 11:32:31 -0800770# A dictionary of prod servers to test.
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800771# Format: server_name: (server_host, server_host_override, errors_allowed)
772# TODO(adelez): implement logic for errors_allowed where if the indicated tests
773# fail, they don't impact the overall test result.
774prod_servers = {
Adele Zhouc4b61e22016-08-25 16:32:06 -0700775 'default': ('216.239.32.254',
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800776 'grpc-test.sandbox.googleapis.com', False),
Adele Zhouc4b61e22016-08-25 16:32:06 -0700777 'gateway_v2': ('216.239.32.254',
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800778 'grpc-test2.sandbox.googleapis.com', True),
murgatroid99090a2512016-02-19 11:32:31 -0800779 'cloud_gateway': ('216.239.32.255', 'grpc-test.sandbox.googleapis.com',
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800780 False),
murgatroid99090a2512016-02-19 11:32:31 -0800781 'cloud_gateway_v2': ('216.239.32.255', 'grpc-test2.sandbox.googleapis.com',
Adele Zhou2480f982016-02-26 14:18:46 -0800782 True),
Adele Zhouc4b61e22016-08-25 16:32:06 -0700783 'gateway_v4': ('216.239.32.254',
murgatroid99e621f132016-04-21 14:28:00 -0700784 'grpc-test4.sandbox.googleapis.com', True),
Adele Zhou2480f982016-02-26 14:18:46 -0800785 'cloud_gateway_v4': ('216.239.32.255', 'grpc-test4.sandbox.googleapis.com',
786 True),
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800787}
788
Jan Tattermusch320bd612015-09-15 12:44:35 -0700789argp = argparse.ArgumentParser(description='Run interop tests.')
790argp.add_argument('-l', '--language',
791 choices=['all'] + sorted(_LANGUAGES),
792 nargs='+',
Jan Tattermusch8266c672015-09-17 09:18:03 -0700793 default=['all'],
794 help='Clients to run.')
Jan Tattermusch210a0ea2015-10-02 15:05:36 -0700795argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int)
Jan Tattermusch8266c672015-09-17 09:18:03 -0700796argp.add_argument('--cloud_to_prod',
797 default=False,
798 action='store_const',
799 const=True,
800 help='Run cloud_to_prod tests.')
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700801argp.add_argument('--cloud_to_prod_auth',
802 default=False,
803 action='store_const',
804 const=True,
805 help='Run cloud_to_prod_auth tests.')
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800806argp.add_argument('--prod_servers',
807 choices=prod_servers.keys(),
808 default=['default'],
809 nargs='+',
810 help=('The servers to run cloud_to_prod and '
811 'cloud_to_prod_auth tests against.'))
Jan Tattermusch8266c672015-09-17 09:18:03 -0700812argp.add_argument('-s', '--server',
813 choices=['all'] + sorted(_SERVERS),
Adele Zhou9088d592016-08-16 17:48:42 -0700814 nargs='+',
Jan Tattermusch8266c672015-09-17 09:18:03 -0700815 help='Run cloud_to_cloud servers in a separate docker ' +
816 'image. Servers can only be started automatically if ' +
817 '--use_docker option is enabled.',
818 default=[])
819argp.add_argument('--override_server',
820 action='append',
Adele Zhoue4c35612015-10-16 15:34:23 -0700821 type=lambda kv: kv.split('='),
Jan Tattermusch8266c672015-09-17 09:18:03 -0700822 help='Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000',
823 default=[])
824argp.add_argument('-t', '--travis',
825 default=False,
826 action='store_const',
827 const=True)
828argp.add_argument('--use_docker',
829 default=False,
830 action='store_const',
831 const=True,
832 help='Run all the interop tests under docker. That provides ' +
833 'additional isolation and prevents the need to install ' +
834 'language specific prerequisites. Only available on Linux.')
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700835argp.add_argument('--allow_flakes',
836 default=False,
837 action='store_const',
838 const=True,
Adele Zhoue4c35612015-10-16 15:34:23 -0700839 help='Allow flaky tests to show as passing (re-runs failed tests up to five times)')
Jan Tattermusch7134a312017-03-07 19:00:25 +0100840argp.add_argument('--manual_run',
841 default=False,
842 action='store_const',
843 const=True,
844 help='Prepare things for running interop tests manually. ' +
845 'Preserve docker images after building them and skip '
846 'actually running the tests. Only print commands to run by ' +
847 'hand.')
Carl Mastrangelode449102015-10-28 11:05:49 -0700848argp.add_argument('--http2_interop',
849 default=False,
850 action='store_const',
851 const=True,
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800852 help='Enable HTTP/2 client edge case testing. (Bad client, good server)')
853argp.add_argument('--http2_badserver_interop',
854 default=False,
855 action='store_const',
856 const=True,
857 help='Enable HTTP/2 server edge case testing. (Good client, bad server)')
Adele Zhou57b02fc2017-01-17 14:18:41 -0800858argp.add_argument('--insecure',
859 default=False,
860 action='store_const',
861 const=True,
862 help='Whether to use secure channel.')
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800863
Jan Tattermusch320bd612015-09-15 12:44:35 -0700864args = argp.parse_args()
865
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700866servers = set(s for s in itertools.chain.from_iterable(_SERVERS
Jan Tattermusch8266c672015-09-17 09:18:03 -0700867 if x == 'all' else [x]
868 for x in args.server))
869
870if args.use_docker:
871 if not args.travis:
siddharthshukla0589e532016-07-07 16:08:01 +0200872 print('Seen --use_docker flag, will run interop tests under docker.')
873 print('')
874 print('IMPORTANT: The changes you are testing need to be locally committed')
875 print('because only the committed changes in the current branch will be')
876 print('copied to the docker environment.')
Jan Tattermusch8266c672015-09-17 09:18:03 -0700877 time.sleep(5)
878
Jan Tattermusch7134a312017-03-07 19:00:25 +0100879if args.manual_run and not args.use_docker:
880 print('--manual_run is only supported with --use_docker option enabled.')
881 sys.exit(1)
882
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700883if not args.use_docker and servers:
siddharthshukla0589e532016-07-07 16:08:01 +0200884 print('Running interop servers is only supported with --use_docker option enabled.')
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700885 sys.exit(1)
Jan Tattermusch8266c672015-09-17 09:18:03 -0700886
Jan Tattermuschf49936a2015-09-16 15:44:26 -0700887languages = set(_LANGUAGES[l]
888 for l in itertools.chain.from_iterable(
Siddharth Shuklad194f592017-03-11 19:12:43 +0100889 six.iterkeys(_LANGUAGES) if x == 'all' else [x]
Adele Zhou429e5602017-02-06 14:57:56 -0800890 for x in args.language))
891
892languages_http2_badserver_interop = set()
893if args.http2_badserver_interop:
894 languages_http2_badserver_interop = set(
895 _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS)
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800896
Carl Mastrangelode449102015-10-28 11:05:49 -0700897http2Interop = Http2Client() if args.http2_interop else None
Makarand Dharmapurikar078e4212017-01-10 15:09:21 -0800898http2InteropServer = Http2Server() if args.http2_badserver_interop else None
Jan Tattermusch320bd612015-09-15 12:44:35 -0700899
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700900docker_images={}
901if args.use_docker:
902 # languages for which to build docker images
Adele Zhou429e5602017-02-06 14:57:56 -0800903 languages_to_build = set(
904 _LANGUAGES[k] for k in set([str(l) for l in languages] + [s for s in servers]))
905 languages_to_build = languages_to_build | languages_http2_badserver_interop
906
Carl Mastrangelode449102015-10-28 11:05:49 -0700907 if args.http2_interop:
908 languages_to_build.add(http2Interop)
Jan Tattermusch8266c672015-09-17 09:18:03 -0700909
Makarand Dharmapurikar078e4212017-01-10 15:09:21 -0800910 if args.http2_badserver_interop:
911 languages_to_build.add(http2InteropServer)
912
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700913 build_jobs = []
914 for l in languages_to_build:
915 job = build_interop_image_jobspec(l)
916 docker_images[str(l)] = job.tag
917 build_jobs.append(job)
Jan Tattermusch8266c672015-09-17 09:18:03 -0700918
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700919 if build_jobs:
920 jobset.message('START', 'Building interop docker images.', do_newline=True)
Adele Zhoue4c35612015-10-16 15:34:23 -0700921 num_failures, _ = jobset.run(
922 build_jobs, newline_on_success=True, maxjobs=args.jobs)
923 if num_failures == 0:
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800924 jobset.message('SUCCESS', 'All docker images built successfully.',
Adele Zhoue4c35612015-10-16 15:34:23 -0700925 do_newline=True)
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700926 else:
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800927 jobset.message('FAILED', 'Failed to build interop docker images.',
Adele Zhoue4c35612015-10-16 15:34:23 -0700928 do_newline=True)
Siddharth Shuklad194f592017-03-11 19:12:43 +0100929 for image in six.itervalues(docker_images):
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700930 dockerjob.remove_image(image, skip_nonexistent=True)
Carl Mastrangelo7a171402015-10-26 14:01:03 -0700931 sys.exit(1)
Jan Tattermusch8266c672015-09-17 09:18:03 -0700932
Jan Tattermusch7134a312017-03-07 19:00:25 +0100933server_manual_cmd_log = [] if args.manual_run else None
934client_manual_cmd_log = [] if args.manual_run else None
935
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700936# Start interop servers.
Jan Tattermusch7134a312017-03-07 19:00:25 +0100937server_jobs = {}
938server_addresses = {}
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700939try:
940 for s in servers:
941 lang = str(s)
Adele Zhou57b02fc2017-01-17 14:18:41 -0800942 spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang),
Jan Tattermusch7134a312017-03-07 19:00:25 +0100943 args.insecure, manual_cmd_log=server_manual_cmd_log)
944 if not args.manual_run:
945 job = dockerjob.DockerJob(spec)
946 server_jobs[lang] = job
947 server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT))
948 else:
949 # don't run the server, set server port to a placeholder value
950 server_addresses[lang] = ('localhost', '${SERVER_PORT}')
Jan Tattermusch8266c672015-09-17 09:18:03 -0700951
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800952 if args.http2_badserver_interop:
953 # launch a HTTP2 server emulator that creates edge cases
Makarand Dharmapurikar078e4212017-01-10 15:09:21 -0800954 lang = str(http2InteropServer)
Jan Tattermusch7134a312017-03-07 19:00:25 +0100955 spec = server_jobspec(http2InteropServer, docker_images.get(lang),
956 manual_cmd_log=server_manual_cmd_log)
957 if not args.manual_run:
958 job = dockerjob.DockerJob(spec)
959 server_jobs[lang] = job
960 else:
961 # don't run the server, set server port to a placeholder value
962 server_addresses[lang] = ('localhost', '${SERVER_PORT}')
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -0800963
Jan Tattermusch91ad0182015-10-01 09:22:03 -0700964 jobs = []
965 if args.cloud_to_prod:
Adele Zhou57b02fc2017-01-17 14:18:41 -0800966 if args.insecure:
967 print('TLS is always enabled for cloud_to_prod scenarios.')
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800968 for server_host_name in args.prod_servers:
969 for language in languages:
970 for test_case in _TEST_CASES:
971 if not test_case in language.unimplemented_test_cases():
972 if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION:
973 test_job = cloud_to_prod_jobspec(
murgatroid99090a2512016-02-19 11:32:31 -0800974 language, test_case, server_host_name,
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800975 prod_servers[server_host_name],
Jan Tattermusch7134a312017-03-07 19:00:25 +0100976 docker_image=docker_images.get(str(language)),
977 manual_cmd_log=client_manual_cmd_log)
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800978 jobs.append(test_job)
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -0800979
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800980 if args.http2_interop:
981 for test_case in _HTTP2_TEST_CASES:
982 test_job = cloud_to_prod_jobspec(
murgatroid99090a2512016-02-19 11:32:31 -0800983 http2Interop, test_case, server_host_name,
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800984 prod_servers[server_host_name],
Jan Tattermusch7134a312017-03-07 19:00:25 +0100985 docker_image=docker_images.get(str(http2Interop)),
986 manual_cmd_log=client_manual_cmd_log)
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800987 jobs.append(test_job)
Jan Tattermusch320bd612015-09-15 12:44:35 -0700988
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -0700989 if args.cloud_to_prod_auth:
Adele Zhou57b02fc2017-01-17 14:18:41 -0800990 if args.insecure:
991 print('TLS is always enabled for cloud_to_prod scenarios.')
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800992 for server_host_name in args.prod_servers:
993 for language in languages:
994 for test_case in _AUTH_TEST_CASES:
995 if not test_case in language.unimplemented_test_cases():
996 test_job = cloud_to_prod_jobspec(
murgatroid99090a2512016-02-19 11:32:31 -0800997 language, test_case, server_host_name,
Adele Zhoub9e66cc2016-02-03 13:12:23 -0800998 prod_servers[server_host_name],
Jan Tattermusch7134a312017-03-07 19:00:25 +0100999 docker_image=docker_images.get(str(language)), auth=True,
1000 manual_cmd_log=client_manual_cmd_log)
Adele Zhoub9e66cc2016-02-03 13:12:23 -08001001 jobs.append(test_job)
Jan Tattermuschfb8c77d2015-10-06 09:33:37 -07001002
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001003 for server in args.override_server:
1004 server_name = server[0]
1005 (server_host, server_port) = server[1].split(':')
1006 server_addresses[server_name] = (server_host, server_port)
Jan Tattermusch320bd612015-09-15 12:44:35 -07001007
siddharthshukla0589e532016-07-07 16:08:01 +02001008 for server_name, server_address in server_addresses.items():
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001009 (server_host, server_port) = server_address
Jan Tattermusch5c7a1c82015-12-09 19:56:51 -08001010 server_language = _LANGUAGES.get(server_name, None)
1011 skip_server = [] # test cases unimplemented by server
1012 if server_language:
1013 skip_server = server_language.unimplemented_test_cases_server()
Adele Zhou429e5602017-02-06 14:57:56 -08001014 for language in languages:
1015 for test_case in _TEST_CASES:
1016 if not test_case in language.unimplemented_test_cases():
1017 if not test_case in skip_server:
1018 test_job = cloud_to_cloud_jobspec(language,
1019 test_case,
1020 server_name,
1021 server_host,
1022 server_port,
1023 docker_image=docker_images.get(str(language)),
Jan Tattermusch7134a312017-03-07 19:00:25 +01001024 insecure=args.insecure,
1025 manual_cmd_log=client_manual_cmd_log)
Adele Zhou429e5602017-02-06 14:57:56 -08001026 jobs.append(test_job)
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -08001027
Carl Mastrangelode449102015-10-28 11:05:49 -07001028 if args.http2_interop:
1029 for test_case in _HTTP2_TEST_CASES:
Carl Mastrangelo3b2e1bd2015-11-06 14:31:55 -08001030 if server_name == "go":
1031 # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434
murgatroid99c3910ca2016-01-06 13:14:23 -08001032 continue
Carl Mastrangelode449102015-10-28 11:05:49 -07001033 test_job = cloud_to_cloud_jobspec(http2Interop,
1034 test_case,
1035 server_name,
1036 server_host,
1037 server_port,
Adele Zhou57b02fc2017-01-17 14:18:41 -08001038 docker_image=docker_images.get(str(http2Interop)),
Jan Tattermusch7134a312017-03-07 19:00:25 +01001039 insecure=args.insecure,
1040 manual_cmd_log=client_manual_cmd_log)
Carl Mastrangelode449102015-10-28 11:05:49 -07001041 jobs.append(test_job)
Jan Tattermusch320bd612015-09-15 12:44:35 -07001042
Adele Zhou8e7e1e32017-02-10 15:48:48 -08001043 if args.http2_badserver_interop:
1044 for language in languages_http2_badserver_interop:
1045 for test_case in _HTTP2_BADSERVER_TEST_CASES:
1046 test_job = cloud_to_cloud_jobspec(language,
1047 test_case,
1048 str(http2InteropServer),
1049 'localhost',
1050 _DEFAULT_SERVER_PORT,
Jan Tattermusch7134a312017-03-07 19:00:25 +01001051 docker_image=docker_images.get(str(language)),
1052 manual_cmd_log=client_manual_cmd_log)
Adele Zhou8e7e1e32017-02-10 15:48:48 -08001053 jobs.append(test_job)
Makarand Dharmapurikard383e4a2017-01-10 13:58:07 -08001054
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001055 if not jobs:
siddharthshukla0589e532016-07-07 16:08:01 +02001056 print('No jobs to run.')
Siddharth Shuklad194f592017-03-11 19:12:43 +01001057 for image in six.itervalues(docker_images):
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001058 dockerjob.remove_image(image, skip_nonexistent=True)
1059 sys.exit(1)
1060
Jan Tattermusch7134a312017-03-07 19:00:25 +01001061 if args.manual_run:
1062 print('All tests will skipped --manual_run option is active.')
1063
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -08001064 num_failures, resultset = jobset.run(jobs, newline_on_success=True,
Jan Tattermusch7134a312017-03-07 19:00:25 +01001065 maxjobs=args.jobs,
1066 skip_jobs=args.manual_run)
Adele Zhoue4c35612015-10-16 15:34:23 -07001067 if num_failures:
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001068 jobset.message('FAILED', 'Some tests failed', do_newline=True)
Adele Zhoue4c35612015-10-16 15:34:23 -07001069 else:
1070 jobset.message('SUCCESS', 'All tests passed', do_newline=True)
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001071
Jan Tattermusch7134a312017-03-07 19:00:25 +01001072 write_cmdlog_maybe(server_manual_cmd_log, 'interop_server_cmds.sh')
1073 write_cmdlog_maybe(client_manual_cmd_log, 'interop_client_cmds.sh')
1074
Adele Zhou3bc7ba42015-11-05 10:21:58 -08001075 report_utils.render_junit_xml_report(resultset, 'report.xml')
Masood Malekghassemi0a9cc3c2015-11-09 12:24:05 -08001076
siddharthshukla0589e532016-07-07 16:08:01 +02001077 for name, job in resultset.items():
Carl Mastrangeloe7f8e8e2015-12-08 17:22:44 -08001078 if "http2" in name:
1079 job[0].http2results = aggregate_http2_results(job[0].message)
1080
Adele Zhou429e5602017-02-06 14:57:56 -08001081 http2_badserver_test_cases = (
1082 _HTTP2_BADSERVER_TEST_CASES if args.http2_badserver_interop else [])
1083
Adele Zhou3bc7ba42015-11-05 10:21:58 -08001084 report_utils.render_interop_html_report(
murgatroid99c3910ca2016-01-06 13:14:23 -08001085 set([str(l) for l in languages]), servers, _TEST_CASES, _AUTH_TEST_CASES,
Adele Zhou429e5602017-02-06 14:57:56 -08001086 _HTTP2_TEST_CASES, http2_badserver_test_cases,
1087 _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS, resultset, num_failures,
murgatroid99090a2512016-02-19 11:32:31 -08001088 args.cloud_to_prod_auth or args.cloud_to_prod, args.prod_servers,
Adele Zhoub9e66cc2016-02-03 13:12:23 -08001089 args.http2_interop)
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001090
1091finally:
1092 # Check if servers are still running.
siddharthshukla0589e532016-07-07 16:08:01 +02001093 for server, job in server_jobs.items():
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001094 if not job.is_running():
siddharthshukla0589e532016-07-07 16:08:01 +02001095 print('Server "%s" has exited prematurely.' % server)
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001096
Siddharth Shuklad194f592017-03-11 19:12:43 +01001097 dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)])
Jan Tattermusch91ad0182015-10-01 09:22:03 -07001098
Siddharth Shuklad194f592017-03-11 19:12:43 +01001099 for image in six.itervalues(docker_images):
Jan Tattermusch7134a312017-03-07 19:00:25 +01001100 if not args.manual_run:
1101 print('Removing docker image %s' % image)
1102 dockerjob.remove_image(image)
1103 else:
1104 print('Preserving docker image: %s' % image)