backport #10100 and #10377
diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template
index 0899e00..b517921 100644
--- a/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template
+++ b/templates/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile.template
@@ -33,7 +33,7 @@
<%include file="../../go_path.include"/>
<%include file="../../python_deps.include"/>
- RUN pip install twisted h2
+ RUN pip install twisted h2==2.6.1 hyper
# Define the default command.
CMD ["bash"]
diff --git a/test/http2_test/http2_server_health_check.py b/test/http2_test/http2_server_health_check.py
new file mode 100644
index 0000000..dd9402b
--- /dev/null
+++ b/test/http2_test/http2_server_health_check.py
@@ -0,0 +1,49 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import hyper
+import sys
+
+# Utility to healthcheck the http2 server. Used when starting the server to
+# verify that the server is live before tests begin.
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--server_host', type=str, default='localhost')
+ parser.add_argument('--server_port', type=int, default=8080)
+ args = parser.parse_args()
+ server_host = args.server_host
+ server_port = args.server_port
+ conn = hyper.HTTP20Connection('%s:%d' % (server_host, server_port))
+ conn.request('POST', '/grpc.testing.TestService/UnaryCall')
+ resp = conn.get_response()
+ if resp.headers.get('grpc-encoding') is None:
+ sys.exit(1)
+ else:
+ sys.exit(0)
diff --git a/test/http2_test/http2_test_server.py b/test/http2_test/http2_test_server.py
index abde343..46c3e00 100644
--- a/test/http2_test/http2_test_server.py
+++ b/test/http2_test/http2_test_server.py
@@ -31,6 +31,7 @@
import argparse
import logging
+import sys
import twisted
import twisted.internet
import twisted.internet.endpoints
@@ -53,9 +54,11 @@
'max_streams': test_max_streams.TestcaseSettingsMaxStreams,
}
+_exit_code = 0
+
class H2Factory(twisted.internet.protocol.Factory):
def __init__(self, testcase):
- logging.info('Creating H2Factory for new connection.')
+ logging.info('Creating H2Factory for new connection (%s)', testcase)
self._num_streams = 0
self._testcase = testcase
@@ -83,6 +86,17 @@
)
return parser.parse_args()
+def listen(endpoint, test_case):
+ deferred = endpoint.listen(H2Factory(test_case))
+ def listen_error(reason):
+ # If listening fails, we stop the reactor and exit the program
+ # with exit code 1.
+ global _exit_code
+ _exit_code = 1
+ logging.error('Listening failed: %s' % reason.value)
+ twisted.internet.reactor.stop()
+ deferred.addErrback(listen_error)
+
def start_test_servers(base_port):
""" Start one server per test case on incrementing port numbers
beginning with base_port """
@@ -92,7 +106,9 @@
logging.warning('serving on port %d : %s'%(portnum, test_case))
endpoint = twisted.internet.endpoints.TCP4ServerEndpoint(
twisted.internet.reactor, portnum, backlog=128)
- endpoint.listen(H2Factory(test_case))
+ # Wait until the reactor is running before calling endpoint.listen().
+ twisted.internet.reactor.callWhenRunning(listen, endpoint, test_case)
+
index += 1
if __name__ == '__main__':
@@ -102,3 +118,4 @@
args = parse_arguments()
start_test_servers(args.base_port)
twisted.internet.reactor.run()
+ sys.exit(_exit_code)
diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
index 42e9edf..094a4e0 100644
--- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile
@@ -47,7 +47,7 @@
RUN pip install virtualenv
RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.2.0 six==1.10.0
-RUN pip install twisted h2
+RUN pip install twisted h2==2.6.1 hyper
# Define the default command.
CMD ["bash"]
diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py
index 0869c5c..709fc12 100755
--- a/tools/run_tests/python_utils/dockerjob.py
+++ b/tools/run_tests/python_utils/dockerjob.py
@@ -70,6 +70,23 @@
(port, cid))
+def wait_for_healthy(cid, shortname, timeout_seconds):
+ """Wait timeout_seconds for the container to become healthy"""
+ started = time.time()
+ while time.time() - started < timeout_seconds:
+ try:
+ output = subprocess.check_output(
+ ['docker', 'inspect', '--format="{{.State.Health.Status}}"', cid],
+ stderr=_DEVNULL)
+ if output.strip('\n') == 'healthy':
+ return
+ except subprocess.CalledProcessError as e:
+ pass
+ time.sleep(1)
+ raise Exception('Timed out waiting for %s (%s) to pass health check' %
+ (shortname, cid))
+
+
def finish_jobs(jobs):
"""Kills given docker containers and waits for corresponding jobs to finish"""
for job in jobs:
@@ -113,6 +130,9 @@
def mapped_port(self, port):
return docker_mapped_port(self._container_name, port)
+ def wait_for_healthy(self, timeout_seconds):
+ wait_for_healthy(self._container_name, self._spec.shortname, timeout_seconds)
+
def kill(self, suppress_failure=False):
"""Sends kill signal to the container."""
if suppress_failure:
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 0d5bec1..007e341 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -607,17 +607,13 @@
common_options = [
'--test_case=%s' % test_case,
'--server_host=%s' % server_host,
+ '--server_port=%s' % server_port,
]
if test_case in _HTTP2_BADSERVER_TEST_CASES:
- # We are running the http2_badserver_interop test. Adjust command line accordingly.
- offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case)
- client_options = common_options + ['--server_port=%s' %
- (int(server_port)+offset)]
- cmdline = bash_cmdline(language.client_cmd_http2interop(client_options))
+ cmdline = bash_cmdline(language.client_cmd_http2interop(common_options))
cwd = language.http2_cwd
else:
- client_options = interop_only_options + common_options + ['--server_port=%s' % server_port]
- cmdline = bash_cmdline(language.client_cmd(client_options))
+ cmdline = bash_cmdline(language.client_cmd(common_options+interop_only_options))
cwd = language.client_cwd
environ = language.global_env()
@@ -653,30 +649,40 @@
language.server_cmd(['--port=%s' % _DEFAULT_SERVER_PORT,
'--use_tls=%s' % ('false' if insecure else 'true')]))
environ = language.global_env()
+ docker_args = ['--name=%s' % container_name]
if language.safename == 'http2':
# we are running the http2 interop server. Open next N ports beginning
# with the server port. These ports are used for http2 interop test
- # (one test case per port). We also attach the docker container running
- # the server to local network, so we don't have to mess with port mapping
- port_args = [
- '-p', str(_DEFAULT_SERVER_PORT+0),
- '-p', str(_DEFAULT_SERVER_PORT+1),
- '-p', str(_DEFAULT_SERVER_PORT+2),
- '-p', str(_DEFAULT_SERVER_PORT+3),
- '-p', str(_DEFAULT_SERVER_PORT+4),
- '-p', str(_DEFAULT_SERVER_PORT+5),
- '-p', str(_DEFAULT_SERVER_PORT+6),
- '--net=host',
+ # (one test case per port).
+ docker_args += list(
+ itertools.chain.from_iterable(('-p', str(_DEFAULT_SERVER_PORT + i))
+ for i in range(
+ len(_HTTP2_BADSERVER_TEST_CASES))))
+ # Enable docker's healthcheck mechanism.
+ # This runs a Python script inside the container every second. The script
+ # pings the http2 server to verify it is ready. The 'health-retries' flag
+ # specifies the number of consecutive failures before docker will report
+ # the container's status as 'unhealthy'. Prior to the first 'health_retries'
+ # failures or the first success, the status will be 'starting'. 'docker ps'
+ # or 'docker inspect' can be used to see the health of the container on the
+ # command line.
+ docker_args += [
+ '--health-cmd=python test/http2_test/http2_server_health_check.py '
+ '--server_host=%s --server_port=%d'
+ % ('localhost', _DEFAULT_SERVER_PORT),
+ '--health-interval=1s',
+ '--health-retries=5',
+ '--health-timeout=10s',
]
+
else:
- port_args = ['-p', str(_DEFAULT_SERVER_PORT)]
+ docker_args += ['-p', str(_DEFAULT_SERVER_PORT)]
docker_cmdline = docker_run_cmdline(cmdline,
image=docker_image,
cwd=language.server_cwd,
environ=environ,
- docker_args=port_args +
- ['--name', container_name])
+ docker_args=docker_args)
server_job = jobset.JobSpec(
cmdline=docker_cmdline,
environ=environ,
@@ -849,7 +855,8 @@
languages_http2_badserver_interop = set()
if args.http2_badserver_interop:
languages_http2_badserver_interop = set(
- _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS)
+ _LANGUAGES[l] for l in _LANGUAGES_FOR_HTTP2_BADSERVER_TESTS
+ if 'all' in args.language or l in args.language)
http2Interop = Http2Client() if args.http2_interop else None
http2InteropServer = Http2Server() if args.http2_badserver_interop else None
@@ -888,8 +895,9 @@
sys.exit(1)
# Start interop servers.
-server_jobs={}
-server_addresses={}
+server_jobs = {}
+server_addresses = {}
+http2_badserver_ports = ()
try:
for s in servers:
lang = str(s)
@@ -899,12 +907,13 @@
server_jobs[lang] = job
server_addresses[lang] = ('localhost', job.mapped_port(_DEFAULT_SERVER_PORT))
+ http2_badserver_job = None
if args.http2_badserver_interop:
# launch a HTTP2 server emulator that creates edge cases
lang = str(http2InteropServer)
spec = server_jobspec(http2InteropServer, docker_images.get(lang))
- job = dockerjob.DockerJob(spec)
- server_jobs[lang] = job
+ http2_badserver_job = dockerjob.DockerJob(spec)
+ server_jobs[lang] = http2_badserver_job
jobs = []
if args.cloud_to_prod:
@@ -981,13 +990,16 @@
jobs.append(test_job)
if args.http2_badserver_interop:
+ http2_badserver_job.wait_for_healthy(timeout_seconds=600)
for language in languages_http2_badserver_interop:
for test_case in _HTTP2_BADSERVER_TEST_CASES:
+ offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case)
+ server_port = http2_badserver_job.mapped_port(_DEFAULT_SERVER_PORT+offset)
test_job = cloud_to_cloud_jobspec(language,
test_case,
str(http2InteropServer),
'localhost',
- _DEFAULT_SERVER_PORT,
+ server_port,
docker_image=docker_images.get(str(language)))
jobs.append(test_job)