Make testing toolchain python 3.x compliant
six is necessary for making these scripts cross compatible
between python 2.x and 3.x
Changes:
Add six to python_deps.include
Include python_deps.include to all Dockerfile templates in test directory
Include python_deps.include to all Dockerfile templates in stress_test directory
Include python_deps.include to all Dockerfile templates in interop_test directory
Replace print statements with print function calls (from futute..)
Replace .iteritems() with .items() wherever necessary
use six.moves to import BaseHTTPServer
Generate new dockerfiles using generate_projects.sh
diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py
index bd1269c..e9267be 100644
--- a/tools/run_tests/artifact_targets.py
+++ b/tools/run_tests/artifact_targets.py
@@ -40,7 +40,7 @@
environ['RUN_COMMAND'] = shell_command
docker_args=[]
- for k,v in environ.iteritems():
+ for k,v in environ.items():
docker_args += ['-e', '%s=%s' % (k, v)]
docker_env = {'DOCKERFILE_DIR': dockerfile_dir,
'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
diff --git a/tools/run_tests/distribtest_targets.py b/tools/run_tests/distribtest_targets.py
index 1a7aa0b..7930f2a 100644
--- a/tools/run_tests/distribtest_targets.py
+++ b/tools/run_tests/distribtest_targets.py
@@ -41,7 +41,7 @@
environ['RELATIVE_COPY_PATH'] = 'test/distrib'
docker_args=[]
- for k,v in environ.iteritems():
+ for k,v in environ.items():
docker_args += ['-e', '%s=%s' % (k, v)]
docker_env = {'DOCKERFILE_DIR': dockerfile_dir,
'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh'}
diff --git a/tools/run_tests/dockerjob.py b/tools/run_tests/dockerjob.py
index e4ca3b7..4a7e61b 100755
--- a/tools/run_tests/dockerjob.py
+++ b/tools/run_tests/dockerjob.py
@@ -29,6 +29,8 @@
"""Helpers to run docker instances as jobs."""
+from __future__ import print_function
+
import jobset
import tempfile
import time
@@ -95,7 +97,7 @@
stderr=subprocess.STDOUT) == 0:
return True
time.sleep(2)
- print 'Failed to remove docker image %s' % image
+ print('Failed to remove docker image %s' % image)
return False
diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py
index 3999537..b6fb631 100755
--- a/tools/run_tests/jobset.py
+++ b/tools/run_tests/jobset.py
@@ -29,6 +29,8 @@
"""Run a group of subprocesses and then finish."""
+from __future__ import print_function
+
import multiprocessing
import os
import platform
@@ -123,8 +125,8 @@
try:
if platform_string() == 'windows' or not sys.stdout.isatty():
if explanatory_text:
- print explanatory_text
- print '%s: %s' % (tag, msg)
+ print(explanatory_text)
+ print('%s: %s' % (tag, msg))
return
sys.stdout.write('%s%s%s\x1b[%d;%dm%s\x1b[0m: %s%s' % (
_BEGINNING_OF_LINE,
@@ -368,7 +370,7 @@
self._travis,
self._add_env)
self._running.add(job)
- if not self.resultset.has_key(job.GetSpec().shortname):
+ if job.GetSpec().shortname not in self.resultset:
self.resultset[job.GetSpec().shortname] = []
return True
diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py
index 39a11a2..ce3f08d 100644
--- a/tools/run_tests/package_targets.py
+++ b/tools/run_tests/package_targets.py
@@ -39,7 +39,7 @@
environ['RUN_COMMAND'] = shell_command
docker_args=[]
- for k,v in environ.iteritems():
+ for k,v in environ.items():
docker_args += ['-e', '%s=%s' % (k, v)]
docker_env = {'DOCKERFILE_DIR': dockerfile_dir,
'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
diff --git a/tools/run_tests/port_server.py b/tools/run_tests/port_server.py
index 83f8e6c..e9b3f7f 100755
--- a/tools/run_tests/port_server.py
+++ b/tools/run_tests/port_server.py
@@ -30,8 +30,10 @@
"""Manage TCP ports for unit tests; started by run_tests.py"""
+from __future__ import print_function
+
import argparse
-import BaseHTTPServer
+from six.moves import BaseHTTPServer
import hashlib
import os
import socket
@@ -46,7 +48,7 @@
if len(sys.argv) == 2 and sys.argv[1] == 'dump_version':
- print _MY_VERSION
+ print(_MY_VERSION)
sys.exit(0)
@@ -62,7 +64,7 @@
sys.stderr = open(args.logfile, 'w')
sys.stdout = sys.stderr
-print 'port server running on port %d' % args.port
+print('port server running on port %d' % args.port)
pool = []
in_use = {}
@@ -152,7 +154,7 @@
self.send_header('Content-Type', 'text/plain')
self.end_headers()
now = time.time()
- self.wfile.write(yaml.dump({'pool': pool, 'in_use': dict((k, now - v) for k, v in in_use.iteritems())}))
+ self.wfile.write(yaml.dump({'pool': pool, 'in_use': dict((k, now - v) for k, v in in_use.items())}))
elif self.path == '/quitquitquit':
self.send_response(200)
self.end_headers()
@@ -164,4 +166,4 @@
httpd.handle_request()
sys.stderr.flush()
-print 'done'
+print('done')
diff --git a/tools/run_tests/report_utils.py b/tools/run_tests/report_utils.py
index 35dcaca..5648a69 100644
--- a/tools/run_tests/report_utils.py
+++ b/tools/run_tests/report_utils.py
@@ -29,6 +29,8 @@
"""Generate XML and HTML test reports."""
+from __future__ import print_function
+
try:
from mako.runtime import Context
from mako.template import Template
@@ -60,7 +62,7 @@
root = ET.Element('testsuites')
testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc',
name='tests')
- for shortname, results in resultset.iteritems():
+ for shortname, results in resultset.items():
for result in results:
xml_test = ET.SubElement(testsuite, 'testcase', name=shortname)
if result.elapsed_time:
@@ -83,10 +85,10 @@
try:
mytemplate = Template(filename=template_file, format_exceptions=True)
except NameError:
- print 'Mako template is not installed. Skipping HTML report generation.'
+ print('Mako template is not installed. Skipping HTML report generation.')
return
except IOError as e:
- print 'Failed to find the template %s: %s' % (template_file, e)
+ print('Failed to find the template %s: %s' % (template_file, e))
return
sorted_test_cases = sorted(test_cases)
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 13a4a49..f9065c5 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -30,6 +30,8 @@
"""Run interop (cross-language) tests in parallel."""
+from __future__ import print_function
+
import argparse
import atexit
import dockerjob
@@ -372,7 +374,7 @@
# turn environ into -e docker args
if environ:
- for k,v in environ.iteritems():
+ for k,v in environ.items():
docker_cmdline += ['-e', '%s=%s' % (k,v)]
# set working directory
@@ -674,15 +676,15 @@
if args.use_docker:
if not args.travis:
- print 'Seen --use_docker flag, will run interop tests under docker.'
- print
- print 'IMPORTANT: The changes you are testing need to be locally committed'
- print 'because only the committed changes in the current branch will be'
- print 'copied to the docker environment.'
+ print('Seen --use_docker flag, will run interop tests under docker.')
+ print('')
+ print('IMPORTANT: The changes you are testing need to be locally committed')
+ print('because only the committed changes in the current branch will be')
+ print('copied to the docker environment.')
time.sleep(5)
if not args.use_docker and servers:
- print 'Running interop servers is only supported with --use_docker option enabled.'
+ print('Running interop servers is only supported with --use_docker option enabled.')
sys.exit(1)
languages = set(_LANGUAGES[l]
@@ -768,7 +770,7 @@
(server_host, server_port) = server[1].split(':')
server_addresses[server_name] = (server_host, server_port)
- for server_name, server_address in server_addresses.iteritems():
+ for server_name, server_address in server_addresses.items():
(server_host, server_port) = server_address
server_language = _LANGUAGES.get(server_name, None)
skip_server = [] # test cases unimplemented by server
@@ -800,7 +802,7 @@
jobs.append(test_job)
if not jobs:
- print 'No jobs to run.'
+ print('No jobs to run.')
for image in docker_images.itervalues():
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
@@ -814,7 +816,7 @@
report_utils.render_junit_xml_report(resultset, 'report.xml')
- for name, job in resultset.iteritems():
+ for name, job in resultset.items():
if "http2" in name:
job[0].http2results = aggregate_http2_results(job[0].message)
@@ -826,12 +828,12 @@
finally:
# Check if servers are still running.
- for server, job in server_jobs.iteritems():
+ for server, job in server_jobs.items():
if not job.is_running():
- print 'Server "%s" has exited prematurely.' % server
+ print('Server "%s" has exited prematurely.' % server)
dockerjob.finish_jobs([j for j in server_jobs.itervalues()])
for image in docker_images.itervalues():
- print 'Removing docker image %s' % image
+ print('Removing docker image %s' % image)
dockerjob.remove_image(image)
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index 14901ca..5fdf7a4 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -30,6 +30,8 @@
"""Run performance tests locally or remotely."""
+from __future__ import print_function
+
import argparse
import itertools
import jobset
@@ -310,7 +312,7 @@
'in the same scenario')
if custom_server_lang:
if not workers_by_lang.get(custom_server_lang, []):
- print 'Warning: Skipping scenario %s as' % scenario_json['name']
+ print('Warning: Skipping scenario %s as' % scenario_json['name'])
print('SERVER_LANGUAGE is set to %s yet the language has '
'not been selected with -l' % custom_server_lang)
continue
@@ -319,7 +321,7 @@
workers[idx] = workers_by_lang[custom_server_lang][idx]
if custom_client_lang:
if not workers_by_lang.get(custom_client_lang, []):
- print 'Warning: Skipping scenario %s as' % scenario_json['name']
+ print('Warning: Skipping scenario %s as' % scenario_json['name'])
print('CLIENT_LANGUAGE is set to %s yet the language has '
'not been selected with -l' % custom_client_lang)
continue
@@ -344,14 +346,14 @@
while any(job.is_running() for job in jobs):
for job in qpsworker_jobs:
if job.is_running():
- print 'QPS worker "%s" is still running.' % job.host_and_port
+ print('QPS worker "%s" is still running.' % job.host_and_port)
if retries > 10:
- print 'Killing all QPS workers.'
+ print('Killing all QPS workers.')
for job in jobs:
job.kill()
retries += 1
time.sleep(3)
- print 'All QPS workers finished.'
+ print('All QPS workers finished.')
argp = argparse.ArgumentParser(description='Run performance tests.')
diff --git a/tools/run_tests/run_stress_tests.py b/tools/run_tests/run_stress_tests.py
index e42ee24..de4a228 100755
--- a/tools/run_tests/run_stress_tests.py
+++ b/tools/run_tests/run_stress_tests.py
@@ -29,6 +29,8 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Run stress test in C++"""
+from __future__ import print_function
+
import argparse
import atexit
import dockerjob
@@ -93,7 +95,7 @@
# turn environ into -e docker args
if environ:
- for k, v in environ.iteritems():
+ for k, v in environ.items():
docker_cmdline += ['-e', '%s=%s' % (k, v)]
# set working directory
@@ -140,7 +142,7 @@
'--num_channels_per_server=%s' % num_channels_per_server,
'--metrics_port=%s' % metrics_port
]))
- print cmdline
+ print(cmdline)
cwd = language.client_cwd
environ = language.global_env()
if docker_image:
@@ -287,7 +289,7 @@
(server_host, server_port) = server[1].split(':')
server_addresses[server_name] = (server_host, server_port)
- for server_name, server_address in server_addresses.iteritems():
+ for server_name, server_address in server_addresses.items():
(server_host, server_port) = server_address
for language in languages:
test_job = cloud_to_cloud_jobspec(
@@ -302,7 +304,7 @@
jobs.append(test_job)
if not jobs:
- print 'No jobs to run.'
+ print('No jobs to run.')
for image in docker_images.itervalues():
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
@@ -317,12 +319,12 @@
finally:
# Check if servers are still running.
- for server, job in server_jobs.iteritems():
+ for server, job in server_jobs.items():
if not job.is_running():
- print 'Server "%s" has exited prematurely.' % server
+ print('Server "%s" has exited prematurely.' % server)
dockerjob.finish_jobs([j for j in server_jobs.itervalues()])
for image in docker_images.itervalues():
- print 'Removing docker image %s' % image
+ print('Removing docker image %s' % image)
dockerjob.remove_image(image)
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index b0e2069..18674b7 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -30,6 +30,8 @@
"""Run tests in parallel."""
+from __future__ import print_function
+
import argparse
import ast
import collections
@@ -48,7 +50,7 @@
import tempfile
import traceback
import time
-import urllib2
+from six.moves import urllib
import uuid
import jobset
@@ -93,7 +95,7 @@
would like to run
"""
actual_environ = self.environ.copy()
- for k, v in environ.iteritems():
+ for k, v in environ.items():
actual_environ[k] = v
return jobset.JobSpec(cmdline=self.tool_prefix + cmdline,
shortname=shortname,
@@ -202,7 +204,7 @@
flaky=target.get('flaky', False),
environ=env))
elif self.args.regex == '.*' or self.platform == 'windows':
- print '\nWARNING: binary not found, skipping', binary
+ print('\nWARNING: binary not found, skipping', binary)
return sorted(out)
def make_targets(self):
@@ -393,7 +395,7 @@
return [self.config.job_spec(
config.run,
timeout_seconds=5*60,
- environ=dict(environment.items() +
+ environ=dict(list(environment.items()) +
[('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]),
shortname='%s.test.%s' % (config.name, suite_name),)
for suite_name in tests_json
@@ -755,7 +757,7 @@
elif arch == 'x64':
return '/p:Platform=x64'
else:
- print 'Architecture %s not supported.' % arch
+ print('Architecture %s not supported.' % arch)
sys.exit(1)
@@ -773,11 +775,11 @@
elif runtime_arch == '32bit' and arch == 'x86':
return
else:
- print 'Architecture %s does not match current runtime architecture.' % arch
+ print('Architecture %s does not match current runtime architecture.' % arch)
sys.exit(1)
else:
if args.arch != 'default':
- print 'Architecture %s not supported on current platform.' % args.arch
+ print('Architecture %s not supported on current platform.' % args.arch)
sys.exit(1)
@@ -791,7 +793,7 @@
elif compiler == 'vs2010':
return 'vsprojects\\build_vs2010.bat'
else:
- print 'Compiler %s not supported.' % compiler
+ print('Compiler %s not supported.' % compiler)
sys.exit(1)
@@ -805,7 +807,7 @@
elif compiler == 'vs2010':
return '/p:PlatformToolset=v100'
else:
- print 'Compiler %s not supported.' % compiler
+ print('Compiler %s not supported.' % compiler)
sys.exit(1)
@@ -816,7 +818,7 @@
elif arch == 'x86':
return 'x86'
else:
- print 'Architecture %s not supported with current settings.' % arch
+ print('Architecture %s not supported with current settings.' % arch)
sys.exit(1)
@@ -932,7 +934,7 @@
branch = spec[1]
cwd = 'third_party/%s' % submodule
def git(cmd, cwd=cwd):
- print 'in %s: git %s' % (cwd, cmd)
+ print('in %s: git %s' % (cwd, cmd))
subprocess.check_call('git %s' % cmd, cwd=cwd, shell=True)
git('fetch')
git('checkout %s' % branch)
@@ -943,8 +945,8 @@
if jobset.platform_string() == 'linux':
subprocess.check_call('tools/buildgen/generate_projects.sh', shell=True)
else:
- print 'WARNING: may need to regenerate projects, but since we are not on'
- print ' Linux this step is being skipped. Compilation MAY fail.'
+ print('WARNING: may need to regenerate projects, but since we are not on')
+ print(' Linux this step is being skipped. Compilation MAY fail.')
# grab config
@@ -971,18 +973,18 @@
language_make_options=[]
if any(language.make_options() for language in languages):
if not 'gcov' in args.config and len(languages) != 1:
- print 'languages with custom make options cannot be built simultaneously with other languages'
+ print('languages with custom make options cannot be built simultaneously with other languages')
sys.exit(1)
else:
language_make_options = next(iter(languages)).make_options()
if args.use_docker:
if not args.travis:
- print 'Seen --use_docker flag, will run tests under docker.'
- print
- print 'IMPORTANT: The changes you are testing need to be locally committed'
- print 'because only the committed changes in the current branch will be'
- print 'copied to the docker environment.'
+ print('Seen --use_docker flag, will run tests under docker.')
+ print('')
+ print('IMPORTANT: The changes you are testing need to be locally committed')
+ print('because only the committed changes in the current branch will be')
+ print('copied to the docker environment.')
time.sleep(5)
dockerfile_dirs = set([l.dockerfile_dir() for l in languages])
@@ -1066,7 +1068,7 @@
for l in languages
for cmdline in l.pre_build_steps()))
if make_targets:
- make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.iteritems())
+ make_commands = itertools.chain.from_iterable(make_jobspec(build_config, list(targets), makefile) for (makefile, targets) in make_targets.items())
build_steps.extend(set(make_commands))
build_steps.extend(set(
jobset.JobSpec(cmdline, environ=build_step_environ(build_config), timeout_seconds=None)
@@ -1083,13 +1085,13 @@
def _shut_down_legacy_server(legacy_server_port):
try:
- version = int(urllib2.urlopen(
+ version = int(urllib.request.urlopen(
'http://localhost:%d/version_number' % legacy_server_port,
timeout=10).read())
except:
pass
else:
- urllib2.urlopen(
+ urllib.request.urlopen(
'http://localhost:%d/quitquitquit' % legacy_server_port).read()
@@ -1099,29 +1101,29 @@
# if not running ==> start a new one
# otherwise, leave it up
try:
- version = int(urllib2.urlopen(
+ version = int(urllib.request.urlopen(
'http://localhost:%d/version_number' % port_server_port,
timeout=10).read())
- print 'detected port server running version %d' % version
+ print('detected port server running version %d' % version)
running = True
except Exception as e:
- print 'failed to detect port server: %s' % sys.exc_info()[0]
- print e.strerror
+ print('failed to detect port server: %s' % sys.exc_info()[0])
+ print(e.strerror)
running = False
if running:
current_version = int(subprocess.check_output(
[sys.executable, os.path.abspath('tools/run_tests/port_server.py'),
'dump_version']))
- print 'my port server is version %d' % current_version
+ print('my port server is version %d' % current_version)
running = (version >= current_version)
if not running:
- print 'port_server version mismatch: killing the old one'
- urllib2.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read()
+ print('port_server version mismatch: killing the old one')
+ urllib.request.urlopen('http://localhost:%d/quitquitquit' % port_server_port).read()
time.sleep(1)
if not running:
fd, logfile = tempfile.mkstemp()
os.close(fd)
- print 'starting port_server, with log file %s' % logfile
+ print('starting port_server, with log file %s' % logfile)
args = [sys.executable, os.path.abspath('tools/run_tests/port_server.py'),
'-p', '%d' % port_server_port, '-l', logfile]
env = dict(os.environ)
@@ -1147,34 +1149,34 @@
waits = 0
while True:
if waits > 10:
- print 'killing port server due to excessive start up waits'
+ print('killing port server due to excessive start up waits')
port_server.kill()
if port_server.poll() is not None:
- print 'port_server failed to start'
+ print('port_server failed to start')
# try one final time: maybe another build managed to start one
time.sleep(1)
try:
- urllib2.urlopen('http://localhost:%d/get' % port_server_port,
+ urllib.request.urlopen('http://localhost:%d/get' % port_server_port,
timeout=1).read()
- print 'last ditch attempt to contact port server succeeded'
+ print('last ditch attempt to contact port server succeeded')
break
except:
traceback.print_exc()
port_log = open(logfile, 'r').read()
- print port_log
+ print(port_log)
sys.exit(1)
try:
- urllib2.urlopen('http://localhost:%d/get' % port_server_port,
+ urllib.request.urlopen('http://localhost:%d/get' % port_server_port,
timeout=1).read()
- print 'port server is up and ready'
+ print('port server is up and ready')
break
except socket.timeout:
- print 'waiting for port_server: timeout'
+ print('waiting for port_server: timeout')
traceback.print_exc();
time.sleep(1)
waits += 1
- except urllib2.URLError:
- print 'waiting for port_server: urlerror'
+ except urllib.error.URLError:
+ print('waiting for port_server: urlerror')
traceback.print_exc();
time.sleep(1)
waits += 1
diff --git a/tools/run_tests/task_runner.py b/tools/run_tests/task_runner.py
index b42aa17..2e3fa44 100755
--- a/tools/run_tests/task_runner.py
+++ b/tools/run_tests/task_runner.py
@@ -30,6 +30,8 @@
"""Runs selected gRPC test/build tasks."""
+from __future__ import print_function
+
import argparse
import atexit
import jobset
@@ -111,7 +113,7 @@
for target in targets:
build_jobs.append(target.build_jobspec())
if not build_jobs:
- print 'Nothing to build.'
+ print('Nothing to build.')
sys.exit(1)
jobset.message('START', 'Building targets.', do_newline=True)