Testing port server

run_tests.py will start a server (if it's not running, or if the running
port server mismatches the 'current' one) that serves ports to use for
tests. The server is left running after run_tests.py finishes, so that
in environments such as Mac and Windows where tests run unshielded from
each other, we don't start jumping on already used ports.
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index fa74949..653b98d 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -32,17 +32,20 @@
 
 import argparse
 import glob
+import hashlib
 import itertools
 import json
 import multiprocessing
 import os
 import platform
+import psutil
 import random
 import re
 import subprocess
 import sys
 import time
 import xml.etree.cElementTree as ET
+import urllib2
 
 import jobset
 import watch_dirs
@@ -522,7 +525,43 @@
         self.parse(json.loads(f.read()))
 
 
-def _build_and_run(check_cancelled, newline_on_success, travis, cache, xml_report=None):
+def _start_port_server(port_server_port):
+  # check if a compatible port server is running
+  # if incompatible (version mismatch) ==> start a new one
+  # if not running ==> start a new one
+  # otherwise, leave it up
+  try:
+    version, _, pid = urllib2.urlopen(
+        'http://localhost:%d/version_and_pid' % port_server_port).read().partition('+')
+    running = True
+  except Exception:
+    running = False
+  if running:
+    with open('tools/run_tests/port_server.py') as f:
+      current_version = hashlib.sha1(f.read()).hexdigest()
+      running = (version == current_version)
+      if not running:
+        psutil.Process(int(pid)).terminate()
+  if not running:
+    port_log = open('portlog.txt', 'w')
+    port_server = subprocess.Popen(
+        ['tools/run_tests/port_server.py', '-p', '%d' % port_server_port],
+        stderr=subprocess.STDOUT,
+        stdout=port_log)
+    # ensure port server is up
+    while True:
+      try:
+        urllib2.urlopen('http://localhost:%d/get' % port_server_port).read()
+        break
+      except urllib2.URLError:
+        time.sleep(0.5)
+      except:
+        port_server.kill()
+        raise
+
+
+def _build_and_run(
+    check_cancelled, newline_on_success, travis, cache, xml_report=None):
   """Do one pass of building & running tests."""
   # build latest sequentially
   if not jobset.run(build_steps, maxjobs=1,
@@ -532,6 +571,8 @@
   # start antagonists
   antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py'])
                  for _ in range(0, args.antagonists)]
+  port_server_port = 9999
+  _start_port_server(port_server_port)
   try:
     infinite_runs = runs_per_test == 0
     # When running on travis, we want out test runs to be as similar as possible
@@ -558,7 +599,8 @@
                       maxjobs=args.jobs,
                       stop_on_failure=args.stop_on_failure,
                       cache=cache if not xml_report else None,
-                      xml_report=testsuite):
+                      xml_report=testsuite,
+                      add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}):
       return 2
   finally:
     for antagonist in antagonists: