Siddharth Shukla | 8e64d90 | 2017-03-12 19:50:18 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 2 | # Copyright 2015, Google Inc. |
| 3 | # All rights reserved. |
| 4 | # |
| 5 | # Redistribution and use in source and binary forms, with or without |
| 6 | # modification, are permitted provided that the following conditions are |
| 7 | # met: |
| 8 | # |
| 9 | # * Redistributions of source code must retain the above copyright |
| 10 | # notice, this list of conditions and the following disclaimer. |
| 11 | # * Redistributions in binary form must reproduce the above |
| 12 | # copyright notice, this list of conditions and the following disclaimer |
| 13 | # in the documentation and/or other materials provided with the |
| 14 | # distribution. |
| 15 | # * Neither the name of Google Inc. nor the names of its |
| 16 | # contributors may be used to endorse or promote products derived from |
| 17 | # this software without specific prior written permission. |
| 18 | # |
| 19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 | |
| 31 | """Manage TCP ports for unit tests; started by run_tests.py""" |
| 32 | |
siddharthshukla | 0589e53 | 2016-07-07 16:08:01 +0200 | [diff] [blame] | 33 | from __future__ import print_function |
| 34 | |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 35 | import argparse |
Craig Tiller | ea525eb | 2017-04-24 17:50:32 +0000 | [diff] [blame] | 36 | from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 37 | import hashlib |
| 38 | import os |
| 39 | import socket |
| 40 | import sys |
| 41 | import time |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 42 | import random |
Craig Tiller | ea525eb | 2017-04-24 17:50:32 +0000 | [diff] [blame] | 43 | from SocketServer import ThreadingMixIn |
| 44 | import threading |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 45 | |
Craig Tiller | fe4939f | 2015-10-06 12:55:36 -0700 | [diff] [blame] | 46 | |
| 47 | # increment this number whenever making a change to ensure that |
| 48 | # the changes are picked up by running CI servers |
| 49 | # note that all changes must be backwards compatible |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 50 | _MY_VERSION = 19 |
Craig Tiller | fe4939f | 2015-10-06 12:55:36 -0700 | [diff] [blame] | 51 | |
| 52 | |
| 53 | if len(sys.argv) == 2 and sys.argv[1] == 'dump_version': |
siddharthshukla | 0589e53 | 2016-07-07 16:08:01 +0200 | [diff] [blame] | 54 | print(_MY_VERSION) |
Craig Tiller | fe4939f | 2015-10-06 12:55:36 -0700 | [diff] [blame] | 55 | sys.exit(0) |
| 56 | |
| 57 | |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 58 | argp = argparse.ArgumentParser(description='Server for httpcli_test') |
| 59 | argp.add_argument('-p', '--port', default=12345, type=int) |
Craig Tiller | f0a293e | 2015-10-12 10:05:50 -0700 | [diff] [blame] | 60 | argp.add_argument('-l', '--logfile', default=None, type=str) |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 61 | args = argp.parse_args() |
| 62 | |
Craig Tiller | f0a293e | 2015-10-12 10:05:50 -0700 | [diff] [blame] | 63 | if args.logfile is not None: |
Craig Tiller | 41a7bf5 | 2015-10-12 12:54:55 -0700 | [diff] [blame] | 64 | sys.stdin.close() |
| 65 | sys.stderr.close() |
| 66 | sys.stdout.close() |
Craig Tiller | f0a293e | 2015-10-12 10:05:50 -0700 | [diff] [blame] | 67 | sys.stderr = open(args.logfile, 'w') |
| 68 | sys.stdout = sys.stderr |
| 69 | |
siddharthshukla | 0589e53 | 2016-07-07 16:08:01 +0200 | [diff] [blame] | 70 | print('port server running on port %d' % args.port) |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 71 | |
| 72 | pool = [] |
| 73 | in_use = {} |
Craig Tiller | ec49524 | 2017-04-24 20:55:43 +0000 | [diff] [blame] | 74 | mu = threading.Lock() |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 75 | |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 76 | def can_connect(port): |
| 77 | s = socket.socket() |
| 78 | try: |
| 79 | s.connect(('localhost', port)) |
| 80 | return True |
| 81 | except socket.error, e: |
| 82 | return False |
| 83 | finally: |
| 84 | s.close() |
| 85 | |
| 86 | def can_bind(port, proto): |
| 87 | s = socket.socket(proto, socket.SOCK_STREAM) |
| 88 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
| 89 | try: |
| 90 | s.bind(('localhost', port)) |
| 91 | return True |
| 92 | except socket.error, e: |
| 93 | return False |
| 94 | finally: |
| 95 | s.close() |
| 96 | |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 97 | |
Craig Tiller | c20b19d | 2015-09-22 13:06:06 -0700 | [diff] [blame] | 98 | def refill_pool(max_timeout, req): |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 99 | """Scan for ports not marked for being in use""" |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 100 | chk = list(range(1025, 32766)) |
| 101 | random.shuffle(chk) |
| 102 | for i in chk: |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 103 | if len(pool) > 100: break |
| 104 | if i in in_use: |
| 105 | age = time.time() - in_use[i] |
Craig Tiller | ae322af | 2015-09-13 18:41:21 +0000 | [diff] [blame] | 106 | if age < max_timeout: |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 107 | continue |
Craig Tiller | c20b19d | 2015-09-22 13:06:06 -0700 | [diff] [blame] | 108 | req.log_message("kill old request %d" % i) |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 109 | del in_use[i] |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 110 | if can_bind(i, socket.AF_INET) and can_bind(i, socket.AF_INET6) and not can_connect(i): |
Craig Tiller | c20b19d | 2015-09-22 13:06:06 -0700 | [diff] [blame] | 111 | req.log_message("found available port %d" % i) |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 112 | pool.append(i) |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 113 | |
| 114 | |
Craig Tiller | c20b19d | 2015-09-22 13:06:06 -0700 | [diff] [blame] | 115 | def allocate_port(req): |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 116 | global pool |
| 117 | global in_use |
Craig Tiller | ec49524 | 2017-04-24 20:55:43 +0000 | [diff] [blame] | 118 | global mu |
| 119 | mu.acquire() |
Craig Tiller | ae322af | 2015-09-13 18:41:21 +0000 | [diff] [blame] | 120 | max_timeout = 600 |
| 121 | while not pool: |
Craig Tiller | c20b19d | 2015-09-22 13:06:06 -0700 | [diff] [blame] | 122 | refill_pool(max_timeout, req) |
Craig Tiller | ae322af | 2015-09-13 18:41:21 +0000 | [diff] [blame] | 123 | if not pool: |
Craig Tiller | c20b19d | 2015-09-22 13:06:06 -0700 | [diff] [blame] | 124 | req.log_message("failed to find ports: retrying soon") |
Craig Tiller | ec49524 | 2017-04-24 20:55:43 +0000 | [diff] [blame] | 125 | mu.release() |
Craig Tiller | ae322af | 2015-09-13 18:41:21 +0000 | [diff] [blame] | 126 | time.sleep(1) |
Craig Tiller | ec49524 | 2017-04-24 20:55:43 +0000 | [diff] [blame] | 127 | mu.acquire() |
Craig Tiller | ae322af | 2015-09-13 18:41:21 +0000 | [diff] [blame] | 128 | max_timeout /= 2 |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 129 | port = pool[0] |
| 130 | pool = pool[1:] |
| 131 | in_use[port] = time.time() |
Craig Tiller | ec49524 | 2017-04-24 20:55:43 +0000 | [diff] [blame] | 132 | mu.release() |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 133 | return port |
| 134 | |
| 135 | |
Craig Tiller | ef12559 | 2015-08-05 07:41:35 -0700 | [diff] [blame] | 136 | keep_running = True |
| 137 | |
| 138 | |
Craig Tiller | ea525eb | 2017-04-24 17:50:32 +0000 | [diff] [blame] | 139 | class Handler(BaseHTTPRequestHandler): |
Ken Payson | 189d685 | 2016-07-09 15:58:18 -0700 | [diff] [blame] | 140 | |
| 141 | def setup(self): |
| 142 | # If the client is unreachable for 5 seconds, close the connection |
| 143 | self.timeout = 5 |
Craig Tiller | ea525eb | 2017-04-24 17:50:32 +0000 | [diff] [blame] | 144 | BaseHTTPRequestHandler.setup(self) |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 145 | |
| 146 | def do_GET(self): |
Craig Tiller | ef12559 | 2015-08-05 07:41:35 -0700 | [diff] [blame] | 147 | global keep_running |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 148 | global mu |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 149 | if self.path == '/get': |
| 150 | # allocate a new port, it will stay bound for ten minutes and until |
| 151 | # it's unused |
| 152 | self.send_response(200) |
| 153 | self.send_header('Content-Type', 'text/plain') |
| 154 | self.end_headers() |
Craig Tiller | c20b19d | 2015-09-22 13:06:06 -0700 | [diff] [blame] | 155 | p = allocate_port(self) |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 156 | self.log_message('allocated port %d' % p) |
| 157 | self.wfile.write('%d' % p) |
Craig Tiller | ae322af | 2015-09-13 18:41:21 +0000 | [diff] [blame] | 158 | elif self.path[0:6] == '/drop/': |
| 159 | self.send_response(200) |
| 160 | self.send_header('Content-Type', 'text/plain') |
| 161 | self.end_headers() |
| 162 | p = int(self.path[6:]) |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 163 | mu.acquire() |
Craig Tiller | d2c3971 | 2015-10-12 11:08:49 -0700 | [diff] [blame] | 164 | if p in in_use: |
| 165 | del in_use[p] |
| 166 | pool.append(p) |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 167 | k = 'known' |
Craig Tiller | d2c3971 | 2015-10-12 11:08:49 -0700 | [diff] [blame] | 168 | else: |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 169 | k = 'unknown' |
| 170 | mu.release() |
| 171 | self.log_message('drop %s port %d' % (k, p)) |
Craig Tiller | fe4939f | 2015-10-06 12:55:36 -0700 | [diff] [blame] | 172 | elif self.path == '/version_number': |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 173 | # fetch a version string and the current process pid |
| 174 | self.send_response(200) |
| 175 | self.send_header('Content-Type', 'text/plain') |
| 176 | self.end_headers() |
Craig Tiller | ef12559 | 2015-08-05 07:41:35 -0700 | [diff] [blame] | 177 | self.wfile.write(_MY_VERSION) |
Craig Tiller | ae322af | 2015-09-13 18:41:21 +0000 | [diff] [blame] | 178 | elif self.path == '/dump': |
Jan Tattermusch | 13e65df | 2015-09-21 09:30:49 -0700 | [diff] [blame] | 179 | # yaml module is not installed on Macs and Windows machines by default |
| 180 | # so we import it lazily (/dump action is only used for debugging) |
| 181 | import yaml |
Craig Tiller | ae322af | 2015-09-13 18:41:21 +0000 | [diff] [blame] | 182 | self.send_response(200) |
| 183 | self.send_header('Content-Type', 'text/plain') |
| 184 | self.end_headers() |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 185 | mu.acquire() |
Craig Tiller | ae322af | 2015-09-13 18:41:21 +0000 | [diff] [blame] | 186 | now = time.time() |
Craig Tiller | e1f5302 | 2017-05-04 19:53:19 +0000 | [diff] [blame^] | 187 | out = yaml.dump({'pool': pool, 'in_use': dict((k, now - v) for k, v in in_use.items())}) |
| 188 | mu.release() |
| 189 | self.wfile.write(out) |
Craig Tiller | fe4939f | 2015-10-06 12:55:36 -0700 | [diff] [blame] | 190 | elif self.path == '/quitquitquit': |
Craig Tiller | ef12559 | 2015-08-05 07:41:35 -0700 | [diff] [blame] | 191 | self.send_response(200) |
| 192 | self.end_headers() |
Craig Tiller | 09ebed7 | 2017-04-25 10:19:10 -0700 | [diff] [blame] | 193 | self.server.shutdown() |
Craig Tiller | ea525eb | 2017-04-24 17:50:32 +0000 | [diff] [blame] | 194 | |
| 195 | class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): |
| 196 | """Handle requests in a separate thread""" |
Craig Tiller | f53d9c8 | 2015-08-04 14:19:43 -0700 | [diff] [blame] | 197 | |
| 198 | |
Craig Tiller | 09ebed7 | 2017-04-25 10:19:10 -0700 | [diff] [blame] | 199 | ThreadedHTTPServer(('', args.port), Handler).serve_forever() |
Craig Tiller | ef12559 | 2015-08-05 07:41:35 -0700 | [diff] [blame] | 200 | |