Benjamin Peterson | 90f5ba5 | 2010-03-11 22:53:45 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 2 | |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 3 | import os |
Barry Warsaw | 820c120 | 2008-06-12 04:06:45 +0000 | [diff] [blame] | 4 | import email |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 5 | import urllib.parse |
| 6 | import urllib.request |
Georg Brandl | 2442015 | 2008-05-26 16:32:26 +0000 | [diff] [blame] | 7 | import http.server |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 8 | import unittest |
| 9 | import hashlib |
Benjamin Peterson | ee8712c | 2008-05-20 21:35:26 +0000 | [diff] [blame] | 10 | from test import support |
Victor Stinner | 45df820 | 2010-04-28 22:31:17 +0000 | [diff] [blame] | 11 | threading = support.import_module('threading') |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 12 | |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 13 | |
| 14 | here = os.path.dirname(__file__) |
| 15 | # Self-signed cert file for 'localhost' |
| 16 | CERT_localhost = os.path.join(here, 'keycert.pem') |
| 17 | # Self-signed cert file for 'fakehostname' |
| 18 | CERT_fakehostname = os.path.join(here, 'keycert2.pem') |
| 19 | |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 20 | # Loopback http server infrastructure |
| 21 | |
Georg Brandl | 2442015 | 2008-05-26 16:32:26 +0000 | [diff] [blame] | 22 | class LoopbackHttpServer(http.server.HTTPServer): |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 23 | """HTTP server w/ a few modifications that make it useful for |
| 24 | loopback testing purposes. |
| 25 | """ |
| 26 | |
| 27 | def __init__(self, server_address, RequestHandlerClass): |
Georg Brandl | 2442015 | 2008-05-26 16:32:26 +0000 | [diff] [blame] | 28 | http.server.HTTPServer.__init__(self, |
| 29 | server_address, |
| 30 | RequestHandlerClass) |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 31 | |
| 32 | # Set the timeout of our listening socket really low so |
| 33 | # that we can stop the server easily. |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 34 | self.socket.settimeout(0.1) |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 35 | |
| 36 | def get_request(self): |
Georg Brandl | 2442015 | 2008-05-26 16:32:26 +0000 | [diff] [blame] | 37 | """HTTPServer method, overridden.""" |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 38 | |
| 39 | request, client_address = self.socket.accept() |
| 40 | |
| 41 | # It's a loopback connection, so setting the timeout |
| 42 | # really low shouldn't affect anything, but should make |
| 43 | # deadlocks less likely to occur. |
| 44 | request.settimeout(10.0) |
| 45 | |
| 46 | return (request, client_address) |
| 47 | |
| 48 | class LoopbackHttpServerThread(threading.Thread): |
| 49 | """Stoppable thread that runs a loopback http server.""" |
| 50 | |
Guido van Rossum | 806c246 | 2007-08-06 23:33:07 +0000 | [diff] [blame] | 51 | def __init__(self, request_handler): |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 52 | threading.Thread.__init__(self) |
Guido van Rossum | 4566c71 | 2007-08-21 03:36:47 +0000 | [diff] [blame] | 53 | self._stop_server = False |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 54 | self.ready = threading.Event() |
Guido van Rossum | 806c246 | 2007-08-06 23:33:07 +0000 | [diff] [blame] | 55 | request_handler.protocol_version = "HTTP/1.0" |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 56 | self.httpd = LoopbackHttpServer(("127.0.0.1", 0), |
Guido van Rossum | 806c246 | 2007-08-06 23:33:07 +0000 | [diff] [blame] | 57 | request_handler) |
| 58 | #print "Serving HTTP on %s port %s" % (self.httpd.server_name, |
| 59 | # self.httpd.server_port) |
| 60 | self.port = self.httpd.server_port |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 61 | |
| 62 | def stop(self): |
| 63 | """Stops the webserver if it's currently running.""" |
| 64 | |
| 65 | # Set the stop flag. |
Guido van Rossum | 4566c71 | 2007-08-21 03:36:47 +0000 | [diff] [blame] | 66 | self._stop_server = True |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 67 | |
| 68 | self.join() |
Antoine Pitrou | b6751dc | 2010-10-30 17:33:22 +0000 | [diff] [blame] | 69 | self.httpd.server_close() |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 70 | |
| 71 | def run(self): |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 72 | self.ready.set() |
Guido van Rossum | 4566c71 | 2007-08-21 03:36:47 +0000 | [diff] [blame] | 73 | while not self._stop_server: |
Guido van Rossum | 806c246 | 2007-08-06 23:33:07 +0000 | [diff] [blame] | 74 | self.httpd.handle_request() |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 75 | |
| 76 | # Authentication infrastructure |
| 77 | |
| 78 | class DigestAuthHandler: |
| 79 | """Handler for performing digest authentication.""" |
| 80 | |
| 81 | def __init__(self): |
| 82 | self._request_num = 0 |
| 83 | self._nonces = [] |
| 84 | self._users = {} |
| 85 | self._realm_name = "Test Realm" |
| 86 | self._qop = "auth" |
| 87 | |
| 88 | def set_qop(self, qop): |
| 89 | self._qop = qop |
| 90 | |
| 91 | def set_users(self, users): |
| 92 | assert isinstance(users, dict) |
| 93 | self._users = users |
| 94 | |
| 95 | def set_realm(self, realm): |
| 96 | self._realm_name = realm |
| 97 | |
| 98 | def _generate_nonce(self): |
| 99 | self._request_num += 1 |
Guido van Rossum | 8136014 | 2007-08-29 14:26:52 +0000 | [diff] [blame] | 100 | nonce = hashlib.md5(str(self._request_num).encode("ascii")).hexdigest() |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 101 | self._nonces.append(nonce) |
| 102 | return nonce |
| 103 | |
| 104 | def _create_auth_dict(self, auth_str): |
| 105 | first_space_index = auth_str.find(" ") |
| 106 | auth_str = auth_str[first_space_index+1:] |
| 107 | |
| 108 | parts = auth_str.split(",") |
| 109 | |
| 110 | auth_dict = {} |
| 111 | for part in parts: |
| 112 | name, value = part.split("=") |
| 113 | name = name.strip() |
| 114 | if value[0] == '"' and value[-1] == '"': |
| 115 | value = value[1:-1] |
| 116 | else: |
| 117 | value = value.strip() |
| 118 | auth_dict[name] = value |
| 119 | return auth_dict |
| 120 | |
| 121 | def _validate_auth(self, auth_dict, password, method, uri): |
| 122 | final_dict = {} |
| 123 | final_dict.update(auth_dict) |
| 124 | final_dict["password"] = password |
| 125 | final_dict["method"] = method |
| 126 | final_dict["uri"] = uri |
| 127 | HA1_str = "%(username)s:%(realm)s:%(password)s" % final_dict |
Guido van Rossum | 8136014 | 2007-08-29 14:26:52 +0000 | [diff] [blame] | 128 | HA1 = hashlib.md5(HA1_str.encode("ascii")).hexdigest() |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 129 | HA2_str = "%(method)s:%(uri)s" % final_dict |
Guido van Rossum | 8136014 | 2007-08-29 14:26:52 +0000 | [diff] [blame] | 130 | HA2 = hashlib.md5(HA2_str.encode("ascii")).hexdigest() |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 131 | final_dict["HA1"] = HA1 |
| 132 | final_dict["HA2"] = HA2 |
| 133 | response_str = "%(HA1)s:%(nonce)s:%(nc)s:" \ |
| 134 | "%(cnonce)s:%(qop)s:%(HA2)s" % final_dict |
Guido van Rossum | 8136014 | 2007-08-29 14:26:52 +0000 | [diff] [blame] | 135 | response = hashlib.md5(response_str.encode("ascii")).hexdigest() |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 136 | |
| 137 | return response == auth_dict["response"] |
| 138 | |
| 139 | def _return_auth_challenge(self, request_handler): |
| 140 | request_handler.send_response(407, "Proxy Authentication Required") |
| 141 | request_handler.send_header("Content-Type", "text/html") |
| 142 | request_handler.send_header( |
| 143 | 'Proxy-Authenticate', 'Digest realm="%s", ' |
| 144 | 'qop="%s",' |
| 145 | 'nonce="%s", ' % \ |
| 146 | (self._realm_name, self._qop, self._generate_nonce())) |
| 147 | # XXX: Not sure if we're supposed to add this next header or |
| 148 | # not. |
| 149 | #request_handler.send_header('Connection', 'close') |
| 150 | request_handler.end_headers() |
Guido van Rossum | 8a392d7 | 2007-11-21 22:09:45 +0000 | [diff] [blame] | 151 | request_handler.wfile.write(b"Proxy Authentication Required.") |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 152 | return False |
| 153 | |
| 154 | def handle_request(self, request_handler): |
| 155 | """Performs digest authentication on the given HTTP request |
| 156 | handler. Returns True if authentication was successful, False |
| 157 | otherwise. |
| 158 | |
| 159 | If no users have been set, then digest auth is effectively |
| 160 | disabled and this method will always return True. |
| 161 | """ |
| 162 | |
| 163 | if len(self._users) == 0: |
| 164 | return True |
| 165 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 166 | if "Proxy-Authorization" not in request_handler.headers: |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 167 | return self._return_auth_challenge(request_handler) |
| 168 | else: |
| 169 | auth_dict = self._create_auth_dict( |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 170 | request_handler.headers["Proxy-Authorization"] |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 171 | ) |
| 172 | if auth_dict["username"] in self._users: |
| 173 | password = self._users[ auth_dict["username"] ] |
| 174 | else: |
| 175 | return self._return_auth_challenge(request_handler) |
| 176 | if not auth_dict.get("nonce") in self._nonces: |
| 177 | return self._return_auth_challenge(request_handler) |
| 178 | else: |
| 179 | self._nonces.remove(auth_dict["nonce"]) |
| 180 | |
| 181 | auth_validated = False |
| 182 | |
| 183 | # MSIE uses short_path in its validation, but Python's |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 184 | # urllib.request uses the full path, so we're going to see if |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 185 | # either of them works here. |
| 186 | |
| 187 | for path in [request_handler.path, request_handler.short_path]: |
| 188 | if self._validate_auth(auth_dict, |
| 189 | password, |
| 190 | request_handler.command, |
| 191 | path): |
| 192 | auth_validated = True |
| 193 | |
| 194 | if not auth_validated: |
| 195 | return self._return_auth_challenge(request_handler) |
| 196 | return True |
| 197 | |
| 198 | # Proxy test infrastructure |
| 199 | |
Georg Brandl | 2442015 | 2008-05-26 16:32:26 +0000 | [diff] [blame] | 200 | class FakeProxyHandler(http.server.BaseHTTPRequestHandler): |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 201 | """This is a 'fake proxy' that makes it look like the entire |
| 202 | internet has gone down due to a sudden zombie invasion. It main |
| 203 | utility is in providing us with authentication support for |
| 204 | testing. |
| 205 | """ |
| 206 | |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 207 | def __init__(self, digest_auth_handler, *args, **kwargs): |
| 208 | # This has to be set before calling our parent's __init__(), which will |
| 209 | # try to call do_GET(). |
| 210 | self.digest_auth_handler = digest_auth_handler |
| 211 | http.server.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 212 | |
| 213 | def log_message(self, format, *args): |
| 214 | # Uncomment the next line for debugging. |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 215 | # sys.stderr.write(format % args) |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 216 | pass |
| 217 | |
| 218 | def do_GET(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 219 | (scm, netloc, path, params, query, fragment) = urllib.parse.urlparse( |
| 220 | self.path, "http") |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 221 | self.short_path = path |
| 222 | if self.digest_auth_handler.handle_request(self): |
| 223 | self.send_response(200, "OK") |
| 224 | self.send_header("Content-Type", "text/html") |
| 225 | self.end_headers() |
Guido van Rossum | 8a392d7 | 2007-11-21 22:09:45 +0000 | [diff] [blame] | 226 | self.wfile.write(bytes("You've reached %s!<BR>" % self.path, |
| 227 | "ascii")) |
| 228 | self.wfile.write(b"Our apologies, but our server is down due to " |
| 229 | b"a sudden zombie invasion.") |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 230 | |
| 231 | # Test cases |
| 232 | |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 233 | class ProxyAuthTests(unittest.TestCase): |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 234 | URL = "http://localhost" |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 235 | |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 236 | USER = "tester" |
| 237 | PASSWD = "test123" |
| 238 | REALM = "TestRealm" |
| 239 | |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 240 | def setUp(self): |
Florent Xicluna | 9b86b9a | 2010-03-19 19:00:44 +0000 | [diff] [blame] | 241 | super(ProxyAuthTests, self).setUp() |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 242 | self.digest_auth_handler = DigestAuthHandler() |
| 243 | self.digest_auth_handler.set_users({self.USER: self.PASSWD}) |
| 244 | self.digest_auth_handler.set_realm(self.REALM) |
| 245 | def create_fake_proxy_handler(*args, **kwargs): |
| 246 | return FakeProxyHandler(self.digest_auth_handler, *args, **kwargs) |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 247 | |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 248 | self.server = LoopbackHttpServerThread(create_fake_proxy_handler) |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 249 | self.server.start() |
| 250 | self.server.ready.wait() |
Guido van Rossum | 806c246 | 2007-08-06 23:33:07 +0000 | [diff] [blame] | 251 | proxy_url = "http://127.0.0.1:%d" % self.server.port |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 252 | handler = urllib.request.ProxyHandler({"http" : proxy_url}) |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 253 | self.proxy_digest_handler = urllib.request.ProxyDigestAuthHandler() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 254 | self.opener = urllib.request.build_opener( |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 255 | handler, self.proxy_digest_handler) |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 256 | |
| 257 | def tearDown(self): |
| 258 | self.server.stop() |
Florent Xicluna | 9b86b9a | 2010-03-19 19:00:44 +0000 | [diff] [blame] | 259 | super(ProxyAuthTests, self).tearDown() |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 260 | |
| 261 | def test_proxy_with_bad_password_raises_httperror(self): |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 262 | self.proxy_digest_handler.add_password(self.REALM, self.URL, |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 263 | self.USER, self.PASSWD+"bad") |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 264 | self.digest_auth_handler.set_qop("auth") |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 265 | self.assertRaises(urllib.error.HTTPError, |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 266 | self.opener.open, |
| 267 | self.URL) |
| 268 | |
| 269 | def test_proxy_with_no_password_raises_httperror(self): |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 270 | self.digest_auth_handler.set_qop("auth") |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 271 | self.assertRaises(urllib.error.HTTPError, |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 272 | self.opener.open, |
| 273 | self.URL) |
| 274 | |
| 275 | def test_proxy_qop_auth_works(self): |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 276 | self.proxy_digest_handler.add_password(self.REALM, self.URL, |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 277 | self.USER, self.PASSWD) |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 278 | self.digest_auth_handler.set_qop("auth") |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 279 | result = self.opener.open(self.URL) |
| 280 | while result.read(): |
| 281 | pass |
| 282 | result.close() |
| 283 | |
| 284 | def test_proxy_qop_auth_int_works_or_throws_urlerror(self): |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 285 | self.proxy_digest_handler.add_password(self.REALM, self.URL, |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 286 | self.USER, self.PASSWD) |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 287 | self.digest_auth_handler.set_qop("auth-int") |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 288 | try: |
| 289 | result = self.opener.open(self.URL) |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 290 | except urllib.error.URLError: |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 291 | # It's okay if we don't support auth-int, but we certainly |
| 292 | # shouldn't receive any kind of exception here other than |
| 293 | # a URLError. |
| 294 | result = None |
| 295 | if result: |
| 296 | while result.read(): |
| 297 | pass |
| 298 | result.close() |
| 299 | |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 300 | |
| 301 | def GetRequestHandler(responses): |
| 302 | |
Georg Brandl | 2442015 | 2008-05-26 16:32:26 +0000 | [diff] [blame] | 303 | class FakeHTTPRequestHandler(http.server.BaseHTTPRequestHandler): |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 304 | |
| 305 | server_version = "TestHTTP/" |
| 306 | requests = [] |
| 307 | headers_received = [] |
| 308 | port = 80 |
| 309 | |
| 310 | def do_GET(self): |
| 311 | body = self.send_head() |
Florent Xicluna | 37d3d9a | 2010-08-08 16:25:27 +0000 | [diff] [blame] | 312 | while body: |
| 313 | done = self.wfile.write(body) |
| 314 | body = body[done:] |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 315 | |
| 316 | def do_POST(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 317 | content_length = self.headers["Content-Length"] |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 318 | post_data = self.rfile.read(int(content_length)) |
| 319 | self.do_GET() |
| 320 | self.requests.append(post_data) |
| 321 | |
| 322 | def send_head(self): |
| 323 | FakeHTTPRequestHandler.headers_received = self.headers |
| 324 | self.requests.append(self.path) |
| 325 | response_code, headers, body = responses.pop(0) |
| 326 | |
| 327 | self.send_response(response_code) |
| 328 | |
| 329 | for (header, value) in headers: |
Antoine Pitrou | b353c12 | 2009-02-11 00:39:14 +0000 | [diff] [blame] | 330 | self.send_header(header, value % {'port':self.port}) |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 331 | if body: |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 332 | self.send_header("Content-type", "text/plain") |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 333 | self.end_headers() |
| 334 | return body |
| 335 | self.end_headers() |
| 336 | |
| 337 | def log_message(self, *args): |
| 338 | pass |
| 339 | |
| 340 | |
| 341 | return FakeHTTPRequestHandler |
| 342 | |
| 343 | |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 344 | class TestUrlopen(unittest.TestCase): |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 345 | """Tests urllib.request.urlopen using the network. |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 346 | |
| 347 | These tests are not exhaustive. Assuming that testing using files does a |
| 348 | good job overall of some of the basic interface features. There are no |
| 349 | tests exercising the optional 'data' and 'proxies' arguments. No tests |
| 350 | for transparent redirection have been written. |
| 351 | """ |
| 352 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 353 | def setUp(self): |
Florent Xicluna | 9b86b9a | 2010-03-19 19:00:44 +0000 | [diff] [blame] | 354 | super(TestUrlopen, self).setUp() |
Senthil Kumaran | 303eb47 | 2012-12-26 01:45:58 -0800 | [diff] [blame] | 355 | # Ignore proxies for localhost tests. |
| 356 | os.environ['NO_PROXY'] = '*' |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 357 | self.server = None |
| 358 | |
| 359 | def tearDown(self): |
| 360 | if self.server is not None: |
| 361 | self.server.stop() |
Florent Xicluna | 9b86b9a | 2010-03-19 19:00:44 +0000 | [diff] [blame] | 362 | super(TestUrlopen, self).tearDown() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 363 | |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 364 | def urlopen(self, url, data=None, **kwargs): |
Antoine Pitrou | b353c12 | 2009-02-11 00:39:14 +0000 | [diff] [blame] | 365 | l = [] |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 366 | f = urllib.request.urlopen(url, data, **kwargs) |
Antoine Pitrou | b353c12 | 2009-02-11 00:39:14 +0000 | [diff] [blame] | 367 | try: |
| 368 | # Exercise various methods |
| 369 | l.extend(f.readlines(200)) |
| 370 | l.append(f.readline()) |
| 371 | l.append(f.read(1024)) |
| 372 | l.append(f.read()) |
| 373 | finally: |
| 374 | f.close() |
| 375 | return b"".join(l) |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 376 | |
| 377 | def start_server(self, responses=None): |
| 378 | if responses is None: |
| 379 | responses = [(200, [], b"we don't care")] |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 380 | handler = GetRequestHandler(responses) |
| 381 | |
| 382 | self.server = LoopbackHttpServerThread(handler) |
| 383 | self.server.start() |
| 384 | self.server.ready.wait() |
| 385 | port = self.server.port |
| 386 | handler.port = port |
| 387 | return handler |
| 388 | |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 389 | def start_https_server(self, responses=None, certfile=CERT_localhost): |
| 390 | if not hasattr(urllib.request, 'HTTPSHandler'): |
| 391 | self.skipTest('ssl support required') |
| 392 | from test.ssl_servers import make_https_server |
| 393 | if responses is None: |
| 394 | responses = [(200, [], b"we care a bit")] |
| 395 | handler = GetRequestHandler(responses) |
| 396 | server = make_https_server(self, certfile=certfile, handler_class=handler) |
| 397 | handler.port = server.port |
| 398 | return handler |
| 399 | |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 400 | def test_redirection(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 401 | expected_response = b"We got here..." |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 402 | responses = [ |
Antoine Pitrou | b353c12 | 2009-02-11 00:39:14 +0000 | [diff] [blame] | 403 | (302, [("Location", "http://localhost:%(port)s/somewhere_else")], |
| 404 | ""), |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 405 | (200, [], expected_response) |
| 406 | ] |
| 407 | |
| 408 | handler = self.start_server(responses) |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 409 | data = self.urlopen("http://localhost:%s/" % handler.port) |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 410 | self.assertEqual(data, expected_response) |
| 411 | self.assertEqual(handler.requests, ["/", "/somewhere_else"]) |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 412 | |
Antoine Pitrou | b353c12 | 2009-02-11 00:39:14 +0000 | [diff] [blame] | 413 | def test_chunked(self): |
| 414 | expected_response = b"hello world" |
| 415 | chunked_start = ( |
| 416 | b'a\r\n' |
| 417 | b'hello worl\r\n' |
| 418 | b'1\r\n' |
| 419 | b'd\r\n' |
| 420 | b'0\r\n' |
| 421 | ) |
| 422 | response = [(200, [("Transfer-Encoding", "chunked")], chunked_start)] |
| 423 | handler = self.start_server(response) |
| 424 | data = self.urlopen("http://localhost:%s/" % handler.port) |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 425 | self.assertEqual(data, expected_response) |
Antoine Pitrou | b353c12 | 2009-02-11 00:39:14 +0000 | [diff] [blame] | 426 | |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 427 | def test_404(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 428 | expected_response = b"Bad bad bad..." |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 429 | handler = self.start_server([(404, [], expected_response)]) |
| 430 | |
| 431 | try: |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 432 | self.urlopen("http://localhost:%s/weeble" % handler.port) |
| 433 | except urllib.error.URLError as f: |
| 434 | data = f.read() |
| 435 | f.close() |
| 436 | else: |
| 437 | self.fail("404 should raise URLError") |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 438 | |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 439 | self.assertEqual(data, expected_response) |
| 440 | self.assertEqual(handler.requests, ["/weeble"]) |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 441 | |
| 442 | def test_200(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 443 | expected_response = b"pycon 2008..." |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 444 | handler = self.start_server([(200, [], expected_response)]) |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 445 | data = self.urlopen("http://localhost:%s/bizarre" % handler.port) |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 446 | self.assertEqual(data, expected_response) |
| 447 | self.assertEqual(handler.requests, ["/bizarre"]) |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 448 | |
| 449 | def test_200_with_parameters(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 450 | expected_response = b"pycon 2008..." |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 451 | handler = self.start_server([(200, [], expected_response)]) |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 452 | data = self.urlopen("http://localhost:%s/bizarre" % handler.port, |
| 453 | b"get=with_feeling") |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 454 | self.assertEqual(data, expected_response) |
| 455 | self.assertEqual(handler.requests, ["/bizarre", b"get=with_feeling"]) |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 456 | |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 457 | def test_https(self): |
| 458 | handler = self.start_https_server() |
| 459 | data = self.urlopen("https://localhost:%s/bizarre" % handler.port) |
| 460 | self.assertEqual(data, b"we care a bit") |
| 461 | |
| 462 | def test_https_with_cafile(self): |
| 463 | handler = self.start_https_server(certfile=CERT_localhost) |
| 464 | import ssl |
| 465 | # Good cert |
| 466 | data = self.urlopen("https://localhost:%s/bizarre" % handler.port, |
| 467 | cafile=CERT_localhost) |
| 468 | self.assertEqual(data, b"we care a bit") |
| 469 | # Bad cert |
| 470 | with self.assertRaises(urllib.error.URLError) as cm: |
| 471 | self.urlopen("https://localhost:%s/bizarre" % handler.port, |
| 472 | cafile=CERT_fakehostname) |
| 473 | # Good cert, but mismatching hostname |
| 474 | handler = self.start_https_server(certfile=CERT_fakehostname) |
| 475 | with self.assertRaises(ssl.CertificateError) as cm: |
| 476 | self.urlopen("https://localhost:%s/bizarre" % handler.port, |
| 477 | cafile=CERT_fakehostname) |
| 478 | |
Antoine Pitrou | de9ac6c | 2012-05-16 21:40:01 +0200 | [diff] [blame] | 479 | def test_https_with_cadefault(self): |
| 480 | handler = self.start_https_server(certfile=CERT_localhost) |
| 481 | # Self-signed cert should fail verification with system certificate store |
| 482 | with self.assertRaises(urllib.error.URLError) as cm: |
| 483 | self.urlopen("https://localhost:%s/bizarre" % handler.port, |
| 484 | cadefault=True) |
| 485 | |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 486 | def test_sending_headers(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 487 | handler = self.start_server() |
| 488 | req = urllib.request.Request("http://localhost:%s/" % handler.port, |
| 489 | headers={"Range": "bytes=20-39"}) |
| 490 | urllib.request.urlopen(req) |
| 491 | self.assertEqual(handler.headers_received["Range"], "bytes=20-39") |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 492 | |
| 493 | def test_basic(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 494 | handler = self.start_server() |
| 495 | open_url = urllib.request.urlopen("http://localhost:%s" % handler.port) |
| 496 | for attr in ("read", "close", "info", "geturl"): |
Benjamin Peterson | c9c0f20 | 2009-06-30 23:06:06 +0000 | [diff] [blame] | 497 | self.assertTrue(hasattr(open_url, attr), "object returned from " |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 498 | "urlopen lacks the %s attribute" % attr) |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 499 | try: |
Benjamin Peterson | c9c0f20 | 2009-06-30 23:06:06 +0000 | [diff] [blame] | 500 | self.assertTrue(open_url.read(), "calling 'read' failed") |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 501 | finally: |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 502 | open_url.close() |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 503 | |
| 504 | def test_info(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 505 | handler = self.start_server() |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 506 | try: |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 507 | open_url = urllib.request.urlopen( |
| 508 | "http://localhost:%s" % handler.port) |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 509 | info_obj = open_url.info() |
Ezio Melotti | e961593 | 2010-01-24 19:26:24 +0000 | [diff] [blame] | 510 | self.assertIsInstance(info_obj, email.message.Message, |
| 511 | "object returned by 'info' is not an " |
| 512 | "instance of email.message.Message") |
Barry Warsaw | 820c120 | 2008-06-12 04:06:45 +0000 | [diff] [blame] | 513 | self.assertEqual(info_obj.get_content_subtype(), "plain") |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 514 | finally: |
| 515 | self.server.stop() |
| 516 | |
| 517 | def test_geturl(self): |
| 518 | # Make sure same URL as opened is returned by geturl. |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 519 | handler = self.start_server() |
| 520 | open_url = urllib.request.urlopen("http://localhost:%s" % handler.port) |
| 521 | url = open_url.geturl() |
| 522 | self.assertEqual(url, "http://localhost:%s" % handler.port) |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 523 | |
| 524 | def test_bad_address(self): |
| 525 | # Make sure proper exception is raised when connecting to a bogus |
| 526 | # address. |
| 527 | self.assertRaises(IOError, |
R. David Murray | 8da3cac | 2009-09-29 14:01:08 +0000 | [diff] [blame] | 528 | # Given that both VeriSign and various ISPs have in |
| 529 | # the past or are presently hijacking various invalid |
| 530 | # domain name requests in an attempt to boost traffic |
| 531 | # to their own sites, finding a domain name to use |
| 532 | # for this test is difficult. RFC2606 leads one to |
| 533 | # believe that '.invalid' should work, but experience |
| 534 | # seemed to indicate otherwise. Single character |
| 535 | # TLDs are likely to remain invalid, so this seems to |
| 536 | # be the best choice. The trailing '.' prevents a |
| 537 | # related problem: The normal DNS resolver appends |
| 538 | # the domain names from the search path if there is |
| 539 | # no '.' the end and, and if one of those domains |
| 540 | # implements a '*' rule a result is returned. |
| 541 | # However, none of this will prevent the test from |
| 542 | # failing if the ISP hijacks all invalid domain |
| 543 | # requests. The real solution would be to be able to |
| 544 | # parameterize the framework with a mock resolver. |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 545 | urllib.request.urlopen, |
R. David Murray | 8da3cac | 2009-09-29 14:01:08 +0000 | [diff] [blame] | 546 | "http://sadflkjsasf.i.nvali.d./") |
Christian Heimes | bbe741d | 2008-03-28 10:53:29 +0000 | [diff] [blame] | 547 | |
Florent Xicluna | 37d3d9a | 2010-08-08 16:25:27 +0000 | [diff] [blame] | 548 | def test_iteration(self): |
| 549 | expected_response = b"pycon 2008..." |
| 550 | handler = self.start_server([(200, [], expected_response)]) |
| 551 | data = urllib.request.urlopen("http://localhost:%s" % handler.port) |
| 552 | for line in data: |
| 553 | self.assertEqual(line, expected_response) |
| 554 | |
| 555 | def test_line_iteration(self): |
| 556 | lines = [b"We\n", b"got\n", b"here\n", b"verylong " * 8192 + b"\n"] |
| 557 | expected_response = b"".join(lines) |
| 558 | handler = self.start_server([(200, [], expected_response)]) |
| 559 | data = urllib.request.urlopen("http://localhost:%s" % handler.port) |
| 560 | for index, line in enumerate(data): |
| 561 | self.assertEqual(line, lines[index], |
| 562 | "Fetched line number %s doesn't match expected:\n" |
| 563 | " Expected length was %s, got %s" % |
| 564 | (index, len(lines[index]), len(line))) |
| 565 | self.assertEqual(index + 1, len(lines)) |
| 566 | |
Antoine Pitrou | 803e6d6 | 2010-10-13 10:36:15 +0000 | [diff] [blame] | 567 | |
| 568 | @support.reap_threads |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 569 | def test_main(): |
Collin Winter | 9a4414d | 2009-05-18 22:32:26 +0000 | [diff] [blame] | 570 | support.run_unittest(ProxyAuthTests, TestUrlopen) |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 571 | |
| 572 | if __name__ == "__main__": |
| 573 | test_main() |