blob: 0316c3f463603e5326c6bf03a4e8e393d529426a [file] [log] [blame]
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001import unittest
Benjamin Petersonee8712c2008-05-20 21:35:26 +00002from test import support
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00003
Christian Heimes05e8be12008-02-23 18:30:17 +00004import os
Guido van Rossum34d19282007-08-09 01:03:29 +00005import io
Georg Brandlf78e02b2008-06-10 17:40:04 +00006import socket
Senthil Kumaran7bc0d872010-12-19 10:49:52 +00007import array
Senthil Kumaran4de00a22011-05-11 21:17:57 +08008import sys
Victor Stinner9a902432014-03-19 17:31:20 +01009try:
10 import ssl
11except ImportError:
12 ssl = None
Jeremy Hyltone3e61042001-05-09 15:50:25 +000013
Jeremy Hylton1afc1692008-06-18 20:49:58 +000014import urllib.request
Ronald Oussorene72e1612011-03-14 18:15:25 -040015# The proxy bypass method imported below has logic specific to the OSX
16# proxy config data structure but is testable on all platforms.
17from urllib.request import Request, OpenerDirector, _proxy_bypass_macosx_sysconf
Senthil Kumaran83070752013-05-24 09:14:12 -070018from urllib.parse import urlparse
guido@google.coma119df92011-03-29 11:41:02 -070019import urllib.error
Jeremy Hyltone3e61042001-05-09 15:50:25 +000020
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +000021# XXX
22# Request
23# CacheFTPHandler (hard to write)
Thomas Wouters477c8d52006-05-27 19:21:47 +000024# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler
Jeremy Hyltone3e61042001-05-09 15:50:25 +000025
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +000026class TrivialTests(unittest.TestCase):
Senthil Kumaran6c5bd402011-11-01 23:20:31 +080027
28 def test___all__(self):
29 # Verify which names are exposed
30 for module in 'request', 'response', 'parse', 'error', 'robotparser':
31 context = {}
32 exec('from urllib.%s import *' % module, context)
33 del context['__builtins__']
Florent Xicluna3dbb1f12011-11-04 22:15:37 +010034 if module == 'request' and os.name == 'nt':
35 u, p = context.pop('url2pathname'), context.pop('pathname2url')
36 self.assertEqual(u.__module__, 'nturl2path')
37 self.assertEqual(p.__module__, 'nturl2path')
Senthil Kumaran6c5bd402011-11-01 23:20:31 +080038 for k, v in context.items():
39 self.assertEqual(v.__module__, 'urllib.%s' % module,
40 "%r is exposed in 'urllib.%s' but defined in %r" %
41 (k, module, v.__module__))
42
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +000043 def test_trivial(self):
44 # A couple trivial tests
Guido van Rossume2ae77b2001-10-24 20:42:55 +000045
Jeremy Hylton1afc1692008-06-18 20:49:58 +000046 self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url')
Tim Peters861adac2001-07-16 20:49:49 +000047
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +000048 # XXX Name hacking to get this to work on Windows.
Jeremy Hylton1afc1692008-06-18 20:49:58 +000049 fname = os.path.abspath(urllib.request.__file__).replace('\\', '/')
Senthil Kumarand587e302010-01-10 17:45:52 +000050
Senthil Kumarand587e302010-01-10 17:45:52 +000051 if os.name == 'nt':
52 file_url = "file:///%s" % fname
53 else:
54 file_url = "file://%s" % fname
55
Jeremy Hylton1afc1692008-06-18 20:49:58 +000056 f = urllib.request.urlopen(file_url)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +000057
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -070058 f.read()
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +000059 f.close()
Tim Petersf5f32b42005-07-17 23:16:17 +000060
Georg Brandle1b13d22005-08-24 22:20:32 +000061 def test_parse_http_list(self):
Jeremy Hylton1afc1692008-06-18 20:49:58 +000062 tests = [
63 ('a,b,c', ['a', 'b', 'c']),
64 ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']),
65 ('a, b, "c", "d", "e,f", g, h',
66 ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']),
67 ('a="b\\"c", d="e\\,f", g="h\\\\i"',
68 ['a="b"c"', 'd="e,f"', 'g="h\\i"'])]
Georg Brandle1b13d22005-08-24 22:20:32 +000069 for string, list in tests:
Florent Xicluna419e3842010-08-08 16:16:07 +000070 self.assertEqual(urllib.request.parse_http_list(string), list)
Georg Brandle1b13d22005-08-24 22:20:32 +000071
Senthil Kumaran843fae92013-03-19 13:43:42 -070072 def test_URLError_reasonstr(self):
73 err = urllib.error.URLError('reason')
74 self.assertIn(err.reason, str(err))
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +000075
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -070076class RequestHdrsTests(unittest.TestCase):
Thomas Wouters00ee7ba2006-08-21 19:07:27 +000077
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -070078 def test_request_headers_dict(self):
79 """
80 The Request.headers dictionary is not a documented interface. It
81 should stay that way, because the complete set of headers are only
82 accessible through the .get_header(), .has_header(), .header_items()
83 interface. However, .headers pre-dates those methods, and so real code
84 will be using the dictionary.
85
86 The introduction in 2.4 of those methods was a mistake for the same
87 reason: code that previously saw all (urllib2 user)-provided headers in
88 .headers now sees only a subset.
89
90 """
91 url = "http://example.com"
92 self.assertEqual(Request(url,
93 headers={"Spam-eggs": "blah"}
94 ).headers["Spam-eggs"], "blah")
95 self.assertEqual(Request(url,
96 headers={"spam-EggS": "blah"}
97 ).headers["Spam-eggs"], "blah")
98
99 def test_request_headers_methods(self):
100 """
101 Note the case normalization of header names here, to
102 .capitalize()-case. This should be preserved for
103 backwards-compatibility. (In the HTTP case, normalization to
104 .title()-case is done by urllib2 before sending headers to
105 http.client).
106
107 Note that e.g. r.has_header("spam-EggS") is currently False, and
108 r.get_header("spam-EggS") returns None, but that could be changed in
109 future.
110
111 Method r.remove_header should remove items both from r.headers and
112 r.unredirected_hdrs dictionaries
113 """
114 url = "http://example.com"
115 req = Request(url, headers={"Spam-eggs": "blah"})
116 self.assertTrue(req.has_header("Spam-eggs"))
117 self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')])
118
119 req.add_header("Foo-Bar", "baz")
120 self.assertEqual(sorted(req.header_items()),
121 [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')])
122 self.assertFalse(req.has_header("Not-there"))
123 self.assertIsNone(req.get_header("Not-there"))
124 self.assertEqual(req.get_header("Not-there", "default"), "default")
125
126 req.remove_header("Spam-eggs")
127 self.assertFalse(req.has_header("Spam-eggs"))
128
129 req.add_unredirected_header("Unredirected-spam", "Eggs")
130 self.assertTrue(req.has_header("Unredirected-spam"))
131
132 req.remove_header("Unredirected-spam")
133 self.assertFalse(req.has_header("Unredirected-spam"))
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000134
135
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700136 def test_password_manager(self):
137 mgr = urllib.request.HTTPPasswordMgr()
138 add = mgr.add_password
139 find_user_pass = mgr.find_user_password
140 add("Some Realm", "http://example.com/", "joe", "password")
141 add("Some Realm", "http://example.com/ni", "ni", "ni")
142 add("c", "http://example.com/foo", "foo", "ni")
143 add("c", "http://example.com/bar", "bar", "nini")
144 add("b", "http://example.com/", "first", "blah")
145 add("b", "http://example.com/", "second", "spam")
146 add("a", "http://example.com", "1", "a")
147 add("Some Realm", "http://c.example.com:3128", "3", "c")
148 add("Some Realm", "d.example.com", "4", "d")
149 add("Some Realm", "e.example.com:3128", "5", "e")
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000150
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700151 self.assertEqual(find_user_pass("Some Realm", "example.com"),
152 ('joe', 'password'))
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000153
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700154 #self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"),
155 # ('ni', 'ni'))
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000156
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700157 self.assertEqual(find_user_pass("Some Realm", "http://example.com"),
158 ('joe', 'password'))
159 self.assertEqual(find_user_pass("Some Realm", "http://example.com/"),
160 ('joe', 'password'))
161 self.assertEqual(
162 find_user_pass("Some Realm", "http://example.com/spam"),
163 ('joe', 'password'))
164 self.assertEqual(
165 find_user_pass("Some Realm", "http://example.com/spam/spam"),
166 ('joe', 'password'))
167 self.assertEqual(find_user_pass("c", "http://example.com/foo"),
168 ('foo', 'ni'))
169 self.assertEqual(find_user_pass("c", "http://example.com/bar"),
170 ('bar', 'nini'))
171 self.assertEqual(find_user_pass("b", "http://example.com/"),
172 ('second', 'spam'))
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000173
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700174 # No special relationship between a.example.com and example.com:
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000175
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700176 self.assertEqual(find_user_pass("a", "http://example.com/"),
177 ('1', 'a'))
178 self.assertEqual(find_user_pass("a", "http://a.example.com/"),
179 (None, None))
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000180
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700181 # Ports:
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000182
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700183 self.assertEqual(find_user_pass("Some Realm", "c.example.com"),
184 (None, None))
185 self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"),
186 ('3', 'c'))
187 self.assertEqual(
188 find_user_pass("Some Realm", "http://c.example.com:3128"),
189 ('3', 'c'))
190 self.assertEqual(find_user_pass("Some Realm", "d.example.com"),
191 ('4', 'd'))
192 self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"),
193 ('5', 'e'))
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000194
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700195 def test_password_manager_default_port(self):
196 """
197 The point to note here is that we can't guess the default port if
198 there's no scheme. This applies to both add_password and
199 find_user_password.
200 """
201 mgr = urllib.request.HTTPPasswordMgr()
202 add = mgr.add_password
203 find_user_pass = mgr.find_user_password
204 add("f", "http://g.example.com:80", "10", "j")
205 add("g", "http://h.example.com", "11", "k")
206 add("h", "i.example.com:80", "12", "l")
207 add("i", "j.example.com", "13", "m")
208 self.assertEqual(find_user_pass("f", "g.example.com:100"),
209 (None, None))
210 self.assertEqual(find_user_pass("f", "g.example.com:80"),
211 ('10', 'j'))
212 self.assertEqual(find_user_pass("f", "g.example.com"),
213 (None, None))
214 self.assertEqual(find_user_pass("f", "http://g.example.com:100"),
215 (None, None))
216 self.assertEqual(find_user_pass("f", "http://g.example.com:80"),
217 ('10', 'j'))
218 self.assertEqual(find_user_pass("f", "http://g.example.com"),
219 ('10', 'j'))
220 self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k'))
221 self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k'))
222 self.assertEqual(find_user_pass("g", "http://h.example.com:80"),
223 ('11', 'k'))
224 self.assertEqual(find_user_pass("h", "i.example.com"), (None, None))
225 self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l'))
226 self.assertEqual(find_user_pass("h", "http://i.example.com:80"),
227 ('12', 'l'))
228 self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm'))
229 self.assertEqual(find_user_pass("i", "j.example.com:80"),
230 (None, None))
231 self.assertEqual(find_user_pass("i", "http://j.example.com"),
232 ('13', 'm'))
233 self.assertEqual(find_user_pass("i", "http://j.example.com:80"),
234 (None, None))
Andrew Svetlovbff98fe2012-11-27 23:06:19 +0200235
Thomas Wouters4d70c3d2006-06-08 14:42:34 +0000236
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000237class MockOpener:
238 addheaders = []
Senthil Kumaranfb8cc2f2009-07-19 02:44:19 +0000239 def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
240 self.req, self.data, self.timeout = req, data, timeout
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000241 def error(self, proto, *args):
242 self.proto, self.args = proto, args
243
244class MockFile:
245 def read(self, count=None): pass
246 def readline(self, count=None): pass
247 def close(self): pass
248
Martin v. Löwis2a6ba902004-05-31 18:22:40 +0000249class MockHeaders(dict):
250 def getheaders(self, name):
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000251 return list(self.values())
Martin v. Löwis2a6ba902004-05-31 18:22:40 +0000252
Guido van Rossum34d19282007-08-09 01:03:29 +0000253class MockResponse(io.StringIO):
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000254 def __init__(self, code, msg, headers, data, url=None):
Guido van Rossum34d19282007-08-09 01:03:29 +0000255 io.StringIO.__init__(self, data)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000256 self.code, self.msg, self.headers, self.url = code, msg, headers, url
257 def info(self):
258 return self.headers
259 def geturl(self):
260 return self.url
261
Martin v. Löwis2a6ba902004-05-31 18:22:40 +0000262class MockCookieJar:
263 def add_cookie_header(self, request):
264 self.ach_req = request
265 def extract_cookies(self, response, request):
266 self.ec_req, self.ec_r = request, response
267
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000268class FakeMethod:
269 def __init__(self, meth_name, action, handle):
270 self.meth_name = meth_name
271 self.handle = handle
272 self.action = action
273 def __call__(self, *args):
274 return self.handle(self.meth_name, self.action, *args)
275
Senthil Kumaran47fff872009-12-20 07:10:31 +0000276class MockHTTPResponse(io.IOBase):
277 def __init__(self, fp, msg, status, reason):
278 self.fp = fp
279 self.msg = msg
280 self.status = status
281 self.reason = reason
282 self.code = 200
283
284 def read(self):
285 return ''
286
287 def info(self):
288 return {}
289
290 def geturl(self):
291 return self.url
292
293
294class MockHTTPClass:
295 def __init__(self):
296 self.level = 0
297 self.req_headers = []
298 self.data = None
299 self.raise_on_endheaders = False
Nadeem Vawdabd26b542012-10-21 17:37:43 +0200300 self.sock = None
Senthil Kumaran47fff872009-12-20 07:10:31 +0000301 self._tunnel_headers = {}
302
303 def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
304 self.host = host
305 self.timeout = timeout
306 return self
307
308 def set_debuglevel(self, level):
309 self.level = level
310
311 def set_tunnel(self, host, port=None, headers=None):
312 self._tunnel_host = host
313 self._tunnel_port = port
314 if headers:
315 self._tunnel_headers = headers
316 else:
317 self._tunnel_headers.clear()
318
Benjamin Peterson3d5b8db2009-12-24 01:14:05 +0000319 def request(self, method, url, body=None, headers=None):
Senthil Kumaran47fff872009-12-20 07:10:31 +0000320 self.method = method
321 self.selector = url
Benjamin Peterson3d5b8db2009-12-24 01:14:05 +0000322 if headers is not None:
323 self.req_headers += headers.items()
Senthil Kumaran47fff872009-12-20 07:10:31 +0000324 self.req_headers.sort()
325 if body:
326 self.data = body
327 if self.raise_on_endheaders:
Andrew Svetlov0832af62012-12-18 23:10:48 +0200328 raise OSError()
Senthil Kumaran47fff872009-12-20 07:10:31 +0000329 def getresponse(self):
330 return MockHTTPResponse(MockFile(), {}, 200, "OK")
331
Victor Stinnera4c45d72011-06-17 14:01:18 +0200332 def close(self):
333 pass
334
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000335class MockHandler:
Thomas Wouters477c8d52006-05-27 19:21:47 +0000336 # useful for testing handler machinery
337 # see add_ordered_mock_handlers() docstring
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000338 handler_order = 500
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000339 def __init__(self, methods):
340 self._define_methods(methods)
341 def _define_methods(self, methods):
342 for spec in methods:
343 if len(spec) == 2: name, action = spec
344 else: name, action = spec, None
345 meth = FakeMethod(name, action, self.handle)
346 setattr(self.__class__, name, meth)
347 def handle(self, fn_name, action, *args, **kwds):
348 self.parent.calls.append((self, fn_name, args, kwds))
349 if action is None:
350 return None
351 elif action == "return self":
352 return self
353 elif action == "return response":
354 res = MockResponse(200, "OK", {}, "")
355 return res
356 elif action == "return request":
357 return Request("http://blah/")
358 elif action.startswith("error"):
359 code = action[action.rfind(" ")+1:]
360 try:
361 code = int(code)
362 except ValueError:
363 pass
364 res = MockResponse(200, "OK", {}, "")
365 return self.parent.error("http", args[0], res, code, "", {})
366 elif action == "raise":
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000367 raise urllib.error.URLError("blah")
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000368 assert False
369 def close(self): pass
370 def add_parent(self, parent):
371 self.parent = parent
372 self.parent.calls = []
373 def __lt__(self, other):
374 if not hasattr(other, "handler_order"):
375 # No handler_order, leave in original order. Yuck.
376 return True
377 return self.handler_order < other.handler_order
378
379def add_ordered_mock_handlers(opener, meth_spec):
380 """Create MockHandlers and add them to an OpenerDirector.
381
382 meth_spec: list of lists of tuples and strings defining methods to define
383 on handlers. eg:
384
385 [["http_error", "ftp_open"], ["http_open"]]
386
387 defines methods .http_error() and .ftp_open() on one handler, and
388 .http_open() on another. These methods just record their arguments and
389 return None. Using a tuple instead of a string causes the method to
390 perform some action (see MockHandler.handle()), eg:
391
392 [["http_error"], [("http_open", "return request")]]
393
394 defines .http_error() on one handler (which simply returns None), and
395 .http_open() on another handler, which returns a Request object.
396
397 """
398 handlers = []
399 count = 0
400 for meths in meth_spec:
401 class MockHandlerSubclass(MockHandler): pass
402 h = MockHandlerSubclass(meths)
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000403 h.handler_order += count
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000404 h.add_parent(opener)
405 count = count + 1
406 handlers.append(h)
407 opener.add_handler(h)
408 return handlers
409
Thomas Wouters477c8d52006-05-27 19:21:47 +0000410def build_test_opener(*handler_instances):
411 opener = OpenerDirector()
412 for h in handler_instances:
413 opener.add_handler(h)
414 return opener
415
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000416class MockHTTPHandler(urllib.request.BaseHandler):
Thomas Wouters477c8d52006-05-27 19:21:47 +0000417 # useful for testing redirections and auth
418 # sends supplied headers and code as first response
419 # sends 200 OK as second response
420 def __init__(self, code, headers):
421 self.code = code
422 self.headers = headers
423 self.reset()
424 def reset(self):
425 self._count = 0
426 self.requests = []
427 def http_open(self, req):
Barry Warsaw820c1202008-06-12 04:06:45 +0000428 import email, http.client, copy
Thomas Wouters477c8d52006-05-27 19:21:47 +0000429 self.requests.append(copy.deepcopy(req))
430 if self._count == 0:
431 self._count = self._count + 1
Georg Brandl24420152008-05-26 16:32:26 +0000432 name = http.client.responses[self.code]
Barry Warsaw820c1202008-06-12 04:06:45 +0000433 msg = email.message_from_string(self.headers)
Thomas Wouters477c8d52006-05-27 19:21:47 +0000434 return self.parent.error(
435 "http", req, MockFile(), self.code, name, msg)
436 else:
437 self.req = req
Barry Warsaw820c1202008-06-12 04:06:45 +0000438 msg = email.message_from_string("\r\n\r\n")
Thomas Wouters477c8d52006-05-27 19:21:47 +0000439 return MockResponse(200, "OK", msg, "", req.get_full_url())
440
Senthil Kumaran47fff872009-12-20 07:10:31 +0000441class MockHTTPSHandler(urllib.request.AbstractHTTPHandler):
442 # Useful for testing the Proxy-Authorization request by verifying the
443 # properties of httpcon
Benjamin Peterson3d5b8db2009-12-24 01:14:05 +0000444
445 def __init__(self):
446 urllib.request.AbstractHTTPHandler.__init__(self)
447 self.httpconn = MockHTTPClass()
448
Senthil Kumaran47fff872009-12-20 07:10:31 +0000449 def https_open(self, req):
450 return self.do_open(self.httpconn, req)
451
Thomas Wouters477c8d52006-05-27 19:21:47 +0000452class MockPasswordManager:
453 def add_password(self, realm, uri, user, password):
454 self.realm = realm
455 self.url = uri
456 self.user = user
457 self.password = password
458 def find_user_password(self, realm, authuri):
459 self.target_realm = realm
460 self.target_url = authuri
461 return self.user, self.password
462
463
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000464class OpenerDirectorTests(unittest.TestCase):
465
Guido van Rossumb5a755e2007-07-18 18:15:48 +0000466 def test_add_non_handler(self):
467 class NonHandler(object):
468 pass
469 self.assertRaises(TypeError,
470 OpenerDirector().add_handler, NonHandler())
471
Thomas Wouters4d70c3d2006-06-08 14:42:34 +0000472 def test_badly_named_methods(self):
473 # test work-around for three methods that accidentally follow the
474 # naming conventions for handler methods
475 # (*_open() / *_request() / *_response())
476
477 # These used to call the accidentally-named methods, causing a
478 # TypeError in real code; here, returning self from these mock
479 # methods would either cause no exception, or AttributeError.
480
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000481 from urllib.error import URLError
Thomas Wouters4d70c3d2006-06-08 14:42:34 +0000482
483 o = OpenerDirector()
484 meth_spec = [
485 [("do_open", "return self"), ("proxy_open", "return self")],
486 [("redirect_request", "return self")],
487 ]
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700488 add_ordered_mock_handlers(o, meth_spec)
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000489 o.add_handler(urllib.request.UnknownHandler())
Thomas Wouters4d70c3d2006-06-08 14:42:34 +0000490 for scheme in "do", "proxy", "redirect":
491 self.assertRaises(URLError, o.open, scheme+"://example.com/")
492
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000493 def test_handled(self):
494 # handler returning non-None means no more handlers will be called
495 o = OpenerDirector()
496 meth_spec = [
497 ["http_open", "ftp_open", "http_error_302"],
498 ["ftp_open"],
499 [("http_open", "return self")],
500 [("http_open", "return self")],
501 ]
502 handlers = add_ordered_mock_handlers(o, meth_spec)
503
504 req = Request("http://example.com/")
505 r = o.open(req)
506 # Second .http_open() gets called, third doesn't, since second returned
507 # non-None. Handlers without .http_open() never get any methods called
508 # on them.
509 # In fact, second mock handler defining .http_open() returns self
510 # (instead of response), which becomes the OpenerDirector's return
511 # value.
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000512 self.assertEqual(r, handlers[2])
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000513 calls = [(handlers[0], "http_open"), (handlers[2], "http_open")]
514 for expected, got in zip(calls, o.calls):
515 handler, name, args, kwds = got
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000516 self.assertEqual((handler, name), expected)
517 self.assertEqual(args, (req,))
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000518
519 def test_handler_order(self):
520 o = OpenerDirector()
521 handlers = []
522 for meths, handler_order in [
523 ([("http_open", "return self")], 500),
524 (["http_open"], 0),
525 ]:
526 class MockHandlerSubclass(MockHandler): pass
527 h = MockHandlerSubclass(meths)
528 h.handler_order = handler_order
529 handlers.append(h)
530 o.add_handler(h)
531
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700532 o.open("http://example.com/")
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000533 # handlers called in reverse order, thanks to their sort order
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000534 self.assertEqual(o.calls[0][0], handlers[1])
535 self.assertEqual(o.calls[1][0], handlers[0])
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000536
537 def test_raise(self):
538 # raising URLError stops processing of request
539 o = OpenerDirector()
540 meth_spec = [
541 [("http_open", "raise")],
542 [("http_open", "return self")],
543 ]
544 handlers = add_ordered_mock_handlers(o, meth_spec)
545
546 req = Request("http://example.com/")
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000547 self.assertRaises(urllib.error.URLError, o.open, req)
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000548 self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})])
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000549
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000550 def test_http_error(self):
551 # XXX http_error_default
552 # http errors are a special case
553 o = OpenerDirector()
554 meth_spec = [
555 [("http_open", "error 302")],
556 [("http_error_400", "raise"), "http_open"],
557 [("http_error_302", "return response"), "http_error_303",
558 "http_error"],
559 [("http_error_302")],
560 ]
561 handlers = add_ordered_mock_handlers(o, meth_spec)
562
563 class Unknown:
564 def __eq__(self, other): return True
565
566 req = Request("http://example.com/")
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700567 o.open(req)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000568 assert len(o.calls) == 2
569 calls = [(handlers[0], "http_open", (req,)),
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000570 (handlers[2], "http_error_302",
571 (req, Unknown(), 302, "", {}))]
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000572 for expected, got in zip(calls, o.calls):
573 handler, method_name, args = expected
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000574 self.assertEqual((handler, method_name), got[:2])
575 self.assertEqual(args, got[2])
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000576
Senthil Kumaran38b968b92012-03-14 13:43:53 -0700577
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000578 def test_processors(self):
579 # *_request / *_response methods get called appropriately
580 o = OpenerDirector()
581 meth_spec = [
582 [("http_request", "return request"),
583 ("http_response", "return response")],
584 [("http_request", "return request"),
585 ("http_response", "return response")],
586 ]
587 handlers = add_ordered_mock_handlers(o, meth_spec)
588
589 req = Request("http://example.com/")
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700590 o.open(req)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000591 # processor methods are called on *all* handlers that define them,
592 # not just the first handler that handles the request
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000593 calls = [
594 (handlers[0], "http_request"), (handlers[1], "http_request"),
595 (handlers[0], "http_response"), (handlers[1], "http_response")]
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000596
597 for i, (handler, name, args, kwds) in enumerate(o.calls):
598 if i < 2:
599 # *_request
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000600 self.assertEqual((handler, name), calls[i])
601 self.assertEqual(len(args), 1)
Ezio Melottie9615932010-01-24 19:26:24 +0000602 self.assertIsInstance(args[0], Request)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000603 else:
604 # *_response
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000605 self.assertEqual((handler, name), calls[i])
606 self.assertEqual(len(args), 2)
Ezio Melottie9615932010-01-24 19:26:24 +0000607 self.assertIsInstance(args[0], Request)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000608 # response from opener.open is None, because there's no
609 # handler that defines http_open to handle it
Serhiy Storchaka25d8aea2014-02-08 14:50:08 +0200610 if args[1] is not None:
611 self.assertIsInstance(args[1], MockResponse)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000612
Tim Peters58eb11c2004-01-18 20:29:55 +0000613def sanepathname2url(path):
Victor Stinner6c6f8512010-08-07 10:09:35 +0000614 try:
Marc-André Lemburg8f36af72011-02-25 15:42:01 +0000615 path.encode("utf-8")
Victor Stinner6c6f8512010-08-07 10:09:35 +0000616 except UnicodeEncodeError:
617 raise unittest.SkipTest("path is not encodable to utf8")
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000618 urlpath = urllib.request.pathname2url(path)
Tim Peters58eb11c2004-01-18 20:29:55 +0000619 if os.name == "nt" and urlpath.startswith("///"):
620 urlpath = urlpath[2:]
621 # XXX don't ask me about the mac...
622 return urlpath
623
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000624class HandlerTests(unittest.TestCase):
625
626 def test_ftp(self):
627 class MockFTPWrapper:
628 def __init__(self, data): self.data = data
629 def retrfile(self, filename, filetype):
630 self.filename, self.filetype = filename, filetype
Guido van Rossum34d19282007-08-09 01:03:29 +0000631 return io.StringIO(self.data), len(self.data)
Nadeem Vawda08f5f7a2011-07-23 14:03:00 +0200632 def close(self): pass
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000633
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000634 class NullFTPHandler(urllib.request.FTPHandler):
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000635 def __init__(self, data): self.data = data
Georg Brandlf78e02b2008-06-10 17:40:04 +0000636 def connect_ftp(self, user, passwd, host, port, dirs,
637 timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000638 self.user, self.passwd = user, passwd
639 self.host, self.port = host, port
640 self.dirs = dirs
641 self.ftpwrapper = MockFTPWrapper(self.data)
642 return self.ftpwrapper
643
Georg Brandlf78e02b2008-06-10 17:40:04 +0000644 import ftplib
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000645 data = "rheum rhaponicum"
646 h = NullFTPHandler(data)
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700647 h.parent = MockOpener()
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000648
Senthil Kumarandaa29d02010-11-18 15:36:41 +0000649 for url, host, port, user, passwd, type_, dirs, filename, mimetype in [
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000650 ("ftp://localhost/foo/bar/baz.html",
Senthil Kumarandaa29d02010-11-18 15:36:41 +0000651 "localhost", ftplib.FTP_PORT, "", "", "I",
652 ["foo", "bar"], "baz.html", "text/html"),
653 ("ftp://parrot@localhost/foo/bar/baz.html",
654 "localhost", ftplib.FTP_PORT, "parrot", "", "I",
655 ["foo", "bar"], "baz.html", "text/html"),
656 ("ftp://%25parrot@localhost/foo/bar/baz.html",
657 "localhost", ftplib.FTP_PORT, "%parrot", "", "I",
658 ["foo", "bar"], "baz.html", "text/html"),
659 ("ftp://%2542parrot@localhost/foo/bar/baz.html",
660 "localhost", ftplib.FTP_PORT, "%42parrot", "", "I",
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000661 ["foo", "bar"], "baz.html", "text/html"),
Kurt B. Kaiser3f7cb5d2004-07-11 17:14:13 +0000662 ("ftp://localhost:80/foo/bar/",
Senthil Kumarandaa29d02010-11-18 15:36:41 +0000663 "localhost", 80, "", "", "D",
Kurt B. Kaiser3f7cb5d2004-07-11 17:14:13 +0000664 ["foo", "bar"], "", None),
665 ("ftp://localhost/baz.gif;type=a",
Senthil Kumarandaa29d02010-11-18 15:36:41 +0000666 "localhost", ftplib.FTP_PORT, "", "", "A",
Kurt B. Kaiser3f7cb5d2004-07-11 17:14:13 +0000667 [], "baz.gif", None), # XXX really this should guess image/gif
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000668 ]:
Guido van Rossumcd16bf62007-06-13 18:07:49 +0000669 req = Request(url)
670 req.timeout = None
671 r = h.ftp_open(req)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000672 # ftp authentication not yet implemented by FTPHandler
Senthil Kumarandaa29d02010-11-18 15:36:41 +0000673 self.assertEqual(h.user, user)
674 self.assertEqual(h.passwd, passwd)
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000675 self.assertEqual(h.host, socket.gethostbyname(host))
676 self.assertEqual(h.port, port)
677 self.assertEqual(h.dirs, dirs)
678 self.assertEqual(h.ftpwrapper.filename, filename)
679 self.assertEqual(h.ftpwrapper.filetype, type_)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000680 headers = r.info()
Kurt B. Kaiser3f7cb5d2004-07-11 17:14:13 +0000681 self.assertEqual(headers.get("Content-type"), mimetype)
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000682 self.assertEqual(int(headers["Content-length"]), len(data))
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000683
684 def test_file(self):
Benjamin Petersona0c0a4a2008-06-12 22:15:50 +0000685 import email.utils, socket
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000686 h = urllib.request.FileHandler()
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000687 o = h.parent = MockOpener()
688
Benjamin Petersonee8712c2008-05-20 21:35:26 +0000689 TESTFN = support.TESTFN
Tim Peters58eb11c2004-01-18 20:29:55 +0000690 urlpath = sanepathname2url(os.path.abspath(TESTFN))
Guido van Rossum6a2ccd02007-07-16 20:51:57 +0000691 towrite = b"hello, world\n"
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000692 urls = [
Tim Peters58eb11c2004-01-18 20:29:55 +0000693 "file://localhost%s" % urlpath,
694 "file://%s" % urlpath,
695 "file://%s%s" % (socket.gethostbyname('localhost'), urlpath),
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000696 ]
697 try:
698 localaddr = socket.gethostbyname(socket.gethostname())
699 except socket.gaierror:
700 localaddr = ''
701 if localaddr:
702 urls.append("file://%s%s" % (localaddr, urlpath))
703
704 for url in urls:
Tim Peters58eb11c2004-01-18 20:29:55 +0000705 f = open(TESTFN, "wb")
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000706 try:
707 try:
708 f.write(towrite)
709 finally:
710 f.close()
711
712 r = h.file_open(Request(url))
713 try:
714 data = r.read()
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000715 headers = r.info()
Senthil Kumaran4fbed102010-05-08 03:29:09 +0000716 respurl = r.geturl()
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000717 finally:
718 r.close()
Tim Peters58eb11c2004-01-18 20:29:55 +0000719 stats = os.stat(TESTFN)
Benjamin Petersona0c0a4a2008-06-12 22:15:50 +0000720 modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000721 finally:
722 os.remove(TESTFN)
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000723 self.assertEqual(data, towrite)
724 self.assertEqual(headers["Content-type"], "text/plain")
725 self.assertEqual(headers["Content-length"], "13")
Tim Peters58eb11c2004-01-18 20:29:55 +0000726 self.assertEqual(headers["Last-modified"], modified)
Senthil Kumaran4fbed102010-05-08 03:29:09 +0000727 self.assertEqual(respurl, url)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000728
729 for url in [
Tim Peters58eb11c2004-01-18 20:29:55 +0000730 "file://localhost:80%s" % urlpath,
Guido van Rossumd8faa362007-04-27 19:54:29 +0000731 "file:///file_does_not_exist.txt",
732 "file://%s:80%s/%s" % (socket.gethostbyname('localhost'),
733 os.getcwd(), TESTFN),
734 "file://somerandomhost.ontheinternet.com%s/%s" %
735 (os.getcwd(), TESTFN),
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000736 ]:
737 try:
Tim Peters58eb11c2004-01-18 20:29:55 +0000738 f = open(TESTFN, "wb")
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000739 try:
740 f.write(towrite)
741 finally:
742 f.close()
743
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000744 self.assertRaises(urllib.error.URLError,
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000745 h.file_open, Request(url))
746 finally:
747 os.remove(TESTFN)
748
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000749 h = urllib.request.FileHandler()
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000750 o = h.parent = MockOpener()
751 # XXXX why does // mean ftp (and /// mean not ftp!), and where
752 # is file: scheme specified? I think this is really a bug, and
753 # what was intended was to distinguish between URLs like:
754 # file:/blah.txt (a file)
755 # file://localhost/blah.txt (a file)
756 # file:///blah.txt (a file)
757 # file://ftp.example.com/blah.txt (an ftp URL)
758 for url, ftp in [
Senthil Kumaran383c32d2010-10-14 11:57:35 +0000759 ("file://ftp.example.com//foo.txt", False),
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000760 ("file://ftp.example.com///foo.txt", False),
761# XXXX bug: fails with OSError, should be URLError
762 ("file://ftp.example.com/foo.txt", False),
Senthil Kumaran383c32d2010-10-14 11:57:35 +0000763 ("file://somehost//foo/something.txt", False),
Senthil Kumaran2ef16322010-07-11 03:12:43 +0000764 ("file://localhost//foo/something.txt", False),
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000765 ]:
766 req = Request(url)
767 try:
768 h.file_open(req)
769 # XXXX remove OSError when bug fixed
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000770 except (urllib.error.URLError, OSError):
Florent Xicluna419e3842010-08-08 16:16:07 +0000771 self.assertFalse(ftp)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000772 else:
Florent Xicluna419e3842010-08-08 16:16:07 +0000773 self.assertIs(o.req, req)
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000774 self.assertEqual(req.type, "ftp")
Łukasz Langad7e81cc2011-01-09 18:18:53 +0000775 self.assertEqual(req.type == "ftp", ftp)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000776
777 def test_http(self):
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000778
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000779 h = urllib.request.AbstractHTTPHandler()
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000780 o = h.parent = MockOpener()
781
782 url = "http://example.com/"
Senthil Kumaran7bc0d872010-12-19 10:49:52 +0000783 for method, data in [("GET", None), ("POST", b"blah")]:
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000784 req = Request(url, data, {"Foo": "bar"})
Guido van Rossumcd16bf62007-06-13 18:07:49 +0000785 req.timeout = None
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000786 req.add_unredirected_header("Spam", "eggs")
787 http = MockHTTPClass()
788 r = h.do_open(http, req)
789
790 # result attributes
791 r.read; r.readline # wrapped MockFile methods
792 r.info; r.geturl # addinfourl methods
793 r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply()
794 hdrs = r.info()
Guido van Rossume2b70bc2006-08-18 22:13:04 +0000795 hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply()
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000796 self.assertEqual(r.geturl(), url)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000797
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000798 self.assertEqual(http.host, "example.com")
799 self.assertEqual(http.level, 0)
800 self.assertEqual(http.method, method)
801 self.assertEqual(http.selector, "/")
802 self.assertEqual(http.req_headers,
Jeremy Hyltonb3ee6f92004-02-24 19:40:35 +0000803 [("Connection", "close"),
804 ("Foo", "bar"), ("Spam", "eggs")])
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000805 self.assertEqual(http.data, data)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000806
Andrew Svetlov0832af62012-12-18 23:10:48 +0200807 # check OSError converted to URLError
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000808 http.raise_on_endheaders = True
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000809 self.assertRaises(urllib.error.URLError, h.do_open, http, req)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000810
Senthil Kumaran29333122011-02-11 11:25:47 +0000811 # Check for TypeError on POST data which is str.
812 req = Request("http://example.com/","badpost")
813 self.assertRaises(TypeError, h.do_request_, req)
814
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000815 # check adding of standard headers
816 o.addheaders = [("Spam", "eggs")]
Senthil Kumaran7bc0d872010-12-19 10:49:52 +0000817 for data in b"", None: # POST, GET
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000818 req = Request("http://example.com/", data)
819 r = MockResponse(200, "OK", {}, "")
Martin v. Löwis2a6ba902004-05-31 18:22:40 +0000820 newreq = h.do_request_(req)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000821 if data is None: # GET
Benjamin Peterson577473f2010-01-19 00:09:57 +0000822 self.assertNotIn("Content-length", req.unredirected_hdrs)
823 self.assertNotIn("Content-type", req.unredirected_hdrs)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000824 else: # POST
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000825 self.assertEqual(req.unredirected_hdrs["Content-length"], "0")
826 self.assertEqual(req.unredirected_hdrs["Content-type"],
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000827 "application/x-www-form-urlencoded")
828 # XXX the details of Host could be better tested
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000829 self.assertEqual(req.unredirected_hdrs["Host"], "example.com")
830 self.assertEqual(req.unredirected_hdrs["Spam"], "eggs")
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000831
832 # don't clobber existing headers
833 req.add_unredirected_header("Content-length", "foo")
834 req.add_unredirected_header("Content-type", "bar")
835 req.add_unredirected_header("Host", "baz")
836 req.add_unredirected_header("Spam", "foo")
Martin v. Löwis2a6ba902004-05-31 18:22:40 +0000837 newreq = h.do_request_(req)
Thomas Wouters00ee7ba2006-08-21 19:07:27 +0000838 self.assertEqual(req.unredirected_hdrs["Content-length"], "foo")
839 self.assertEqual(req.unredirected_hdrs["Content-type"], "bar")
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000840 self.assertEqual(req.unredirected_hdrs["Host"], "baz")
841 self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000842
Senthil Kumaran7bc0d872010-12-19 10:49:52 +0000843 # Check iterable body support
844 def iterable_body():
845 yield b"one"
846 yield b"two"
847 yield b"three"
848
849 for headers in {}, {"Content-Length": 11}:
850 req = Request("http://example.com/", iterable_body(), headers)
851 if not headers:
852 # Having an iterable body without a Content-Length should
853 # raise an exception
854 self.assertRaises(ValueError, h.do_request_, req)
855 else:
856 newreq = h.do_request_(req)
857
Senthil Kumaran29333122011-02-11 11:25:47 +0000858 # A file object.
859 # Test only Content-Length attribute of request.
Senthil Kumaran7bc0d872010-12-19 10:49:52 +0000860
Senthil Kumaran29333122011-02-11 11:25:47 +0000861 file_obj = io.BytesIO()
862 file_obj.write(b"Something\nSomething\nSomething\n")
Senthil Kumaran7bc0d872010-12-19 10:49:52 +0000863
864 for headers in {}, {"Content-Length": 30}:
865 req = Request("http://example.com/", file_obj, headers)
866 if not headers:
867 # Having an iterable body without a Content-Length should
868 # raise an exception
869 self.assertRaises(ValueError, h.do_request_, req)
870 else:
871 newreq = h.do_request_(req)
872 self.assertEqual(int(newreq.get_header('Content-length')),30)
873
874 file_obj.close()
875
876 # array.array Iterable - Content Length is calculated
877
878 iterable_array = array.array("I",[1,2,3,4])
879
880 for headers in {}, {"Content-Length": 16}:
881 req = Request("http://example.com/", iterable_array, headers)
882 newreq = h.do_request_(req)
883 self.assertEqual(int(newreq.get_header('Content-length')),16)
Senthil Kumaran7bc0d872010-12-19 10:49:52 +0000884
Facundo Batista72dc1ea2008-08-16 14:44:32 +0000885 def test_http_doubleslash(self):
886 # Checks the presence of any unnecessary double slash in url does not
887 # break anything. Previously, a double slash directly after the host
Ezio Melottie130a522011-10-19 10:58:56 +0300888 # could cause incorrect parsing.
Facundo Batista72dc1ea2008-08-16 14:44:32 +0000889 h = urllib.request.AbstractHTTPHandler()
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700890 h.parent = MockOpener()
Facundo Batista72dc1ea2008-08-16 14:44:32 +0000891
Senthil Kumaran7bc0d872010-12-19 10:49:52 +0000892 data = b""
Facundo Batista72dc1ea2008-08-16 14:44:32 +0000893 ds_urls = [
894 "http://example.com/foo/bar/baz.html",
895 "http://example.com//foo/bar/baz.html",
896 "http://example.com/foo//bar/baz.html",
897 "http://example.com/foo/bar//baz.html"
898 ]
899
900 for ds_url in ds_urls:
901 ds_req = Request(ds_url, data)
902
903 # Check whether host is determined correctly if there is no proxy
904 np_ds_req = h.do_request_(ds_req)
905 self.assertEqual(np_ds_req.unredirected_hdrs["Host"],"example.com")
906
907 # Check whether host is determined correctly if there is a proxy
908 ds_req.set_proxy("someproxy:3128",None)
909 p_ds_req = h.do_request_(ds_req)
910 self.assertEqual(p_ds_req.unredirected_hdrs["Host"],"example.com")
911
Senthil Kumaran52380922013-04-25 05:45:48 -0700912 def test_full_url_setter(self):
913 # Checks to ensure that components are set correctly after setting the
914 # full_url of a Request object
915
916 urls = [
917 'http://example.com?foo=bar#baz',
918 'http://example.com?foo=bar&spam=eggs#bash',
919 'http://example.com',
920 ]
921
922 # testing a reusable request instance, but the url parameter is
923 # required, so just use a dummy one to instantiate
924 r = Request('http://example.com')
925 for url in urls:
926 r.full_url = url
Senthil Kumaran83070752013-05-24 09:14:12 -0700927 parsed = urlparse(url)
928
Senthil Kumaran52380922013-04-25 05:45:48 -0700929 self.assertEqual(r.get_full_url(), url)
Senthil Kumaran83070752013-05-24 09:14:12 -0700930 # full_url setter uses splittag to split into components.
931 # splittag sets the fragment as None while urlparse sets it to ''
932 self.assertEqual(r.fragment or '', parsed.fragment)
933 self.assertEqual(urlparse(r.get_full_url()).query, parsed.query)
Senthil Kumaran52380922013-04-25 05:45:48 -0700934
935 def test_full_url_deleter(self):
936 r = Request('http://www.example.com')
937 del r.full_url
938 self.assertIsNone(r.full_url)
939 self.assertIsNone(r.fragment)
940 self.assertEqual(r.selector, '')
941
Senthil Kumaranc2958622010-11-22 04:48:26 +0000942 def test_fixpath_in_weirdurls(self):
943 # Issue4493: urllib2 to supply '/' when to urls where path does not
944 # start with'/'
945
946 h = urllib.request.AbstractHTTPHandler()
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700947 h.parent = MockOpener()
Senthil Kumaranc2958622010-11-22 04:48:26 +0000948
949 weird_url = 'http://www.python.org?getspam'
950 req = Request(weird_url)
951 newreq = h.do_request_(req)
952 self.assertEqual(newreq.host,'www.python.org')
953 self.assertEqual(newreq.selector,'/?getspam')
954
955 url_without_path = 'http://www.python.org'
956 req = Request(url_without_path)
957 newreq = h.do_request_(req)
958 self.assertEqual(newreq.host,'www.python.org')
959 self.assertEqual(newreq.selector,'')
960
Facundo Batista72dc1ea2008-08-16 14:44:32 +0000961
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000962 def test_errors(self):
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000963 h = urllib.request.HTTPErrorProcessor()
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000964 o = h.parent = MockOpener()
965
966 url = "http://example.com/"
967 req = Request(url)
Guido van Rossumd8faa362007-04-27 19:54:29 +0000968 # all 2xx are passed through
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000969 r = MockResponse(200, "OK", {}, "", url)
970 newr = h.http_response(req, r)
Florent Xicluna419e3842010-08-08 16:16:07 +0000971 self.assertIs(r, newr)
972 self.assertFalse(hasattr(o, "proto")) # o.error not called
Guido van Rossumd8faa362007-04-27 19:54:29 +0000973 r = MockResponse(202, "Accepted", {}, "", url)
974 newr = h.http_response(req, r)
Florent Xicluna419e3842010-08-08 16:16:07 +0000975 self.assertIs(r, newr)
976 self.assertFalse(hasattr(o, "proto")) # o.error not called
Guido van Rossumd8faa362007-04-27 19:54:29 +0000977 r = MockResponse(206, "Partial content", {}, "", url)
978 newr = h.http_response(req, r)
Florent Xicluna419e3842010-08-08 16:16:07 +0000979 self.assertIs(r, newr)
980 self.assertFalse(hasattr(o, "proto")) # o.error not called
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000981 # anything else calls o.error (and MockOpener returns None, here)
Guido van Rossumd8faa362007-04-27 19:54:29 +0000982 r = MockResponse(502, "Bad gateway", {}, "", url)
Florent Xicluna419e3842010-08-08 16:16:07 +0000983 self.assertIsNone(h.http_response(req, r))
Jeremy Hyltondf38ea92003-12-17 20:42:38 +0000984 self.assertEqual(o.proto, "http") # o.error called
Guido van Rossumd8faa362007-04-27 19:54:29 +0000985 self.assertEqual(o.args, (req, r, 502, "Bad gateway", {}))
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +0000986
Martin v. Löwis2a6ba902004-05-31 18:22:40 +0000987 def test_cookies(self):
988 cj = MockCookieJar()
Jeremy Hylton1afc1692008-06-18 20:49:58 +0000989 h = urllib.request.HTTPCookieProcessor(cj)
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -0700990 h.parent = MockOpener()
Martin v. Löwis2a6ba902004-05-31 18:22:40 +0000991
992 req = Request("http://example.com/")
993 r = MockResponse(200, "OK", {}, "")
994 newreq = h.http_request(req)
Florent Xicluna419e3842010-08-08 16:16:07 +0000995 self.assertIs(cj.ach_req, req)
996 self.assertIs(cj.ach_req, newreq)
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -0700997 self.assertEqual(req.origin_req_host, "example.com")
998 self.assertFalse(req.unverifiable)
Martin v. Löwis2a6ba902004-05-31 18:22:40 +0000999 newr = h.http_response(req, r)
Florent Xicluna419e3842010-08-08 16:16:07 +00001000 self.assertIs(cj.ec_req, req)
1001 self.assertIs(cj.ec_r, r)
1002 self.assertIs(r, newr)
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001003
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001004 def test_redirect(self):
1005 from_url = "http://example.com/a.html"
1006 to_url = "http://example.com/b.html"
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001007 h = urllib.request.HTTPRedirectHandler()
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001008 o = h.parent = MockOpener()
1009
1010 # ordinary redirect behaviour
1011 for code in 301, 302, 303, 307:
1012 for data in None, "blah\nblah\n":
1013 method = getattr(h, "http_error_%s" % code)
1014 req = Request(from_url, data)
Senthil Kumaranfb8cc2f2009-07-19 02:44:19 +00001015 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001016 req.add_header("Nonsense", "viking=withhold")
Christian Heimes77c02eb2008-02-09 02:18:51 +00001017 if data is not None:
1018 req.add_header("Content-Length", str(len(data)))
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001019 req.add_unredirected_header("Spam", "spam")
1020 try:
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001021 method(req, MockFile(), code, "Blah",
1022 MockHeaders({"location": to_url}))
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001023 except urllib.error.HTTPError:
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001024 # 307 in response to POST requires user OK
Serhiy Storchaka25d8aea2014-02-08 14:50:08 +02001025 self.assertEqual(code, 307)
1026 self.assertIsNotNone(data)
Jeremy Hyltondf38ea92003-12-17 20:42:38 +00001027 self.assertEqual(o.req.get_full_url(), to_url)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001028 try:
Jeremy Hyltondf38ea92003-12-17 20:42:38 +00001029 self.assertEqual(o.req.get_method(), "GET")
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001030 except AttributeError:
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001031 self.assertFalse(o.req.data)
Christian Heimes77c02eb2008-02-09 02:18:51 +00001032
1033 # now it's a GET, there should not be headers regarding content
1034 # (possibly dragged from before being a POST)
1035 headers = [x.lower() for x in o.req.headers]
Benjamin Peterson577473f2010-01-19 00:09:57 +00001036 self.assertNotIn("content-length", headers)
1037 self.assertNotIn("content-type", headers)
Christian Heimes77c02eb2008-02-09 02:18:51 +00001038
Jeremy Hyltondf38ea92003-12-17 20:42:38 +00001039 self.assertEqual(o.req.headers["Nonsense"],
1040 "viking=withhold")
Benjamin Peterson577473f2010-01-19 00:09:57 +00001041 self.assertNotIn("Spam", o.req.headers)
1042 self.assertNotIn("Spam", o.req.unredirected_hdrs)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001043
1044 # loop detection
1045 req = Request(from_url)
Senthil Kumaranfb8cc2f2009-07-19 02:44:19 +00001046 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001047 def redirect(h, req, url=to_url):
1048 h.http_error_302(req, MockFile(), 302, "Blah",
1049 MockHeaders({"location": url}))
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001050 # Note that the *original* request shares the same record of
1051 # redirections with the sub-requests caused by the redirections.
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001052
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001053 # detect infinite loop redirect of a URL to itself
1054 req = Request(from_url, origin_req_host="example.com")
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001055 count = 0
Senthil Kumaranfb8cc2f2009-07-19 02:44:19 +00001056 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001057 try:
1058 while 1:
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001059 redirect(h, req, "http://example.com/")
1060 count = count + 1
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001061 except urllib.error.HTTPError:
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001062 # don't stop until max_repeats, because cookies may introduce state
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001063 self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats)
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001064
1065 # detect endless non-repeating chain of redirects
1066 req = Request(from_url, origin_req_host="example.com")
1067 count = 0
Senthil Kumaranfb8cc2f2009-07-19 02:44:19 +00001068 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001069 try:
1070 while 1:
1071 redirect(h, req, "http://example.com/%d" % count)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001072 count = count + 1
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001073 except urllib.error.HTTPError:
Jeremy Hyltondf38ea92003-12-17 20:42:38 +00001074 self.assertEqual(count,
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001075 urllib.request.HTTPRedirectHandler.max_redirections)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001076
guido@google.coma119df92011-03-29 11:41:02 -07001077
1078 def test_invalid_redirect(self):
1079 from_url = "http://example.com/a.html"
1080 valid_schemes = ['http','https','ftp']
1081 invalid_schemes = ['file','imap','ldap']
1082 schemeless_url = "example.com/b.html"
1083 h = urllib.request.HTTPRedirectHandler()
1084 o = h.parent = MockOpener()
1085 req = Request(from_url)
1086 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1087
1088 for scheme in invalid_schemes:
1089 invalid_url = scheme + '://' + schemeless_url
1090 self.assertRaises(urllib.error.HTTPError, h.http_error_302,
1091 req, MockFile(), 302, "Security Loophole",
1092 MockHeaders({"location": invalid_url}))
1093
1094 for scheme in valid_schemes:
1095 valid_url = scheme + '://' + schemeless_url
1096 h.http_error_302(req, MockFile(), 302, "That's fine",
1097 MockHeaders({"location": valid_url}))
1098 self.assertEqual(o.req.get_full_url(), valid_url)
1099
Senthil Kumaran6497aa32012-01-04 13:46:59 +08001100 def test_relative_redirect(self):
1101 from_url = "http://example.com/a.html"
1102 relative_url = "/b.html"
1103 h = urllib.request.HTTPRedirectHandler()
1104 o = h.parent = MockOpener()
1105 req = Request(from_url)
1106 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1107
1108 valid_url = urllib.parse.urljoin(from_url,relative_url)
1109 h.http_error_302(req, MockFile(), 302, "That's fine",
1110 MockHeaders({"location": valid_url}))
1111 self.assertEqual(o.req.get_full_url(), valid_url)
1112
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001113 def test_cookie_redirect(self):
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001114 # cookies shouldn't leak into redirected requests
Georg Brandl24420152008-05-26 16:32:26 +00001115 from http.cookiejar import CookieJar
1116 from test.test_http_cookiejar import interact_netscape
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001117
1118 cj = CookieJar()
1119 interact_netscape(cj, "http://www.example.com/", "spam=eggs")
Thomas Wouters477c8d52006-05-27 19:21:47 +00001120 hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n")
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001121 hdeh = urllib.request.HTTPDefaultErrorHandler()
1122 hrh = urllib.request.HTTPRedirectHandler()
1123 cp = urllib.request.HTTPCookieProcessor(cj)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001124 o = build_test_opener(hh, hdeh, hrh, cp)
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001125 o.open("http://www.example.com/")
Florent Xicluna419e3842010-08-08 16:16:07 +00001126 self.assertFalse(hh.req.has_header("Cookie"))
Martin v. Löwis2a6ba902004-05-31 18:22:40 +00001127
Senthil Kumaran26430412011-04-13 07:01:19 +08001128 def test_redirect_fragment(self):
1129 redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n'
1130 hh = MockHTTPHandler(302, 'Location: ' + redirected_url)
1131 hdeh = urllib.request.HTTPDefaultErrorHandler()
1132 hrh = urllib.request.HTTPRedirectHandler()
1133 o = build_test_opener(hh, hdeh, hrh)
1134 fp = o.open('http://www.example.com')
1135 self.assertEqual(fp.geturl(), redirected_url.strip())
1136
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001137 def test_proxy(self):
1138 o = OpenerDirector()
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001139 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001140 o.add_handler(ph)
1141 meth_spec = [
1142 [("http_open", "return response")]
1143 ]
1144 handlers = add_ordered_mock_handlers(o, meth_spec)
1145
1146 req = Request("http://acme.example.com/")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001147 self.assertEqual(req.host, "acme.example.com")
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001148 o.open(req)
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001149 self.assertEqual(req.host, "proxy.example.com:3128")
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001150
1151 self.assertEqual([(handlers[0], "http_open")],
1152 [tup[0:2] for tup in o.calls])
1153
Senthil Kumaran7bb04972009-10-11 04:58:55 +00001154 def test_proxy_no_proxy(self):
1155 os.environ['no_proxy'] = 'python.org'
1156 o = OpenerDirector()
1157 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
1158 o.add_handler(ph)
1159 req = Request("http://www.perl.org/")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001160 self.assertEqual(req.host, "www.perl.org")
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001161 o.open(req)
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001162 self.assertEqual(req.host, "proxy.example.com")
Senthil Kumaran7bb04972009-10-11 04:58:55 +00001163 req = Request("http://www.python.org")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001164 self.assertEqual(req.host, "www.python.org")
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001165 o.open(req)
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001166 self.assertEqual(req.host, "www.python.org")
Senthil Kumaran7bb04972009-10-11 04:58:55 +00001167 del os.environ['no_proxy']
1168
Ronald Oussorene72e1612011-03-14 18:15:25 -04001169 def test_proxy_no_proxy_all(self):
1170 os.environ['no_proxy'] = '*'
1171 o = OpenerDirector()
1172 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
1173 o.add_handler(ph)
1174 req = Request("http://www.python.org")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001175 self.assertEqual(req.host, "www.python.org")
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001176 o.open(req)
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001177 self.assertEqual(req.host, "www.python.org")
Ronald Oussorene72e1612011-03-14 18:15:25 -04001178 del os.environ['no_proxy']
1179
Senthil Kumaran7bb04972009-10-11 04:58:55 +00001180
Senthil Kumaran97f0c6b2009-07-25 04:24:38 +00001181 def test_proxy_https(self):
1182 o = OpenerDirector()
1183 ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128"))
1184 o.add_handler(ph)
1185 meth_spec = [
1186 [("https_open", "return response")]
1187 ]
1188 handlers = add_ordered_mock_handlers(o, meth_spec)
1189
1190 req = Request("https://www.example.com/")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001191 self.assertEqual(req.host, "www.example.com")
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001192 o.open(req)
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001193 self.assertEqual(req.host, "proxy.example.com:3128")
Senthil Kumaran97f0c6b2009-07-25 04:24:38 +00001194 self.assertEqual([(handlers[0], "https_open")],
1195 [tup[0:2] for tup in o.calls])
1196
Senthil Kumaran47fff872009-12-20 07:10:31 +00001197 def test_proxy_https_proxy_authorization(self):
1198 o = OpenerDirector()
1199 ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128'))
1200 o.add_handler(ph)
1201 https_handler = MockHTTPSHandler()
1202 o.add_handler(https_handler)
1203 req = Request("https://www.example.com/")
1204 req.add_header("Proxy-Authorization","FooBar")
1205 req.add_header("User-Agent","Grail")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001206 self.assertEqual(req.host, "www.example.com")
Senthil Kumaran47fff872009-12-20 07:10:31 +00001207 self.assertIsNone(req._tunnel_host)
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001208 o.open(req)
Senthil Kumaran47fff872009-12-20 07:10:31 +00001209 # Verify Proxy-Authorization gets tunneled to request.
1210 # httpsconn req_headers do not have the Proxy-Authorization header but
1211 # the req will have.
Ezio Melottib58e0bd2010-01-23 15:40:09 +00001212 self.assertNotIn(("Proxy-Authorization","FooBar"),
Senthil Kumaran47fff872009-12-20 07:10:31 +00001213 https_handler.httpconn.req_headers)
Ezio Melottib58e0bd2010-01-23 15:40:09 +00001214 self.assertIn(("User-Agent","Grail"),
1215 https_handler.httpconn.req_headers)
Senthil Kumaran47fff872009-12-20 07:10:31 +00001216 self.assertIsNotNone(req._tunnel_host)
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001217 self.assertEqual(req.host, "proxy.example.com:3128")
Senthil Kumaran47fff872009-12-20 07:10:31 +00001218 self.assertEqual(req.get_header("Proxy-authorization"),"FooBar")
Senthil Kumaran97f0c6b2009-07-25 04:24:38 +00001219
Senthil Kumaran4de00a22011-05-11 21:17:57 +08001220 # TODO: This should be only for OSX
1221 @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX")
Ronald Oussorene72e1612011-03-14 18:15:25 -04001222 def test_osx_proxy_bypass(self):
1223 bypass = {
1224 'exclude_simple': False,
1225 'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10',
1226 '10.0/16']
1227 }
1228 # Check hosts that should trigger the proxy bypass
1229 for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1',
1230 '10.0.0.1'):
1231 self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass),
1232 'expected bypass of %s to be True' % host)
1233 # Check hosts that should not trigger the proxy bypass
R David Murrayfdbe9182014-03-15 12:00:14 -04001234 for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1',
1235 'notinbypass'):
Ronald Oussorene72e1612011-03-14 18:15:25 -04001236 self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass),
1237 'expected bypass of %s to be False' % host)
1238
1239 # Check the exclude_simple flag
1240 bypass = {'exclude_simple': True, 'exceptions': []}
1241 self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass))
1242
Christian Heimes4fbc72b2008-03-22 00:47:35 +00001243 def test_basic_auth(self, quote_char='"'):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001244 opener = OpenerDirector()
1245 password_manager = MockPasswordManager()
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001246 auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001247 realm = "ACME Widget Store"
1248 http_handler = MockHTTPHandler(
Christian Heimes4fbc72b2008-03-22 00:47:35 +00001249 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' %
1250 (quote_char, realm, quote_char) )
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001251 opener.add_handler(auth_handler)
1252 opener.add_handler(http_handler)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001253 self._test_basic_auth(opener, auth_handler, "Authorization",
1254 realm, http_handler, password_manager,
1255 "http://acme.example.com/protected",
1256 "http://acme.example.com/protected",
1257 )
1258
Christian Heimes4fbc72b2008-03-22 00:47:35 +00001259 def test_basic_auth_with_single_quoted_realm(self):
1260 self.test_basic_auth(quote_char="'")
1261
Senthil Kumaran34f3fcc2012-05-15 22:30:25 +08001262 def test_basic_auth_with_unquoted_realm(self):
1263 opener = OpenerDirector()
1264 password_manager = MockPasswordManager()
1265 auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
1266 realm = "ACME Widget Store"
1267 http_handler = MockHTTPHandler(
1268 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm)
1269 opener.add_handler(auth_handler)
1270 opener.add_handler(http_handler)
Senthil Kumaran0ea91cb2012-05-15 23:59:42 +08001271 with self.assertWarns(UserWarning):
1272 self._test_basic_auth(opener, auth_handler, "Authorization",
1273 realm, http_handler, password_manager,
1274 "http://acme.example.com/protected",
1275 "http://acme.example.com/protected",
1276 )
Senthil Kumaran34f3fcc2012-05-15 22:30:25 +08001277
Thomas Wouters477c8d52006-05-27 19:21:47 +00001278 def test_proxy_basic_auth(self):
1279 opener = OpenerDirector()
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001280 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
Thomas Wouters477c8d52006-05-27 19:21:47 +00001281 opener.add_handler(ph)
1282 password_manager = MockPasswordManager()
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001283 auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001284 realm = "ACME Networks"
1285 http_handler = MockHTTPHandler(
1286 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001287 opener.add_handler(auth_handler)
1288 opener.add_handler(http_handler)
Thomas Wouters00ee7ba2006-08-21 19:07:27 +00001289 self._test_basic_auth(opener, auth_handler, "Proxy-authorization",
Thomas Wouters477c8d52006-05-27 19:21:47 +00001290 realm, http_handler, password_manager,
1291 "http://acme.example.com:3128/protected",
1292 "proxy.example.com:3128",
1293 )
1294
1295 def test_basic_and_digest_auth_handlers(self):
Andrew Svetlov7bd61cb2012-12-19 22:49:25 +02001296 # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40*
Thomas Wouters477c8d52006-05-27 19:21:47 +00001297 # response (http://python.org/sf/1479302), where it should instead
1298 # return None to allow another handler (especially
1299 # HTTPBasicAuthHandler) to handle the response.
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001300
1301 # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must
1302 # try digest first (since it's the strongest auth scheme), so we record
1303 # order of calls here to check digest comes first:
1304 class RecordingOpenerDirector(OpenerDirector):
1305 def __init__(self):
1306 OpenerDirector.__init__(self)
1307 self.recorded = []
1308 def record(self, info):
1309 self.recorded.append(info)
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001310 class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001311 def http_error_401(self, *args, **kwds):
1312 self.parent.record("digest")
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001313 urllib.request.HTTPDigestAuthHandler.http_error_401(self,
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001314 *args, **kwds)
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001315 class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001316 def http_error_401(self, *args, **kwds):
1317 self.parent.record("basic")
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001318 urllib.request.HTTPBasicAuthHandler.http_error_401(self,
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001319 *args, **kwds)
1320
1321 opener = RecordingOpenerDirector()
Thomas Wouters477c8d52006-05-27 19:21:47 +00001322 password_manager = MockPasswordManager()
1323 digest_handler = TestDigestAuthHandler(password_manager)
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001324 basic_handler = TestBasicAuthHandler(password_manager)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001325 realm = "ACME Networks"
1326 http_handler = MockHTTPHandler(
1327 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001328 opener.add_handler(basic_handler)
1329 opener.add_handler(digest_handler)
1330 opener.add_handler(http_handler)
1331
1332 # check basic auth isn't blocked by digest handler failing
Thomas Wouters477c8d52006-05-27 19:21:47 +00001333 self._test_basic_auth(opener, basic_handler, "Authorization",
1334 realm, http_handler, password_manager,
1335 "http://acme.example.com/protected",
1336 "http://acme.example.com/protected",
1337 )
Thomas Wouters4d70c3d2006-06-08 14:42:34 +00001338 # check digest was tried before basic (twice, because
1339 # _test_basic_auth called .open() twice)
1340 self.assertEqual(opener.recorded, ["digest", "basic"]*2)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001341
Senthil Kumaran4de00a22011-05-11 21:17:57 +08001342 def test_unsupported_auth_digest_handler(self):
1343 opener = OpenerDirector()
1344 # While using DigestAuthHandler
1345 digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None)
1346 http_handler = MockHTTPHandler(
1347 401, 'WWW-Authenticate: Kerberos\r\n\r\n')
1348 opener.add_handler(digest_auth_handler)
1349 opener.add_handler(http_handler)
1350 self.assertRaises(ValueError,opener.open,"http://www.example.com")
1351
1352 def test_unsupported_auth_basic_handler(self):
1353 # While using BasicAuthHandler
1354 opener = OpenerDirector()
1355 basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None)
1356 http_handler = MockHTTPHandler(
1357 401, 'WWW-Authenticate: NTLM\r\n\r\n')
1358 opener.add_handler(basic_auth_handler)
1359 opener.add_handler(http_handler)
1360 self.assertRaises(ValueError,opener.open,"http://www.example.com")
1361
Thomas Wouters477c8d52006-05-27 19:21:47 +00001362 def _test_basic_auth(self, opener, auth_handler, auth_header,
1363 realm, http_handler, password_manager,
1364 request_url, protected_url):
Christian Heimes05e8be12008-02-23 18:30:17 +00001365 import base64
Thomas Wouters477c8d52006-05-27 19:21:47 +00001366 user, password = "wile", "coyote"
Thomas Wouters477c8d52006-05-27 19:21:47 +00001367
1368 # .add_password() fed through to password manager
1369 auth_handler.add_password(realm, request_url, user, password)
1370 self.assertEqual(realm, password_manager.realm)
1371 self.assertEqual(request_url, password_manager.url)
1372 self.assertEqual(user, password_manager.user)
1373 self.assertEqual(password, password_manager.password)
1374
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001375 opener.open(request_url)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001376
1377 # should have asked the password manager for the username/password
1378 self.assertEqual(password_manager.target_realm, realm)
1379 self.assertEqual(password_manager.target_url, protected_url)
1380
1381 # expect one request without authorization, then one with
1382 self.assertEqual(len(http_handler.requests), 2)
1383 self.assertFalse(http_handler.requests[0].has_header(auth_header))
Guido van Rossum98b349f2007-08-27 21:47:52 +00001384 userpass = bytes('%s:%s' % (user, password), "ascii")
Guido van Rossum98297ee2007-11-06 21:34:58 +00001385 auth_hdr_value = ('Basic ' +
Georg Brandl706824f2009-06-04 09:42:55 +00001386 base64.encodebytes(userpass).strip().decode())
Thomas Wouters477c8d52006-05-27 19:21:47 +00001387 self.assertEqual(http_handler.requests[1].get_header(auth_header),
1388 auth_hdr_value)
Senthil Kumaranca2fc9e2010-02-24 16:53:16 +00001389 self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header],
1390 auth_hdr_value)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001391 # if the password manager can't find a password, the handler won't
1392 # handle the HTTP auth error
1393 password_manager.user = password_manager.password = None
1394 http_handler.reset()
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001395 opener.open(request_url)
Thomas Wouters477c8d52006-05-27 19:21:47 +00001396 self.assertEqual(len(http_handler.requests), 1)
1397 self.assertFalse(http_handler.requests[0].has_header(auth_header))
1398
Senthil Kumaran4de00a22011-05-11 21:17:57 +08001399
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001400class MiscTests(unittest.TestCase):
1401
Senthil Kumarane9853da2013-03-19 12:07:43 -07001402 def opener_has_handler(self, opener, handler_class):
1403 self.assertTrue(any(h.__class__ == handler_class
1404 for h in opener.handlers))
1405
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001406 def test_build_opener(self):
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001407 class MyHTTPHandler(urllib.request.HTTPHandler): pass
1408 class FooHandler(urllib.request.BaseHandler):
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001409 def foo_open(self): pass
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001410 class BarHandler(urllib.request.BaseHandler):
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001411 def bar_open(self): pass
1412
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001413 build_opener = urllib.request.build_opener
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001414
1415 o = build_opener(FooHandler, BarHandler)
1416 self.opener_has_handler(o, FooHandler)
1417 self.opener_has_handler(o, BarHandler)
1418
1419 # can take a mix of classes and instances
1420 o = build_opener(FooHandler, BarHandler())
1421 self.opener_has_handler(o, FooHandler)
1422 self.opener_has_handler(o, BarHandler)
1423
1424 # subclasses of default handlers override default handlers
1425 o = build_opener(MyHTTPHandler)
1426 self.opener_has_handler(o, MyHTTPHandler)
1427
1428 # a particular case of overriding: default handlers can be passed
1429 # in explicitly
1430 o = build_opener()
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001431 self.opener_has_handler(o, urllib.request.HTTPHandler)
1432 o = build_opener(urllib.request.HTTPHandler)
1433 self.opener_has_handler(o, urllib.request.HTTPHandler)
1434 o = build_opener(urllib.request.HTTPHandler())
1435 self.opener_has_handler(o, urllib.request.HTTPHandler)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001436
Christian Heimes81ee3ef2008-05-04 22:42:01 +00001437 # Issue2670: multiple handlers sharing the same base class
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001438 class MyOtherHTTPHandler(urllib.request.HTTPHandler): pass
Christian Heimes81ee3ef2008-05-04 22:42:01 +00001439 o = build_opener(MyHTTPHandler, MyOtherHTTPHandler)
1440 self.opener_has_handler(o, MyHTTPHandler)
1441 self.opener_has_handler(o, MyOtherHTTPHandler)
1442
Brett Cannon80512de2013-01-25 22:27:21 -05001443 @unittest.skipUnless(support.is_resource_enabled('network'),
1444 'test requires network access')
Victor Stinner9a902432014-03-19 17:31:20 +01001445 @unittest.skipIf(ssl is None,
1446 'test requires the ssl module')
Andrew Svetlovbff98fe2012-11-27 23:06:19 +02001447 def test_issue16464(self):
1448 opener = urllib.request.build_opener()
1449 request = urllib.request.Request("http://www.python.org/~jeremy/")
1450 self.assertEqual(None, request.data)
1451
1452 opener.open(request, "1".encode("us-ascii"))
1453 self.assertEqual(b"1", request.data)
1454 self.assertEqual("1", request.get_header("Content-length"))
1455
1456 opener.open(request, "1234567890".encode("us-ascii"))
1457 self.assertEqual(b"1234567890", request.data)
1458 self.assertEqual("10", request.get_header("Content-length"))
1459
Senthil Kumarane9853da2013-03-19 12:07:43 -07001460 def test_HTTPError_interface(self):
1461 """
1462 Issue 13211 reveals that HTTPError didn't implement the URLError
1463 interface even though HTTPError is a subclass of URLError.
Senthil Kumarane9853da2013-03-19 12:07:43 -07001464 """
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001465 msg = 'something bad happened'
1466 url = code = fp = None
1467 hdrs = 'Content-Length: 42'
1468 err = urllib.error.HTTPError(url, code, msg, hdrs, fp)
1469 self.assertTrue(hasattr(err, 'reason'))
1470 self.assertEqual(err.reason, 'something bad happened')
1471 self.assertTrue(hasattr(err, 'headers'))
1472 self.assertEqual(err.headers, 'Content-Length: 42')
1473 expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg)
1474 self.assertEqual(str(err), expected_errmsg)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001475
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001476class RequestTests(unittest.TestCase):
Jason R. Coombs4a652422013-09-08 13:03:40 -04001477 class PutRequest(Request):
1478 method='PUT'
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001479
1480 def setUp(self):
1481 self.get = Request("http://www.python.org/~jeremy/")
1482 self.post = Request("http://www.python.org/~jeremy/",
1483 "data",
1484 headers={"X-Test": "test"})
Jason R. Coombs4a652422013-09-08 13:03:40 -04001485 self.head = Request("http://www.python.org/~jeremy/", method='HEAD')
1486 self.put = self.PutRequest("http://www.python.org/~jeremy/")
1487 self.force_post = self.PutRequest("http://www.python.org/~jeremy/",
1488 method="POST")
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001489
1490 def test_method(self):
1491 self.assertEqual("POST", self.post.get_method())
1492 self.assertEqual("GET", self.get.get_method())
Senthil Kumaran0b5463f2013-09-09 23:13:06 -07001493 self.assertEqual("HEAD", self.head.get_method())
Jason R. Coombs4a652422013-09-08 13:03:40 -04001494 self.assertEqual("PUT", self.put.get_method())
1495 self.assertEqual("POST", self.force_post.get_method())
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001496
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001497 def test_data(self):
1498 self.assertFalse(self.get.data)
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001499 self.assertEqual("GET", self.get.get_method())
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001500 self.get.data = "spam"
1501 self.assertTrue(self.get.data)
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001502 self.assertEqual("POST", self.get.get_method())
1503
Andrew Svetlovbff98fe2012-11-27 23:06:19 +02001504 # issue 16464
1505 # if we change data we need to remove content-length header
1506 # (cause it's most probably calculated for previous value)
1507 def test_setting_data_should_remove_content_length(self):
R David Murray9cc7d452013-03-20 00:10:51 -04001508 self.assertNotIn("Content-length", self.get.unredirected_hdrs)
Andrew Svetlovbff98fe2012-11-27 23:06:19 +02001509 self.get.add_unredirected_header("Content-length", 42)
1510 self.assertEqual(42, self.get.unredirected_hdrs["Content-length"])
1511 self.get.data = "spam"
R David Murray9cc7d452013-03-20 00:10:51 -04001512 self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1513
1514 # issue 17485 same for deleting data.
1515 def test_deleting_data_should_remove_content_length(self):
1516 self.assertNotIn("Content-length", self.get.unredirected_hdrs)
1517 self.get.data = 'foo'
1518 self.get.add_unredirected_header("Content-length", 3)
1519 self.assertEqual(3, self.get.unredirected_hdrs["Content-length"])
1520 del self.get.data
1521 self.assertNotIn("Content-length", self.get.unredirected_hdrs)
Andrew Svetlovbff98fe2012-11-27 23:06:19 +02001522
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001523 def test_get_full_url(self):
1524 self.assertEqual("http://www.python.org/~jeremy/",
1525 self.get.get_full_url())
1526
1527 def test_selector(self):
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001528 self.assertEqual("/~jeremy/", self.get.selector)
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001529 req = Request("http://www.python.org/")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001530 self.assertEqual("/", req.selector)
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001531
1532 def test_get_type(self):
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001533 self.assertEqual("http", self.get.type)
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001534
1535 def test_get_host(self):
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001536 self.assertEqual("www.python.org", self.get.host)
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001537
1538 def test_get_host_unquote(self):
1539 req = Request("http://www.%70ython.org/")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001540 self.assertEqual("www.python.org", req.host)
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001541
1542 def test_proxy(self):
Florent Xicluna419e3842010-08-08 16:16:07 +00001543 self.assertFalse(self.get.has_proxy())
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001544 self.get.set_proxy("www.perl.org", "http")
Benjamin Petersonc9c0f202009-06-30 23:06:06 +00001545 self.assertTrue(self.get.has_proxy())
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001546 self.assertEqual("www.python.org", self.get.origin_req_host)
1547 self.assertEqual("www.perl.org", self.get.host)
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001548
Senthil Kumarand95cc752010-08-08 11:27:53 +00001549 def test_wrapped_url(self):
1550 req = Request("<URL:http://www.python.org>")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001551 self.assertEqual("www.python.org", req.host)
Senthil Kumarand95cc752010-08-08 11:27:53 +00001552
Senthil Kumaran26430412011-04-13 07:01:19 +08001553 def test_url_fragment(self):
Senthil Kumarand95cc752010-08-08 11:27:53 +00001554 req = Request("http://www.python.org/?qs=query#fragment=true")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001555 self.assertEqual("/?qs=query", req.selector)
Senthil Kumarand95cc752010-08-08 11:27:53 +00001556 req = Request("http://www.python.org/#fun=true")
Senthil Kumaran77ebfcc2012-08-20 13:43:59 -07001557 self.assertEqual("/", req.selector)
Senthil Kumarand95cc752010-08-08 11:27:53 +00001558
Senthil Kumaran26430412011-04-13 07:01:19 +08001559 # Issue 11703: geturl() omits fragment in the original URL.
1560 url = 'http://docs.python.org/library/urllib2.html#OK'
1561 req = Request(url)
1562 self.assertEqual(req.get_full_url(), url)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001563
Senthil Kumaran83070752013-05-24 09:14:12 -07001564 def test_url_fullurl_get_full_url(self):
1565 urls = ['http://docs.python.org',
1566 'http://docs.python.org/library/urllib2.html#OK',
1567 'http://www.python.org/?qs=query#fragment=true' ]
1568 for url in urls:
1569 req = Request(url)
1570 self.assertEqual(req.get_full_url(), req.full_url)
1571
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001572def test_main(verbose=None):
Thomas Wouters477c8d52006-05-27 19:21:47 +00001573 from test import test_urllib2
Benjamin Petersonee8712c2008-05-20 21:35:26 +00001574 support.run_doctest(test_urllib2, verbose)
Jeremy Hylton1afc1692008-06-18 20:49:58 +00001575 support.run_doctest(urllib.request, verbose)
Andrew M. Kuchlingbd3200f2004-06-29 13:15:46 +00001576 tests = (TrivialTests,
1577 OpenerDirectorTests,
1578 HandlerTests,
Benjamin Peterson6ebe78f2008-12-21 00:06:59 +00001579 MiscTests,
Senthil Kumaranfa1b02a2013-04-08 22:24:17 -07001580 RequestTests,
1581 RequestHdrsTests)
Benjamin Petersonee8712c2008-05-20 21:35:26 +00001582 support.run_unittest(*tests)
Jeremy Hyltonc1be59f2003-12-14 05:27:34 +00001583
1584if __name__ == "__main__":
1585 test_main(verbose=True)