Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1 | import unittest |
Benjamin Peterson | ee8712c | 2008-05-20 21:35:26 +0000 | [diff] [blame] | 2 | from test import support |
Serhiy Storchaka | f54c350 | 2014-09-06 21:41:39 +0300 | [diff] [blame] | 3 | from test import test_urllib |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 4 | |
Christian Heimes | 05e8be1 | 2008-02-23 18:30:17 +0000 | [diff] [blame] | 5 | import os |
Guido van Rossum | 34d1928 | 2007-08-09 01:03:29 +0000 | [diff] [blame] | 6 | import io |
Georg Brandl | f78e02b | 2008-06-10 17:40:04 +0000 | [diff] [blame] | 7 | import socket |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 8 | import array |
Senthil Kumaran | 4de00a2 | 2011-05-11 21:17:57 +0800 | [diff] [blame] | 9 | import sys |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 10 | import tempfile |
| 11 | import subprocess |
Jeremy Hylton | e3e6104 | 2001-05-09 15:50:25 +0000 | [diff] [blame] | 12 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 13 | import urllib.request |
Ronald Oussoren | e72e161 | 2011-03-14 18:15:25 -0400 | [diff] [blame] | 14 | # The proxy bypass method imported below has logic specific to the OSX |
| 15 | # proxy config data structure but is testable on all platforms. |
R David Murray | 4c7f995 | 2015-04-16 16:36:18 -0400 | [diff] [blame] | 16 | from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, |
| 17 | HTTPPasswordMgrWithPriorAuth, _parse_proxy, |
Berker Peksag | e88dd1c | 2016-03-06 16:16:40 +0200 | [diff] [blame] | 18 | _proxy_bypass_macosx_sysconf, |
| 19 | AbstractDigestAuthHandler) |
Senthil Kumaran | 8307075 | 2013-05-24 09:14:12 -0700 | [diff] [blame] | 20 | from urllib.parse import urlparse |
guido@google.com | a119df9 | 2011-03-29 11:41:02 -0700 | [diff] [blame] | 21 | import urllib.error |
Serhiy Storchaka | f54c350 | 2014-09-06 21:41:39 +0300 | [diff] [blame] | 22 | import http.client |
Jeremy Hylton | e3e6104 | 2001-05-09 15:50:25 +0000 | [diff] [blame] | 23 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 24 | # XXX |
| 25 | # Request |
| 26 | # CacheFTPHandler (hard to write) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 27 | # parse_keqv_list, parse_http_list, HTTPDigestAuthHandler |
Jeremy Hylton | e3e6104 | 2001-05-09 15:50:25 +0000 | [diff] [blame] | 28 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 29 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 30 | class TrivialTests(unittest.TestCase): |
Senthil Kumaran | 6c5bd40 | 2011-11-01 23:20:31 +0800 | [diff] [blame] | 31 | |
| 32 | def test___all__(self): |
| 33 | # Verify which names are exposed |
| 34 | for module in 'request', 'response', 'parse', 'error', 'robotparser': |
| 35 | context = {} |
| 36 | exec('from urllib.%s import *' % module, context) |
| 37 | del context['__builtins__'] |
Florent Xicluna | 3dbb1f1 | 2011-11-04 22:15:37 +0100 | [diff] [blame] | 38 | if module == 'request' and os.name == 'nt': |
| 39 | u, p = context.pop('url2pathname'), context.pop('pathname2url') |
| 40 | self.assertEqual(u.__module__, 'nturl2path') |
| 41 | self.assertEqual(p.__module__, 'nturl2path') |
Senthil Kumaran | 6c5bd40 | 2011-11-01 23:20:31 +0800 | [diff] [blame] | 42 | for k, v in context.items(): |
| 43 | self.assertEqual(v.__module__, 'urllib.%s' % module, |
| 44 | "%r is exposed in 'urllib.%s' but defined in %r" % |
| 45 | (k, module, v.__module__)) |
| 46 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 47 | def test_trivial(self): |
| 48 | # A couple trivial tests |
Guido van Rossum | e2ae77b | 2001-10-24 20:42:55 +0000 | [diff] [blame] | 49 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 50 | self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url') |
Tim Peters | 861adac | 2001-07-16 20:49:49 +0000 | [diff] [blame] | 51 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 52 | # XXX Name hacking to get this to work on Windows. |
Serhiy Storchaka | 5106d04 | 2015-01-26 10:26:14 +0200 | [diff] [blame] | 53 | fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/') |
Senthil Kumaran | d587e30 | 2010-01-10 17:45:52 +0000 | [diff] [blame] | 54 | |
Senthil Kumaran | d587e30 | 2010-01-10 17:45:52 +0000 | [diff] [blame] | 55 | if os.name == 'nt': |
| 56 | file_url = "file:///%s" % fname |
| 57 | else: |
| 58 | file_url = "file://%s" % fname |
| 59 | |
Serhiy Storchaka | 5b10b98 | 2019-03-05 10:06:26 +0200 | [diff] [blame] | 60 | with urllib.request.urlopen(file_url) as f: |
| 61 | f.read() |
Tim Peters | f5f32b4 | 2005-07-17 23:16:17 +0000 | [diff] [blame] | 62 | |
Georg Brandl | e1b13d2 | 2005-08-24 22:20:32 +0000 | [diff] [blame] | 63 | def test_parse_http_list(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 64 | tests = [ |
| 65 | ('a,b,c', ['a', 'b', 'c']), |
| 66 | ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']), |
| 67 | ('a, b, "c", "d", "e,f", g, h', |
| 68 | ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']), |
| 69 | ('a="b\\"c", d="e\\,f", g="h\\\\i"', |
| 70 | ['a="b"c"', 'd="e,f"', 'g="h\\i"'])] |
Georg Brandl | e1b13d2 | 2005-08-24 22:20:32 +0000 | [diff] [blame] | 71 | for string, list in tests: |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 72 | self.assertEqual(urllib.request.parse_http_list(string), list) |
Georg Brandl | e1b13d2 | 2005-08-24 22:20:32 +0000 | [diff] [blame] | 73 | |
Senthil Kumaran | 843fae9 | 2013-03-19 13:43:42 -0700 | [diff] [blame] | 74 | def test_URLError_reasonstr(self): |
| 75 | err = urllib.error.URLError('reason') |
| 76 | self.assertIn(err.reason, str(err)) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 77 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 78 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 79 | class RequestHdrsTests(unittest.TestCase): |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 80 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 81 | def test_request_headers_dict(self): |
| 82 | """ |
| 83 | The Request.headers dictionary is not a documented interface. It |
| 84 | should stay that way, because the complete set of headers are only |
| 85 | accessible through the .get_header(), .has_header(), .header_items() |
| 86 | interface. However, .headers pre-dates those methods, and so real code |
| 87 | will be using the dictionary. |
| 88 | |
| 89 | The introduction in 2.4 of those methods was a mistake for the same |
| 90 | reason: code that previously saw all (urllib2 user)-provided headers in |
| 91 | .headers now sees only a subset. |
| 92 | |
| 93 | """ |
| 94 | url = "http://example.com" |
| 95 | self.assertEqual(Request(url, |
| 96 | headers={"Spam-eggs": "blah"} |
| 97 | ).headers["Spam-eggs"], "blah") |
| 98 | self.assertEqual(Request(url, |
| 99 | headers={"spam-EggS": "blah"} |
| 100 | ).headers["Spam-eggs"], "blah") |
| 101 | |
| 102 | def test_request_headers_methods(self): |
| 103 | """ |
| 104 | Note the case normalization of header names here, to |
| 105 | .capitalize()-case. This should be preserved for |
| 106 | backwards-compatibility. (In the HTTP case, normalization to |
| 107 | .title()-case is done by urllib2 before sending headers to |
| 108 | http.client). |
| 109 | |
| 110 | Note that e.g. r.has_header("spam-EggS") is currently False, and |
| 111 | r.get_header("spam-EggS") returns None, but that could be changed in |
| 112 | future. |
| 113 | |
| 114 | Method r.remove_header should remove items both from r.headers and |
| 115 | r.unredirected_hdrs dictionaries |
| 116 | """ |
| 117 | url = "http://example.com" |
| 118 | req = Request(url, headers={"Spam-eggs": "blah"}) |
| 119 | self.assertTrue(req.has_header("Spam-eggs")) |
| 120 | self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')]) |
| 121 | |
| 122 | req.add_header("Foo-Bar", "baz") |
| 123 | self.assertEqual(sorted(req.header_items()), |
| 124 | [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')]) |
| 125 | self.assertFalse(req.has_header("Not-there")) |
| 126 | self.assertIsNone(req.get_header("Not-there")) |
| 127 | self.assertEqual(req.get_header("Not-there", "default"), "default") |
| 128 | |
| 129 | req.remove_header("Spam-eggs") |
| 130 | self.assertFalse(req.has_header("Spam-eggs")) |
| 131 | |
| 132 | req.add_unredirected_header("Unredirected-spam", "Eggs") |
| 133 | self.assertTrue(req.has_header("Unredirected-spam")) |
| 134 | |
| 135 | req.remove_header("Unredirected-spam") |
| 136 | self.assertFalse(req.has_header("Unredirected-spam")) |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 137 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 138 | def test_password_manager(self): |
| 139 | mgr = urllib.request.HTTPPasswordMgr() |
| 140 | add = mgr.add_password |
| 141 | find_user_pass = mgr.find_user_password |
Senthil Kumaran | 1f5425f | 2017-03-31 22:27:27 -0700 | [diff] [blame] | 142 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 143 | add("Some Realm", "http://example.com/", "joe", "password") |
| 144 | add("Some Realm", "http://example.com/ni", "ni", "ni") |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 145 | add("Some Realm", "http://c.example.com:3128", "3", "c") |
| 146 | add("Some Realm", "d.example.com", "4", "d") |
| 147 | add("Some Realm", "e.example.com:3128", "5", "e") |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 148 | |
Senthil Kumaran | 1f5425f | 2017-03-31 22:27:27 -0700 | [diff] [blame] | 149 | # For the same realm, password set the highest path is the winner. |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 150 | self.assertEqual(find_user_pass("Some Realm", "example.com"), |
| 151 | ('joe', 'password')) |
Senthil Kumaran | 1f5425f | 2017-03-31 22:27:27 -0700 | [diff] [blame] | 152 | self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), |
| 153 | ('joe', 'password')) |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 154 | self.assertEqual(find_user_pass("Some Realm", "http://example.com"), |
| 155 | ('joe', 'password')) |
| 156 | self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), |
| 157 | ('joe', 'password')) |
Senthil Kumaran | 1f5425f | 2017-03-31 22:27:27 -0700 | [diff] [blame] | 158 | self.assertEqual(find_user_pass("Some Realm", |
| 159 | "http://example.com/spam"), |
| 160 | ('joe', 'password')) |
| 161 | |
| 162 | self.assertEqual(find_user_pass("Some Realm", |
| 163 | "http://example.com/spam/spam"), |
| 164 | ('joe', 'password')) |
| 165 | |
| 166 | # You can have different passwords for different paths. |
| 167 | |
| 168 | add("c", "http://example.com/foo", "foo", "ni") |
| 169 | add("c", "http://example.com/bar", "bar", "nini") |
| 170 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 171 | self.assertEqual(find_user_pass("c", "http://example.com/foo"), |
| 172 | ('foo', 'ni')) |
Senthil Kumaran | 1f5425f | 2017-03-31 22:27:27 -0700 | [diff] [blame] | 173 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 174 | self.assertEqual(find_user_pass("c", "http://example.com/bar"), |
| 175 | ('bar', 'nini')) |
Senthil Kumaran | 1f5425f | 2017-03-31 22:27:27 -0700 | [diff] [blame] | 176 | |
| 177 | # For the same path, newer password should be considered. |
| 178 | |
| 179 | add("b", "http://example.com/", "first", "blah") |
| 180 | add("b", "http://example.com/", "second", "spam") |
| 181 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 182 | self.assertEqual(find_user_pass("b", "http://example.com/"), |
| 183 | ('second', 'spam')) |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 184 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 185 | # No special relationship between a.example.com and example.com: |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 186 | |
Senthil Kumaran | 1f5425f | 2017-03-31 22:27:27 -0700 | [diff] [blame] | 187 | add("a", "http://example.com", "1", "a") |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 188 | self.assertEqual(find_user_pass("a", "http://example.com/"), |
| 189 | ('1', 'a')) |
Senthil Kumaran | 1f5425f | 2017-03-31 22:27:27 -0700 | [diff] [blame] | 190 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 191 | self.assertEqual(find_user_pass("a", "http://a.example.com/"), |
| 192 | (None, None)) |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 193 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 194 | # Ports: |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 195 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 196 | self.assertEqual(find_user_pass("Some Realm", "c.example.com"), |
| 197 | (None, None)) |
| 198 | self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"), |
| 199 | ('3', 'c')) |
| 200 | self.assertEqual( |
| 201 | find_user_pass("Some Realm", "http://c.example.com:3128"), |
| 202 | ('3', 'c')) |
| 203 | self.assertEqual(find_user_pass("Some Realm", "d.example.com"), |
| 204 | ('4', 'd')) |
| 205 | self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"), |
| 206 | ('5', 'e')) |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 207 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 208 | def test_password_manager_default_port(self): |
| 209 | """ |
| 210 | The point to note here is that we can't guess the default port if |
| 211 | there's no scheme. This applies to both add_password and |
| 212 | find_user_password. |
| 213 | """ |
| 214 | mgr = urllib.request.HTTPPasswordMgr() |
| 215 | add = mgr.add_password |
| 216 | find_user_pass = mgr.find_user_password |
| 217 | add("f", "http://g.example.com:80", "10", "j") |
| 218 | add("g", "http://h.example.com", "11", "k") |
| 219 | add("h", "i.example.com:80", "12", "l") |
| 220 | add("i", "j.example.com", "13", "m") |
| 221 | self.assertEqual(find_user_pass("f", "g.example.com:100"), |
| 222 | (None, None)) |
| 223 | self.assertEqual(find_user_pass("f", "g.example.com:80"), |
| 224 | ('10', 'j')) |
| 225 | self.assertEqual(find_user_pass("f", "g.example.com"), |
| 226 | (None, None)) |
| 227 | self.assertEqual(find_user_pass("f", "http://g.example.com:100"), |
| 228 | (None, None)) |
| 229 | self.assertEqual(find_user_pass("f", "http://g.example.com:80"), |
| 230 | ('10', 'j')) |
| 231 | self.assertEqual(find_user_pass("f", "http://g.example.com"), |
| 232 | ('10', 'j')) |
| 233 | self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k')) |
| 234 | self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k')) |
| 235 | self.assertEqual(find_user_pass("g", "http://h.example.com:80"), |
| 236 | ('11', 'k')) |
| 237 | self.assertEqual(find_user_pass("h", "i.example.com"), (None, None)) |
| 238 | self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l')) |
| 239 | self.assertEqual(find_user_pass("h", "http://i.example.com:80"), |
| 240 | ('12', 'l')) |
| 241 | self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm')) |
| 242 | self.assertEqual(find_user_pass("i", "j.example.com:80"), |
| 243 | (None, None)) |
| 244 | self.assertEqual(find_user_pass("i", "http://j.example.com"), |
| 245 | ('13', 'm')) |
| 246 | self.assertEqual(find_user_pass("i", "http://j.example.com:80"), |
| 247 | (None, None)) |
Andrew Svetlov | bff98fe | 2012-11-27 23:06:19 +0200 | [diff] [blame] | 248 | |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 249 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 250 | class MockOpener: |
| 251 | addheaders = [] |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 252 | |
Senthil Kumaran | fb8cc2f | 2009-07-19 02:44:19 +0000 | [diff] [blame] | 253 | def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): |
| 254 | self.req, self.data, self.timeout = req, data, timeout |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 255 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 256 | def error(self, proto, *args): |
| 257 | self.proto, self.args = proto, args |
| 258 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 259 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 260 | class MockFile: |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 261 | def read(self, count=None): |
| 262 | pass |
| 263 | |
| 264 | def readline(self, count=None): |
| 265 | pass |
| 266 | |
| 267 | def close(self): |
| 268 | pass |
| 269 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 270 | |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 271 | class MockHeaders(dict): |
| 272 | def getheaders(self, name): |
Guido van Rossum | cc2b016 | 2007-02-11 06:12:03 +0000 | [diff] [blame] | 273 | return list(self.values()) |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 274 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 275 | |
Guido van Rossum | 34d1928 | 2007-08-09 01:03:29 +0000 | [diff] [blame] | 276 | class MockResponse(io.StringIO): |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 277 | def __init__(self, code, msg, headers, data, url=None): |
Guido van Rossum | 34d1928 | 2007-08-09 01:03:29 +0000 | [diff] [blame] | 278 | io.StringIO.__init__(self, data) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 279 | self.code, self.msg, self.headers, self.url = code, msg, headers, url |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 280 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 281 | def info(self): |
| 282 | return self.headers |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 283 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 284 | def geturl(self): |
| 285 | return self.url |
| 286 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 287 | |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 288 | class MockCookieJar: |
| 289 | def add_cookie_header(self, request): |
| 290 | self.ach_req = request |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 291 | |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 292 | def extract_cookies(self, response, request): |
| 293 | self.ec_req, self.ec_r = request, response |
| 294 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 295 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 296 | class FakeMethod: |
| 297 | def __init__(self, meth_name, action, handle): |
| 298 | self.meth_name = meth_name |
| 299 | self.handle = handle |
| 300 | self.action = action |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 301 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 302 | def __call__(self, *args): |
| 303 | return self.handle(self.meth_name, self.action, *args) |
| 304 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 305 | |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 306 | class MockHTTPResponse(io.IOBase): |
| 307 | def __init__(self, fp, msg, status, reason): |
| 308 | self.fp = fp |
| 309 | self.msg = msg |
| 310 | self.status = status |
| 311 | self.reason = reason |
| 312 | self.code = 200 |
| 313 | |
| 314 | def read(self): |
| 315 | return '' |
| 316 | |
| 317 | def info(self): |
| 318 | return {} |
| 319 | |
| 320 | def geturl(self): |
| 321 | return self.url |
| 322 | |
| 323 | |
| 324 | class MockHTTPClass: |
| 325 | def __init__(self): |
| 326 | self.level = 0 |
| 327 | self.req_headers = [] |
| 328 | self.data = None |
| 329 | self.raise_on_endheaders = False |
Nadeem Vawda | bd26b54 | 2012-10-21 17:37:43 +0200 | [diff] [blame] | 330 | self.sock = None |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 331 | self._tunnel_headers = {} |
| 332 | |
| 333 | def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): |
| 334 | self.host = host |
| 335 | self.timeout = timeout |
| 336 | return self |
| 337 | |
| 338 | def set_debuglevel(self, level): |
| 339 | self.level = level |
| 340 | |
| 341 | def set_tunnel(self, host, port=None, headers=None): |
| 342 | self._tunnel_host = host |
| 343 | self._tunnel_port = port |
| 344 | if headers: |
| 345 | self._tunnel_headers = headers |
| 346 | else: |
| 347 | self._tunnel_headers.clear() |
| 348 | |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 349 | def request(self, method, url, body=None, headers=None, *, |
| 350 | encode_chunked=False): |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 351 | self.method = method |
| 352 | self.selector = url |
Benjamin Peterson | 3d5b8db | 2009-12-24 01:14:05 +0000 | [diff] [blame] | 353 | if headers is not None: |
| 354 | self.req_headers += headers.items() |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 355 | self.req_headers.sort() |
| 356 | if body: |
| 357 | self.data = body |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 358 | self.encode_chunked = encode_chunked |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 359 | if self.raise_on_endheaders: |
Andrew Svetlov | 0832af6 | 2012-12-18 23:10:48 +0200 | [diff] [blame] | 360 | raise OSError() |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 361 | |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 362 | def getresponse(self): |
| 363 | return MockHTTPResponse(MockFile(), {}, 200, "OK") |
| 364 | |
Victor Stinner | a4c45d7 | 2011-06-17 14:01:18 +0200 | [diff] [blame] | 365 | def close(self): |
| 366 | pass |
| 367 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 368 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 369 | class MockHandler: |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 370 | # useful for testing handler machinery |
| 371 | # see add_ordered_mock_handlers() docstring |
Thomas Wouters | 49fd7fa | 2006-04-21 10:40:58 +0000 | [diff] [blame] | 372 | handler_order = 500 |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 373 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 374 | def __init__(self, methods): |
| 375 | self._define_methods(methods) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 376 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 377 | def _define_methods(self, methods): |
| 378 | for spec in methods: |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 379 | if len(spec) == 2: |
| 380 | name, action = spec |
| 381 | else: |
| 382 | name, action = spec, None |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 383 | meth = FakeMethod(name, action, self.handle) |
| 384 | setattr(self.__class__, name, meth) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 385 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 386 | def handle(self, fn_name, action, *args, **kwds): |
| 387 | self.parent.calls.append((self, fn_name, args, kwds)) |
| 388 | if action is None: |
| 389 | return None |
| 390 | elif action == "return self": |
| 391 | return self |
| 392 | elif action == "return response": |
| 393 | res = MockResponse(200, "OK", {}, "") |
| 394 | return res |
| 395 | elif action == "return request": |
| 396 | return Request("http://blah/") |
| 397 | elif action.startswith("error"): |
| 398 | code = action[action.rfind(" ")+1:] |
| 399 | try: |
| 400 | code = int(code) |
| 401 | except ValueError: |
| 402 | pass |
| 403 | res = MockResponse(200, "OK", {}, "") |
| 404 | return self.parent.error("http", args[0], res, code, "", {}) |
| 405 | elif action == "raise": |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 406 | raise urllib.error.URLError("blah") |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 407 | assert False |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 408 | |
| 409 | def close(self): |
| 410 | pass |
| 411 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 412 | def add_parent(self, parent): |
| 413 | self.parent = parent |
| 414 | self.parent.calls = [] |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 415 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 416 | def __lt__(self, other): |
| 417 | if not hasattr(other, "handler_order"): |
| 418 | # No handler_order, leave in original order. Yuck. |
| 419 | return True |
| 420 | return self.handler_order < other.handler_order |
| 421 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 422 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 423 | def add_ordered_mock_handlers(opener, meth_spec): |
| 424 | """Create MockHandlers and add them to an OpenerDirector. |
| 425 | |
| 426 | meth_spec: list of lists of tuples and strings defining methods to define |
| 427 | on handlers. eg: |
| 428 | |
| 429 | [["http_error", "ftp_open"], ["http_open"]] |
| 430 | |
| 431 | defines methods .http_error() and .ftp_open() on one handler, and |
| 432 | .http_open() on another. These methods just record their arguments and |
| 433 | return None. Using a tuple instead of a string causes the method to |
| 434 | perform some action (see MockHandler.handle()), eg: |
| 435 | |
| 436 | [["http_error"], [("http_open", "return request")]] |
| 437 | |
| 438 | defines .http_error() on one handler (which simply returns None), and |
| 439 | .http_open() on another handler, which returns a Request object. |
| 440 | |
| 441 | """ |
| 442 | handlers = [] |
| 443 | count = 0 |
| 444 | for meths in meth_spec: |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 445 | class MockHandlerSubclass(MockHandler): |
| 446 | pass |
| 447 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 448 | h = MockHandlerSubclass(meths) |
Thomas Wouters | 49fd7fa | 2006-04-21 10:40:58 +0000 | [diff] [blame] | 449 | h.handler_order += count |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 450 | h.add_parent(opener) |
| 451 | count = count + 1 |
| 452 | handlers.append(h) |
| 453 | opener.add_handler(h) |
| 454 | return handlers |
| 455 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 456 | |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 457 | def build_test_opener(*handler_instances): |
| 458 | opener = OpenerDirector() |
| 459 | for h in handler_instances: |
| 460 | opener.add_handler(h) |
| 461 | return opener |
| 462 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 463 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 464 | class MockHTTPHandler(urllib.request.BaseHandler): |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 465 | # useful for testing redirections and auth |
| 466 | # sends supplied headers and code as first response |
| 467 | # sends 200 OK as second response |
| 468 | def __init__(self, code, headers): |
| 469 | self.code = code |
| 470 | self.headers = headers |
| 471 | self.reset() |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 472 | |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 473 | def reset(self): |
| 474 | self._count = 0 |
| 475 | self.requests = [] |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 476 | |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 477 | def http_open(self, req): |
Martin Panter | ce6e068 | 2016-05-16 01:07:13 +0000 | [diff] [blame] | 478 | import email, copy |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 479 | self.requests.append(copy.deepcopy(req)) |
| 480 | if self._count == 0: |
| 481 | self._count = self._count + 1 |
Georg Brandl | 2442015 | 2008-05-26 16:32:26 +0000 | [diff] [blame] | 482 | name = http.client.responses[self.code] |
Barry Warsaw | 820c120 | 2008-06-12 04:06:45 +0000 | [diff] [blame] | 483 | msg = email.message_from_string(self.headers) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 484 | return self.parent.error( |
| 485 | "http", req, MockFile(), self.code, name, msg) |
| 486 | else: |
| 487 | self.req = req |
Barry Warsaw | 820c120 | 2008-06-12 04:06:45 +0000 | [diff] [blame] | 488 | msg = email.message_from_string("\r\n\r\n") |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 489 | return MockResponse(200, "OK", msg, "", req.get_full_url()) |
| 490 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 491 | |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 492 | class MockHTTPSHandler(urllib.request.AbstractHTTPHandler): |
| 493 | # Useful for testing the Proxy-Authorization request by verifying the |
| 494 | # properties of httpcon |
Benjamin Peterson | 3d5b8db | 2009-12-24 01:14:05 +0000 | [diff] [blame] | 495 | |
Senthil Kumaran | 9642eed | 2016-05-13 01:32:42 -0700 | [diff] [blame] | 496 | def __init__(self, debuglevel=0): |
| 497 | urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=debuglevel) |
Benjamin Peterson | 3d5b8db | 2009-12-24 01:14:05 +0000 | [diff] [blame] | 498 | self.httpconn = MockHTTPClass() |
| 499 | |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 500 | def https_open(self, req): |
| 501 | return self.do_open(self.httpconn, req) |
| 502 | |
R David Murray | 4c7f995 | 2015-04-16 16:36:18 -0400 | [diff] [blame] | 503 | |
| 504 | class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): |
| 505 | # useful for testing auth |
| 506 | # sends supplied code response |
| 507 | # checks if auth header is specified in request |
| 508 | def __init__(self, code): |
| 509 | self.code = code |
| 510 | self.has_auth_header = False |
| 511 | |
| 512 | def reset(self): |
| 513 | self.has_auth_header = False |
| 514 | |
| 515 | def http_open(self, req): |
| 516 | if req.has_header('Authorization'): |
| 517 | self.has_auth_header = True |
| 518 | name = http.client.responses[self.code] |
| 519 | return MockResponse(self.code, name, MockFile(), "", req.get_full_url()) |
| 520 | |
| 521 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 522 | |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 523 | class MockPasswordManager: |
| 524 | def add_password(self, realm, uri, user, password): |
| 525 | self.realm = realm |
| 526 | self.url = uri |
| 527 | self.user = user |
| 528 | self.password = password |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 529 | |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 530 | def find_user_password(self, realm, authuri): |
| 531 | self.target_realm = realm |
| 532 | self.target_url = authuri |
| 533 | return self.user, self.password |
| 534 | |
| 535 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 536 | class OpenerDirectorTests(unittest.TestCase): |
| 537 | |
Guido van Rossum | b5a755e | 2007-07-18 18:15:48 +0000 | [diff] [blame] | 538 | def test_add_non_handler(self): |
| 539 | class NonHandler(object): |
| 540 | pass |
| 541 | self.assertRaises(TypeError, |
| 542 | OpenerDirector().add_handler, NonHandler()) |
| 543 | |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 544 | def test_badly_named_methods(self): |
| 545 | # test work-around for three methods that accidentally follow the |
| 546 | # naming conventions for handler methods |
| 547 | # (*_open() / *_request() / *_response()) |
| 548 | |
| 549 | # These used to call the accidentally-named methods, causing a |
| 550 | # TypeError in real code; here, returning self from these mock |
| 551 | # methods would either cause no exception, or AttributeError. |
| 552 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 553 | from urllib.error import URLError |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 554 | |
| 555 | o = OpenerDirector() |
| 556 | meth_spec = [ |
| 557 | [("do_open", "return self"), ("proxy_open", "return self")], |
| 558 | [("redirect_request", "return self")], |
| 559 | ] |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 560 | add_ordered_mock_handlers(o, meth_spec) |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 561 | o.add_handler(urllib.request.UnknownHandler()) |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 562 | for scheme in "do", "proxy", "redirect": |
| 563 | self.assertRaises(URLError, o.open, scheme+"://example.com/") |
| 564 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 565 | def test_handled(self): |
| 566 | # handler returning non-None means no more handlers will be called |
| 567 | o = OpenerDirector() |
| 568 | meth_spec = [ |
| 569 | ["http_open", "ftp_open", "http_error_302"], |
| 570 | ["ftp_open"], |
| 571 | [("http_open", "return self")], |
| 572 | [("http_open", "return self")], |
| 573 | ] |
| 574 | handlers = add_ordered_mock_handlers(o, meth_spec) |
| 575 | |
| 576 | req = Request("http://example.com/") |
| 577 | r = o.open(req) |
| 578 | # Second .http_open() gets called, third doesn't, since second returned |
| 579 | # non-None. Handlers without .http_open() never get any methods called |
| 580 | # on them. |
| 581 | # In fact, second mock handler defining .http_open() returns self |
| 582 | # (instead of response), which becomes the OpenerDirector's return |
| 583 | # value. |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 584 | self.assertEqual(r, handlers[2]) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 585 | calls = [(handlers[0], "http_open"), (handlers[2], "http_open")] |
| 586 | for expected, got in zip(calls, o.calls): |
| 587 | handler, name, args, kwds = got |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 588 | self.assertEqual((handler, name), expected) |
| 589 | self.assertEqual(args, (req,)) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 590 | |
| 591 | def test_handler_order(self): |
| 592 | o = OpenerDirector() |
| 593 | handlers = [] |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 594 | for meths, handler_order in [([("http_open", "return self")], 500), |
| 595 | (["http_open"], 0)]: |
| 596 | class MockHandlerSubclass(MockHandler): |
| 597 | pass |
| 598 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 599 | h = MockHandlerSubclass(meths) |
| 600 | h.handler_order = handler_order |
| 601 | handlers.append(h) |
| 602 | o.add_handler(h) |
| 603 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 604 | o.open("http://example.com/") |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 605 | # handlers called in reverse order, thanks to their sort order |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 606 | self.assertEqual(o.calls[0][0], handlers[1]) |
| 607 | self.assertEqual(o.calls[1][0], handlers[0]) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 608 | |
| 609 | def test_raise(self): |
| 610 | # raising URLError stops processing of request |
| 611 | o = OpenerDirector() |
| 612 | meth_spec = [ |
| 613 | [("http_open", "raise")], |
| 614 | [("http_open", "return self")], |
| 615 | ] |
| 616 | handlers = add_ordered_mock_handlers(o, meth_spec) |
| 617 | |
| 618 | req = Request("http://example.com/") |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 619 | self.assertRaises(urllib.error.URLError, o.open, req) |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 620 | self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 621 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 622 | def test_http_error(self): |
| 623 | # XXX http_error_default |
| 624 | # http errors are a special case |
| 625 | o = OpenerDirector() |
| 626 | meth_spec = [ |
| 627 | [("http_open", "error 302")], |
| 628 | [("http_error_400", "raise"), "http_open"], |
| 629 | [("http_error_302", "return response"), "http_error_303", |
| 630 | "http_error"], |
| 631 | [("http_error_302")], |
| 632 | ] |
| 633 | handlers = add_ordered_mock_handlers(o, meth_spec) |
| 634 | |
| 635 | class Unknown: |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 636 | def __eq__(self, other): |
| 637 | return True |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 638 | |
| 639 | req = Request("http://example.com/") |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 640 | o.open(req) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 641 | assert len(o.calls) == 2 |
| 642 | calls = [(handlers[0], "http_open", (req,)), |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 643 | (handlers[2], "http_error_302", |
| 644 | (req, Unknown(), 302, "", {}))] |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 645 | for expected, got in zip(calls, o.calls): |
| 646 | handler, method_name, args = expected |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 647 | self.assertEqual((handler, method_name), got[:2]) |
| 648 | self.assertEqual(args, got[2]) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 649 | |
| 650 | def test_processors(self): |
| 651 | # *_request / *_response methods get called appropriately |
| 652 | o = OpenerDirector() |
| 653 | meth_spec = [ |
| 654 | [("http_request", "return request"), |
| 655 | ("http_response", "return response")], |
| 656 | [("http_request", "return request"), |
| 657 | ("http_response", "return response")], |
| 658 | ] |
| 659 | handlers = add_ordered_mock_handlers(o, meth_spec) |
| 660 | |
| 661 | req = Request("http://example.com/") |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 662 | o.open(req) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 663 | # processor methods are called on *all* handlers that define them, |
| 664 | # not just the first handler that handles the request |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 665 | calls = [ |
| 666 | (handlers[0], "http_request"), (handlers[1], "http_request"), |
| 667 | (handlers[0], "http_response"), (handlers[1], "http_response")] |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 668 | |
| 669 | for i, (handler, name, args, kwds) in enumerate(o.calls): |
| 670 | if i < 2: |
| 671 | # *_request |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 672 | self.assertEqual((handler, name), calls[i]) |
| 673 | self.assertEqual(len(args), 1) |
Ezio Melotti | e961593 | 2010-01-24 19:26:24 +0000 | [diff] [blame] | 674 | self.assertIsInstance(args[0], Request) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 675 | else: |
| 676 | # *_response |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 677 | self.assertEqual((handler, name), calls[i]) |
| 678 | self.assertEqual(len(args), 2) |
Ezio Melotti | e961593 | 2010-01-24 19:26:24 +0000 | [diff] [blame] | 679 | self.assertIsInstance(args[0], Request) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 680 | # response from opener.open is None, because there's no |
| 681 | # handler that defines http_open to handle it |
Serhiy Storchaka | 25d8aea | 2014-02-08 14:50:08 +0200 | [diff] [blame] | 682 | if args[1] is not None: |
| 683 | self.assertIsInstance(args[1], MockResponse) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 684 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 685 | |
Tim Peters | 58eb11c | 2004-01-18 20:29:55 +0000 | [diff] [blame] | 686 | def sanepathname2url(path): |
Victor Stinner | 6c6f851 | 2010-08-07 10:09:35 +0000 | [diff] [blame] | 687 | try: |
Marc-André Lemburg | 8f36af7 | 2011-02-25 15:42:01 +0000 | [diff] [blame] | 688 | path.encode("utf-8") |
Victor Stinner | 6c6f851 | 2010-08-07 10:09:35 +0000 | [diff] [blame] | 689 | except UnicodeEncodeError: |
| 690 | raise unittest.SkipTest("path is not encodable to utf8") |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 691 | urlpath = urllib.request.pathname2url(path) |
Tim Peters | 58eb11c | 2004-01-18 20:29:55 +0000 | [diff] [blame] | 692 | if os.name == "nt" and urlpath.startswith("///"): |
| 693 | urlpath = urlpath[2:] |
| 694 | # XXX don't ask me about the mac... |
| 695 | return urlpath |
| 696 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 697 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 698 | class HandlerTests(unittest.TestCase): |
| 699 | |
| 700 | def test_ftp(self): |
| 701 | class MockFTPWrapper: |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 702 | def __init__(self, data): |
| 703 | self.data = data |
| 704 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 705 | def retrfile(self, filename, filetype): |
| 706 | self.filename, self.filetype = filename, filetype |
Guido van Rossum | 34d1928 | 2007-08-09 01:03:29 +0000 | [diff] [blame] | 707 | return io.StringIO(self.data), len(self.data) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 708 | |
| 709 | def close(self): |
| 710 | pass |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 711 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 712 | class NullFTPHandler(urllib.request.FTPHandler): |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 713 | def __init__(self, data): |
| 714 | self.data = data |
| 715 | |
Georg Brandl | f78e02b | 2008-06-10 17:40:04 +0000 | [diff] [blame] | 716 | def connect_ftp(self, user, passwd, host, port, dirs, |
| 717 | timeout=socket._GLOBAL_DEFAULT_TIMEOUT): |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 718 | self.user, self.passwd = user, passwd |
| 719 | self.host, self.port = host, port |
| 720 | self.dirs = dirs |
| 721 | self.ftpwrapper = MockFTPWrapper(self.data) |
| 722 | return self.ftpwrapper |
| 723 | |
Georg Brandl | f78e02b | 2008-06-10 17:40:04 +0000 | [diff] [blame] | 724 | import ftplib |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 725 | data = "rheum rhaponicum" |
| 726 | h = NullFTPHandler(data) |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 727 | h.parent = MockOpener() |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 728 | |
Senthil Kumaran | daa29d0 | 2010-11-18 15:36:41 +0000 | [diff] [blame] | 729 | for url, host, port, user, passwd, type_, dirs, filename, mimetype in [ |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 730 | ("ftp://localhost/foo/bar/baz.html", |
Senthil Kumaran | daa29d0 | 2010-11-18 15:36:41 +0000 | [diff] [blame] | 731 | "localhost", ftplib.FTP_PORT, "", "", "I", |
| 732 | ["foo", "bar"], "baz.html", "text/html"), |
| 733 | ("ftp://parrot@localhost/foo/bar/baz.html", |
| 734 | "localhost", ftplib.FTP_PORT, "parrot", "", "I", |
| 735 | ["foo", "bar"], "baz.html", "text/html"), |
| 736 | ("ftp://%25parrot@localhost/foo/bar/baz.html", |
| 737 | "localhost", ftplib.FTP_PORT, "%parrot", "", "I", |
| 738 | ["foo", "bar"], "baz.html", "text/html"), |
| 739 | ("ftp://%2542parrot@localhost/foo/bar/baz.html", |
| 740 | "localhost", ftplib.FTP_PORT, "%42parrot", "", "I", |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 741 | ["foo", "bar"], "baz.html", "text/html"), |
Kurt B. Kaiser | 3f7cb5d | 2004-07-11 17:14:13 +0000 | [diff] [blame] | 742 | ("ftp://localhost:80/foo/bar/", |
Senthil Kumaran | daa29d0 | 2010-11-18 15:36:41 +0000 | [diff] [blame] | 743 | "localhost", 80, "", "", "D", |
Kurt B. Kaiser | 3f7cb5d | 2004-07-11 17:14:13 +0000 | [diff] [blame] | 744 | ["foo", "bar"], "", None), |
| 745 | ("ftp://localhost/baz.gif;type=a", |
Senthil Kumaran | daa29d0 | 2010-11-18 15:36:41 +0000 | [diff] [blame] | 746 | "localhost", ftplib.FTP_PORT, "", "", "A", |
Kurt B. Kaiser | 3f7cb5d | 2004-07-11 17:14:13 +0000 | [diff] [blame] | 747 | [], "baz.gif", None), # XXX really this should guess image/gif |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 748 | ]: |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 749 | req = Request(url) |
| 750 | req.timeout = None |
| 751 | r = h.ftp_open(req) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 752 | # ftp authentication not yet implemented by FTPHandler |
Senthil Kumaran | daa29d0 | 2010-11-18 15:36:41 +0000 | [diff] [blame] | 753 | self.assertEqual(h.user, user) |
| 754 | self.assertEqual(h.passwd, passwd) |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 755 | self.assertEqual(h.host, socket.gethostbyname(host)) |
| 756 | self.assertEqual(h.port, port) |
| 757 | self.assertEqual(h.dirs, dirs) |
| 758 | self.assertEqual(h.ftpwrapper.filename, filename) |
| 759 | self.assertEqual(h.ftpwrapper.filetype, type_) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 760 | headers = r.info() |
Kurt B. Kaiser | 3f7cb5d | 2004-07-11 17:14:13 +0000 | [diff] [blame] | 761 | self.assertEqual(headers.get("Content-type"), mimetype) |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 762 | self.assertEqual(int(headers["Content-length"]), len(data)) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 763 | |
| 764 | def test_file(self): |
Senthil Kumaran | bc07ac5 | 2014-07-22 00:15:20 -0700 | [diff] [blame] | 765 | import email.utils |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 766 | h = urllib.request.FileHandler() |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 767 | o = h.parent = MockOpener() |
| 768 | |
Benjamin Peterson | ee8712c | 2008-05-20 21:35:26 +0000 | [diff] [blame] | 769 | TESTFN = support.TESTFN |
Tim Peters | 58eb11c | 2004-01-18 20:29:55 +0000 | [diff] [blame] | 770 | urlpath = sanepathname2url(os.path.abspath(TESTFN)) |
Guido van Rossum | 6a2ccd0 | 2007-07-16 20:51:57 +0000 | [diff] [blame] | 771 | towrite = b"hello, world\n" |
Thomas Wouters | 49fd7fa | 2006-04-21 10:40:58 +0000 | [diff] [blame] | 772 | urls = [ |
Tim Peters | 58eb11c | 2004-01-18 20:29:55 +0000 | [diff] [blame] | 773 | "file://localhost%s" % urlpath, |
| 774 | "file://%s" % urlpath, |
| 775 | "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), |
Thomas Wouters | 49fd7fa | 2006-04-21 10:40:58 +0000 | [diff] [blame] | 776 | ] |
| 777 | try: |
| 778 | localaddr = socket.gethostbyname(socket.gethostname()) |
| 779 | except socket.gaierror: |
| 780 | localaddr = '' |
| 781 | if localaddr: |
| 782 | urls.append("file://%s%s" % (localaddr, urlpath)) |
| 783 | |
| 784 | for url in urls: |
Tim Peters | 58eb11c | 2004-01-18 20:29:55 +0000 | [diff] [blame] | 785 | f = open(TESTFN, "wb") |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 786 | try: |
| 787 | try: |
| 788 | f.write(towrite) |
| 789 | finally: |
| 790 | f.close() |
| 791 | |
| 792 | r = h.file_open(Request(url)) |
| 793 | try: |
| 794 | data = r.read() |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 795 | headers = r.info() |
Senthil Kumaran | 4fbed10 | 2010-05-08 03:29:09 +0000 | [diff] [blame] | 796 | respurl = r.geturl() |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 797 | finally: |
| 798 | r.close() |
Tim Peters | 58eb11c | 2004-01-18 20:29:55 +0000 | [diff] [blame] | 799 | stats = os.stat(TESTFN) |
Benjamin Peterson | a0c0a4a | 2008-06-12 22:15:50 +0000 | [diff] [blame] | 800 | modified = email.utils.formatdate(stats.st_mtime, usegmt=True) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 801 | finally: |
| 802 | os.remove(TESTFN) |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 803 | self.assertEqual(data, towrite) |
| 804 | self.assertEqual(headers["Content-type"], "text/plain") |
| 805 | self.assertEqual(headers["Content-length"], "13") |
Tim Peters | 58eb11c | 2004-01-18 20:29:55 +0000 | [diff] [blame] | 806 | self.assertEqual(headers["Last-modified"], modified) |
Senthil Kumaran | 4fbed10 | 2010-05-08 03:29:09 +0000 | [diff] [blame] | 807 | self.assertEqual(respurl, url) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 808 | |
| 809 | for url in [ |
Tim Peters | 58eb11c | 2004-01-18 20:29:55 +0000 | [diff] [blame] | 810 | "file://localhost:80%s" % urlpath, |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 811 | "file:///file_does_not_exist.txt", |
Senthil Kumaran | bc07ac5 | 2014-07-22 00:15:20 -0700 | [diff] [blame] | 812 | "file://not-a-local-host.com//dir/file.txt", |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 813 | "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), |
| 814 | os.getcwd(), TESTFN), |
| 815 | "file://somerandomhost.ontheinternet.com%s/%s" % |
| 816 | (os.getcwd(), TESTFN), |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 817 | ]: |
| 818 | try: |
Tim Peters | 58eb11c | 2004-01-18 20:29:55 +0000 | [diff] [blame] | 819 | f = open(TESTFN, "wb") |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 820 | try: |
| 821 | f.write(towrite) |
| 822 | finally: |
| 823 | f.close() |
| 824 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 825 | self.assertRaises(urllib.error.URLError, |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 826 | h.file_open, Request(url)) |
| 827 | finally: |
| 828 | os.remove(TESTFN) |
| 829 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 830 | h = urllib.request.FileHandler() |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 831 | o = h.parent = MockOpener() |
| 832 | # XXXX why does // mean ftp (and /// mean not ftp!), and where |
| 833 | # is file: scheme specified? I think this is really a bug, and |
| 834 | # what was intended was to distinguish between URLs like: |
| 835 | # file:/blah.txt (a file) |
| 836 | # file://localhost/blah.txt (a file) |
| 837 | # file:///blah.txt (a file) |
| 838 | # file://ftp.example.com/blah.txt (an ftp URL) |
| 839 | for url, ftp in [ |
Senthil Kumaran | 383c32d | 2010-10-14 11:57:35 +0000 | [diff] [blame] | 840 | ("file://ftp.example.com//foo.txt", False), |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 841 | ("file://ftp.example.com///foo.txt", False), |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 842 | ("file://ftp.example.com/foo.txt", False), |
Senthil Kumaran | 383c32d | 2010-10-14 11:57:35 +0000 | [diff] [blame] | 843 | ("file://somehost//foo/something.txt", False), |
Senthil Kumaran | 2ef1632 | 2010-07-11 03:12:43 +0000 | [diff] [blame] | 844 | ("file://localhost//foo/something.txt", False), |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 845 | ]: |
| 846 | req = Request(url) |
| 847 | try: |
| 848 | h.file_open(req) |
Senthil Kumaran | ed3dd1c | 2017-03-30 22:43:05 -0700 | [diff] [blame] | 849 | except urllib.error.URLError: |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 850 | self.assertFalse(ftp) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 851 | else: |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 852 | self.assertIs(o.req, req) |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 853 | self.assertEqual(req.type, "ftp") |
Łukasz Langa | d7e81cc | 2011-01-09 18:18:53 +0000 | [diff] [blame] | 854 | self.assertEqual(req.type == "ftp", ftp) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 855 | |
| 856 | def test_http(self): |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 857 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 858 | h = urllib.request.AbstractHTTPHandler() |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 859 | o = h.parent = MockOpener() |
| 860 | |
| 861 | url = "http://example.com/" |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 862 | for method, data in [("GET", None), ("POST", b"blah")]: |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 863 | req = Request(url, data, {"Foo": "bar"}) |
Guido van Rossum | cd16bf6 | 2007-06-13 18:07:49 +0000 | [diff] [blame] | 864 | req.timeout = None |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 865 | req.add_unredirected_header("Spam", "eggs") |
| 866 | http = MockHTTPClass() |
| 867 | r = h.do_open(http, req) |
| 868 | |
| 869 | # result attributes |
| 870 | r.read; r.readline # wrapped MockFile methods |
| 871 | r.info; r.geturl # addinfourl methods |
| 872 | r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply() |
| 873 | hdrs = r.info() |
Guido van Rossum | e2b70bc | 2006-08-18 22:13:04 +0000 | [diff] [blame] | 874 | hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply() |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 875 | self.assertEqual(r.geturl(), url) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 876 | |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 877 | self.assertEqual(http.host, "example.com") |
| 878 | self.assertEqual(http.level, 0) |
| 879 | self.assertEqual(http.method, method) |
| 880 | self.assertEqual(http.selector, "/") |
| 881 | self.assertEqual(http.req_headers, |
Jeremy Hylton | b3ee6f9 | 2004-02-24 19:40:35 +0000 | [diff] [blame] | 882 | [("Connection", "close"), |
| 883 | ("Foo", "bar"), ("Spam", "eggs")]) |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 884 | self.assertEqual(http.data, data) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 885 | |
Andrew Svetlov | 0832af6 | 2012-12-18 23:10:48 +0200 | [diff] [blame] | 886 | # check OSError converted to URLError |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 887 | http.raise_on_endheaders = True |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 888 | self.assertRaises(urllib.error.URLError, h.do_open, http, req) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 889 | |
Senthil Kumaran | 2933312 | 2011-02-11 11:25:47 +0000 | [diff] [blame] | 890 | # Check for TypeError on POST data which is str. |
| 891 | req = Request("http://example.com/","badpost") |
| 892 | self.assertRaises(TypeError, h.do_request_, req) |
| 893 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 894 | # check adding of standard headers |
| 895 | o.addheaders = [("Spam", "eggs")] |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 896 | for data in b"", None: # POST, GET |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 897 | req = Request("http://example.com/", data) |
| 898 | r = MockResponse(200, "OK", {}, "") |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 899 | newreq = h.do_request_(req) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 900 | if data is None: # GET |
Benjamin Peterson | 577473f | 2010-01-19 00:09:57 +0000 | [diff] [blame] | 901 | self.assertNotIn("Content-length", req.unredirected_hdrs) |
| 902 | self.assertNotIn("Content-type", req.unredirected_hdrs) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 903 | else: # POST |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 904 | self.assertEqual(req.unredirected_hdrs["Content-length"], "0") |
| 905 | self.assertEqual(req.unredirected_hdrs["Content-type"], |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 906 | "application/x-www-form-urlencoded") |
| 907 | # XXX the details of Host could be better tested |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 908 | self.assertEqual(req.unredirected_hdrs["Host"], "example.com") |
| 909 | self.assertEqual(req.unredirected_hdrs["Spam"], "eggs") |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 910 | |
| 911 | # don't clobber existing headers |
| 912 | req.add_unredirected_header("Content-length", "foo") |
| 913 | req.add_unredirected_header("Content-type", "bar") |
| 914 | req.add_unredirected_header("Host", "baz") |
| 915 | req.add_unredirected_header("Spam", "foo") |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 916 | newreq = h.do_request_(req) |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 917 | self.assertEqual(req.unredirected_hdrs["Content-length"], "foo") |
| 918 | self.assertEqual(req.unredirected_hdrs["Content-type"], "bar") |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 919 | self.assertEqual(req.unredirected_hdrs["Host"], "baz") |
| 920 | self.assertEqual(req.unredirected_hdrs["Spam"], "foo") |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 921 | |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 922 | def test_http_body_file(self): |
Martin Panter | ef91bb2 | 2016-08-27 01:39:26 +0000 | [diff] [blame] | 923 | # A regular file - chunked encoding is used unless Content Length is |
| 924 | # already set. |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 925 | |
| 926 | h = urllib.request.AbstractHTTPHandler() |
| 927 | o = h.parent = MockOpener() |
| 928 | |
| 929 | file_obj = tempfile.NamedTemporaryFile(mode='w+b', delete=False) |
| 930 | file_path = file_obj.name |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 931 | file_obj.close() |
Martin Panter | ef91bb2 | 2016-08-27 01:39:26 +0000 | [diff] [blame] | 932 | self.addCleanup(os.unlink, file_path) |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 933 | |
Martin Panter | ef91bb2 | 2016-08-27 01:39:26 +0000 | [diff] [blame] | 934 | with open(file_path, "rb") as f: |
| 935 | req = Request("http://example.com/", f, {}) |
| 936 | newreq = h.do_request_(req) |
| 937 | te = newreq.get_header('Transfer-encoding') |
| 938 | self.assertEqual(te, "chunked") |
| 939 | self.assertFalse(newreq.has_header('Content-length')) |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 940 | |
Martin Panter | ef91bb2 | 2016-08-27 01:39:26 +0000 | [diff] [blame] | 941 | with open(file_path, "rb") as f: |
| 942 | req = Request("http://example.com/", f, {"Content-Length": 30}) |
| 943 | newreq = h.do_request_(req) |
| 944 | self.assertEqual(int(newreq.get_header('Content-length')), 30) |
| 945 | self.assertFalse(newreq.has_header("Transfer-encoding")) |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 946 | |
| 947 | def test_http_body_fileobj(self): |
Martin Panter | ef91bb2 | 2016-08-27 01:39:26 +0000 | [diff] [blame] | 948 | # A file object - chunked encoding is used |
| 949 | # unless Content Length is already set. |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 950 | # (Note that there are some subtle differences to a regular |
| 951 | # file, that is why we are testing both cases.) |
| 952 | |
| 953 | h = urllib.request.AbstractHTTPHandler() |
| 954 | o = h.parent = MockOpener() |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 955 | file_obj = io.BytesIO() |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 956 | |
Martin Panter | ef91bb2 | 2016-08-27 01:39:26 +0000 | [diff] [blame] | 957 | req = Request("http://example.com/", file_obj, {}) |
| 958 | newreq = h.do_request_(req) |
| 959 | self.assertEqual(newreq.get_header('Transfer-encoding'), 'chunked') |
| 960 | self.assertFalse(newreq.has_header('Content-length')) |
| 961 | |
| 962 | headers = {"Content-Length": 30} |
| 963 | req = Request("http://example.com/", file_obj, headers) |
| 964 | newreq = h.do_request_(req) |
| 965 | self.assertEqual(int(newreq.get_header('Content-length')), 30) |
| 966 | self.assertFalse(newreq.has_header("Transfer-encoding")) |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 967 | |
| 968 | file_obj.close() |
| 969 | |
| 970 | def test_http_body_pipe(self): |
| 971 | # A file reading from a pipe. |
| 972 | # A pipe cannot be seek'ed. There is no way to determine the |
| 973 | # content length up front. Thus, do_request_() should fall |
| 974 | # back to Transfer-encoding chunked. |
| 975 | |
| 976 | h = urllib.request.AbstractHTTPHandler() |
| 977 | o = h.parent = MockOpener() |
| 978 | |
Martin Panter | ef91bb2 | 2016-08-27 01:39:26 +0000 | [diff] [blame] | 979 | cmd = [sys.executable, "-c", r"pass"] |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 980 | for headers in {}, {"Content-Length": 30}: |
| 981 | with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: |
| 982 | req = Request("http://example.com/", proc.stdout, headers) |
| 983 | newreq = h.do_request_(req) |
| 984 | if not headers: |
| 985 | self.assertEqual(newreq.get_header('Content-length'), None) |
| 986 | self.assertEqual(newreq.get_header('Transfer-encoding'), |
| 987 | 'chunked') |
| 988 | else: |
| 989 | self.assertEqual(int(newreq.get_header('Content-length')), |
| 990 | 30) |
| 991 | |
| 992 | def test_http_body_iterable(self): |
| 993 | # Generic iterable. There is no way to determine the content |
| 994 | # length up front. Fall back to Transfer-encoding chunked. |
| 995 | |
| 996 | h = urllib.request.AbstractHTTPHandler() |
| 997 | o = h.parent = MockOpener() |
| 998 | |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 999 | def iterable_body(): |
| 1000 | yield b"one" |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 1001 | |
| 1002 | for headers in {}, {"Content-Length": 11}: |
| 1003 | req = Request("http://example.com/", iterable_body(), headers) |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 1004 | newreq = h.do_request_(req) |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 1005 | if not headers: |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 1006 | self.assertEqual(newreq.get_header('Content-length'), None) |
| 1007 | self.assertEqual(newreq.get_header('Transfer-encoding'), |
| 1008 | 'chunked') |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 1009 | else: |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 1010 | self.assertEqual(int(newreq.get_header('Content-length')), 11) |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 1011 | |
Martin Panter | ef91bb2 | 2016-08-27 01:39:26 +0000 | [diff] [blame] | 1012 | def test_http_body_empty_seq(self): |
| 1013 | # Zero-length iterable body should be treated like any other iterable |
| 1014 | h = urllib.request.AbstractHTTPHandler() |
| 1015 | h.parent = MockOpener() |
| 1016 | req = h.do_request_(Request("http://example.com/", ())) |
| 1017 | self.assertEqual(req.get_header("Transfer-encoding"), "chunked") |
| 1018 | self.assertFalse(req.has_header("Content-length")) |
| 1019 | |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 1020 | def test_http_body_array(self): |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 1021 | # array.array Iterable - Content Length is calculated |
| 1022 | |
Martin Panter | 3c0d0ba | 2016-08-24 06:33:33 +0000 | [diff] [blame] | 1023 | h = urllib.request.AbstractHTTPHandler() |
| 1024 | o = h.parent = MockOpener() |
| 1025 | |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 1026 | iterable_array = array.array("I",[1,2,3,4]) |
| 1027 | |
| 1028 | for headers in {}, {"Content-Length": 16}: |
| 1029 | req = Request("http://example.com/", iterable_array, headers) |
| 1030 | newreq = h.do_request_(req) |
| 1031 | self.assertEqual(int(newreq.get_header('Content-length')),16) |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 1032 | |
Senthil Kumaran | 9642eed | 2016-05-13 01:32:42 -0700 | [diff] [blame] | 1033 | def test_http_handler_debuglevel(self): |
| 1034 | o = OpenerDirector() |
| 1035 | h = MockHTTPSHandler(debuglevel=1) |
| 1036 | o.add_handler(h) |
| 1037 | o.open("https://www.example.com") |
| 1038 | self.assertEqual(h._debuglevel, 1) |
| 1039 | |
Facundo Batista | 72dc1ea | 2008-08-16 14:44:32 +0000 | [diff] [blame] | 1040 | def test_http_doubleslash(self): |
| 1041 | # Checks the presence of any unnecessary double slash in url does not |
| 1042 | # break anything. Previously, a double slash directly after the host |
Ezio Melotti | e130a52 | 2011-10-19 10:58:56 +0300 | [diff] [blame] | 1043 | # could cause incorrect parsing. |
Facundo Batista | 72dc1ea | 2008-08-16 14:44:32 +0000 | [diff] [blame] | 1044 | h = urllib.request.AbstractHTTPHandler() |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1045 | h.parent = MockOpener() |
Facundo Batista | 72dc1ea | 2008-08-16 14:44:32 +0000 | [diff] [blame] | 1046 | |
Senthil Kumaran | 7bc0d87 | 2010-12-19 10:49:52 +0000 | [diff] [blame] | 1047 | data = b"" |
Facundo Batista | 72dc1ea | 2008-08-16 14:44:32 +0000 | [diff] [blame] | 1048 | ds_urls = [ |
| 1049 | "http://example.com/foo/bar/baz.html", |
| 1050 | "http://example.com//foo/bar/baz.html", |
| 1051 | "http://example.com/foo//bar/baz.html", |
| 1052 | "http://example.com/foo/bar//baz.html" |
| 1053 | ] |
| 1054 | |
| 1055 | for ds_url in ds_urls: |
| 1056 | ds_req = Request(ds_url, data) |
| 1057 | |
| 1058 | # Check whether host is determined correctly if there is no proxy |
| 1059 | np_ds_req = h.do_request_(ds_req) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1060 | self.assertEqual(np_ds_req.unredirected_hdrs["Host"], "example.com") |
Facundo Batista | 72dc1ea | 2008-08-16 14:44:32 +0000 | [diff] [blame] | 1061 | |
| 1062 | # Check whether host is determined correctly if there is a proxy |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1063 | ds_req.set_proxy("someproxy:3128", None) |
Facundo Batista | 72dc1ea | 2008-08-16 14:44:32 +0000 | [diff] [blame] | 1064 | p_ds_req = h.do_request_(ds_req) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1065 | self.assertEqual(p_ds_req.unredirected_hdrs["Host"], "example.com") |
Facundo Batista | 72dc1ea | 2008-08-16 14:44:32 +0000 | [diff] [blame] | 1066 | |
Senthil Kumaran | 5238092 | 2013-04-25 05:45:48 -0700 | [diff] [blame] | 1067 | def test_full_url_setter(self): |
| 1068 | # Checks to ensure that components are set correctly after setting the |
| 1069 | # full_url of a Request object |
| 1070 | |
| 1071 | urls = [ |
| 1072 | 'http://example.com?foo=bar#baz', |
| 1073 | 'http://example.com?foo=bar&spam=eggs#bash', |
| 1074 | 'http://example.com', |
| 1075 | ] |
| 1076 | |
| 1077 | # testing a reusable request instance, but the url parameter is |
| 1078 | # required, so just use a dummy one to instantiate |
| 1079 | r = Request('http://example.com') |
| 1080 | for url in urls: |
| 1081 | r.full_url = url |
Senthil Kumaran | 8307075 | 2013-05-24 09:14:12 -0700 | [diff] [blame] | 1082 | parsed = urlparse(url) |
| 1083 | |
Senthil Kumaran | 5238092 | 2013-04-25 05:45:48 -0700 | [diff] [blame] | 1084 | self.assertEqual(r.get_full_url(), url) |
Senthil Kumaran | 8307075 | 2013-05-24 09:14:12 -0700 | [diff] [blame] | 1085 | # full_url setter uses splittag to split into components. |
| 1086 | # splittag sets the fragment as None while urlparse sets it to '' |
| 1087 | self.assertEqual(r.fragment or '', parsed.fragment) |
| 1088 | self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) |
Senthil Kumaran | 5238092 | 2013-04-25 05:45:48 -0700 | [diff] [blame] | 1089 | |
| 1090 | def test_full_url_deleter(self): |
| 1091 | r = Request('http://www.example.com') |
| 1092 | del r.full_url |
| 1093 | self.assertIsNone(r.full_url) |
| 1094 | self.assertIsNone(r.fragment) |
| 1095 | self.assertEqual(r.selector, '') |
| 1096 | |
Senthil Kumaran | c295862 | 2010-11-22 04:48:26 +0000 | [diff] [blame] | 1097 | def test_fixpath_in_weirdurls(self): |
| 1098 | # Issue4493: urllib2 to supply '/' when to urls where path does not |
| 1099 | # start with'/' |
| 1100 | |
| 1101 | h = urllib.request.AbstractHTTPHandler() |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1102 | h.parent = MockOpener() |
Senthil Kumaran | c295862 | 2010-11-22 04:48:26 +0000 | [diff] [blame] | 1103 | |
| 1104 | weird_url = 'http://www.python.org?getspam' |
| 1105 | req = Request(weird_url) |
| 1106 | newreq = h.do_request_(req) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1107 | self.assertEqual(newreq.host, 'www.python.org') |
| 1108 | self.assertEqual(newreq.selector, '/?getspam') |
Senthil Kumaran | c295862 | 2010-11-22 04:48:26 +0000 | [diff] [blame] | 1109 | |
| 1110 | url_without_path = 'http://www.python.org' |
| 1111 | req = Request(url_without_path) |
| 1112 | newreq = h.do_request_(req) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1113 | self.assertEqual(newreq.host, 'www.python.org') |
| 1114 | self.assertEqual(newreq.selector, '') |
Facundo Batista | 72dc1ea | 2008-08-16 14:44:32 +0000 | [diff] [blame] | 1115 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1116 | def test_errors(self): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1117 | h = urllib.request.HTTPErrorProcessor() |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1118 | o = h.parent = MockOpener() |
| 1119 | |
| 1120 | url = "http://example.com/" |
| 1121 | req = Request(url) |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 1122 | # all 2xx are passed through |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1123 | r = MockResponse(200, "OK", {}, "", url) |
| 1124 | newr = h.http_response(req, r) |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 1125 | self.assertIs(r, newr) |
| 1126 | self.assertFalse(hasattr(o, "proto")) # o.error not called |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 1127 | r = MockResponse(202, "Accepted", {}, "", url) |
| 1128 | newr = h.http_response(req, r) |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 1129 | self.assertIs(r, newr) |
| 1130 | self.assertFalse(hasattr(o, "proto")) # o.error not called |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 1131 | r = MockResponse(206, "Partial content", {}, "", url) |
| 1132 | newr = h.http_response(req, r) |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 1133 | self.assertIs(r, newr) |
| 1134 | self.assertFalse(hasattr(o, "proto")) # o.error not called |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1135 | # anything else calls o.error (and MockOpener returns None, here) |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 1136 | r = MockResponse(502, "Bad gateway", {}, "", url) |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 1137 | self.assertIsNone(h.http_response(req, r)) |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 1138 | self.assertEqual(o.proto, "http") # o.error called |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 1139 | self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1140 | |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1141 | def test_cookies(self): |
| 1142 | cj = MockCookieJar() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1143 | h = urllib.request.HTTPCookieProcessor(cj) |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1144 | h.parent = MockOpener() |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1145 | |
| 1146 | req = Request("http://example.com/") |
| 1147 | r = MockResponse(200, "OK", {}, "") |
| 1148 | newreq = h.http_request(req) |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 1149 | self.assertIs(cj.ach_req, req) |
| 1150 | self.assertIs(cj.ach_req, newreq) |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1151 | self.assertEqual(req.origin_req_host, "example.com") |
| 1152 | self.assertFalse(req.unverifiable) |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1153 | newr = h.http_response(req, r) |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 1154 | self.assertIs(cj.ec_req, req) |
| 1155 | self.assertIs(cj.ec_r, r) |
| 1156 | self.assertIs(r, newr) |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1157 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1158 | def test_redirect(self): |
| 1159 | from_url = "http://example.com/a.html" |
| 1160 | to_url = "http://example.com/b.html" |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1161 | h = urllib.request.HTTPRedirectHandler() |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1162 | o = h.parent = MockOpener() |
| 1163 | |
| 1164 | # ordinary redirect behaviour |
| 1165 | for code in 301, 302, 303, 307: |
| 1166 | for data in None, "blah\nblah\n": |
| 1167 | method = getattr(h, "http_error_%s" % code) |
| 1168 | req = Request(from_url, data) |
Senthil Kumaran | fb8cc2f | 2009-07-19 02:44:19 +0000 | [diff] [blame] | 1169 | req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1170 | req.add_header("Nonsense", "viking=withhold") |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 1171 | if data is not None: |
| 1172 | req.add_header("Content-Length", str(len(data))) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1173 | req.add_unredirected_header("Spam", "spam") |
| 1174 | try: |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1175 | method(req, MockFile(), code, "Blah", |
| 1176 | MockHeaders({"location": to_url})) |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1177 | except urllib.error.HTTPError: |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1178 | # 307 in response to POST requires user OK |
Serhiy Storchaka | 25d8aea | 2014-02-08 14:50:08 +0200 | [diff] [blame] | 1179 | self.assertEqual(code, 307) |
| 1180 | self.assertIsNotNone(data) |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 1181 | self.assertEqual(o.req.get_full_url(), to_url) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1182 | try: |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 1183 | self.assertEqual(o.req.get_method(), "GET") |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1184 | except AttributeError: |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1185 | self.assertFalse(o.req.data) |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 1186 | |
| 1187 | # now it's a GET, there should not be headers regarding content |
| 1188 | # (possibly dragged from before being a POST) |
| 1189 | headers = [x.lower() for x in o.req.headers] |
Benjamin Peterson | 577473f | 2010-01-19 00:09:57 +0000 | [diff] [blame] | 1190 | self.assertNotIn("content-length", headers) |
| 1191 | self.assertNotIn("content-type", headers) |
Christian Heimes | 77c02eb | 2008-02-09 02:18:51 +0000 | [diff] [blame] | 1192 | |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 1193 | self.assertEqual(o.req.headers["Nonsense"], |
| 1194 | "viking=withhold") |
Benjamin Peterson | 577473f | 2010-01-19 00:09:57 +0000 | [diff] [blame] | 1195 | self.assertNotIn("Spam", o.req.headers) |
| 1196 | self.assertNotIn("Spam", o.req.unredirected_hdrs) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1197 | |
| 1198 | # loop detection |
| 1199 | req = Request(from_url) |
Senthil Kumaran | fb8cc2f | 2009-07-19 02:44:19 +0000 | [diff] [blame] | 1200 | req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1201 | |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1202 | def redirect(h, req, url=to_url): |
| 1203 | h.http_error_302(req, MockFile(), 302, "Blah", |
| 1204 | MockHeaders({"location": url})) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1205 | # Note that the *original* request shares the same record of |
| 1206 | # redirections with the sub-requests caused by the redirections. |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1207 | |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1208 | # detect infinite loop redirect of a URL to itself |
| 1209 | req = Request(from_url, origin_req_host="example.com") |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1210 | count = 0 |
Senthil Kumaran | fb8cc2f | 2009-07-19 02:44:19 +0000 | [diff] [blame] | 1211 | req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1212 | try: |
| 1213 | while 1: |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1214 | redirect(h, req, "http://example.com/") |
| 1215 | count = count + 1 |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1216 | except urllib.error.HTTPError: |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1217 | # don't stop until max_repeats, because cookies may introduce state |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1218 | self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats) |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1219 | |
| 1220 | # detect endless non-repeating chain of redirects |
| 1221 | req = Request(from_url, origin_req_host="example.com") |
| 1222 | count = 0 |
Senthil Kumaran | fb8cc2f | 2009-07-19 02:44:19 +0000 | [diff] [blame] | 1223 | req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1224 | try: |
| 1225 | while 1: |
| 1226 | redirect(h, req, "http://example.com/%d" % count) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1227 | count = count + 1 |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1228 | except urllib.error.HTTPError: |
Jeremy Hylton | df38ea9 | 2003-12-17 20:42:38 +0000 | [diff] [blame] | 1229 | self.assertEqual(count, |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1230 | urllib.request.HTTPRedirectHandler.max_redirections) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1231 | |
guido@google.com | a119df9 | 2011-03-29 11:41:02 -0700 | [diff] [blame] | 1232 | def test_invalid_redirect(self): |
| 1233 | from_url = "http://example.com/a.html" |
| 1234 | valid_schemes = ['http','https','ftp'] |
| 1235 | invalid_schemes = ['file','imap','ldap'] |
| 1236 | schemeless_url = "example.com/b.html" |
| 1237 | h = urllib.request.HTTPRedirectHandler() |
| 1238 | o = h.parent = MockOpener() |
| 1239 | req = Request(from_url) |
| 1240 | req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
| 1241 | |
| 1242 | for scheme in invalid_schemes: |
| 1243 | invalid_url = scheme + '://' + schemeless_url |
| 1244 | self.assertRaises(urllib.error.HTTPError, h.http_error_302, |
| 1245 | req, MockFile(), 302, "Security Loophole", |
| 1246 | MockHeaders({"location": invalid_url})) |
| 1247 | |
| 1248 | for scheme in valid_schemes: |
| 1249 | valid_url = scheme + '://' + schemeless_url |
| 1250 | h.http_error_302(req, MockFile(), 302, "That's fine", |
| 1251 | MockHeaders({"location": valid_url})) |
| 1252 | self.assertEqual(o.req.get_full_url(), valid_url) |
| 1253 | |
Senthil Kumaran | 6497aa3 | 2012-01-04 13:46:59 +0800 | [diff] [blame] | 1254 | def test_relative_redirect(self): |
| 1255 | from_url = "http://example.com/a.html" |
| 1256 | relative_url = "/b.html" |
| 1257 | h = urllib.request.HTTPRedirectHandler() |
| 1258 | o = h.parent = MockOpener() |
| 1259 | req = Request(from_url) |
| 1260 | req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
| 1261 | |
| 1262 | valid_url = urllib.parse.urljoin(from_url,relative_url) |
| 1263 | h.http_error_302(req, MockFile(), 302, "That's fine", |
| 1264 | MockHeaders({"location": valid_url})) |
| 1265 | self.assertEqual(o.req.get_full_url(), valid_url) |
| 1266 | |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1267 | def test_cookie_redirect(self): |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1268 | # cookies shouldn't leak into redirected requests |
Georg Brandl | 2442015 | 2008-05-26 16:32:26 +0000 | [diff] [blame] | 1269 | from http.cookiejar import CookieJar |
| 1270 | from test.test_http_cookiejar import interact_netscape |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1271 | |
| 1272 | cj = CookieJar() |
| 1273 | interact_netscape(cj, "http://www.example.com/", "spam=eggs") |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1274 | hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n") |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1275 | hdeh = urllib.request.HTTPDefaultErrorHandler() |
| 1276 | hrh = urllib.request.HTTPRedirectHandler() |
| 1277 | cp = urllib.request.HTTPCookieProcessor(cj) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1278 | o = build_test_opener(hh, hdeh, hrh, cp) |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1279 | o.open("http://www.example.com/") |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 1280 | self.assertFalse(hh.req.has_header("Cookie")) |
Martin v. Löwis | 2a6ba90 | 2004-05-31 18:22:40 +0000 | [diff] [blame] | 1281 | |
Senthil Kumaran | 2643041 | 2011-04-13 07:01:19 +0800 | [diff] [blame] | 1282 | def test_redirect_fragment(self): |
| 1283 | redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' |
| 1284 | hh = MockHTTPHandler(302, 'Location: ' + redirected_url) |
| 1285 | hdeh = urllib.request.HTTPDefaultErrorHandler() |
| 1286 | hrh = urllib.request.HTTPRedirectHandler() |
| 1287 | o = build_test_opener(hh, hdeh, hrh) |
| 1288 | fp = o.open('http://www.example.com') |
| 1289 | self.assertEqual(fp.geturl(), redirected_url.strip()) |
| 1290 | |
Martin Panter | ce6e068 | 2016-05-16 01:07:13 +0000 | [diff] [blame] | 1291 | def test_redirect_no_path(self): |
| 1292 | # Issue 14132: Relative redirect strips original path |
| 1293 | real_class = http.client.HTTPConnection |
| 1294 | response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n" |
| 1295 | http.client.HTTPConnection = test_urllib.fakehttp(response1) |
| 1296 | self.addCleanup(setattr, http.client, "HTTPConnection", real_class) |
| 1297 | urls = iter(("/path", "/path?query")) |
| 1298 | def request(conn, method, url, *pos, **kw): |
| 1299 | self.assertEqual(url, next(urls)) |
| 1300 | real_class.request(conn, method, url, *pos, **kw) |
| 1301 | # Change response for subsequent connection |
| 1302 | conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!" |
| 1303 | http.client.HTTPConnection.request = request |
| 1304 | fp = urllib.request.urlopen("http://python.org/path") |
| 1305 | self.assertEqual(fp.geturl(), "http://python.org/path?query") |
| 1306 | |
Martin Panter | e6f0609 | 2016-05-16 01:14:20 +0000 | [diff] [blame] | 1307 | def test_redirect_encoding(self): |
| 1308 | # Some characters in the redirect target may need special handling, |
| 1309 | # but most ASCII characters should be treated as already encoded |
| 1310 | class Handler(urllib.request.HTTPHandler): |
| 1311 | def http_open(self, req): |
| 1312 | result = self.do_open(self.connection, req) |
| 1313 | self.last_buf = self.connection.buf |
| 1314 | # Set up a normal response for the next request |
| 1315 | self.connection = test_urllib.fakehttp( |
| 1316 | b'HTTP/1.1 200 OK\r\n' |
| 1317 | b'Content-Length: 3\r\n' |
| 1318 | b'\r\n' |
| 1319 | b'123' |
| 1320 | ) |
| 1321 | return result |
| 1322 | handler = Handler() |
| 1323 | opener = urllib.request.build_opener(handler) |
| 1324 | tests = ( |
| 1325 | (b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'), |
| 1326 | (b'/spaced%20path/', b'/spaced%20path/'), |
| 1327 | (b'/spaced path/', b'/spaced%20path/'), |
| 1328 | (b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'), |
| 1329 | ) |
| 1330 | for [location, result] in tests: |
| 1331 | with self.subTest(repr(location)): |
| 1332 | handler.connection = test_urllib.fakehttp( |
| 1333 | b'HTTP/1.1 302 Redirect\r\n' |
| 1334 | b'Location: ' + location + b'\r\n' |
| 1335 | b'\r\n' |
| 1336 | ) |
| 1337 | response = opener.open('http://example.com/') |
| 1338 | expected = b'GET ' + result + b' ' |
| 1339 | request = handler.last_buf |
| 1340 | self.assertTrue(request.startswith(expected), repr(request)) |
| 1341 | |
Thomas Wouters | 49fd7fa | 2006-04-21 10:40:58 +0000 | [diff] [blame] | 1342 | def test_proxy(self): |
| 1343 | o = OpenerDirector() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1344 | ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) |
Thomas Wouters | 49fd7fa | 2006-04-21 10:40:58 +0000 | [diff] [blame] | 1345 | o.add_handler(ph) |
| 1346 | meth_spec = [ |
| 1347 | [("http_open", "return response")] |
| 1348 | ] |
| 1349 | handlers = add_ordered_mock_handlers(o, meth_spec) |
| 1350 | |
| 1351 | req = Request("http://acme.example.com/") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1352 | self.assertEqual(req.host, "acme.example.com") |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1353 | o.open(req) |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1354 | self.assertEqual(req.host, "proxy.example.com:3128") |
Thomas Wouters | 49fd7fa | 2006-04-21 10:40:58 +0000 | [diff] [blame] | 1355 | |
| 1356 | self.assertEqual([(handlers[0], "http_open")], |
| 1357 | [tup[0:2] for tup in o.calls]) |
| 1358 | |
Senthil Kumaran | 7bb0497 | 2009-10-11 04:58:55 +0000 | [diff] [blame] | 1359 | def test_proxy_no_proxy(self): |
| 1360 | os.environ['no_proxy'] = 'python.org' |
| 1361 | o = OpenerDirector() |
| 1362 | ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) |
| 1363 | o.add_handler(ph) |
| 1364 | req = Request("http://www.perl.org/") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1365 | self.assertEqual(req.host, "www.perl.org") |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1366 | o.open(req) |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1367 | self.assertEqual(req.host, "proxy.example.com") |
Senthil Kumaran | 7bb0497 | 2009-10-11 04:58:55 +0000 | [diff] [blame] | 1368 | req = Request("http://www.python.org") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1369 | self.assertEqual(req.host, "www.python.org") |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1370 | o.open(req) |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1371 | self.assertEqual(req.host, "www.python.org") |
Senthil Kumaran | 7bb0497 | 2009-10-11 04:58:55 +0000 | [diff] [blame] | 1372 | del os.environ['no_proxy'] |
| 1373 | |
Ronald Oussoren | e72e161 | 2011-03-14 18:15:25 -0400 | [diff] [blame] | 1374 | def test_proxy_no_proxy_all(self): |
| 1375 | os.environ['no_proxy'] = '*' |
| 1376 | o = OpenerDirector() |
| 1377 | ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) |
| 1378 | o.add_handler(ph) |
| 1379 | req = Request("http://www.python.org") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1380 | self.assertEqual(req.host, "www.python.org") |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1381 | o.open(req) |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1382 | self.assertEqual(req.host, "www.python.org") |
Ronald Oussoren | e72e161 | 2011-03-14 18:15:25 -0400 | [diff] [blame] | 1383 | del os.environ['no_proxy'] |
| 1384 | |
Senthil Kumaran | 97f0c6b | 2009-07-25 04:24:38 +0000 | [diff] [blame] | 1385 | def test_proxy_https(self): |
| 1386 | o = OpenerDirector() |
| 1387 | ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")) |
| 1388 | o.add_handler(ph) |
| 1389 | meth_spec = [ |
| 1390 | [("https_open", "return response")] |
| 1391 | ] |
| 1392 | handlers = add_ordered_mock_handlers(o, meth_spec) |
| 1393 | |
| 1394 | req = Request("https://www.example.com/") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1395 | self.assertEqual(req.host, "www.example.com") |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1396 | o.open(req) |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1397 | self.assertEqual(req.host, "proxy.example.com:3128") |
Senthil Kumaran | 97f0c6b | 2009-07-25 04:24:38 +0000 | [diff] [blame] | 1398 | self.assertEqual([(handlers[0], "https_open")], |
| 1399 | [tup[0:2] for tup in o.calls]) |
| 1400 | |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 1401 | def test_proxy_https_proxy_authorization(self): |
| 1402 | o = OpenerDirector() |
| 1403 | ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128')) |
| 1404 | o.add_handler(ph) |
| 1405 | https_handler = MockHTTPSHandler() |
| 1406 | o.add_handler(https_handler) |
| 1407 | req = Request("https://www.example.com/") |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1408 | req.add_header("Proxy-Authorization", "FooBar") |
| 1409 | req.add_header("User-Agent", "Grail") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1410 | self.assertEqual(req.host, "www.example.com") |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 1411 | self.assertIsNone(req._tunnel_host) |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1412 | o.open(req) |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 1413 | # Verify Proxy-Authorization gets tunneled to request. |
| 1414 | # httpsconn req_headers do not have the Proxy-Authorization header but |
| 1415 | # the req will have. |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1416 | self.assertNotIn(("Proxy-Authorization", "FooBar"), |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 1417 | https_handler.httpconn.req_headers) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1418 | self.assertIn(("User-Agent", "Grail"), |
Ezio Melotti | b58e0bd | 2010-01-23 15:40:09 +0000 | [diff] [blame] | 1419 | https_handler.httpconn.req_headers) |
Senthil Kumaran | 47fff87 | 2009-12-20 07:10:31 +0000 | [diff] [blame] | 1420 | self.assertIsNotNone(req._tunnel_host) |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1421 | self.assertEqual(req.host, "proxy.example.com:3128") |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1422 | self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") |
Senthil Kumaran | 97f0c6b | 2009-07-25 04:24:38 +0000 | [diff] [blame] | 1423 | |
Senthil Kumaran | 4de00a2 | 2011-05-11 21:17:57 +0800 | [diff] [blame] | 1424 | @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") |
Ronald Oussoren | e72e161 | 2011-03-14 18:15:25 -0400 | [diff] [blame] | 1425 | def test_osx_proxy_bypass(self): |
| 1426 | bypass = { |
| 1427 | 'exclude_simple': False, |
| 1428 | 'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10', |
| 1429 | '10.0/16'] |
| 1430 | } |
| 1431 | # Check hosts that should trigger the proxy bypass |
| 1432 | for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1', |
| 1433 | '10.0.0.1'): |
| 1434 | self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass), |
| 1435 | 'expected bypass of %s to be True' % host) |
| 1436 | # Check hosts that should not trigger the proxy bypass |
R David Murray | fdbe918 | 2014-03-15 12:00:14 -0400 | [diff] [blame] | 1437 | for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1', |
| 1438 | 'notinbypass'): |
Ronald Oussoren | e72e161 | 2011-03-14 18:15:25 -0400 | [diff] [blame] | 1439 | self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass), |
| 1440 | 'expected bypass of %s to be False' % host) |
| 1441 | |
| 1442 | # Check the exclude_simple flag |
| 1443 | bypass = {'exclude_simple': True, 'exceptions': []} |
| 1444 | self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass)) |
| 1445 | |
Christian Heimes | 4fbc72b | 2008-03-22 00:47:35 +0000 | [diff] [blame] | 1446 | def test_basic_auth(self, quote_char='"'): |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1447 | opener = OpenerDirector() |
| 1448 | password_manager = MockPasswordManager() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1449 | auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1450 | realm = "ACME Widget Store" |
| 1451 | http_handler = MockHTTPHandler( |
Christian Heimes | 4fbc72b | 2008-03-22 00:47:35 +0000 | [diff] [blame] | 1452 | 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' % |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1453 | (quote_char, realm, quote_char)) |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1454 | opener.add_handler(auth_handler) |
| 1455 | opener.add_handler(http_handler) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1456 | self._test_basic_auth(opener, auth_handler, "Authorization", |
| 1457 | realm, http_handler, password_manager, |
| 1458 | "http://acme.example.com/protected", |
| 1459 | "http://acme.example.com/protected", |
| 1460 | ) |
| 1461 | |
Christian Heimes | 4fbc72b | 2008-03-22 00:47:35 +0000 | [diff] [blame] | 1462 | def test_basic_auth_with_single_quoted_realm(self): |
| 1463 | self.test_basic_auth(quote_char="'") |
| 1464 | |
Senthil Kumaran | 34f3fcc | 2012-05-15 22:30:25 +0800 | [diff] [blame] | 1465 | def test_basic_auth_with_unquoted_realm(self): |
| 1466 | opener = OpenerDirector() |
| 1467 | password_manager = MockPasswordManager() |
| 1468 | auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) |
| 1469 | realm = "ACME Widget Store" |
| 1470 | http_handler = MockHTTPHandler( |
| 1471 | 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm) |
| 1472 | opener.add_handler(auth_handler) |
| 1473 | opener.add_handler(http_handler) |
Senthil Kumaran | 0ea91cb | 2012-05-15 23:59:42 +0800 | [diff] [blame] | 1474 | with self.assertWarns(UserWarning): |
| 1475 | self._test_basic_auth(opener, auth_handler, "Authorization", |
| 1476 | realm, http_handler, password_manager, |
| 1477 | "http://acme.example.com/protected", |
| 1478 | "http://acme.example.com/protected", |
| 1479 | ) |
Senthil Kumaran | 34f3fcc | 2012-05-15 22:30:25 +0800 | [diff] [blame] | 1480 | |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1481 | def test_proxy_basic_auth(self): |
| 1482 | opener = OpenerDirector() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1483 | ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1484 | opener.add_handler(ph) |
| 1485 | password_manager = MockPasswordManager() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1486 | auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1487 | realm = "ACME Networks" |
| 1488 | http_handler = MockHTTPHandler( |
| 1489 | 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1490 | opener.add_handler(auth_handler) |
| 1491 | opener.add_handler(http_handler) |
Thomas Wouters | 00ee7ba | 2006-08-21 19:07:27 +0000 | [diff] [blame] | 1492 | self._test_basic_auth(opener, auth_handler, "Proxy-authorization", |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1493 | realm, http_handler, password_manager, |
| 1494 | "http://acme.example.com:3128/protected", |
| 1495 | "proxy.example.com:3128", |
| 1496 | ) |
| 1497 | |
| 1498 | def test_basic_and_digest_auth_handlers(self): |
Andrew Svetlov | 7bd61cb | 2012-12-19 22:49:25 +0200 | [diff] [blame] | 1499 | # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1500 | # response (http://python.org/sf/1479302), where it should instead |
| 1501 | # return None to allow another handler (especially |
| 1502 | # HTTPBasicAuthHandler) to handle the response. |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1503 | |
| 1504 | # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must |
| 1505 | # try digest first (since it's the strongest auth scheme), so we record |
| 1506 | # order of calls here to check digest comes first: |
| 1507 | class RecordingOpenerDirector(OpenerDirector): |
| 1508 | def __init__(self): |
| 1509 | OpenerDirector.__init__(self) |
| 1510 | self.recorded = [] |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1511 | |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1512 | def record(self, info): |
| 1513 | self.recorded.append(info) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1514 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1515 | class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler): |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1516 | def http_error_401(self, *args, **kwds): |
| 1517 | self.parent.record("digest") |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1518 | urllib.request.HTTPDigestAuthHandler.http_error_401(self, |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1519 | *args, **kwds) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1520 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1521 | class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler): |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1522 | def http_error_401(self, *args, **kwds): |
| 1523 | self.parent.record("basic") |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1524 | urllib.request.HTTPBasicAuthHandler.http_error_401(self, |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1525 | *args, **kwds) |
| 1526 | |
| 1527 | opener = RecordingOpenerDirector() |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1528 | password_manager = MockPasswordManager() |
| 1529 | digest_handler = TestDigestAuthHandler(password_manager) |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1530 | basic_handler = TestBasicAuthHandler(password_manager) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1531 | realm = "ACME Networks" |
| 1532 | http_handler = MockHTTPHandler( |
| 1533 | 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1534 | opener.add_handler(basic_handler) |
| 1535 | opener.add_handler(digest_handler) |
| 1536 | opener.add_handler(http_handler) |
| 1537 | |
| 1538 | # check basic auth isn't blocked by digest handler failing |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1539 | self._test_basic_auth(opener, basic_handler, "Authorization", |
| 1540 | realm, http_handler, password_manager, |
| 1541 | "http://acme.example.com/protected", |
| 1542 | "http://acme.example.com/protected", |
| 1543 | ) |
Thomas Wouters | 4d70c3d | 2006-06-08 14:42:34 +0000 | [diff] [blame] | 1544 | # check digest was tried before basic (twice, because |
| 1545 | # _test_basic_auth called .open() twice) |
| 1546 | self.assertEqual(opener.recorded, ["digest", "basic"]*2) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1547 | |
Senthil Kumaran | 4de00a2 | 2011-05-11 21:17:57 +0800 | [diff] [blame] | 1548 | def test_unsupported_auth_digest_handler(self): |
| 1549 | opener = OpenerDirector() |
| 1550 | # While using DigestAuthHandler |
| 1551 | digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) |
| 1552 | http_handler = MockHTTPHandler( |
| 1553 | 401, 'WWW-Authenticate: Kerberos\r\n\r\n') |
| 1554 | opener.add_handler(digest_auth_handler) |
| 1555 | opener.add_handler(http_handler) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1556 | self.assertRaises(ValueError, opener.open, "http://www.example.com") |
Senthil Kumaran | 4de00a2 | 2011-05-11 21:17:57 +0800 | [diff] [blame] | 1557 | |
| 1558 | def test_unsupported_auth_basic_handler(self): |
| 1559 | # While using BasicAuthHandler |
| 1560 | opener = OpenerDirector() |
| 1561 | basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) |
| 1562 | http_handler = MockHTTPHandler( |
| 1563 | 401, 'WWW-Authenticate: NTLM\r\n\r\n') |
| 1564 | opener.add_handler(basic_auth_handler) |
| 1565 | opener.add_handler(http_handler) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1566 | self.assertRaises(ValueError, opener.open, "http://www.example.com") |
Senthil Kumaran | 4de00a2 | 2011-05-11 21:17:57 +0800 | [diff] [blame] | 1567 | |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1568 | def _test_basic_auth(self, opener, auth_handler, auth_header, |
| 1569 | realm, http_handler, password_manager, |
| 1570 | request_url, protected_url): |
Christian Heimes | 05e8be1 | 2008-02-23 18:30:17 +0000 | [diff] [blame] | 1571 | import base64 |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1572 | user, password = "wile", "coyote" |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1573 | |
| 1574 | # .add_password() fed through to password manager |
| 1575 | auth_handler.add_password(realm, request_url, user, password) |
| 1576 | self.assertEqual(realm, password_manager.realm) |
| 1577 | self.assertEqual(request_url, password_manager.url) |
| 1578 | self.assertEqual(user, password_manager.user) |
| 1579 | self.assertEqual(password, password_manager.password) |
| 1580 | |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1581 | opener.open(request_url) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1582 | |
| 1583 | # should have asked the password manager for the username/password |
| 1584 | self.assertEqual(password_manager.target_realm, realm) |
| 1585 | self.assertEqual(password_manager.target_url, protected_url) |
| 1586 | |
| 1587 | # expect one request without authorization, then one with |
| 1588 | self.assertEqual(len(http_handler.requests), 2) |
| 1589 | self.assertFalse(http_handler.requests[0].has_header(auth_header)) |
Guido van Rossum | 98b349f | 2007-08-27 21:47:52 +0000 | [diff] [blame] | 1590 | userpass = bytes('%s:%s' % (user, password), "ascii") |
Guido van Rossum | 98297ee | 2007-11-06 21:34:58 +0000 | [diff] [blame] | 1591 | auth_hdr_value = ('Basic ' + |
Georg Brandl | 706824f | 2009-06-04 09:42:55 +0000 | [diff] [blame] | 1592 | base64.encodebytes(userpass).strip().decode()) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1593 | self.assertEqual(http_handler.requests[1].get_header(auth_header), |
| 1594 | auth_hdr_value) |
Senthil Kumaran | ca2fc9e | 2010-02-24 16:53:16 +0000 | [diff] [blame] | 1595 | self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header], |
| 1596 | auth_hdr_value) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1597 | # if the password manager can't find a password, the handler won't |
| 1598 | # handle the HTTP auth error |
| 1599 | password_manager.user = password_manager.password = None |
| 1600 | http_handler.reset() |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1601 | opener.open(request_url) |
Thomas Wouters | 477c8d5 | 2006-05-27 19:21:47 +0000 | [diff] [blame] | 1602 | self.assertEqual(len(http_handler.requests), 1) |
| 1603 | self.assertFalse(http_handler.requests[0].has_header(auth_header)) |
| 1604 | |
R David Murray | 4c7f995 | 2015-04-16 16:36:18 -0400 | [diff] [blame] | 1605 | def test_basic_prior_auth_auto_send(self): |
| 1606 | # Assume already authenticated if is_authenticated=True |
| 1607 | # for APIs like Github that don't return 401 |
| 1608 | |
| 1609 | user, password = "wile", "coyote" |
| 1610 | request_url = "http://acme.example.com/protected" |
| 1611 | |
| 1612 | http_handler = MockHTTPHandlerCheckAuth(200) |
| 1613 | |
| 1614 | pwd_manager = HTTPPasswordMgrWithPriorAuth() |
| 1615 | auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) |
| 1616 | auth_prior_handler.add_password( |
| 1617 | None, request_url, user, password, is_authenticated=True) |
| 1618 | |
| 1619 | is_auth = pwd_manager.is_authenticated(request_url) |
| 1620 | self.assertTrue(is_auth) |
| 1621 | |
| 1622 | opener = OpenerDirector() |
| 1623 | opener.add_handler(auth_prior_handler) |
| 1624 | opener.add_handler(http_handler) |
| 1625 | |
| 1626 | opener.open(request_url) |
| 1627 | |
| 1628 | # expect request to be sent with auth header |
| 1629 | self.assertTrue(http_handler.has_auth_header) |
| 1630 | |
| 1631 | def test_basic_prior_auth_send_after_first_success(self): |
| 1632 | # Auto send auth header after authentication is successful once |
| 1633 | |
| 1634 | user, password = 'wile', 'coyote' |
| 1635 | request_url = 'http://acme.example.com/protected' |
| 1636 | realm = 'ACME' |
| 1637 | |
| 1638 | pwd_manager = HTTPPasswordMgrWithPriorAuth() |
| 1639 | auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) |
| 1640 | auth_prior_handler.add_password(realm, request_url, user, password) |
| 1641 | |
| 1642 | is_auth = pwd_manager.is_authenticated(request_url) |
| 1643 | self.assertFalse(is_auth) |
| 1644 | |
| 1645 | opener = OpenerDirector() |
| 1646 | opener.add_handler(auth_prior_handler) |
| 1647 | |
| 1648 | http_handler = MockHTTPHandler( |
| 1649 | 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None) |
| 1650 | opener.add_handler(http_handler) |
| 1651 | |
| 1652 | opener.open(request_url) |
| 1653 | |
| 1654 | is_auth = pwd_manager.is_authenticated(request_url) |
| 1655 | self.assertTrue(is_auth) |
| 1656 | |
| 1657 | http_handler = MockHTTPHandlerCheckAuth(200) |
| 1658 | self.assertFalse(http_handler.has_auth_header) |
| 1659 | |
| 1660 | opener = OpenerDirector() |
| 1661 | opener.add_handler(auth_prior_handler) |
| 1662 | opener.add_handler(http_handler) |
| 1663 | |
| 1664 | # After getting 200 from MockHTTPHandler |
| 1665 | # Next request sends header in the first request |
| 1666 | opener.open(request_url) |
| 1667 | |
| 1668 | # expect request to be sent with auth header |
| 1669 | self.assertTrue(http_handler.has_auth_header) |
| 1670 | |
Serhiy Storchaka | f54c350 | 2014-09-06 21:41:39 +0300 | [diff] [blame] | 1671 | def test_http_closed(self): |
| 1672 | """Test the connection is cleaned up when the response is closed""" |
| 1673 | for (transfer, data) in ( |
| 1674 | ("Connection: close", b"data"), |
| 1675 | ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"), |
| 1676 | ("Content-Length: 4", b"data"), |
| 1677 | ): |
| 1678 | header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer) |
| 1679 | conn = test_urllib.fakehttp(header.encode() + data) |
| 1680 | handler = urllib.request.AbstractHTTPHandler() |
| 1681 | req = Request("http://dummy/") |
| 1682 | req.timeout = None |
| 1683 | with handler.do_open(conn, req) as resp: |
| 1684 | resp.read() |
| 1685 | self.assertTrue(conn.fakesock.closed, |
| 1686 | "Connection not closed with {!r}".format(transfer)) |
| 1687 | |
| 1688 | def test_invalid_closed(self): |
| 1689 | """Test the connection is cleaned up after an invalid response""" |
| 1690 | conn = test_urllib.fakehttp(b"") |
| 1691 | handler = urllib.request.AbstractHTTPHandler() |
| 1692 | req = Request("http://dummy/") |
| 1693 | req.timeout = None |
| 1694 | with self.assertRaises(http.client.BadStatusLine): |
| 1695 | handler.do_open(conn, req) |
| 1696 | self.assertTrue(conn.fakesock.closed, "Connection not closed") |
| 1697 | |
Senthil Kumaran | 4de00a2 | 2011-05-11 21:17:57 +0800 | [diff] [blame] | 1698 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1699 | class MiscTests(unittest.TestCase): |
| 1700 | |
Senthil Kumaran | e9853da | 2013-03-19 12:07:43 -0700 | [diff] [blame] | 1701 | def opener_has_handler(self, opener, handler_class): |
| 1702 | self.assertTrue(any(h.__class__ == handler_class |
| 1703 | for h in opener.handlers)) |
| 1704 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1705 | def test_build_opener(self): |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1706 | class MyHTTPHandler(urllib.request.HTTPHandler): |
| 1707 | pass |
| 1708 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1709 | class FooHandler(urllib.request.BaseHandler): |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1710 | def foo_open(self): |
| 1711 | pass |
| 1712 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1713 | class BarHandler(urllib.request.BaseHandler): |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1714 | def bar_open(self): |
| 1715 | pass |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1716 | |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1717 | build_opener = urllib.request.build_opener |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1718 | |
| 1719 | o = build_opener(FooHandler, BarHandler) |
| 1720 | self.opener_has_handler(o, FooHandler) |
| 1721 | self.opener_has_handler(o, BarHandler) |
| 1722 | |
| 1723 | # can take a mix of classes and instances |
| 1724 | o = build_opener(FooHandler, BarHandler()) |
| 1725 | self.opener_has_handler(o, FooHandler) |
| 1726 | self.opener_has_handler(o, BarHandler) |
| 1727 | |
| 1728 | # subclasses of default handlers override default handlers |
| 1729 | o = build_opener(MyHTTPHandler) |
| 1730 | self.opener_has_handler(o, MyHTTPHandler) |
| 1731 | |
| 1732 | # a particular case of overriding: default handlers can be passed |
| 1733 | # in explicitly |
| 1734 | o = build_opener() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 1735 | self.opener_has_handler(o, urllib.request.HTTPHandler) |
| 1736 | o = build_opener(urllib.request.HTTPHandler) |
| 1737 | self.opener_has_handler(o, urllib.request.HTTPHandler) |
| 1738 | o = build_opener(urllib.request.HTTPHandler()) |
| 1739 | self.opener_has_handler(o, urllib.request.HTTPHandler) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1740 | |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 1741 | # Issue2670: multiple handlers sharing the same base class |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1742 | class MyOtherHTTPHandler(urllib.request.HTTPHandler): |
| 1743 | pass |
| 1744 | |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 1745 | o = build_opener(MyHTTPHandler, MyOtherHTTPHandler) |
| 1746 | self.opener_has_handler(o, MyHTTPHandler) |
| 1747 | self.opener_has_handler(o, MyOtherHTTPHandler) |
| 1748 | |
Brett Cannon | 80512de | 2013-01-25 22:27:21 -0500 | [diff] [blame] | 1749 | @unittest.skipUnless(support.is_resource_enabled('network'), |
| 1750 | 'test requires network access') |
Andrew Svetlov | bff98fe | 2012-11-27 23:06:19 +0200 | [diff] [blame] | 1751 | def test_issue16464(self): |
Berker Peksag | bcdfc6a | 2015-03-02 06:01:01 +0200 | [diff] [blame] | 1752 | with support.transient_internet("http://www.example.com/"): |
| 1753 | opener = urllib.request.build_opener() |
| 1754 | request = urllib.request.Request("http://www.example.com/") |
| 1755 | self.assertEqual(None, request.data) |
Andrew Svetlov | bff98fe | 2012-11-27 23:06:19 +0200 | [diff] [blame] | 1756 | |
Berker Peksag | bcdfc6a | 2015-03-02 06:01:01 +0200 | [diff] [blame] | 1757 | opener.open(request, "1".encode("us-ascii")) |
| 1758 | self.assertEqual(b"1", request.data) |
| 1759 | self.assertEqual("1", request.get_header("Content-length")) |
Andrew Svetlov | bff98fe | 2012-11-27 23:06:19 +0200 | [diff] [blame] | 1760 | |
Berker Peksag | bcdfc6a | 2015-03-02 06:01:01 +0200 | [diff] [blame] | 1761 | opener.open(request, "1234567890".encode("us-ascii")) |
| 1762 | self.assertEqual(b"1234567890", request.data) |
| 1763 | self.assertEqual("10", request.get_header("Content-length")) |
Andrew Svetlov | bff98fe | 2012-11-27 23:06:19 +0200 | [diff] [blame] | 1764 | |
Senthil Kumaran | e9853da | 2013-03-19 12:07:43 -0700 | [diff] [blame] | 1765 | def test_HTTPError_interface(self): |
| 1766 | """ |
| 1767 | Issue 13211 reveals that HTTPError didn't implement the URLError |
| 1768 | interface even though HTTPError is a subclass of URLError. |
Senthil Kumaran | e9853da | 2013-03-19 12:07:43 -0700 | [diff] [blame] | 1769 | """ |
Senthil Kumaran | fa1b02a | 2013-04-08 22:24:17 -0700 | [diff] [blame] | 1770 | msg = 'something bad happened' |
| 1771 | url = code = fp = None |
| 1772 | hdrs = 'Content-Length: 42' |
| 1773 | err = urllib.error.HTTPError(url, code, msg, hdrs, fp) |
| 1774 | self.assertTrue(hasattr(err, 'reason')) |
| 1775 | self.assertEqual(err.reason, 'something bad happened') |
| 1776 | self.assertTrue(hasattr(err, 'headers')) |
| 1777 | self.assertEqual(err.headers, 'Content-Length: 42') |
| 1778 | expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) |
| 1779 | self.assertEqual(str(err), expected_errmsg) |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1780 | expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) |
| 1781 | self.assertEqual(repr(err), expected_errmsg) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1782 | |
Senthil Kumaran | d8e24f1 | 2014-04-14 16:32:20 -0400 | [diff] [blame] | 1783 | def test_parse_proxy(self): |
| 1784 | parse_proxy_test_cases = [ |
| 1785 | ('proxy.example.com', |
| 1786 | (None, None, None, 'proxy.example.com')), |
| 1787 | ('proxy.example.com:3128', |
| 1788 | (None, None, None, 'proxy.example.com:3128')), |
| 1789 | ('proxy.example.com', (None, None, None, 'proxy.example.com')), |
| 1790 | ('proxy.example.com:3128', |
| 1791 | (None, None, None, 'proxy.example.com:3128')), |
| 1792 | # The authority component may optionally include userinfo |
| 1793 | # (assumed to be # username:password): |
| 1794 | ('joe:password@proxy.example.com', |
| 1795 | (None, 'joe', 'password', 'proxy.example.com')), |
| 1796 | ('joe:password@proxy.example.com:3128', |
| 1797 | (None, 'joe', 'password', 'proxy.example.com:3128')), |
| 1798 | #Examples with URLS |
| 1799 | ('http://proxy.example.com/', |
| 1800 | ('http', None, None, 'proxy.example.com')), |
| 1801 | ('http://proxy.example.com:3128/', |
| 1802 | ('http', None, None, 'proxy.example.com:3128')), |
| 1803 | ('http://joe:password@proxy.example.com/', |
| 1804 | ('http', 'joe', 'password', 'proxy.example.com')), |
| 1805 | ('http://joe:password@proxy.example.com:3128', |
| 1806 | ('http', 'joe', 'password', 'proxy.example.com:3128')), |
| 1807 | # Everything after the authority is ignored |
| 1808 | ('ftp://joe:password@proxy.example.com/rubbish:3128', |
| 1809 | ('ftp', 'joe', 'password', 'proxy.example.com')), |
| 1810 | # Test for no trailing '/' case |
| 1811 | ('http://joe:password@proxy.example.com', |
| 1812 | ('http', 'joe', 'password', 'proxy.example.com')) |
| 1813 | ] |
| 1814 | |
| 1815 | for tc, expected in parse_proxy_test_cases: |
| 1816 | self.assertEqual(_parse_proxy(tc), expected) |
| 1817 | |
| 1818 | self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), |
| 1819 | |
Berker Peksag | e88dd1c | 2016-03-06 16:16:40 +0200 | [diff] [blame] | 1820 | def test_unsupported_algorithm(self): |
| 1821 | handler = AbstractDigestAuthHandler() |
| 1822 | with self.assertRaises(ValueError) as exc: |
| 1823 | handler.get_algorithm_impls('invalid') |
| 1824 | self.assertEqual( |
| 1825 | str(exc.exception), |
| 1826 | "Unsupported digest authentication algorithm 'invalid'" |
| 1827 | ) |
| 1828 | |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1829 | |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1830 | class RequestTests(unittest.TestCase): |
Jason R. Coombs | 4a65242 | 2013-09-08 13:03:40 -0400 | [diff] [blame] | 1831 | class PutRequest(Request): |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1832 | method = 'PUT' |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1833 | |
| 1834 | def setUp(self): |
| 1835 | self.get = Request("http://www.python.org/~jeremy/") |
| 1836 | self.post = Request("http://www.python.org/~jeremy/", |
| 1837 | "data", |
| 1838 | headers={"X-Test": "test"}) |
Jason R. Coombs | 4a65242 | 2013-09-08 13:03:40 -0400 | [diff] [blame] | 1839 | self.head = Request("http://www.python.org/~jeremy/", method='HEAD') |
| 1840 | self.put = self.PutRequest("http://www.python.org/~jeremy/") |
| 1841 | self.force_post = self.PutRequest("http://www.python.org/~jeremy/", |
| 1842 | method="POST") |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1843 | |
| 1844 | def test_method(self): |
| 1845 | self.assertEqual("POST", self.post.get_method()) |
| 1846 | self.assertEqual("GET", self.get.get_method()) |
Senthil Kumaran | 0b5463f | 2013-09-09 23:13:06 -0700 | [diff] [blame] | 1847 | self.assertEqual("HEAD", self.head.get_method()) |
Jason R. Coombs | 4a65242 | 2013-09-08 13:03:40 -0400 | [diff] [blame] | 1848 | self.assertEqual("PUT", self.put.get_method()) |
| 1849 | self.assertEqual("POST", self.force_post.get_method()) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1850 | |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1851 | def test_data(self): |
| 1852 | self.assertFalse(self.get.data) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1853 | self.assertEqual("GET", self.get.get_method()) |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1854 | self.get.data = "spam" |
| 1855 | self.assertTrue(self.get.data) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1856 | self.assertEqual("POST", self.get.get_method()) |
| 1857 | |
Andrew Svetlov | bff98fe | 2012-11-27 23:06:19 +0200 | [diff] [blame] | 1858 | # issue 16464 |
| 1859 | # if we change data we need to remove content-length header |
| 1860 | # (cause it's most probably calculated for previous value) |
| 1861 | def test_setting_data_should_remove_content_length(self): |
R David Murray | 9cc7d45 | 2013-03-20 00:10:51 -0400 | [diff] [blame] | 1862 | self.assertNotIn("Content-length", self.get.unredirected_hdrs) |
Andrew Svetlov | bff98fe | 2012-11-27 23:06:19 +0200 | [diff] [blame] | 1863 | self.get.add_unredirected_header("Content-length", 42) |
| 1864 | self.assertEqual(42, self.get.unredirected_hdrs["Content-length"]) |
| 1865 | self.get.data = "spam" |
R David Murray | 9cc7d45 | 2013-03-20 00:10:51 -0400 | [diff] [blame] | 1866 | self.assertNotIn("Content-length", self.get.unredirected_hdrs) |
| 1867 | |
| 1868 | # issue 17485 same for deleting data. |
| 1869 | def test_deleting_data_should_remove_content_length(self): |
| 1870 | self.assertNotIn("Content-length", self.get.unredirected_hdrs) |
| 1871 | self.get.data = 'foo' |
| 1872 | self.get.add_unredirected_header("Content-length", 3) |
| 1873 | self.assertEqual(3, self.get.unredirected_hdrs["Content-length"]) |
| 1874 | del self.get.data |
| 1875 | self.assertNotIn("Content-length", self.get.unredirected_hdrs) |
Andrew Svetlov | bff98fe | 2012-11-27 23:06:19 +0200 | [diff] [blame] | 1876 | |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1877 | def test_get_full_url(self): |
| 1878 | self.assertEqual("http://www.python.org/~jeremy/", |
| 1879 | self.get.get_full_url()) |
| 1880 | |
| 1881 | def test_selector(self): |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1882 | self.assertEqual("/~jeremy/", self.get.selector) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1883 | req = Request("http://www.python.org/") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1884 | self.assertEqual("/", req.selector) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1885 | |
| 1886 | def test_get_type(self): |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1887 | self.assertEqual("http", self.get.type) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1888 | |
| 1889 | def test_get_host(self): |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1890 | self.assertEqual("www.python.org", self.get.host) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1891 | |
| 1892 | def test_get_host_unquote(self): |
| 1893 | req = Request("http://www.%70ython.org/") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1894 | self.assertEqual("www.python.org", req.host) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1895 | |
| 1896 | def test_proxy(self): |
Florent Xicluna | 419e384 | 2010-08-08 16:16:07 +0000 | [diff] [blame] | 1897 | self.assertFalse(self.get.has_proxy()) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1898 | self.get.set_proxy("www.perl.org", "http") |
Benjamin Peterson | c9c0f20 | 2009-06-30 23:06:06 +0000 | [diff] [blame] | 1899 | self.assertTrue(self.get.has_proxy()) |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1900 | self.assertEqual("www.python.org", self.get.origin_req_host) |
| 1901 | self.assertEqual("www.perl.org", self.get.host) |
Benjamin Peterson | 6ebe78f | 2008-12-21 00:06:59 +0000 | [diff] [blame] | 1902 | |
Senthil Kumaran | d95cc75 | 2010-08-08 11:27:53 +0000 | [diff] [blame] | 1903 | def test_wrapped_url(self): |
| 1904 | req = Request("<URL:http://www.python.org>") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1905 | self.assertEqual("www.python.org", req.host) |
Senthil Kumaran | d95cc75 | 2010-08-08 11:27:53 +0000 | [diff] [blame] | 1906 | |
Senthil Kumaran | 2643041 | 2011-04-13 07:01:19 +0800 | [diff] [blame] | 1907 | def test_url_fragment(self): |
Senthil Kumaran | d95cc75 | 2010-08-08 11:27:53 +0000 | [diff] [blame] | 1908 | req = Request("http://www.python.org/?qs=query#fragment=true") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1909 | self.assertEqual("/?qs=query", req.selector) |
Senthil Kumaran | d95cc75 | 2010-08-08 11:27:53 +0000 | [diff] [blame] | 1910 | req = Request("http://www.python.org/#fun=true") |
Senthil Kumaran | 77ebfcc | 2012-08-20 13:43:59 -0700 | [diff] [blame] | 1911 | self.assertEqual("/", req.selector) |
Senthil Kumaran | d95cc75 | 2010-08-08 11:27:53 +0000 | [diff] [blame] | 1912 | |
Senthil Kumaran | 2643041 | 2011-04-13 07:01:19 +0800 | [diff] [blame] | 1913 | # Issue 11703: geturl() omits fragment in the original URL. |
| 1914 | url = 'http://docs.python.org/library/urllib2.html#OK' |
| 1915 | req = Request(url) |
| 1916 | self.assertEqual(req.get_full_url(), url) |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1917 | |
Senthil Kumaran | 8307075 | 2013-05-24 09:14:12 -0700 | [diff] [blame] | 1918 | def test_url_fullurl_get_full_url(self): |
| 1919 | urls = ['http://docs.python.org', |
| 1920 | 'http://docs.python.org/library/urllib2.html#OK', |
Facundo Batista | 244afcf | 2015-04-22 18:35:54 -0300 | [diff] [blame] | 1921 | 'http://www.python.org/?qs=query#fragment=true'] |
Senthil Kumaran | 8307075 | 2013-05-24 09:14:12 -0700 | [diff] [blame] | 1922 | for url in urls: |
| 1923 | req = Request(url) |
| 1924 | self.assertEqual(req.get_full_url(), req.full_url) |
| 1925 | |
Jeremy Hylton | c1be59f | 2003-12-14 05:27:34 +0000 | [diff] [blame] | 1926 | |
| 1927 | if __name__ == "__main__": |
Berker Peksag | bcdfc6a | 2015-03-02 06:01:01 +0200 | [diff] [blame] | 1928 | unittest.main() |