blob: cecfbb0f0deb4415711d841789c0ca2253c26949 [file] [log] [blame]
Guido van Rossume7b146f2000-02-04 15:28:42 +00001"""Open an arbitrary URL.
2
3See the following document for more info on URLs:
4"Names and Addresses, URIs, URLs, URNs, URCs", at
5http://www.w3.org/pub/WWW/Addressing/Overview.html
6
7See also the HTTP spec (from which the error codes are derived):
8"HTTP - Hypertext Transfer Protocol", at
9http://www.w3.org/pub/WWW/Protocols/
10
11Related standards and specs:
12- RFC1808: the "relative URL" spec. (authoritative status)
13- RFC1738 - the "URL standard". (authoritative status)
14- RFC1630 - the "URI spec". (informational status)
15
16The object returned by URLopener().open(file) will differ per
17protocol. All you know is that is has methods read(), readline(),
18readlines(), fileno(), close() and info(). The read*(), fileno()
Fredrik Lundhb49f88b2000-09-24 18:51:25 +000019and close() methods work like those of open files.
Guido van Rossume7b146f2000-02-04 15:28:42 +000020The info() method returns a mimetools.Message object which can be
21used to query various info about the object, if available.
22(mimetools.Message objects are queried with the getheader() method.)
23"""
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000024
Guido van Rossum7c395db1994-07-04 22:14:49 +000025import string
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000026import socket
Jack Jansendc3e3f61995-12-15 13:22:13 +000027import os
Guido van Rossumf0713d32001-08-09 17:43:35 +000028import time
Guido van Rossum3c8484e1996-11-20 22:02:24 +000029import sys
Brett Cannon69200fa2004-03-23 21:26:39 +000030from urlparse import urljoin as basejoin
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000031
Skip Montanaro40fc1602001-03-01 04:27:19 +000032__all__ = ["urlopen", "URLopener", "FancyURLopener", "urlretrieve",
33 "urlcleanup", "quote", "quote_plus", "unquote", "unquote_plus",
Skip Montanaro44d5e0c2001-03-13 19:47:16 +000034 "urlencode", "url2pathname", "pathname2url", "splittag",
35 "localhost", "thishost", "ftperrors", "basejoin", "unwrap",
36 "splittype", "splithost", "splituser", "splitpasswd", "splitport",
37 "splitnport", "splitquery", "splitattr", "splitvalue",
Brett Cannond75f0432007-05-16 22:42:29 +000038 "getproxies"]
Skip Montanaro40fc1602001-03-01 04:27:19 +000039
Martin v. Löwis3e865952006-01-24 15:51:21 +000040__version__ = '1.17' # XXX This version is not always updated :-(
Guido van Rossumf668d171997-06-06 21:11:11 +000041
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000042MAXFTPCACHE = 10 # Trim the ftp cache beyond this size
Guido van Rossum6cb15a01995-06-22 19:00:13 +000043
Jack Jansendc3e3f61995-12-15 13:22:13 +000044# Helper for non-unix systems
45if os.name == 'mac':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000046 from macurl2path import url2pathname, pathname2url
Guido van Rossum7e7ca0b1998-03-26 21:01:39 +000047elif os.name == 'nt':
Fredrik Lundhb49f88b2000-09-24 18:51:25 +000048 from nturl2path import url2pathname, pathname2url
Guido van Rossumd74fb6b2001-03-02 06:43:49 +000049elif os.name == 'riscos':
50 from rourl2path import url2pathname, pathname2url
Jack Jansendc3e3f61995-12-15 13:22:13 +000051else:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000052 def url2pathname(pathname):
Georg Brandlc0b24732005-12-26 22:53:56 +000053 """OS-specific conversion from a relative URL of the 'file' scheme
54 to a file system path; not recommended for general use."""
Guido van Rossum367ac801999-03-12 14:31:10 +000055 return unquote(pathname)
Georg Brandlc0b24732005-12-26 22:53:56 +000056
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000057 def pathname2url(pathname):
Georg Brandlc0b24732005-12-26 22:53:56 +000058 """OS-specific conversion from a file system path to a relative URL
59 of the 'file' scheme; not recommended for general use."""
Guido van Rossum367ac801999-03-12 14:31:10 +000060 return quote(pathname)
Guido van Rossum33add0a1998-12-18 15:25:22 +000061
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000062# This really consists of two pieces:
63# (1) a class which handles opening of all sorts of URLs
64# (plus assorted utilities etc.)
65# (2) a set of functions for parsing URLs
66# XXX Should these be separated out into different modules?
67
68
69# Shortcut for basic usage
70_urlopener = None
Fred Drakedf6eca72002-04-04 20:41:34 +000071def urlopen(url, data=None, proxies=None):
Skip Montanaro79f1c172000-08-22 03:00:52 +000072 """urlopen(url [, data]) -> open file-like object"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000073 global _urlopener
Fred Drakedf6eca72002-04-04 20:41:34 +000074 if proxies is not None:
75 opener = FancyURLopener(proxies=proxies)
76 elif not _urlopener:
77 opener = FancyURLopener()
78 _urlopener = opener
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000079 else:
Fred Drakedf6eca72002-04-04 20:41:34 +000080 opener = _urlopener
81 if data is None:
82 return opener.open(url)
83 else:
84 return opener.open(url, data)
Fred Drake316a7932000-08-24 01:01:26 +000085def urlretrieve(url, filename=None, reporthook=None, data=None):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000086 global _urlopener
87 if not _urlopener:
88 _urlopener = FancyURLopener()
Fred Drake316a7932000-08-24 01:01:26 +000089 return _urlopener.retrieve(url, filename, reporthook, data)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000090def urlcleanup():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000091 if _urlopener:
92 _urlopener.cleanup()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000093
Georg Brandlb9256022005-08-24 18:46:39 +000094# exception raised when downloaded size does not match content-length
95class ContentTooShortError(IOError):
96 def __init__(self, message, content):
97 IOError.__init__(self, message)
98 self.content = content
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000099
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000100ftpcache = {}
101class URLopener:
Guido van Rossume7b146f2000-02-04 15:28:42 +0000102 """Class to open URLs.
103 This is a class rather than just a subroutine because we may need
104 more than one set of global protocol-specific options.
105 Note -- this is a base class for those who don't want the
106 automatic handling of errors type 302 (relocated) and 401
107 (authorization needed)."""
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000108
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000109 __tempfiles = None
Guido van Rossum29e77811996-11-27 19:39:58 +0000110
Guido van Rossumba311382000-08-24 16:18:04 +0000111 version = "Python-urllib/%s" % __version__
112
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000113 # Constructor
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000114 def __init__(self, proxies=None, **x509):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000115 if proxies is None:
116 proxies = getproxies()
117 assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
118 self.proxies = proxies
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000119 self.key_file = x509.get('key_file')
120 self.cert_file = x509.get('cert_file')
Georg Brandl0619a322006-07-26 07:40:17 +0000121 self.addheaders = [('User-Agent', self.version)]
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000122 self.__tempfiles = []
123 self.__unlink = os.unlink # See cleanup()
124 self.tempcache = None
125 # Undocumented feature: if you assign {} to tempcache,
126 # it is used to cache files retrieved with
127 # self.retrieve(). This is not enabled by default
128 # since it does not work for changing documents (and I
129 # haven't got the logic to check expiration headers
130 # yet).
131 self.ftpcache = ftpcache
132 # Undocumented feature: you can use a different
133 # ftp cache by assigning to the .ftpcache member;
134 # in case you want logically independent URL openers
135 # XXX This is not threadsafe. Bah.
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000136
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000137 def __del__(self):
138 self.close()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000139
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000140 def close(self):
141 self.cleanup()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000142
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000143 def cleanup(self):
144 # This code sometimes runs when the rest of this module
145 # has already been deleted, so it can't use any globals
146 # or import anything.
147 if self.__tempfiles:
148 for file in self.__tempfiles:
149 try:
150 self.__unlink(file)
Martin v. Löwis58682b72001-08-11 15:02:57 +0000151 except OSError:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000152 pass
153 del self.__tempfiles[:]
154 if self.tempcache:
155 self.tempcache.clear()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000156
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000157 def addheader(self, *args):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000158 """Add a header to be used by the HTTP interface only
159 e.g. u.addheader('Accept', 'sound/basic')"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000160 self.addheaders.append(args)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000161
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000162 # External interface
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000163 def open(self, fullurl, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000164 """Use URLopener().open(file) instead of open(file, 'r')."""
Martin v. Löwis1d994332000-12-03 18:30:10 +0000165 fullurl = unwrap(toBytes(fullurl))
Raymond Hettinger54f02222002-06-01 14:18:47 +0000166 if self.tempcache and fullurl in self.tempcache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000167 filename, headers = self.tempcache[fullurl]
168 fp = open(filename, 'rb')
169 return addinfourl(fp, headers, fullurl)
Martin v. Löwis1d994332000-12-03 18:30:10 +0000170 urltype, url = splittype(fullurl)
171 if not urltype:
172 urltype = 'file'
Raymond Hettinger54f02222002-06-01 14:18:47 +0000173 if urltype in self.proxies:
Martin v. Löwis1d994332000-12-03 18:30:10 +0000174 proxy = self.proxies[urltype]
175 urltype, proxyhost = splittype(proxy)
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000176 host, selector = splithost(proxyhost)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000177 url = (host, fullurl) # Signal special case to open_*()
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000178 else:
179 proxy = None
Martin v. Löwis1d994332000-12-03 18:30:10 +0000180 name = 'open_' + urltype
181 self.type = urltype
Brett Cannonaaeffaf2004-03-23 23:50:17 +0000182 name = name.replace('-', '_')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000183 if not hasattr(self, name):
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000184 if proxy:
185 return self.open_unknown_proxy(proxy, fullurl, data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000186 else:
187 return self.open_unknown(fullurl, data)
188 try:
189 if data is None:
190 return getattr(self, name)(url)
191 else:
192 return getattr(self, name)(url, data)
193 except socket.error, msg:
194 raise IOError, ('socket error', msg), sys.exc_info()[2]
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000195
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000196 def open_unknown(self, fullurl, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000197 """Overridable interface to open unknown URL type."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000198 type, url = splittype(fullurl)
199 raise IOError, ('url error', 'unknown url type', type)
Guido van Rossumca445401995-08-29 19:19:12 +0000200
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000201 def open_unknown_proxy(self, proxy, fullurl, data=None):
202 """Overridable interface to open unknown URL type."""
203 type, url = splittype(fullurl)
204 raise IOError, ('url error', 'invalid proxy for %s' % type, proxy)
205
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000206 # External interface
Sjoerd Mullenderd7b86f02000-08-25 11:23:36 +0000207 def retrieve(self, url, filename=None, reporthook=None, data=None):
Brett Cannon7d618c72003-04-24 02:43:20 +0000208 """retrieve(url) returns (filename, headers) for a local object
Guido van Rossume7b146f2000-02-04 15:28:42 +0000209 or (tempfilename, headers) for a remote object."""
Martin v. Löwis1d994332000-12-03 18:30:10 +0000210 url = unwrap(toBytes(url))
Raymond Hettinger54f02222002-06-01 14:18:47 +0000211 if self.tempcache and url in self.tempcache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000212 return self.tempcache[url]
213 type, url1 = splittype(url)
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000214 if filename is None and (not type or type == 'file'):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000215 try:
216 fp = self.open_local_file(url1)
217 hdrs = fp.info()
218 del fp
219 return url2pathname(splithost(url1)[1]), hdrs
220 except IOError, msg:
221 pass
Fred Drake316a7932000-08-24 01:01:26 +0000222 fp = self.open(url, data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000223 headers = fp.info()
Guido van Rossum3b0a3292002-08-09 16:38:32 +0000224 if filename:
225 tfp = open(filename, 'wb')
226 else:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000227 import tempfile
228 garbage, path = splittype(url)
229 garbage, path = splithost(path or "")
230 path, garbage = splitquery(path or "")
231 path, garbage = splitattr(path or "")
232 suffix = os.path.splitext(path)[1]
Guido van Rossum3b0a3292002-08-09 16:38:32 +0000233 (fd, filename) = tempfile.mkstemp(suffix)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000234 self.__tempfiles.append(filename)
Jeremy Hylton3bd6fde2002-10-11 14:36:24 +0000235 tfp = os.fdopen(fd, 'wb')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000236 result = filename, headers
237 if self.tempcache is not None:
238 self.tempcache[url] = result
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000239 bs = 1024*8
240 size = -1
Georg Brandlb9256022005-08-24 18:46:39 +0000241 read = 0
Georg Brandl5a650a22005-08-26 08:51:34 +0000242 blocknum = 0
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000243 if reporthook:
Raymond Hettinger54f02222002-06-01 14:18:47 +0000244 if "content-length" in headers:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000245 size = int(headers["Content-Length"])
Georg Brandl5a650a22005-08-26 08:51:34 +0000246 reporthook(blocknum, bs, size)
247 while 1:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000248 block = fp.read(bs)
Georg Brandl5a650a22005-08-26 08:51:34 +0000249 if block == "":
250 break
Georg Brandlb9256022005-08-24 18:46:39 +0000251 read += len(block)
Georg Brandl5a650a22005-08-26 08:51:34 +0000252 tfp.write(block)
Georg Brandlb9256022005-08-24 18:46:39 +0000253 blocknum += 1
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000254 if reporthook:
255 reporthook(blocknum, bs, size)
256 fp.close()
257 tfp.close()
258 del fp
259 del tfp
Georg Brandlb9256022005-08-24 18:46:39 +0000260
261 # raise exception if actual size does not match content-length header
262 if size >= 0 and read < size:
263 raise ContentTooShortError("retrieval incomplete: got only %i out "
264 "of %i bytes" % (read, size), result)
265
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000266 return result
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000267
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000268 # Each method named open_<type> knows how to open that type of URL
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000269
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000270 def open_http(self, url, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000271 """Use HTTP protocol."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000272 import httplib
273 user_passwd = None
Martin v. Löwis3e865952006-01-24 15:51:21 +0000274 proxy_passwd= None
Walter Dörwald65230a22002-06-03 15:58:32 +0000275 if isinstance(url, str):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000276 host, selector = splithost(url)
277 if host:
278 user_passwd, host = splituser(host)
279 host = unquote(host)
280 realhost = host
281 else:
282 host, selector = url
Martin v. Löwis3e865952006-01-24 15:51:21 +0000283 # check whether the proxy contains authorization information
284 proxy_passwd, host = splituser(host)
285 # now we proceed with the url we want to obtain
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000286 urltype, rest = splittype(selector)
287 url = rest
288 user_passwd = None
Guido van Rossumb2493f82000-12-15 15:01:37 +0000289 if urltype.lower() != 'http':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000290 realhost = None
291 else:
292 realhost, rest = splithost(rest)
293 if realhost:
294 user_passwd, realhost = splituser(realhost)
295 if user_passwd:
296 selector = "%s://%s%s" % (urltype, realhost, rest)
Tim Peters55c12d42001-08-09 18:04:14 +0000297 if proxy_bypass(realhost):
298 host = realhost
299
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000300 #print "proxy via http:", host, selector
301 if not host: raise IOError, ('http error', 'no host given')
Tim Peters92037a12006-01-24 22:44:08 +0000302
Martin v. Löwis3e865952006-01-24 15:51:21 +0000303 if proxy_passwd:
304 import base64
Andrew M. Kuchling872dba42006-10-27 17:11:23 +0000305 proxy_auth = base64.b64encode(proxy_passwd).strip()
Martin v. Löwis3e865952006-01-24 15:51:21 +0000306 else:
307 proxy_auth = None
308
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000309 if user_passwd:
310 import base64
Andrew M. Kuchling872dba42006-10-27 17:11:23 +0000311 auth = base64.b64encode(user_passwd).strip()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000312 else:
313 auth = None
314 h = httplib.HTTP(host)
315 if data is not None:
316 h.putrequest('POST', selector)
Georg Brandl0619a322006-07-26 07:40:17 +0000317 h.putheader('Content-Type', 'application/x-www-form-urlencoded')
318 h.putheader('Content-Length', '%d' % len(data))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000319 else:
320 h.putrequest('GET', selector)
Martin v. Löwis3e865952006-01-24 15:51:21 +0000321 if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000322 if auth: h.putheader('Authorization', 'Basic %s' % auth)
323 if realhost: h.putheader('Host', realhost)
Guido van Rossum68468eb2003-02-27 20:14:51 +0000324 for args in self.addheaders: h.putheader(*args)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000325 h.endheaders()
326 if data is not None:
Fred Drakeec3dfde2001-07-04 05:18:29 +0000327 h.send(data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000328 errcode, errmsg, headers = h.getreply()
Neal Norwitzce55e212007-03-20 08:14:57 +0000329 fp = h.getfile()
Georg Brandlf66b6032007-03-14 08:27:52 +0000330 if errcode == -1:
Neal Norwitzce55e212007-03-20 08:14:57 +0000331 if fp: fp.close()
Georg Brandlf66b6032007-03-14 08:27:52 +0000332 # something went wrong with the HTTP status line
333 raise IOError, ('http protocol error', 0,
334 'got a bad status line', None)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000335 if errcode == 200:
336 return addinfourl(fp, headers, "http:" + url)
337 else:
338 if data is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000339 return self.http_error(url, fp, errcode, errmsg, headers)
Guido van Rossum29aab751999-03-09 19:31:21 +0000340 else:
341 return self.http_error(url, fp, errcode, errmsg, headers, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000342
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000343 def http_error(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000344 """Handle http errors.
345 Derived class can override this, or provide specific handlers
346 named http_error_DDD where DDD is the 3-digit error code."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000347 # First check if there's a specific handler for this error
348 name = 'http_error_%d' % errcode
349 if hasattr(self, name):
350 method = getattr(self, name)
351 if data is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000352 result = method(url, fp, errcode, errmsg, headers)
Jeremy Hyltonb30f52a1999-02-25 16:14:58 +0000353 else:
354 result = method(url, fp, errcode, errmsg, headers, data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000355 if result: return result
Jeremy Hyltonb30f52a1999-02-25 16:14:58 +0000356 return self.http_error_default(url, fp, errcode, errmsg, headers)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000357
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000358 def http_error_default(self, url, fp, errcode, errmsg, headers):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000359 """Default error handler: close the connection and raise IOError."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000360 void = fp.read()
361 fp.close()
362 raise IOError, ('http error', errcode, errmsg, headers)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000363
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000364 if hasattr(socket, "ssl"):
Andrew M. Kuchling141e9892000-04-23 02:53:11 +0000365 def open_https(self, url, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000366 """Use HTTPS protocol."""
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000367 import httplib
Fred Drake567ca8e2000-08-21 21:42:42 +0000368 user_passwd = None
Martin v. Löwis3e865952006-01-24 15:51:21 +0000369 proxy_passwd = None
Walter Dörwald65230a22002-06-03 15:58:32 +0000370 if isinstance(url, str):
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000371 host, selector = splithost(url)
Fred Drake567ca8e2000-08-21 21:42:42 +0000372 if host:
373 user_passwd, host = splituser(host)
374 host = unquote(host)
375 realhost = host
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000376 else:
377 host, selector = url
Martin v. Löwis3e865952006-01-24 15:51:21 +0000378 # here, we determine, whether the proxy contains authorization information
379 proxy_passwd, host = splituser(host)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000380 urltype, rest = splittype(selector)
Fred Drake567ca8e2000-08-21 21:42:42 +0000381 url = rest
382 user_passwd = None
Guido van Rossumb2493f82000-12-15 15:01:37 +0000383 if urltype.lower() != 'https':
Fred Drake567ca8e2000-08-21 21:42:42 +0000384 realhost = None
385 else:
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000386 realhost, rest = splithost(rest)
Fred Drake567ca8e2000-08-21 21:42:42 +0000387 if realhost:
388 user_passwd, realhost = splituser(realhost)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000389 if user_passwd:
390 selector = "%s://%s%s" % (urltype, realhost, rest)
Andrew M. Kuchling7ad47922000-06-10 01:41:48 +0000391 #print "proxy via https:", host, selector
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000392 if not host: raise IOError, ('https error', 'no host given')
Martin v. Löwis3e865952006-01-24 15:51:21 +0000393 if proxy_passwd:
394 import base64
Andrew M. Kuchling872dba42006-10-27 17:11:23 +0000395 proxy_auth = base64.b64encode(proxy_passwd).strip()
Martin v. Löwis3e865952006-01-24 15:51:21 +0000396 else:
397 proxy_auth = None
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000398 if user_passwd:
399 import base64
Andrew M. Kuchling872dba42006-10-27 17:11:23 +0000400 auth = base64.b64encode(user_passwd).strip()
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000401 else:
402 auth = None
403 h = httplib.HTTPS(host, 0,
404 key_file=self.key_file,
405 cert_file=self.cert_file)
Andrew M. Kuchling141e9892000-04-23 02:53:11 +0000406 if data is not None:
407 h.putrequest('POST', selector)
Georg Brandl0619a322006-07-26 07:40:17 +0000408 h.putheader('Content-Type',
Andrew M. Kuchling141e9892000-04-23 02:53:11 +0000409 'application/x-www-form-urlencoded')
Georg Brandl0619a322006-07-26 07:40:17 +0000410 h.putheader('Content-Length', '%d' % len(data))
Andrew M. Kuchling141e9892000-04-23 02:53:11 +0000411 else:
412 h.putrequest('GET', selector)
Andrew M. Kuchling52278572006-12-19 15:11:41 +0000413 if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
414 if auth: h.putheader('Authorization', 'Basic %s' % auth)
Fred Drake567ca8e2000-08-21 21:42:42 +0000415 if realhost: h.putheader('Host', realhost)
Guido van Rossum68468eb2003-02-27 20:14:51 +0000416 for args in self.addheaders: h.putheader(*args)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000417 h.endheaders()
Andrew M. Kuchling43c5af02000-04-24 14:17:06 +0000418 if data is not None:
Fred Drakeec3dfde2001-07-04 05:18:29 +0000419 h.send(data)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000420 errcode, errmsg, headers = h.getreply()
Neal Norwitzce55e212007-03-20 08:14:57 +0000421 fp = h.getfile()
Georg Brandlf66b6032007-03-14 08:27:52 +0000422 if errcode == -1:
Neal Norwitzce55e212007-03-20 08:14:57 +0000423 if fp: fp.close()
Georg Brandlf66b6032007-03-14 08:27:52 +0000424 # something went wrong with the HTTP status line
425 raise IOError, ('http protocol error', 0,
426 'got a bad status line', None)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000427 if errcode == 200:
Guido van Rossumb931bf32001-12-08 17:09:07 +0000428 return addinfourl(fp, headers, "https:" + url)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000429 else:
Fred Drake567ca8e2000-08-21 21:42:42 +0000430 if data is None:
431 return self.http_error(url, fp, errcode, errmsg, headers)
432 else:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000433 return self.http_error(url, fp, errcode, errmsg, headers,
434 data)
Fred Drake567ca8e2000-08-21 21:42:42 +0000435
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000436 def open_file(self, url):
Neal Norwitzc5d0dbd2006-04-09 04:00:49 +0000437 """Use local file or FTP depending on form of URL."""
Martin v. Löwis3e865952006-01-24 15:51:21 +0000438 if not isinstance(url, str):
439 raise IOError, ('file error', 'proxy support for file protocol currently not implemented')
Jack Jansen4ef11032002-09-12 20:14:04 +0000440 if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000441 return self.open_ftp(url)
442 else:
443 return self.open_local_file(url)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000444
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000445 def open_local_file(self, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000446 """Use local file."""
Georg Brandl5a096e12007-01-22 19:40:21 +0000447 import mimetypes, mimetools, email.utils
Raymond Hettingera6172712004-12-31 19:15:26 +0000448 try:
449 from cStringIO import StringIO
450 except ImportError:
451 from StringIO import StringIO
Guido van Rossumf0713d32001-08-09 17:43:35 +0000452 host, file = splithost(url)
453 localname = url2pathname(file)
Guido van Rossuma2da3052002-04-15 00:25:01 +0000454 try:
455 stats = os.stat(localname)
456 except OSError, e:
457 raise IOError(e.errno, e.strerror, e.filename)
Walter Dörwald92b48b72002-03-22 17:30:38 +0000458 size = stats.st_size
Georg Brandl5a096e12007-01-22 19:40:21 +0000459 modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000460 mtype = mimetypes.guess_type(url)[0]
Raymond Hettingera6172712004-12-31 19:15:26 +0000461 headers = mimetools.Message(StringIO(
Guido van Rossumf0713d32001-08-09 17:43:35 +0000462 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
463 (mtype or 'text/plain', size, modified)))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000464 if not host:
Guido van Rossum336a2011999-06-24 15:27:36 +0000465 urlfile = file
466 if file[:1] == '/':
467 urlfile = 'file://' + file
Guido van Rossumf0713d32001-08-09 17:43:35 +0000468 return addinfourl(open(localname, 'rb'),
Guido van Rossum336a2011999-06-24 15:27:36 +0000469 headers, urlfile)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000470 host, port = splitport(host)
471 if not port \
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000472 and socket.gethostbyname(host) in (localhost(), thishost()):
Guido van Rossum336a2011999-06-24 15:27:36 +0000473 urlfile = file
474 if file[:1] == '/':
475 urlfile = 'file://' + file
Guido van Rossumf0713d32001-08-09 17:43:35 +0000476 return addinfourl(open(localname, 'rb'),
Guido van Rossum336a2011999-06-24 15:27:36 +0000477 headers, urlfile)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000478 raise IOError, ('local file error', 'not on local host')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000479
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000480 def open_ftp(self, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000481 """Use FTP protocol."""
Martin v. Löwis3e865952006-01-24 15:51:21 +0000482 if not isinstance(url, str):
483 raise IOError, ('ftp error', 'proxy support for ftp protocol currently not implemented')
Raymond Hettingera6172712004-12-31 19:15:26 +0000484 import mimetypes, mimetools
485 try:
486 from cStringIO import StringIO
487 except ImportError:
488 from StringIO import StringIO
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000489 host, path = splithost(url)
490 if not host: raise IOError, ('ftp error', 'no host given')
491 host, port = splitport(host)
492 user, host = splituser(host)
493 if user: user, passwd = splitpasswd(user)
494 else: passwd = None
495 host = unquote(host)
496 user = unquote(user or '')
497 passwd = unquote(passwd or '')
498 host = socket.gethostbyname(host)
499 if not port:
500 import ftplib
501 port = ftplib.FTP_PORT
502 else:
503 port = int(port)
504 path, attrs = splitattr(path)
505 path = unquote(path)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000506 dirs = path.split('/')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000507 dirs, file = dirs[:-1], dirs[-1]
508 if dirs and not dirs[0]: dirs = dirs[1:]
Guido van Rossum5e006a31999-08-18 17:40:33 +0000509 if dirs and not dirs[0]: dirs[0] = '/'
Guido van Rossumb2493f82000-12-15 15:01:37 +0000510 key = user, host, port, '/'.join(dirs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000511 # XXX thread unsafe!
512 if len(self.ftpcache) > MAXFTPCACHE:
513 # Prune the cache, rather arbitrarily
514 for k in self.ftpcache.keys():
515 if k != key:
516 v = self.ftpcache[k]
517 del self.ftpcache[k]
518 v.close()
519 try:
Raymond Hettinger54f02222002-06-01 14:18:47 +0000520 if not key in self.ftpcache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000521 self.ftpcache[key] = \
522 ftpwrapper(user, passwd, host, port, dirs)
523 if not file: type = 'D'
524 else: type = 'I'
525 for attr in attrs:
526 attr, value = splitvalue(attr)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000527 if attr.lower() == 'type' and \
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000528 value in ('a', 'A', 'i', 'I', 'd', 'D'):
Guido van Rossumb2493f82000-12-15 15:01:37 +0000529 type = value.upper()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000530 (fp, retrlen) = self.ftpcache[key].retrfile(file, type)
Guido van Rossum88e0b5b2001-08-23 13:38:15 +0000531 mtype = mimetypes.guess_type("ftp:" + url)[0]
532 headers = ""
533 if mtype:
534 headers += "Content-Type: %s\n" % mtype
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000535 if retrlen is not None and retrlen >= 0:
Guido van Rossum88e0b5b2001-08-23 13:38:15 +0000536 headers += "Content-Length: %d\n" % retrlen
Raymond Hettingera6172712004-12-31 19:15:26 +0000537 headers = mimetools.Message(StringIO(headers))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000538 return addinfourl(fp, headers, "ftp:" + url)
539 except ftperrors(), msg:
540 raise IOError, ('ftp error', msg), sys.exc_info()[2]
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000541
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000542 def open_data(self, url, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000543 """Use "data" URL."""
Martin v. Löwis3e865952006-01-24 15:51:21 +0000544 if not isinstance(url, str):
545 raise IOError, ('data error', 'proxy support for data protocol currently not implemented')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000546 # ignore POSTed data
547 #
548 # syntax of data URLs:
549 # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
550 # mediatype := [ type "/" subtype ] *( ";" parameter )
551 # data := *urlchar
552 # parameter := attribute "=" value
Raymond Hettingera6172712004-12-31 19:15:26 +0000553 import mimetools
554 try:
555 from cStringIO import StringIO
556 except ImportError:
557 from StringIO import StringIO
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000558 try:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000559 [type, data] = url.split(',', 1)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000560 except ValueError:
561 raise IOError, ('data error', 'bad data URL')
562 if not type:
563 type = 'text/plain;charset=US-ASCII'
Guido van Rossumb2493f82000-12-15 15:01:37 +0000564 semi = type.rfind(';')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000565 if semi >= 0 and '=' not in type[semi:]:
566 encoding = type[semi+1:]
567 type = type[:semi]
568 else:
569 encoding = ''
570 msg = []
571 msg.append('Date: %s'%time.strftime('%a, %d %b %Y %T GMT',
572 time.gmtime(time.time())))
573 msg.append('Content-type: %s' % type)
574 if encoding == 'base64':
575 import base64
576 data = base64.decodestring(data)
577 else:
578 data = unquote(data)
Georg Brandl0619a322006-07-26 07:40:17 +0000579 msg.append('Content-Length: %d' % len(data))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000580 msg.append('')
581 msg.append(data)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000582 msg = '\n'.join(msg)
Raymond Hettingera6172712004-12-31 19:15:26 +0000583 f = StringIO(msg)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000584 headers = mimetools.Message(f, 0)
Georg Brandl1f663572005-11-26 16:50:44 +0000585 #f.fileno = None # needed for addinfourl
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000586 return addinfourl(f, headers, url)
Guido van Rossum6d4d1c21998-03-12 14:32:55 +0000587
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000588
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000589class FancyURLopener(URLopener):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000590 """Derived class with handlers for errors we can handle (perhaps)."""
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000591
Neal Norwitz60e04cd2002-06-11 13:38:51 +0000592 def __init__(self, *args, **kwargs):
Guido van Rossum68468eb2003-02-27 20:14:51 +0000593 URLopener.__init__(self, *args, **kwargs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000594 self.auth_cache = {}
Skip Montanaroc3e11d62001-02-15 16:56:36 +0000595 self.tries = 0
596 self.maxtries = 10
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000597
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000598 def http_error_default(self, url, fp, errcode, errmsg, headers):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000599 """Default error handling -- don't raise an exception."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000600 return addinfourl(fp, headers, "http:" + url)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000601
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000602 def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000603 """Error 302 -- relocated (temporarily)."""
Skip Montanaroc3e11d62001-02-15 16:56:36 +0000604 self.tries += 1
605 if self.maxtries and self.tries >= self.maxtries:
606 if hasattr(self, "http_error_500"):
607 meth = self.http_error_500
608 else:
609 meth = self.http_error_default
610 self.tries = 0
611 return meth(url, fp, 500,
612 "Internal Server Error: Redirect Recursion", headers)
613 result = self.redirect_internal(url, fp, errcode, errmsg, headers,
614 data)
615 self.tries = 0
616 return result
617
618 def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
Raymond Hettinger54f02222002-06-01 14:18:47 +0000619 if 'location' in headers:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000620 newurl = headers['location']
Raymond Hettinger54f02222002-06-01 14:18:47 +0000621 elif 'uri' in headers:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000622 newurl = headers['uri']
623 else:
624 return
625 void = fp.read()
626 fp.close()
Guido van Rossum3527f591999-03-29 20:23:41 +0000627 # In case the server sent a relative URL, join with original:
Moshe Zadka5d87d472001-04-09 14:54:21 +0000628 newurl = basejoin(self.type + ":" + url, newurl)
Guido van Rossumfa19f7c2003-05-16 01:46:51 +0000629 return self.open(newurl)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000630
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000631 def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000632 """Error 301 -- also relocated (permanently)."""
633 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
Guido van Rossume6ad8911996-09-10 17:02:56 +0000634
Raymond Hettinger024aaa12003-04-24 15:32:12 +0000635 def http_error_303(self, url, fp, errcode, errmsg, headers, data=None):
636 """Error 303 -- also relocated (essentially identical to 302)."""
637 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
638
Guido van Rossumfa19f7c2003-05-16 01:46:51 +0000639 def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):
640 """Error 307 -- relocated, but turn POST into error."""
641 if data is None:
642 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
643 else:
644 return self.http_error_default(url, fp, errcode, errmsg, headers)
645
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000646 def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000647 """Error 401 -- authentication required.
Martin v. Löwis3e865952006-01-24 15:51:21 +0000648 This function supports Basic authentication only."""
Raymond Hettinger54f02222002-06-01 14:18:47 +0000649 if not 'www-authenticate' in headers:
Tim Peters85ba6732001-02-28 08:26:44 +0000650 URLopener.http_error_default(self, url, fp,
Fred Drakec680ae82001-10-13 18:37:07 +0000651 errcode, errmsg, headers)
Moshe Zadkae99bd172001-02-27 06:27:04 +0000652 stuff = headers['www-authenticate']
653 import re
654 match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
655 if not match:
Tim Peters85ba6732001-02-28 08:26:44 +0000656 URLopener.http_error_default(self, url, fp,
Moshe Zadkae99bd172001-02-27 06:27:04 +0000657 errcode, errmsg, headers)
658 scheme, realm = match.groups()
659 if scheme.lower() != 'basic':
Tim Peters85ba6732001-02-28 08:26:44 +0000660 URLopener.http_error_default(self, url, fp,
Moshe Zadkae99bd172001-02-27 06:27:04 +0000661 errcode, errmsg, headers)
662 name = 'retry_' + self.type + '_basic_auth'
663 if data is None:
664 return getattr(self,name)(url, realm)
665 else:
666 return getattr(self,name)(url, realm, data)
Tim Peters92037a12006-01-24 22:44:08 +0000667
Martin v. Löwis3e865952006-01-24 15:51:21 +0000668 def http_error_407(self, url, fp, errcode, errmsg, headers, data=None):
669 """Error 407 -- proxy authentication required.
670 This function supports Basic authentication only."""
671 if not 'proxy-authenticate' in headers:
672 URLopener.http_error_default(self, url, fp,
673 errcode, errmsg, headers)
674 stuff = headers['proxy-authenticate']
675 import re
676 match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
677 if not match:
678 URLopener.http_error_default(self, url, fp,
679 errcode, errmsg, headers)
680 scheme, realm = match.groups()
681 if scheme.lower() != 'basic':
682 URLopener.http_error_default(self, url, fp,
683 errcode, errmsg, headers)
684 name = 'retry_proxy_' + self.type + '_basic_auth'
685 if data is None:
686 return getattr(self,name)(url, realm)
687 else:
688 return getattr(self,name)(url, realm, data)
Tim Peters92037a12006-01-24 22:44:08 +0000689
Martin v. Löwis3e865952006-01-24 15:51:21 +0000690 def retry_proxy_http_basic_auth(self, url, realm, data=None):
691 host, selector = splithost(url)
692 newurl = 'http://' + host + selector
693 proxy = self.proxies['http']
694 urltype, proxyhost = splittype(proxy)
695 proxyhost, proxyselector = splithost(proxyhost)
696 i = proxyhost.find('@') + 1
697 proxyhost = proxyhost[i:]
698 user, passwd = self.get_user_passwd(proxyhost, realm, i)
699 if not (user or passwd): return None
700 proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
701 self.proxies['http'] = 'http://' + proxyhost + proxyselector
702 if data is None:
703 return self.open(newurl)
704 else:
705 return self.open(newurl, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000706
Martin v. Löwis3e865952006-01-24 15:51:21 +0000707 def retry_proxy_https_basic_auth(self, url, realm, data=None):
708 host, selector = splithost(url)
709 newurl = 'https://' + host + selector
710 proxy = self.proxies['https']
711 urltype, proxyhost = splittype(proxy)
712 proxyhost, proxyselector = splithost(proxyhost)
713 i = proxyhost.find('@') + 1
714 proxyhost = proxyhost[i:]
715 user, passwd = self.get_user_passwd(proxyhost, realm, i)
716 if not (user or passwd): return None
717 proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
718 self.proxies['https'] = 'https://' + proxyhost + proxyselector
719 if data is None:
720 return self.open(newurl)
721 else:
722 return self.open(newurl, data)
Tim Peters92037a12006-01-24 22:44:08 +0000723
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000724 def retry_http_basic_auth(self, url, realm, data=None):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000725 host, selector = splithost(url)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000726 i = host.find('@') + 1
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000727 host = host[i:]
728 user, passwd = self.get_user_passwd(host, realm, i)
729 if not (user or passwd): return None
Guido van Rossumafc4f042001-01-15 18:31:13 +0000730 host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000731 newurl = 'http://' + host + selector
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000732 if data is None:
733 return self.open(newurl)
734 else:
735 return self.open(newurl, data)
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000736
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000737 def retry_https_basic_auth(self, url, realm, data=None):
Tim Peterse1190062001-01-15 03:34:38 +0000738 host, selector = splithost(url)
739 i = host.find('@') + 1
740 host = host[i:]
741 user, passwd = self.get_user_passwd(host, realm, i)
742 if not (user or passwd): return None
Guido van Rossumafc4f042001-01-15 18:31:13 +0000743 host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
Martin v. Löwis3e865952006-01-24 15:51:21 +0000744 newurl = 'https://' + host + selector
745 if data is None:
746 return self.open(newurl)
747 else:
748 return self.open(newurl, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000749
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000750 def get_user_passwd(self, host, realm, clear_cache = 0):
Guido van Rossumb2493f82000-12-15 15:01:37 +0000751 key = realm + '@' + host.lower()
Raymond Hettinger54f02222002-06-01 14:18:47 +0000752 if key in self.auth_cache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000753 if clear_cache:
754 del self.auth_cache[key]
755 else:
756 return self.auth_cache[key]
757 user, passwd = self.prompt_user_passwd(host, realm)
758 if user or passwd: self.auth_cache[key] = (user, passwd)
759 return user, passwd
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000760
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000761 def prompt_user_passwd(self, host, realm):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000762 """Override this in a GUI environment!"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000763 import getpass
764 try:
765 user = raw_input("Enter username for %s at %s: " % (realm,
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000766 host))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000767 passwd = getpass.getpass("Enter password for %s in %s at %s: " %
768 (user, realm, host))
769 return user, passwd
770 except KeyboardInterrupt:
771 print
772 return None, None
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000773
774
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000775# Utility functions
776
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000777_localhost = None
778def localhost():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000779 """Return the IP address of the magic hostname 'localhost'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000780 global _localhost
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000781 if _localhost is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000782 _localhost = socket.gethostbyname('localhost')
783 return _localhost
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000784
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000785_thishost = None
786def thishost():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000787 """Return the IP address of the current host."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000788 global _thishost
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000789 if _thishost is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000790 _thishost = socket.gethostbyname(socket.gethostname())
791 return _thishost
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000792
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000793_ftperrors = None
794def ftperrors():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000795 """Return the set of errors raised by the FTP class."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000796 global _ftperrors
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000797 if _ftperrors is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000798 import ftplib
799 _ftperrors = ftplib.all_errors
800 return _ftperrors
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000801
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000802_noheaders = None
803def noheaders():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000804 """Return an empty mimetools.Message object."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000805 global _noheaders
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000806 if _noheaders is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000807 import mimetools
Raymond Hettingera6172712004-12-31 19:15:26 +0000808 try:
809 from cStringIO import StringIO
810 except ImportError:
811 from StringIO import StringIO
812 _noheaders = mimetools.Message(StringIO(), 0)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000813 _noheaders.fp.close() # Recycle file descriptor
814 return _noheaders
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000815
816
817# Utility classes
818
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000819class ftpwrapper:
Guido van Rossume7b146f2000-02-04 15:28:42 +0000820 """Class used by open_ftp() for cache of open FTP connections."""
821
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000822 def __init__(self, user, passwd, host, port, dirs):
823 self.user = user
824 self.passwd = passwd
825 self.host = host
826 self.port = port
827 self.dirs = dirs
828 self.init()
Guido van Rossume7b146f2000-02-04 15:28:42 +0000829
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000830 def init(self):
831 import ftplib
832 self.busy = 0
833 self.ftp = ftplib.FTP()
834 self.ftp.connect(self.host, self.port)
835 self.ftp.login(self.user, self.passwd)
836 for dir in self.dirs:
837 self.ftp.cwd(dir)
Guido van Rossume7b146f2000-02-04 15:28:42 +0000838
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000839 def retrfile(self, file, type):
840 import ftplib
841 self.endtransfer()
842 if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
843 else: cmd = 'TYPE ' + type; isdir = 0
844 try:
845 self.ftp.voidcmd(cmd)
846 except ftplib.all_errors:
847 self.init()
848 self.ftp.voidcmd(cmd)
849 conn = None
850 if file and not isdir:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000851 # Try to retrieve as a file
852 try:
853 cmd = 'RETR ' + file
854 conn = self.ftp.ntransfercmd(cmd)
855 except ftplib.error_perm, reason:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000856 if str(reason)[:3] != '550':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000857 raise IOError, ('ftp error', reason), sys.exc_info()[2]
858 if not conn:
859 # Set transfer mode to ASCII!
860 self.ftp.voidcmd('TYPE A')
861 # Try a directory listing
862 if file: cmd = 'LIST ' + file
863 else: cmd = 'LIST'
864 conn = self.ftp.ntransfercmd(cmd)
865 self.busy = 1
866 # Pass back both a suitably decorated object and a retrieval length
867 return (addclosehook(conn[0].makefile('rb'),
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000868 self.endtransfer), conn[1])
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000869 def endtransfer(self):
870 if not self.busy:
871 return
872 self.busy = 0
873 try:
874 self.ftp.voidresp()
875 except ftperrors():
876 pass
Guido van Rossume7b146f2000-02-04 15:28:42 +0000877
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000878 def close(self):
879 self.endtransfer()
880 try:
881 self.ftp.close()
882 except ftperrors():
883 pass
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000884
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000885class addbase:
Guido van Rossume7b146f2000-02-04 15:28:42 +0000886 """Base class for addinfo and addclosehook."""
887
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000888 def __init__(self, fp):
889 self.fp = fp
890 self.read = self.fp.read
891 self.readline = self.fp.readline
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000892 if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
Georg Brandl1f663572005-11-26 16:50:44 +0000893 if hasattr(self.fp, "fileno"):
894 self.fileno = self.fp.fileno
895 else:
896 self.fileno = lambda: None
Raymond Hettinger42182eb2003-03-09 05:33:33 +0000897 if hasattr(self.fp, "__iter__"):
898 self.__iter__ = self.fp.__iter__
899 if hasattr(self.fp, "next"):
900 self.next = self.fp.next
Guido van Rossume7b146f2000-02-04 15:28:42 +0000901
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000902 def __repr__(self):
Walter Dörwald70a6b492004-02-12 17:35:32 +0000903 return '<%s at %r whose fp = %r>' % (self.__class__.__name__,
904 id(self), self.fp)
Guido van Rossume7b146f2000-02-04 15:28:42 +0000905
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000906 def close(self):
907 self.read = None
908 self.readline = None
909 self.readlines = None
910 self.fileno = None
911 if self.fp: self.fp.close()
912 self.fp = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000913
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000914class addclosehook(addbase):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000915 """Class to add a close hook to an open file."""
916
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000917 def __init__(self, fp, closehook, *hookargs):
918 addbase.__init__(self, fp)
919 self.closehook = closehook
920 self.hookargs = hookargs
Guido van Rossume7b146f2000-02-04 15:28:42 +0000921
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000922 def close(self):
Guido van Rossumc580dae2000-05-24 13:21:46 +0000923 addbase.close(self)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000924 if self.closehook:
Guido van Rossum68468eb2003-02-27 20:14:51 +0000925 self.closehook(*self.hookargs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000926 self.closehook = None
927 self.hookargs = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000928
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000929class addinfo(addbase):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000930 """class to add an info() method to an open file."""
931
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000932 def __init__(self, fp, headers):
933 addbase.__init__(self, fp)
934 self.headers = headers
Guido van Rossume7b146f2000-02-04 15:28:42 +0000935
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000936 def info(self):
937 return self.headers
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000938
Guido van Rossume6ad8911996-09-10 17:02:56 +0000939class addinfourl(addbase):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000940 """class to add info() and geturl() methods to an open file."""
941
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000942 def __init__(self, fp, headers, url):
943 addbase.__init__(self, fp)
944 self.headers = headers
945 self.url = url
Guido van Rossume7b146f2000-02-04 15:28:42 +0000946
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000947 def info(self):
948 return self.headers
Guido van Rossume7b146f2000-02-04 15:28:42 +0000949
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000950 def geturl(self):
951 return self.url
Guido van Rossume6ad8911996-09-10 17:02:56 +0000952
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000953
Guido van Rossum7c395db1994-07-04 22:14:49 +0000954# Utilities to parse URLs (most of these return None for missing parts):
Sjoerd Mullendere0371b81995-11-10 10:36:07 +0000955# unwrap('<URL:type://host/path>') --> 'type://host/path'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000956# splittype('type:opaquestring') --> 'type', 'opaquestring'
957# splithost('//host[:port]/path') --> 'host[:port]', '/path'
Guido van Rossum7c395db1994-07-04 22:14:49 +0000958# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
959# splitpasswd('user:passwd') -> 'user', 'passwd'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000960# splitport('host:port') --> 'host', 'port'
961# splitquery('/path?query') --> '/path', 'query'
962# splittag('/path#tag') --> '/path', 'tag'
Guido van Rossum7c395db1994-07-04 22:14:49 +0000963# splitattr('/path;attr1=value1;attr2=value2;...') ->
964# '/path', ['attr1=value1', 'attr2=value2', ...]
965# splitvalue('attr=value') --> 'attr', 'value'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000966# unquote('abc%20def') -> 'abc def'
967# quote('abc def') -> 'abc%20def')
968
Walter Dörwald65230a22002-06-03 15:58:32 +0000969try:
970 unicode
971except NameError:
Guido van Rossum4b46c0a2002-05-24 17:58:05 +0000972 def _is_unicode(x):
973 return 0
Walter Dörwald65230a22002-06-03 15:58:32 +0000974else:
975 def _is_unicode(x):
976 return isinstance(x, unicode)
Guido van Rossum4b46c0a2002-05-24 17:58:05 +0000977
Martin v. Löwis1d994332000-12-03 18:30:10 +0000978def toBytes(url):
979 """toBytes(u"URL") --> 'URL'."""
980 # Most URL schemes require ASCII. If that changes, the conversion
981 # can be relaxed
Guido van Rossum4b46c0a2002-05-24 17:58:05 +0000982 if _is_unicode(url):
Martin v. Löwis1d994332000-12-03 18:30:10 +0000983 try:
984 url = url.encode("ASCII")
985 except UnicodeError:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000986 raise UnicodeError("URL " + repr(url) +
987 " contains non-ASCII characters")
Martin v. Löwis1d994332000-12-03 18:30:10 +0000988 return url
989
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000990def unwrap(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000991 """unwrap('<URL:type://host/path>') --> 'type://host/path'."""
Guido van Rossumb2493f82000-12-15 15:01:37 +0000992 url = url.strip()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000993 if url[:1] == '<' and url[-1:] == '>':
Guido van Rossumb2493f82000-12-15 15:01:37 +0000994 url = url[1:-1].strip()
995 if url[:4] == 'URL:': url = url[4:].strip()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000996 return url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000997
Guido van Rossum332e1441997-09-29 23:23:46 +0000998_typeprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000999def splittype(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001000 """splittype('type:opaquestring') --> 'type', 'opaquestring'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001001 global _typeprog
1002 if _typeprog is None:
1003 import re
1004 _typeprog = re.compile('^([^/:]+):')
Guido van Rossum332e1441997-09-29 23:23:46 +00001005
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001006 match = _typeprog.match(url)
1007 if match:
1008 scheme = match.group(1)
Fred Drake9e94afd2000-07-01 07:03:30 +00001009 return scheme.lower(), url[len(scheme) + 1:]
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001010 return None, url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001011
Guido van Rossum332e1441997-09-29 23:23:46 +00001012_hostprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001013def splithost(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001014 """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001015 global _hostprog
1016 if _hostprog is None:
1017 import re
Georg Brandl1c168d82006-03-26 20:59:38 +00001018 _hostprog = re.compile('^//([^/?]*)(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001019
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001020 match = _hostprog.match(url)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001021 if match: return match.group(1, 2)
1022 return None, url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001023
Guido van Rossum332e1441997-09-29 23:23:46 +00001024_userprog = None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001025def splituser(host):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001026 """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001027 global _userprog
1028 if _userprog is None:
1029 import re
Raymond Hettingerf2e45dd2002-08-18 20:08:56 +00001030 _userprog = re.compile('^(.*)@(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001031
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001032 match = _userprog.match(host)
Fred Drake567ca8e2000-08-21 21:42:42 +00001033 if match: return map(unquote, match.group(1, 2))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001034 return None, host
Guido van Rossum7c395db1994-07-04 22:14:49 +00001035
Guido van Rossum332e1441997-09-29 23:23:46 +00001036_passwdprog = None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001037def splitpasswd(user):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001038 """splitpasswd('user:passwd') -> 'user', 'passwd'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001039 global _passwdprog
1040 if _passwdprog is None:
1041 import re
1042 _passwdprog = re.compile('^([^:]*):(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001043
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001044 match = _passwdprog.match(user)
1045 if match: return match.group(1, 2)
1046 return user, None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001047
Guido van Rossume7b146f2000-02-04 15:28:42 +00001048# splittag('/path#tag') --> '/path', 'tag'
Guido van Rossum332e1441997-09-29 23:23:46 +00001049_portprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001050def splitport(host):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001051 """splitport('host:port') --> 'host', 'port'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001052 global _portprog
1053 if _portprog is None:
1054 import re
1055 _portprog = re.compile('^(.*):([0-9]+)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001056
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001057 match = _portprog.match(host)
1058 if match: return match.group(1, 2)
1059 return host, None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001060
Guido van Rossum332e1441997-09-29 23:23:46 +00001061_nportprog = None
Guido van Rossum53725a21996-06-13 19:12:35 +00001062def splitnport(host, defport=-1):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001063 """Split host and port, returning numeric port.
1064 Return given default port if no ':' found; defaults to -1.
1065 Return numerical port if a valid number are found after ':'.
1066 Return None if ':' but not a valid number."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001067 global _nportprog
1068 if _nportprog is None:
1069 import re
1070 _nportprog = re.compile('^(.*):(.*)$')
Guido van Rossum7e7ca0b1998-03-26 21:01:39 +00001071
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001072 match = _nportprog.match(host)
1073 if match:
1074 host, port = match.group(1, 2)
1075 try:
Guido van Rossumb2493f82000-12-15 15:01:37 +00001076 if not port: raise ValueError, "no digits"
1077 nport = int(port)
1078 except ValueError:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001079 nport = None
1080 return host, nport
1081 return host, defport
Guido van Rossum53725a21996-06-13 19:12:35 +00001082
Guido van Rossum332e1441997-09-29 23:23:46 +00001083_queryprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001084def splitquery(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001085 """splitquery('/path?query') --> '/path', 'query'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001086 global _queryprog
1087 if _queryprog is None:
1088 import re
1089 _queryprog = re.compile('^(.*)\?([^?]*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001090
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001091 match = _queryprog.match(url)
1092 if match: return match.group(1, 2)
1093 return url, None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001094
Guido van Rossum332e1441997-09-29 23:23:46 +00001095_tagprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001096def splittag(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001097 """splittag('/path#tag') --> '/path', 'tag'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001098 global _tagprog
1099 if _tagprog is None:
1100 import re
1101 _tagprog = re.compile('^(.*)#([^#]*)$')
Guido van Rossum7e7ca0b1998-03-26 21:01:39 +00001102
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001103 match = _tagprog.match(url)
1104 if match: return match.group(1, 2)
1105 return url, None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001106
Guido van Rossum7c395db1994-07-04 22:14:49 +00001107def splitattr(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001108 """splitattr('/path;attr1=value1;attr2=value2;...') ->
1109 '/path', ['attr1=value1', 'attr2=value2', ...]."""
Guido van Rossumb2493f82000-12-15 15:01:37 +00001110 words = url.split(';')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001111 return words[0], words[1:]
Guido van Rossum7c395db1994-07-04 22:14:49 +00001112
Guido van Rossum332e1441997-09-29 23:23:46 +00001113_valueprog = None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001114def splitvalue(attr):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001115 """splitvalue('attr=value') --> 'attr', 'value'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001116 global _valueprog
1117 if _valueprog is None:
1118 import re
1119 _valueprog = re.compile('^([^=]*)=(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001120
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001121 match = _valueprog.match(attr)
1122 if match: return match.group(1, 2)
1123 return attr, None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001124
Raymond Hettinger803ce802005-09-10 06:49:04 +00001125_hextochr = dict(('%02x' % i, chr(i)) for i in range(256))
1126_hextochr.update(('%02X' % i, chr(i)) for i in range(256))
1127
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001128def unquote(s):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001129 """unquote('abc%20def') -> 'abc def'."""
Raymond Hettinger803ce802005-09-10 06:49:04 +00001130 res = s.split('%')
1131 for i in xrange(1, len(res)):
1132 item = res[i]
1133 try:
1134 res[i] = _hextochr[item[:2]] + item[2:]
1135 except KeyError:
1136 res[i] = '%' + item
Raymond Hettinger4b0f20d2005-10-15 16:41:53 +00001137 except UnicodeDecodeError:
1138 res[i] = unichr(int(item[:2], 16)) + item[2:]
Guido van Rossumb2493f82000-12-15 15:01:37 +00001139 return "".join(res)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001140
Guido van Rossum0564e121996-12-13 14:47:36 +00001141def unquote_plus(s):
Skip Montanaro79f1c172000-08-22 03:00:52 +00001142 """unquote('%7e/abc+def') -> '~/abc def'"""
Brett Cannonaaeffaf2004-03-23 23:50:17 +00001143 s = s.replace('+', ' ')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001144 return unquote(s)
Guido van Rossum0564e121996-12-13 14:47:36 +00001145
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001146always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
Jeremy Hylton6102e292000-08-31 15:48:10 +00001147 'abcdefghijklmnopqrstuvwxyz'
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001148 '0123456789' '_.-')
Raymond Hettinger199d2f72005-09-09 22:27:13 +00001149_safemaps = {}
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001150
Guido van Rossum7c395db1994-07-04 22:14:49 +00001151def quote(s, safe = '/'):
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001152 """quote('abc def') -> 'abc%20def'
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001153
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001154 Each part of a URL, e.g. the path info, the query, etc., has a
1155 different set of reserved characters that must be quoted.
1156
1157 RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
1158 the following reserved characters.
1159
1160 reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
1161 "$" | ","
1162
1163 Each of these characters is reserved in some component of a URL,
1164 but not necessarily in all of them.
1165
1166 By default, the quote function is intended for quoting the path
1167 section of a URL. Thus, it will not encode '/'. This character
1168 is reserved, but in typical usage the quote function is being
1169 called on a path where the existing slash characters are used as
1170 reserved characters.
1171 """
Raymond Hettinger199d2f72005-09-09 22:27:13 +00001172 cachekey = (safe, always_safe)
1173 try:
1174 safe_map = _safemaps[cachekey]
1175 except KeyError:
1176 safe += always_safe
1177 safe_map = {}
1178 for i in range(256):
1179 c = chr(i)
1180 safe_map[c] = (c in safe) and c or ('%%%02X' % i)
1181 _safemaps[cachekey] = safe_map
1182 res = map(safe_map.__getitem__, s)
Guido van Rossumb2493f82000-12-15 15:01:37 +00001183 return ''.join(res)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001184
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001185def quote_plus(s, safe = ''):
1186 """Quote the query fragment of a URL; replacing ' ' with '+'"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001187 if ' ' in s:
Raymond Hettingercf6b6322005-09-10 18:17:54 +00001188 s = quote(s, safe + ' ')
1189 return s.replace(' ', '+')
1190 return quote(s, safe)
Guido van Rossum0564e121996-12-13 14:47:36 +00001191
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001192def urlencode(query,doseq=0):
1193 """Encode a sequence of two-element tuples or dictionary into a URL query string.
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001194
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001195 If any values in the query arg are sequences and doseq is true, each
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001196 sequence element is converted to a separate parameter.
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001197
1198 If the query arg is a sequence of two-element tuples, the order of the
1199 parameters in the output will match the order of parameters in the
1200 input.
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001201 """
Tim Peters658cba62001-02-09 20:06:00 +00001202
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001203 if hasattr(query,"items"):
1204 # mapping objects
1205 query = query.items()
1206 else:
1207 # it's a bother at times that strings and string-like objects are
1208 # sequences...
1209 try:
1210 # non-sequence items should not work with len()
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001211 # non-empty strings will fail this
Walter Dörwald65230a22002-06-03 15:58:32 +00001212 if len(query) and not isinstance(query[0], tuple):
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001213 raise TypeError
1214 # zero-length sequences of all types will get here and succeed,
1215 # but that's a minor nit - since the original implementation
1216 # allowed empty dicts that type of behavior probably should be
1217 # preserved for consistency
1218 except TypeError:
1219 ty,va,tb = sys.exc_info()
1220 raise TypeError, "not a valid non-string sequence or mapping object", tb
1221
Guido van Rossume7b146f2000-02-04 15:28:42 +00001222 l = []
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001223 if not doseq:
1224 # preserve old behavior
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001225 for k, v in query:
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001226 k = quote_plus(str(k))
1227 v = quote_plus(str(v))
1228 l.append(k + '=' + v)
1229 else:
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001230 for k, v in query:
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001231 k = quote_plus(str(k))
Walter Dörwald65230a22002-06-03 15:58:32 +00001232 if isinstance(v, str):
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001233 v = quote_plus(v)
1234 l.append(k + '=' + v)
Guido van Rossum4b46c0a2002-05-24 17:58:05 +00001235 elif _is_unicode(v):
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001236 # is there a reasonable way to convert to ASCII?
1237 # encode generates a string, but "replace" or "ignore"
1238 # lose information and "strict" can raise UnicodeError
1239 v = quote_plus(v.encode("ASCII","replace"))
1240 l.append(k + '=' + v)
1241 else:
1242 try:
1243 # is this a sufficient test for sequence-ness?
1244 x = len(v)
1245 except TypeError:
1246 # not a sequence
1247 v = quote_plus(str(v))
1248 l.append(k + '=' + v)
1249 else:
1250 # loop over the sequence
1251 for elt in v:
1252 l.append(k + '=' + quote_plus(str(elt)))
Guido van Rossumb2493f82000-12-15 15:01:37 +00001253 return '&'.join(l)
Guido van Rossum810a3391998-07-22 21:33:23 +00001254
Guido van Rossum442e7201996-03-20 15:33:11 +00001255# Proxy handling
Mark Hammond4f570b92000-07-26 07:04:38 +00001256def getproxies_environment():
1257 """Return a dictionary of scheme -> proxy server URL mappings.
1258
1259 Scan the environment for variables named <scheme>_proxy;
1260 this seems to be the standard convention. If you need a
1261 different way, you can pass a proxies dictionary to the
1262 [Fancy]URLopener constructor.
1263
1264 """
1265 proxies = {}
1266 for name, value in os.environ.items():
Guido van Rossumb2493f82000-12-15 15:01:37 +00001267 name = name.lower()
Mark Hammond4f570b92000-07-26 07:04:38 +00001268 if value and name[-6:] == '_proxy':
1269 proxies[name[:-6]] = value
1270 return proxies
1271
Jack Jansen11d9b062004-07-16 11:45:00 +00001272if sys.platform == 'darwin':
1273 def getproxies_internetconfig():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001274 """Return a dictionary of scheme -> proxy server URL mappings.
Guido van Rossum442e7201996-03-20 15:33:11 +00001275
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001276 By convention the mac uses Internet Config to store
1277 proxies. An HTTP proxy, for instance, is stored under
1278 the HttpProxy key.
Guido van Rossum442e7201996-03-20 15:33:11 +00001279
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001280 """
1281 try:
1282 import ic
1283 except ImportError:
1284 return {}
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001285
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001286 try:
1287 config = ic.IC()
1288 except ic.error:
1289 return {}
1290 proxies = {}
1291 # HTTP:
Raymond Hettinger54f02222002-06-01 14:18:47 +00001292 if 'UseHTTPProxy' in config and config['UseHTTPProxy']:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001293 try:
1294 value = config['HTTPProxyHost']
1295 except ic.error:
1296 pass
1297 else:
1298 proxies['http'] = 'http://%s' % value
1299 # FTP: XXXX To be done.
1300 # Gopher: XXXX To be done.
1301 return proxies
Mark Hammond4f570b92000-07-26 07:04:38 +00001302
Tim Peters55c12d42001-08-09 18:04:14 +00001303 def proxy_bypass(x):
1304 return 0
1305
Jack Jansen11d9b062004-07-16 11:45:00 +00001306 def getproxies():
1307 return getproxies_environment() or getproxies_internetconfig()
Tim Peters182b5ac2004-07-18 06:16:08 +00001308
Mark Hammond4f570b92000-07-26 07:04:38 +00001309elif os.name == 'nt':
1310 def getproxies_registry():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001311 """Return a dictionary of scheme -> proxy server URL mappings.
Mark Hammond4f570b92000-07-26 07:04:38 +00001312
1313 Win32 uses the registry to store proxies.
1314
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001315 """
1316 proxies = {}
Mark Hammond4f570b92000-07-26 07:04:38 +00001317 try:
1318 import _winreg
1319 except ImportError:
1320 # Std module, so should be around - but you never know!
1321 return proxies
1322 try:
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001323 internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1324 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
Mark Hammond4f570b92000-07-26 07:04:38 +00001325 proxyEnable = _winreg.QueryValueEx(internetSettings,
1326 'ProxyEnable')[0]
1327 if proxyEnable:
1328 # Returned as Unicode but problems if not converted to ASCII
1329 proxyServer = str(_winreg.QueryValueEx(internetSettings,
1330 'ProxyServer')[0])
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001331 if '=' in proxyServer:
1332 # Per-protocol settings
Mark Hammond4f570b92000-07-26 07:04:38 +00001333 for p in proxyServer.split(';'):
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001334 protocol, address = p.split('=', 1)
Guido van Rossumb955d6c2002-03-31 23:38:48 +00001335 # See if address has a type:// prefix
Guido van Rossum64e5aa92002-04-02 14:38:16 +00001336 import re
1337 if not re.match('^([^/:]+)://', address):
Guido van Rossumb955d6c2002-03-31 23:38:48 +00001338 address = '%s://%s' % (protocol, address)
1339 proxies[protocol] = address
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001340 else:
1341 # Use one setting for all protocols
1342 if proxyServer[:5] == 'http:':
1343 proxies['http'] = proxyServer
1344 else:
1345 proxies['http'] = 'http://%s' % proxyServer
1346 proxies['ftp'] = 'ftp://%s' % proxyServer
Mark Hammond4f570b92000-07-26 07:04:38 +00001347 internetSettings.Close()
1348 except (WindowsError, ValueError, TypeError):
1349 # Either registry key not found etc, or the value in an
1350 # unexpected format.
1351 # proxies already set up to be empty so nothing to do
1352 pass
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001353 return proxies
Guido van Rossum442e7201996-03-20 15:33:11 +00001354
Mark Hammond4f570b92000-07-26 07:04:38 +00001355 def getproxies():
1356 """Return a dictionary of scheme -> proxy server URL mappings.
1357
1358 Returns settings gathered from the environment, if specified,
1359 or the registry.
1360
1361 """
1362 return getproxies_environment() or getproxies_registry()
Tim Peters55c12d42001-08-09 18:04:14 +00001363
1364 def proxy_bypass(host):
1365 try:
1366 import _winreg
1367 import re
Tim Peters55c12d42001-08-09 18:04:14 +00001368 except ImportError:
1369 # Std modules, so should be around - but you never know!
1370 return 0
1371 try:
1372 internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1373 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
1374 proxyEnable = _winreg.QueryValueEx(internetSettings,
1375 'ProxyEnable')[0]
1376 proxyOverride = str(_winreg.QueryValueEx(internetSettings,
1377 'ProxyOverride')[0])
1378 # ^^^^ Returned as Unicode but problems if not converted to ASCII
1379 except WindowsError:
1380 return 0
1381 if not proxyEnable or not proxyOverride:
1382 return 0
1383 # try to make a host list from name and IP address.
Georg Brandl1f636702006-02-18 23:10:23 +00001384 rawHost, port = splitport(host)
1385 host = [rawHost]
Tim Peters55c12d42001-08-09 18:04:14 +00001386 try:
Georg Brandl1f636702006-02-18 23:10:23 +00001387 addr = socket.gethostbyname(rawHost)
1388 if addr != rawHost:
Tim Peters55c12d42001-08-09 18:04:14 +00001389 host.append(addr)
1390 except socket.error:
1391 pass
Georg Brandl1f636702006-02-18 23:10:23 +00001392 try:
1393 fqdn = socket.getfqdn(rawHost)
1394 if fqdn != rawHost:
1395 host.append(fqdn)
1396 except socket.error:
1397 pass
Tim Peters55c12d42001-08-09 18:04:14 +00001398 # make a check value list from the registry entry: replace the
1399 # '<local>' string by the localhost entry and the corresponding
1400 # canonical entry.
1401 proxyOverride = proxyOverride.split(';')
1402 i = 0
1403 while i < len(proxyOverride):
1404 if proxyOverride[i] == '<local>':
1405 proxyOverride[i:i+1] = ['localhost',
1406 '127.0.0.1',
1407 socket.gethostname(),
1408 socket.gethostbyname(
1409 socket.gethostname())]
1410 i += 1
1411 # print proxyOverride
1412 # now check if we match one of the registry values.
1413 for test in proxyOverride:
Tim Petersab9ba272001-08-09 21:40:30 +00001414 test = test.replace(".", r"\.") # mask dots
1415 test = test.replace("*", r".*") # change glob sequence
1416 test = test.replace("?", r".") # change glob char
Tim Peters55c12d42001-08-09 18:04:14 +00001417 for val in host:
1418 # print "%s <--> %s" %( test, val )
1419 if re.match(test, val, re.I):
1420 return 1
1421 return 0
1422
Mark Hammond4f570b92000-07-26 07:04:38 +00001423else:
1424 # By default use environment variables
1425 getproxies = getproxies_environment
1426
Tim Peters55c12d42001-08-09 18:04:14 +00001427 def proxy_bypass(host):
1428 return 0
Guido van Rossum442e7201996-03-20 15:33:11 +00001429
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001430# Test and time quote() and unquote()
1431def test1():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001432 s = ''
1433 for i in range(256): s = s + chr(i)
1434 s = s*4
1435 t0 = time.time()
1436 qs = quote(s)
1437 uqs = unquote(qs)
1438 t1 = time.time()
1439 if uqs != s:
1440 print 'Wrong!'
Walter Dörwald70a6b492004-02-12 17:35:32 +00001441 print repr(s)
1442 print repr(qs)
1443 print repr(uqs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001444 print round(t1 - t0, 3), 'sec'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001445
1446
Guido van Rossum9ab96d41998-09-28 14:07:00 +00001447def reporthook(blocknum, blocksize, totalsize):
1448 # Report during remote transfers
Guido van Rossumb2493f82000-12-15 15:01:37 +00001449 print "Block number: %d, Block size: %d, Total size: %d" % (
1450 blocknum, blocksize, totalsize)
Guido van Rossum9ab96d41998-09-28 14:07:00 +00001451
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001452# Test program
Guido van Rossum23490151998-06-25 02:39:00 +00001453def test(args=[]):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001454 if not args:
1455 args = [
1456 '/etc/passwd',
1457 'file:/etc/passwd',
1458 'file://localhost/etc/passwd',
Collin Winter071d1ae2007-03-12 01:55:54 +00001459 'ftp://ftp.gnu.org/pub/README',
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001460 'http://www.python.org/index.html',
1461 ]
Guido van Rossum09c8b6c1999-12-07 21:37:17 +00001462 if hasattr(URLopener, "open_https"):
1463 args.append('https://synergy.as.cmu.edu/~geek/')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001464 try:
1465 for url in args:
1466 print '-'*10, url, '-'*10
1467 fn, h = urlretrieve(url, None, reporthook)
Guido van Rossumb2493f82000-12-15 15:01:37 +00001468 print fn
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001469 if h:
1470 print '======'
1471 for k in h.keys(): print k + ':', h[k]
1472 print '======'
1473 fp = open(fn, 'rb')
1474 data = fp.read()
1475 del fp
1476 if '\r' in data:
1477 table = string.maketrans("", "")
Guido van Rossumb2493f82000-12-15 15:01:37 +00001478 data = data.translate(table, "\r")
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001479 print data
1480 fn, h = None, None
1481 print '-'*40
1482 finally:
1483 urlcleanup()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001484
Guido van Rossum23490151998-06-25 02:39:00 +00001485def main():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001486 import getopt, sys
1487 try:
1488 opts, args = getopt.getopt(sys.argv[1:], "th")
1489 except getopt.error, msg:
1490 print msg
1491 print "Use -h for help"
1492 return
1493 t = 0
1494 for o, a in opts:
1495 if o == '-t':
1496 t = t + 1
1497 if o == '-h':
1498 print "Usage: python urllib.py [-t] [url ...]"
1499 print "-t runs self-test;",
1500 print "otherwise, contents of urls are printed"
1501 return
1502 if t:
1503 if t > 1:
1504 test1()
1505 test(args)
1506 else:
1507 if not args:
1508 print "Use -h for help"
1509 for url in args:
1510 print urlopen(url).read(),
Guido van Rossum23490151998-06-25 02:39:00 +00001511
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001512# Run test program when run as a script
1513if __name__ == '__main__':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001514 main()