blob: d65c0b033837e662fbbf8ad3479107267adb51c1 [file] [log] [blame]
Guido van Rossume7b146f2000-02-04 15:28:42 +00001"""Open an arbitrary URL.
2
3See the following document for more info on URLs:
4"Names and Addresses, URIs, URLs, URNs, URCs", at
5http://www.w3.org/pub/WWW/Addressing/Overview.html
6
7See also the HTTP spec (from which the error codes are derived):
8"HTTP - Hypertext Transfer Protocol", at
9http://www.w3.org/pub/WWW/Protocols/
10
11Related standards and specs:
12- RFC1808: the "relative URL" spec. (authoritative status)
13- RFC1738 - the "URL standard". (authoritative status)
14- RFC1630 - the "URI spec". (informational status)
15
16The object returned by URLopener().open(file) will differ per
17protocol. All you know is that is has methods read(), readline(),
18readlines(), fileno(), close() and info(). The read*(), fileno()
Fredrik Lundhb49f88b2000-09-24 18:51:25 +000019and close() methods work like those of open files.
Guido van Rossume7b146f2000-02-04 15:28:42 +000020The info() method returns a mimetools.Message object which can be
21used to query various info about the object, if available.
22(mimetools.Message objects are queried with the getheader() method.)
23"""
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000024
Guido van Rossum7c395db1994-07-04 22:14:49 +000025import string
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000026import socket
Jack Jansendc3e3f61995-12-15 13:22:13 +000027import os
Guido van Rossumf0713d32001-08-09 17:43:35 +000028import time
Guido van Rossum3c8484e1996-11-20 22:02:24 +000029import sys
Brett Cannon69200fa2004-03-23 21:26:39 +000030from urlparse import urljoin as basejoin
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000031
Skip Montanaro40fc1602001-03-01 04:27:19 +000032__all__ = ["urlopen", "URLopener", "FancyURLopener", "urlretrieve",
33 "urlcleanup", "quote", "quote_plus", "unquote", "unquote_plus",
Skip Montanaro44d5e0c2001-03-13 19:47:16 +000034 "urlencode", "url2pathname", "pathname2url", "splittag",
35 "localhost", "thishost", "ftperrors", "basejoin", "unwrap",
36 "splittype", "splithost", "splituser", "splitpasswd", "splitport",
37 "splitnport", "splitquery", "splitattr", "splitvalue",
38 "splitgophertype", "getproxies"]
Skip Montanaro40fc1602001-03-01 04:27:19 +000039
Martin v. Löwis3e865952006-01-24 15:51:21 +000040__version__ = '1.17' # XXX This version is not always updated :-(
Guido van Rossumf668d171997-06-06 21:11:11 +000041
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000042MAXFTPCACHE = 10 # Trim the ftp cache beyond this size
Guido van Rossum6cb15a01995-06-22 19:00:13 +000043
Jack Jansendc3e3f61995-12-15 13:22:13 +000044# Helper for non-unix systems
45if os.name == 'mac':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000046 from macurl2path import url2pathname, pathname2url
Guido van Rossum7e7ca0b1998-03-26 21:01:39 +000047elif os.name == 'nt':
Fredrik Lundhb49f88b2000-09-24 18:51:25 +000048 from nturl2path import url2pathname, pathname2url
Guido van Rossumd74fb6b2001-03-02 06:43:49 +000049elif os.name == 'riscos':
50 from rourl2path import url2pathname, pathname2url
Jack Jansendc3e3f61995-12-15 13:22:13 +000051else:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000052 def url2pathname(pathname):
Georg Brandlc0b24732005-12-26 22:53:56 +000053 """OS-specific conversion from a relative URL of the 'file' scheme
54 to a file system path; not recommended for general use."""
Guido van Rossum367ac801999-03-12 14:31:10 +000055 return unquote(pathname)
Georg Brandlc0b24732005-12-26 22:53:56 +000056
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000057 def pathname2url(pathname):
Georg Brandlc0b24732005-12-26 22:53:56 +000058 """OS-specific conversion from a file system path to a relative URL
59 of the 'file' scheme; not recommended for general use."""
Guido van Rossum367ac801999-03-12 14:31:10 +000060 return quote(pathname)
Guido van Rossum33add0a1998-12-18 15:25:22 +000061
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000062# This really consists of two pieces:
63# (1) a class which handles opening of all sorts of URLs
64# (plus assorted utilities etc.)
65# (2) a set of functions for parsing URLs
66# XXX Should these be separated out into different modules?
67
68
69# Shortcut for basic usage
70_urlopener = None
Fred Drakedf6eca72002-04-04 20:41:34 +000071def urlopen(url, data=None, proxies=None):
Skip Montanaro79f1c172000-08-22 03:00:52 +000072 """urlopen(url [, data]) -> open file-like object"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000073 global _urlopener
Fred Drakedf6eca72002-04-04 20:41:34 +000074 if proxies is not None:
75 opener = FancyURLopener(proxies=proxies)
76 elif not _urlopener:
77 opener = FancyURLopener()
78 _urlopener = opener
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000079 else:
Fred Drakedf6eca72002-04-04 20:41:34 +000080 opener = _urlopener
81 if data is None:
82 return opener.open(url)
83 else:
84 return opener.open(url, data)
Fred Drake316a7932000-08-24 01:01:26 +000085def urlretrieve(url, filename=None, reporthook=None, data=None):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000086 global _urlopener
87 if not _urlopener:
88 _urlopener = FancyURLopener()
Fred Drake316a7932000-08-24 01:01:26 +000089 return _urlopener.retrieve(url, filename, reporthook, data)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000090def urlcleanup():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000091 if _urlopener:
92 _urlopener.cleanup()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000093
Georg Brandlb9256022005-08-24 18:46:39 +000094# exception raised when downloaded size does not match content-length
95class ContentTooShortError(IOError):
96 def __init__(self, message, content):
97 IOError.__init__(self, message)
98 self.content = content
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000099
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000100ftpcache = {}
101class URLopener:
Guido van Rossume7b146f2000-02-04 15:28:42 +0000102 """Class to open URLs.
103 This is a class rather than just a subroutine because we may need
104 more than one set of global protocol-specific options.
105 Note -- this is a base class for those who don't want the
106 automatic handling of errors type 302 (relocated) and 401
107 (authorization needed)."""
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000108
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000109 __tempfiles = None
Guido van Rossum29e77811996-11-27 19:39:58 +0000110
Guido van Rossumba311382000-08-24 16:18:04 +0000111 version = "Python-urllib/%s" % __version__
112
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000113 # Constructor
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000114 def __init__(self, proxies=None, **x509):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000115 if proxies is None:
116 proxies = getproxies()
117 assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
118 self.proxies = proxies
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000119 self.key_file = x509.get('key_file')
120 self.cert_file = x509.get('cert_file')
Guido van Rossumba311382000-08-24 16:18:04 +0000121 self.addheaders = [('User-agent', self.version)]
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000122 self.__tempfiles = []
123 self.__unlink = os.unlink # See cleanup()
124 self.tempcache = None
125 # Undocumented feature: if you assign {} to tempcache,
126 # it is used to cache files retrieved with
127 # self.retrieve(). This is not enabled by default
128 # since it does not work for changing documents (and I
129 # haven't got the logic to check expiration headers
130 # yet).
131 self.ftpcache = ftpcache
132 # Undocumented feature: you can use a different
133 # ftp cache by assigning to the .ftpcache member;
134 # in case you want logically independent URL openers
135 # XXX This is not threadsafe. Bah.
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000136
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000137 def __del__(self):
138 self.close()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000139
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000140 def close(self):
141 self.cleanup()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000142
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000143 def cleanup(self):
144 # This code sometimes runs when the rest of this module
145 # has already been deleted, so it can't use any globals
146 # or import anything.
147 if self.__tempfiles:
148 for file in self.__tempfiles:
149 try:
150 self.__unlink(file)
Martin v. Löwis58682b72001-08-11 15:02:57 +0000151 except OSError:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000152 pass
153 del self.__tempfiles[:]
154 if self.tempcache:
155 self.tempcache.clear()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000156
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000157 def addheader(self, *args):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000158 """Add a header to be used by the HTTP interface only
159 e.g. u.addheader('Accept', 'sound/basic')"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000160 self.addheaders.append(args)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000161
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000162 # External interface
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000163 def open(self, fullurl, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000164 """Use URLopener().open(file) instead of open(file, 'r')."""
Martin v. Löwis1d994332000-12-03 18:30:10 +0000165 fullurl = unwrap(toBytes(fullurl))
Raymond Hettinger54f02222002-06-01 14:18:47 +0000166 if self.tempcache and fullurl in self.tempcache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000167 filename, headers = self.tempcache[fullurl]
168 fp = open(filename, 'rb')
169 return addinfourl(fp, headers, fullurl)
Martin v. Löwis1d994332000-12-03 18:30:10 +0000170 urltype, url = splittype(fullurl)
171 if not urltype:
172 urltype = 'file'
Raymond Hettinger54f02222002-06-01 14:18:47 +0000173 if urltype in self.proxies:
Martin v. Löwis1d994332000-12-03 18:30:10 +0000174 proxy = self.proxies[urltype]
175 urltype, proxyhost = splittype(proxy)
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000176 host, selector = splithost(proxyhost)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000177 url = (host, fullurl) # Signal special case to open_*()
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000178 else:
179 proxy = None
Martin v. Löwis1d994332000-12-03 18:30:10 +0000180 name = 'open_' + urltype
181 self.type = urltype
Brett Cannonaaeffaf2004-03-23 23:50:17 +0000182 name = name.replace('-', '_')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000183 if not hasattr(self, name):
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000184 if proxy:
185 return self.open_unknown_proxy(proxy, fullurl, data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000186 else:
187 return self.open_unknown(fullurl, data)
188 try:
189 if data is None:
190 return getattr(self, name)(url)
191 else:
192 return getattr(self, name)(url, data)
193 except socket.error, msg:
194 raise IOError, ('socket error', msg), sys.exc_info()[2]
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000195
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000196 def open_unknown(self, fullurl, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000197 """Overridable interface to open unknown URL type."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000198 type, url = splittype(fullurl)
199 raise IOError, ('url error', 'unknown url type', type)
Guido van Rossumca445401995-08-29 19:19:12 +0000200
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000201 def open_unknown_proxy(self, proxy, fullurl, data=None):
202 """Overridable interface to open unknown URL type."""
203 type, url = splittype(fullurl)
204 raise IOError, ('url error', 'invalid proxy for %s' % type, proxy)
205
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000206 # External interface
Sjoerd Mullenderd7b86f02000-08-25 11:23:36 +0000207 def retrieve(self, url, filename=None, reporthook=None, data=None):
Brett Cannon7d618c72003-04-24 02:43:20 +0000208 """retrieve(url) returns (filename, headers) for a local object
Guido van Rossume7b146f2000-02-04 15:28:42 +0000209 or (tempfilename, headers) for a remote object."""
Martin v. Löwis1d994332000-12-03 18:30:10 +0000210 url = unwrap(toBytes(url))
Raymond Hettinger54f02222002-06-01 14:18:47 +0000211 if self.tempcache and url in self.tempcache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000212 return self.tempcache[url]
213 type, url1 = splittype(url)
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000214 if filename is None and (not type or type == 'file'):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000215 try:
216 fp = self.open_local_file(url1)
217 hdrs = fp.info()
218 del fp
219 return url2pathname(splithost(url1)[1]), hdrs
220 except IOError, msg:
221 pass
Fred Drake316a7932000-08-24 01:01:26 +0000222 fp = self.open(url, data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000223 headers = fp.info()
Guido van Rossum3b0a3292002-08-09 16:38:32 +0000224 if filename:
225 tfp = open(filename, 'wb')
226 else:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000227 import tempfile
228 garbage, path = splittype(url)
229 garbage, path = splithost(path or "")
230 path, garbage = splitquery(path or "")
231 path, garbage = splitattr(path or "")
232 suffix = os.path.splitext(path)[1]
Guido van Rossum3b0a3292002-08-09 16:38:32 +0000233 (fd, filename) = tempfile.mkstemp(suffix)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000234 self.__tempfiles.append(filename)
Jeremy Hylton3bd6fde2002-10-11 14:36:24 +0000235 tfp = os.fdopen(fd, 'wb')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000236 result = filename, headers
237 if self.tempcache is not None:
238 self.tempcache[url] = result
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000239 bs = 1024*8
240 size = -1
Georg Brandlb9256022005-08-24 18:46:39 +0000241 read = 0
Georg Brandl5a650a22005-08-26 08:51:34 +0000242 blocknum = 0
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000243 if reporthook:
Raymond Hettinger54f02222002-06-01 14:18:47 +0000244 if "content-length" in headers:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000245 size = int(headers["Content-Length"])
Georg Brandl5a650a22005-08-26 08:51:34 +0000246 reporthook(blocknum, bs, size)
247 while 1:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000248 block = fp.read(bs)
Georg Brandl5a650a22005-08-26 08:51:34 +0000249 if block == "":
250 break
Georg Brandlb9256022005-08-24 18:46:39 +0000251 read += len(block)
Georg Brandl5a650a22005-08-26 08:51:34 +0000252 tfp.write(block)
Georg Brandlb9256022005-08-24 18:46:39 +0000253 blocknum += 1
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000254 if reporthook:
255 reporthook(blocknum, bs, size)
256 fp.close()
257 tfp.close()
258 del fp
259 del tfp
Georg Brandlb9256022005-08-24 18:46:39 +0000260
261 # raise exception if actual size does not match content-length header
262 if size >= 0 and read < size:
263 raise ContentTooShortError("retrieval incomplete: got only %i out "
264 "of %i bytes" % (read, size), result)
265
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000266 return result
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000267
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000268 # Each method named open_<type> knows how to open that type of URL
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000269
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000270 def open_http(self, url, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000271 """Use HTTP protocol."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000272 import httplib
273 user_passwd = None
Martin v. Löwis3e865952006-01-24 15:51:21 +0000274 proxy_passwd= None
Walter Dörwald65230a22002-06-03 15:58:32 +0000275 if isinstance(url, str):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000276 host, selector = splithost(url)
277 if host:
278 user_passwd, host = splituser(host)
279 host = unquote(host)
280 realhost = host
281 else:
282 host, selector = url
Martin v. Löwis3e865952006-01-24 15:51:21 +0000283 # check whether the proxy contains authorization information
284 proxy_passwd, host = splituser(host)
285 # now we proceed with the url we want to obtain
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000286 urltype, rest = splittype(selector)
287 url = rest
288 user_passwd = None
Guido van Rossumb2493f82000-12-15 15:01:37 +0000289 if urltype.lower() != 'http':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000290 realhost = None
291 else:
292 realhost, rest = splithost(rest)
293 if realhost:
294 user_passwd, realhost = splituser(realhost)
295 if user_passwd:
296 selector = "%s://%s%s" % (urltype, realhost, rest)
Tim Peters55c12d42001-08-09 18:04:14 +0000297 if proxy_bypass(realhost):
298 host = realhost
299
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000300 #print "proxy via http:", host, selector
301 if not host: raise IOError, ('http error', 'no host given')
Tim Peters92037a12006-01-24 22:44:08 +0000302
Martin v. Löwis3e865952006-01-24 15:51:21 +0000303 if proxy_passwd:
304 import base64
305 proxy_auth = base64.encodestring(proxy_passwd).strip()
306 else:
307 proxy_auth = None
308
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000309 if user_passwd:
310 import base64
Guido van Rossumb2493f82000-12-15 15:01:37 +0000311 auth = base64.encodestring(user_passwd).strip()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000312 else:
313 auth = None
314 h = httplib.HTTP(host)
315 if data is not None:
316 h.putrequest('POST', selector)
317 h.putheader('Content-type', 'application/x-www-form-urlencoded')
318 h.putheader('Content-length', '%d' % len(data))
319 else:
320 h.putrequest('GET', selector)
Martin v. Löwis3e865952006-01-24 15:51:21 +0000321 if proxy_auth: h.putheader('Proxy-Authorization', 'Basic %s' % proxy_auth)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000322 if auth: h.putheader('Authorization', 'Basic %s' % auth)
323 if realhost: h.putheader('Host', realhost)
Guido van Rossum68468eb2003-02-27 20:14:51 +0000324 for args in self.addheaders: h.putheader(*args)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000325 h.endheaders()
326 if data is not None:
Fred Drakeec3dfde2001-07-04 05:18:29 +0000327 h.send(data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000328 errcode, errmsg, headers = h.getreply()
329 fp = h.getfile()
330 if errcode == 200:
331 return addinfourl(fp, headers, "http:" + url)
332 else:
333 if data is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000334 return self.http_error(url, fp, errcode, errmsg, headers)
Guido van Rossum29aab751999-03-09 19:31:21 +0000335 else:
336 return self.http_error(url, fp, errcode, errmsg, headers, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000337
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000338 def http_error(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000339 """Handle http errors.
340 Derived class can override this, or provide specific handlers
341 named http_error_DDD where DDD is the 3-digit error code."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000342 # First check if there's a specific handler for this error
343 name = 'http_error_%d' % errcode
344 if hasattr(self, name):
345 method = getattr(self, name)
346 if data is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000347 result = method(url, fp, errcode, errmsg, headers)
Jeremy Hyltonb30f52a1999-02-25 16:14:58 +0000348 else:
349 result = method(url, fp, errcode, errmsg, headers, data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000350 if result: return result
Jeremy Hyltonb30f52a1999-02-25 16:14:58 +0000351 return self.http_error_default(url, fp, errcode, errmsg, headers)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000352
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000353 def http_error_default(self, url, fp, errcode, errmsg, headers):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000354 """Default error handler: close the connection and raise IOError."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000355 void = fp.read()
356 fp.close()
357 raise IOError, ('http error', errcode, errmsg, headers)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000358
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000359 if hasattr(socket, "ssl"):
Andrew M. Kuchling141e9892000-04-23 02:53:11 +0000360 def open_https(self, url, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000361 """Use HTTPS protocol."""
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000362 import httplib
Fred Drake567ca8e2000-08-21 21:42:42 +0000363 user_passwd = None
Martin v. Löwis3e865952006-01-24 15:51:21 +0000364 proxy_passwd = None
Walter Dörwald65230a22002-06-03 15:58:32 +0000365 if isinstance(url, str):
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000366 host, selector = splithost(url)
Fred Drake567ca8e2000-08-21 21:42:42 +0000367 if host:
368 user_passwd, host = splituser(host)
369 host = unquote(host)
370 realhost = host
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000371 else:
372 host, selector = url
Martin v. Löwis3e865952006-01-24 15:51:21 +0000373 # here, we determine, whether the proxy contains authorization information
374 proxy_passwd, host = splituser(host)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000375 urltype, rest = splittype(selector)
Fred Drake567ca8e2000-08-21 21:42:42 +0000376 url = rest
377 user_passwd = None
Guido van Rossumb2493f82000-12-15 15:01:37 +0000378 if urltype.lower() != 'https':
Fred Drake567ca8e2000-08-21 21:42:42 +0000379 realhost = None
380 else:
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000381 realhost, rest = splithost(rest)
Fred Drake567ca8e2000-08-21 21:42:42 +0000382 if realhost:
383 user_passwd, realhost = splituser(realhost)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000384 if user_passwd:
385 selector = "%s://%s%s" % (urltype, realhost, rest)
Andrew M. Kuchling7ad47922000-06-10 01:41:48 +0000386 #print "proxy via https:", host, selector
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000387 if not host: raise IOError, ('https error', 'no host given')
Martin v. Löwis3e865952006-01-24 15:51:21 +0000388 if proxy_passwd:
389 import base64
390 proxy_auth = base64.encodestring(proxy_passwd).strip()
391 else:
392 proxy_auth = None
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000393 if user_passwd:
394 import base64
Guido van Rossumb2493f82000-12-15 15:01:37 +0000395 auth = base64.encodestring(user_passwd).strip()
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000396 else:
397 auth = None
398 h = httplib.HTTPS(host, 0,
399 key_file=self.key_file,
400 cert_file=self.cert_file)
Andrew M. Kuchling141e9892000-04-23 02:53:11 +0000401 if data is not None:
402 h.putrequest('POST', selector)
403 h.putheader('Content-type',
404 'application/x-www-form-urlencoded')
405 h.putheader('Content-length', '%d' % len(data))
406 else:
407 h.putrequest('GET', selector)
Martin v. Löwis3e865952006-01-24 15:51:21 +0000408 if proxy_auth: h.putheader('Proxy-Authorization: Basic %s' % proxy_auth)
409 if auth: h.putheader('Authorization: Basic %s' % auth)
Fred Drake567ca8e2000-08-21 21:42:42 +0000410 if realhost: h.putheader('Host', realhost)
Guido van Rossum68468eb2003-02-27 20:14:51 +0000411 for args in self.addheaders: h.putheader(*args)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000412 h.endheaders()
Andrew M. Kuchling43c5af02000-04-24 14:17:06 +0000413 if data is not None:
Fred Drakeec3dfde2001-07-04 05:18:29 +0000414 h.send(data)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000415 errcode, errmsg, headers = h.getreply()
416 fp = h.getfile()
417 if errcode == 200:
Guido van Rossumb931bf32001-12-08 17:09:07 +0000418 return addinfourl(fp, headers, "https:" + url)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000419 else:
Fred Drake567ca8e2000-08-21 21:42:42 +0000420 if data is None:
421 return self.http_error(url, fp, errcode, errmsg, headers)
422 else:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000423 return self.http_error(url, fp, errcode, errmsg, headers,
424 data)
Fred Drake567ca8e2000-08-21 21:42:42 +0000425
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000426 def open_gopher(self, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000427 """Use Gopher protocol."""
Martin v. Löwis3e865952006-01-24 15:51:21 +0000428 if not isinstance(url, str):
429 raise IOError, ('gopher error', 'proxy support for gopher protocol currently not implemented')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000430 import gopherlib
431 host, selector = splithost(url)
432 if not host: raise IOError, ('gopher error', 'no host given')
433 host = unquote(host)
434 type, selector = splitgophertype(selector)
435 selector, query = splitquery(selector)
436 selector = unquote(selector)
437 if query:
438 query = unquote(query)
439 fp = gopherlib.send_query(selector, query, host)
440 else:
441 fp = gopherlib.send_selector(selector, host)
442 return addinfourl(fp, noheaders(), "gopher:" + url)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000443
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000444 def open_file(self, url):
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000445 """Use local file or FTP depending on form of URL."""
Martin v. Löwis3e865952006-01-24 15:51:21 +0000446 if not isinstance(url, str):
447 raise IOError, ('file error', 'proxy support for file protocol currently not implemented')
Jack Jansen4ef11032002-09-12 20:14:04 +0000448 if url[:2] == '//' and url[2:3] != '/' and url[2:12].lower() != 'localhost/':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000449 return self.open_ftp(url)
450 else:
451 return self.open_local_file(url)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000452
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000453 def open_local_file(self, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000454 """Use local file."""
Raymond Hettingera6172712004-12-31 19:15:26 +0000455 import mimetypes, mimetools, email.Utils
456 try:
457 from cStringIO import StringIO
458 except ImportError:
459 from StringIO import StringIO
Guido van Rossumf0713d32001-08-09 17:43:35 +0000460 host, file = splithost(url)
461 localname = url2pathname(file)
Guido van Rossuma2da3052002-04-15 00:25:01 +0000462 try:
463 stats = os.stat(localname)
464 except OSError, e:
465 raise IOError(e.errno, e.strerror, e.filename)
Walter Dörwald92b48b72002-03-22 17:30:38 +0000466 size = stats.st_size
Anthony Baxter3dd9e462004-10-11 13:53:08 +0000467 modified = email.Utils.formatdate(stats.st_mtime, usegmt=True)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000468 mtype = mimetypes.guess_type(url)[0]
Raymond Hettingera6172712004-12-31 19:15:26 +0000469 headers = mimetools.Message(StringIO(
Guido van Rossumf0713d32001-08-09 17:43:35 +0000470 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
471 (mtype or 'text/plain', size, modified)))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000472 if not host:
Guido van Rossum336a2011999-06-24 15:27:36 +0000473 urlfile = file
474 if file[:1] == '/':
475 urlfile = 'file://' + file
Guido van Rossumf0713d32001-08-09 17:43:35 +0000476 return addinfourl(open(localname, 'rb'),
Guido van Rossum336a2011999-06-24 15:27:36 +0000477 headers, urlfile)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000478 host, port = splitport(host)
479 if not port \
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000480 and socket.gethostbyname(host) in (localhost(), thishost()):
Guido van Rossum336a2011999-06-24 15:27:36 +0000481 urlfile = file
482 if file[:1] == '/':
483 urlfile = 'file://' + file
Guido van Rossumf0713d32001-08-09 17:43:35 +0000484 return addinfourl(open(localname, 'rb'),
Guido van Rossum336a2011999-06-24 15:27:36 +0000485 headers, urlfile)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000486 raise IOError, ('local file error', 'not on local host')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000487
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000488 def open_ftp(self, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000489 """Use FTP protocol."""
Martin v. Löwis3e865952006-01-24 15:51:21 +0000490 if not isinstance(url, str):
491 raise IOError, ('ftp error', 'proxy support for ftp protocol currently not implemented')
Raymond Hettingera6172712004-12-31 19:15:26 +0000492 import mimetypes, mimetools
493 try:
494 from cStringIO import StringIO
495 except ImportError:
496 from StringIO import StringIO
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000497 host, path = splithost(url)
498 if not host: raise IOError, ('ftp error', 'no host given')
499 host, port = splitport(host)
500 user, host = splituser(host)
501 if user: user, passwd = splitpasswd(user)
502 else: passwd = None
503 host = unquote(host)
504 user = unquote(user or '')
505 passwd = unquote(passwd or '')
506 host = socket.gethostbyname(host)
507 if not port:
508 import ftplib
509 port = ftplib.FTP_PORT
510 else:
511 port = int(port)
512 path, attrs = splitattr(path)
513 path = unquote(path)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000514 dirs = path.split('/')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000515 dirs, file = dirs[:-1], dirs[-1]
516 if dirs and not dirs[0]: dirs = dirs[1:]
Guido van Rossum5e006a31999-08-18 17:40:33 +0000517 if dirs and not dirs[0]: dirs[0] = '/'
Guido van Rossumb2493f82000-12-15 15:01:37 +0000518 key = user, host, port, '/'.join(dirs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000519 # XXX thread unsafe!
520 if len(self.ftpcache) > MAXFTPCACHE:
521 # Prune the cache, rather arbitrarily
522 for k in self.ftpcache.keys():
523 if k != key:
524 v = self.ftpcache[k]
525 del self.ftpcache[k]
526 v.close()
527 try:
Raymond Hettinger54f02222002-06-01 14:18:47 +0000528 if not key in self.ftpcache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000529 self.ftpcache[key] = \
530 ftpwrapper(user, passwd, host, port, dirs)
531 if not file: type = 'D'
532 else: type = 'I'
533 for attr in attrs:
534 attr, value = splitvalue(attr)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000535 if attr.lower() == 'type' and \
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000536 value in ('a', 'A', 'i', 'I', 'd', 'D'):
Guido van Rossumb2493f82000-12-15 15:01:37 +0000537 type = value.upper()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000538 (fp, retrlen) = self.ftpcache[key].retrfile(file, type)
Guido van Rossum88e0b5b2001-08-23 13:38:15 +0000539 mtype = mimetypes.guess_type("ftp:" + url)[0]
540 headers = ""
541 if mtype:
542 headers += "Content-Type: %s\n" % mtype
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000543 if retrlen is not None and retrlen >= 0:
Guido van Rossum88e0b5b2001-08-23 13:38:15 +0000544 headers += "Content-Length: %d\n" % retrlen
Raymond Hettingera6172712004-12-31 19:15:26 +0000545 headers = mimetools.Message(StringIO(headers))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000546 return addinfourl(fp, headers, "ftp:" + url)
547 except ftperrors(), msg:
548 raise IOError, ('ftp error', msg), sys.exc_info()[2]
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000549
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000550 def open_data(self, url, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000551 """Use "data" URL."""
Martin v. Löwis3e865952006-01-24 15:51:21 +0000552 if not isinstance(url, str):
553 raise IOError, ('data error', 'proxy support for data protocol currently not implemented')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000554 # ignore POSTed data
555 #
556 # syntax of data URLs:
557 # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
558 # mediatype := [ type "/" subtype ] *( ";" parameter )
559 # data := *urlchar
560 # parameter := attribute "=" value
Raymond Hettingera6172712004-12-31 19:15:26 +0000561 import mimetools
562 try:
563 from cStringIO import StringIO
564 except ImportError:
565 from StringIO import StringIO
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000566 try:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000567 [type, data] = url.split(',', 1)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000568 except ValueError:
569 raise IOError, ('data error', 'bad data URL')
570 if not type:
571 type = 'text/plain;charset=US-ASCII'
Guido van Rossumb2493f82000-12-15 15:01:37 +0000572 semi = type.rfind(';')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000573 if semi >= 0 and '=' not in type[semi:]:
574 encoding = type[semi+1:]
575 type = type[:semi]
576 else:
577 encoding = ''
578 msg = []
579 msg.append('Date: %s'%time.strftime('%a, %d %b %Y %T GMT',
580 time.gmtime(time.time())))
581 msg.append('Content-type: %s' % type)
582 if encoding == 'base64':
583 import base64
584 data = base64.decodestring(data)
585 else:
586 data = unquote(data)
587 msg.append('Content-length: %d' % len(data))
588 msg.append('')
589 msg.append(data)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000590 msg = '\n'.join(msg)
Raymond Hettingera6172712004-12-31 19:15:26 +0000591 f = StringIO(msg)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000592 headers = mimetools.Message(f, 0)
Georg Brandl1f663572005-11-26 16:50:44 +0000593 #f.fileno = None # needed for addinfourl
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000594 return addinfourl(f, headers, url)
Guido van Rossum6d4d1c21998-03-12 14:32:55 +0000595
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000596
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000597class FancyURLopener(URLopener):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000598 """Derived class with handlers for errors we can handle (perhaps)."""
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000599
Neal Norwitz60e04cd2002-06-11 13:38:51 +0000600 def __init__(self, *args, **kwargs):
Guido van Rossum68468eb2003-02-27 20:14:51 +0000601 URLopener.__init__(self, *args, **kwargs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000602 self.auth_cache = {}
Skip Montanaroc3e11d62001-02-15 16:56:36 +0000603 self.tries = 0
604 self.maxtries = 10
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000605
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000606 def http_error_default(self, url, fp, errcode, errmsg, headers):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000607 """Default error handling -- don't raise an exception."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000608 return addinfourl(fp, headers, "http:" + url)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000609
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000610 def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000611 """Error 302 -- relocated (temporarily)."""
Skip Montanaroc3e11d62001-02-15 16:56:36 +0000612 self.tries += 1
613 if self.maxtries and self.tries >= self.maxtries:
614 if hasattr(self, "http_error_500"):
615 meth = self.http_error_500
616 else:
617 meth = self.http_error_default
618 self.tries = 0
619 return meth(url, fp, 500,
620 "Internal Server Error: Redirect Recursion", headers)
621 result = self.redirect_internal(url, fp, errcode, errmsg, headers,
622 data)
623 self.tries = 0
624 return result
625
626 def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
Raymond Hettinger54f02222002-06-01 14:18:47 +0000627 if 'location' in headers:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000628 newurl = headers['location']
Raymond Hettinger54f02222002-06-01 14:18:47 +0000629 elif 'uri' in headers:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000630 newurl = headers['uri']
631 else:
632 return
633 void = fp.read()
634 fp.close()
Guido van Rossum3527f591999-03-29 20:23:41 +0000635 # In case the server sent a relative URL, join with original:
Moshe Zadka5d87d472001-04-09 14:54:21 +0000636 newurl = basejoin(self.type + ":" + url, newurl)
Guido van Rossumfa19f7c2003-05-16 01:46:51 +0000637 return self.open(newurl)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000638
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000639 def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000640 """Error 301 -- also relocated (permanently)."""
641 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
Guido van Rossume6ad8911996-09-10 17:02:56 +0000642
Raymond Hettinger024aaa12003-04-24 15:32:12 +0000643 def http_error_303(self, url, fp, errcode, errmsg, headers, data=None):
644 """Error 303 -- also relocated (essentially identical to 302)."""
645 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
646
Guido van Rossumfa19f7c2003-05-16 01:46:51 +0000647 def http_error_307(self, url, fp, errcode, errmsg, headers, data=None):
648 """Error 307 -- relocated, but turn POST into error."""
649 if data is None:
650 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
651 else:
652 return self.http_error_default(url, fp, errcode, errmsg, headers)
653
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000654 def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000655 """Error 401 -- authentication required.
Martin v. Löwis3e865952006-01-24 15:51:21 +0000656 This function supports Basic authentication only."""
Raymond Hettinger54f02222002-06-01 14:18:47 +0000657 if not 'www-authenticate' in headers:
Tim Peters85ba6732001-02-28 08:26:44 +0000658 URLopener.http_error_default(self, url, fp,
Fred Drakec680ae82001-10-13 18:37:07 +0000659 errcode, errmsg, headers)
Moshe Zadkae99bd172001-02-27 06:27:04 +0000660 stuff = headers['www-authenticate']
661 import re
662 match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
663 if not match:
Tim Peters85ba6732001-02-28 08:26:44 +0000664 URLopener.http_error_default(self, url, fp,
Moshe Zadkae99bd172001-02-27 06:27:04 +0000665 errcode, errmsg, headers)
666 scheme, realm = match.groups()
667 if scheme.lower() != 'basic':
Tim Peters85ba6732001-02-28 08:26:44 +0000668 URLopener.http_error_default(self, url, fp,
Moshe Zadkae99bd172001-02-27 06:27:04 +0000669 errcode, errmsg, headers)
670 name = 'retry_' + self.type + '_basic_auth'
671 if data is None:
672 return getattr(self,name)(url, realm)
673 else:
674 return getattr(self,name)(url, realm, data)
Tim Peters92037a12006-01-24 22:44:08 +0000675
Martin v. Löwis3e865952006-01-24 15:51:21 +0000676 def http_error_407(self, url, fp, errcode, errmsg, headers, data=None):
677 """Error 407 -- proxy authentication required.
678 This function supports Basic authentication only."""
679 if not 'proxy-authenticate' in headers:
680 URLopener.http_error_default(self, url, fp,
681 errcode, errmsg, headers)
682 stuff = headers['proxy-authenticate']
683 import re
684 match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
685 if not match:
686 URLopener.http_error_default(self, url, fp,
687 errcode, errmsg, headers)
688 scheme, realm = match.groups()
689 if scheme.lower() != 'basic':
690 URLopener.http_error_default(self, url, fp,
691 errcode, errmsg, headers)
692 name = 'retry_proxy_' + self.type + '_basic_auth'
693 if data is None:
694 return getattr(self,name)(url, realm)
695 else:
696 return getattr(self,name)(url, realm, data)
Tim Peters92037a12006-01-24 22:44:08 +0000697
Martin v. Löwis3e865952006-01-24 15:51:21 +0000698 def retry_proxy_http_basic_auth(self, url, realm, data=None):
699 host, selector = splithost(url)
700 newurl = 'http://' + host + selector
701 proxy = self.proxies['http']
702 urltype, proxyhost = splittype(proxy)
703 proxyhost, proxyselector = splithost(proxyhost)
704 i = proxyhost.find('@') + 1
705 proxyhost = proxyhost[i:]
706 user, passwd = self.get_user_passwd(proxyhost, realm, i)
707 if not (user or passwd): return None
708 proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
709 self.proxies['http'] = 'http://' + proxyhost + proxyselector
710 if data is None:
711 return self.open(newurl)
712 else:
713 return self.open(newurl, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000714
Martin v. Löwis3e865952006-01-24 15:51:21 +0000715 def retry_proxy_https_basic_auth(self, url, realm, data=None):
716 host, selector = splithost(url)
717 newurl = 'https://' + host + selector
718 proxy = self.proxies['https']
719 urltype, proxyhost = splittype(proxy)
720 proxyhost, proxyselector = splithost(proxyhost)
721 i = proxyhost.find('@') + 1
722 proxyhost = proxyhost[i:]
723 user, passwd = self.get_user_passwd(proxyhost, realm, i)
724 if not (user or passwd): return None
725 proxyhost = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + proxyhost
726 self.proxies['https'] = 'https://' + proxyhost + proxyselector
727 if data is None:
728 return self.open(newurl)
729 else:
730 return self.open(newurl, data)
Tim Peters92037a12006-01-24 22:44:08 +0000731
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000732 def retry_http_basic_auth(self, url, realm, data=None):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000733 host, selector = splithost(url)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000734 i = host.find('@') + 1
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000735 host = host[i:]
736 user, passwd = self.get_user_passwd(host, realm, i)
737 if not (user or passwd): return None
Guido van Rossumafc4f042001-01-15 18:31:13 +0000738 host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000739 newurl = 'http://' + host + selector
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000740 if data is None:
741 return self.open(newurl)
742 else:
743 return self.open(newurl, data)
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000744
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000745 def retry_https_basic_auth(self, url, realm, data=None):
Tim Peterse1190062001-01-15 03:34:38 +0000746 host, selector = splithost(url)
747 i = host.find('@') + 1
748 host = host[i:]
749 user, passwd = self.get_user_passwd(host, realm, i)
750 if not (user or passwd): return None
Guido van Rossumafc4f042001-01-15 18:31:13 +0000751 host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
Martin v. Löwis3e865952006-01-24 15:51:21 +0000752 newurl = 'https://' + host + selector
753 if data is None:
754 return self.open(newurl)
755 else:
756 return self.open(newurl, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000757
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000758 def get_user_passwd(self, host, realm, clear_cache = 0):
Guido van Rossumb2493f82000-12-15 15:01:37 +0000759 key = realm + '@' + host.lower()
Raymond Hettinger54f02222002-06-01 14:18:47 +0000760 if key in self.auth_cache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000761 if clear_cache:
762 del self.auth_cache[key]
763 else:
764 return self.auth_cache[key]
765 user, passwd = self.prompt_user_passwd(host, realm)
766 if user or passwd: self.auth_cache[key] = (user, passwd)
767 return user, passwd
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000768
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000769 def prompt_user_passwd(self, host, realm):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000770 """Override this in a GUI environment!"""
Neal Norwitzce96f692006-03-17 06:49:51 +0000771 import getpass, sys
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000772 try:
Neal Norwitzce96f692006-03-17 06:49:51 +0000773 sys.stdout.write("Enter username for %s at %s: " % (realm, host))
774 sys.stdout.flush()
775 user = sys.stdin.readline()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000776 passwd = getpass.getpass("Enter password for %s in %s at %s: " %
777 (user, realm, host))
778 return user, passwd
779 except KeyboardInterrupt:
780 print
781 return None, None
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000782
783
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000784# Utility functions
785
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000786_localhost = None
787def localhost():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000788 """Return the IP address of the magic hostname 'localhost'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000789 global _localhost
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000790 if _localhost is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000791 _localhost = socket.gethostbyname('localhost')
792 return _localhost
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000793
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000794_thishost = None
795def thishost():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000796 """Return the IP address of the current host."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000797 global _thishost
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000798 if _thishost is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000799 _thishost = socket.gethostbyname(socket.gethostname())
800 return _thishost
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000801
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000802_ftperrors = None
803def ftperrors():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000804 """Return the set of errors raised by the FTP class."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000805 global _ftperrors
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000806 if _ftperrors is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000807 import ftplib
808 _ftperrors = ftplib.all_errors
809 return _ftperrors
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000810
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000811_noheaders = None
812def noheaders():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000813 """Return an empty mimetools.Message object."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000814 global _noheaders
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000815 if _noheaders is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000816 import mimetools
Raymond Hettingera6172712004-12-31 19:15:26 +0000817 try:
818 from cStringIO import StringIO
819 except ImportError:
820 from StringIO import StringIO
821 _noheaders = mimetools.Message(StringIO(), 0)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000822 _noheaders.fp.close() # Recycle file descriptor
823 return _noheaders
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000824
825
826# Utility classes
827
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000828class ftpwrapper:
Guido van Rossume7b146f2000-02-04 15:28:42 +0000829 """Class used by open_ftp() for cache of open FTP connections."""
830
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000831 def __init__(self, user, passwd, host, port, dirs):
832 self.user = user
833 self.passwd = passwd
834 self.host = host
835 self.port = port
836 self.dirs = dirs
837 self.init()
Guido van Rossume7b146f2000-02-04 15:28:42 +0000838
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000839 def init(self):
840 import ftplib
841 self.busy = 0
842 self.ftp = ftplib.FTP()
843 self.ftp.connect(self.host, self.port)
844 self.ftp.login(self.user, self.passwd)
845 for dir in self.dirs:
846 self.ftp.cwd(dir)
Guido van Rossume7b146f2000-02-04 15:28:42 +0000847
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000848 def retrfile(self, file, type):
849 import ftplib
850 self.endtransfer()
851 if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
852 else: cmd = 'TYPE ' + type; isdir = 0
853 try:
854 self.ftp.voidcmd(cmd)
855 except ftplib.all_errors:
856 self.init()
857 self.ftp.voidcmd(cmd)
858 conn = None
859 if file and not isdir:
860 # Use nlst to see if the file exists at all
861 try:
862 self.ftp.nlst(file)
863 except ftplib.error_perm, reason:
864 raise IOError, ('ftp error', reason), sys.exc_info()[2]
865 # Restore the transfer mode!
866 self.ftp.voidcmd(cmd)
867 # Try to retrieve as a file
868 try:
869 cmd = 'RETR ' + file
870 conn = self.ftp.ntransfercmd(cmd)
871 except ftplib.error_perm, reason:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000872 if str(reason)[:3] != '550':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000873 raise IOError, ('ftp error', reason), sys.exc_info()[2]
874 if not conn:
875 # Set transfer mode to ASCII!
876 self.ftp.voidcmd('TYPE A')
877 # Try a directory listing
878 if file: cmd = 'LIST ' + file
879 else: cmd = 'LIST'
880 conn = self.ftp.ntransfercmd(cmd)
881 self.busy = 1
882 # Pass back both a suitably decorated object and a retrieval length
883 return (addclosehook(conn[0].makefile('rb'),
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000884 self.endtransfer), conn[1])
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000885 def endtransfer(self):
886 if not self.busy:
887 return
888 self.busy = 0
889 try:
890 self.ftp.voidresp()
891 except ftperrors():
892 pass
Guido van Rossume7b146f2000-02-04 15:28:42 +0000893
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000894 def close(self):
895 self.endtransfer()
896 try:
897 self.ftp.close()
898 except ftperrors():
899 pass
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000900
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000901class addbase:
Guido van Rossume7b146f2000-02-04 15:28:42 +0000902 """Base class for addinfo and addclosehook."""
903
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000904 def __init__(self, fp):
905 self.fp = fp
906 self.read = self.fp.read
907 self.readline = self.fp.readline
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000908 if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
Georg Brandl1f663572005-11-26 16:50:44 +0000909 if hasattr(self.fp, "fileno"):
910 self.fileno = self.fp.fileno
911 else:
912 self.fileno = lambda: None
Raymond Hettinger42182eb2003-03-09 05:33:33 +0000913 if hasattr(self.fp, "__iter__"):
914 self.__iter__ = self.fp.__iter__
915 if hasattr(self.fp, "next"):
916 self.next = self.fp.next
Guido van Rossume7b146f2000-02-04 15:28:42 +0000917
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000918 def __repr__(self):
Walter Dörwald70a6b492004-02-12 17:35:32 +0000919 return '<%s at %r whose fp = %r>' % (self.__class__.__name__,
920 id(self), self.fp)
Guido van Rossume7b146f2000-02-04 15:28:42 +0000921
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000922 def close(self):
923 self.read = None
924 self.readline = None
925 self.readlines = None
926 self.fileno = None
927 if self.fp: self.fp.close()
928 self.fp = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000929
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000930class addclosehook(addbase):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000931 """Class to add a close hook to an open file."""
932
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000933 def __init__(self, fp, closehook, *hookargs):
934 addbase.__init__(self, fp)
935 self.closehook = closehook
936 self.hookargs = hookargs
Guido van Rossume7b146f2000-02-04 15:28:42 +0000937
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000938 def close(self):
Guido van Rossumc580dae2000-05-24 13:21:46 +0000939 addbase.close(self)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000940 if self.closehook:
Guido van Rossum68468eb2003-02-27 20:14:51 +0000941 self.closehook(*self.hookargs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000942 self.closehook = None
943 self.hookargs = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000944
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000945class addinfo(addbase):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000946 """class to add an info() method to an open file."""
947
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000948 def __init__(self, fp, headers):
949 addbase.__init__(self, fp)
950 self.headers = headers
Guido van Rossume7b146f2000-02-04 15:28:42 +0000951
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000952 def info(self):
953 return self.headers
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000954
Guido van Rossume6ad8911996-09-10 17:02:56 +0000955class addinfourl(addbase):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000956 """class to add info() and geturl() methods to an open file."""
957
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000958 def __init__(self, fp, headers, url):
959 addbase.__init__(self, fp)
960 self.headers = headers
961 self.url = url
Guido van Rossume7b146f2000-02-04 15:28:42 +0000962
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000963 def info(self):
964 return self.headers
Guido van Rossume7b146f2000-02-04 15:28:42 +0000965
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000966 def geturl(self):
967 return self.url
Guido van Rossume6ad8911996-09-10 17:02:56 +0000968
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000969
Guido van Rossum7c395db1994-07-04 22:14:49 +0000970# Utilities to parse URLs (most of these return None for missing parts):
Sjoerd Mullendere0371b81995-11-10 10:36:07 +0000971# unwrap('<URL:type://host/path>') --> 'type://host/path'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000972# splittype('type:opaquestring') --> 'type', 'opaquestring'
973# splithost('//host[:port]/path') --> 'host[:port]', '/path'
Guido van Rossum7c395db1994-07-04 22:14:49 +0000974# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
975# splitpasswd('user:passwd') -> 'user', 'passwd'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000976# splitport('host:port') --> 'host', 'port'
977# splitquery('/path?query') --> '/path', 'query'
978# splittag('/path#tag') --> '/path', 'tag'
Guido van Rossum7c395db1994-07-04 22:14:49 +0000979# splitattr('/path;attr1=value1;attr2=value2;...') ->
980# '/path', ['attr1=value1', 'attr2=value2', ...]
981# splitvalue('attr=value') --> 'attr', 'value'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000982# splitgophertype('/Xselector') --> 'X', 'selector'
983# unquote('abc%20def') -> 'abc def'
984# quote('abc def') -> 'abc%20def')
985
Walter Dörwald65230a22002-06-03 15:58:32 +0000986try:
987 unicode
988except NameError:
Guido van Rossum4b46c0a2002-05-24 17:58:05 +0000989 def _is_unicode(x):
990 return 0
Walter Dörwald65230a22002-06-03 15:58:32 +0000991else:
992 def _is_unicode(x):
993 return isinstance(x, unicode)
Guido van Rossum4b46c0a2002-05-24 17:58:05 +0000994
Martin v. Löwis1d994332000-12-03 18:30:10 +0000995def toBytes(url):
996 """toBytes(u"URL") --> 'URL'."""
997 # Most URL schemes require ASCII. If that changes, the conversion
998 # can be relaxed
Guido van Rossum4b46c0a2002-05-24 17:58:05 +0000999 if _is_unicode(url):
Martin v. Löwis1d994332000-12-03 18:30:10 +00001000 try:
1001 url = url.encode("ASCII")
1002 except UnicodeError:
Guido van Rossumb2493f82000-12-15 15:01:37 +00001003 raise UnicodeError("URL " + repr(url) +
1004 " contains non-ASCII characters")
Martin v. Löwis1d994332000-12-03 18:30:10 +00001005 return url
1006
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001007def unwrap(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001008 """unwrap('<URL:type://host/path>') --> 'type://host/path'."""
Guido van Rossumb2493f82000-12-15 15:01:37 +00001009 url = url.strip()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001010 if url[:1] == '<' and url[-1:] == '>':
Guido van Rossumb2493f82000-12-15 15:01:37 +00001011 url = url[1:-1].strip()
1012 if url[:4] == 'URL:': url = url[4:].strip()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001013 return url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001014
Guido van Rossum332e1441997-09-29 23:23:46 +00001015_typeprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001016def splittype(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001017 """splittype('type:opaquestring') --> 'type', 'opaquestring'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001018 global _typeprog
1019 if _typeprog is None:
1020 import re
1021 _typeprog = re.compile('^([^/:]+):')
Guido van Rossum332e1441997-09-29 23:23:46 +00001022
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001023 match = _typeprog.match(url)
1024 if match:
1025 scheme = match.group(1)
Fred Drake9e94afd2000-07-01 07:03:30 +00001026 return scheme.lower(), url[len(scheme) + 1:]
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001027 return None, url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001028
Guido van Rossum332e1441997-09-29 23:23:46 +00001029_hostprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001030def splithost(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001031 """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001032 global _hostprog
1033 if _hostprog is None:
1034 import re
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001035 _hostprog = re.compile('^//([^/?]*)(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001036
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001037 match = _hostprog.match(url)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001038 if match: return match.group(1, 2)
1039 return None, url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001040
Guido van Rossum332e1441997-09-29 23:23:46 +00001041_userprog = None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001042def splituser(host):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001043 """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001044 global _userprog
1045 if _userprog is None:
1046 import re
Raymond Hettingerf2e45dd2002-08-18 20:08:56 +00001047 _userprog = re.compile('^(.*)@(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001048
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001049 match = _userprog.match(host)
Fred Drake567ca8e2000-08-21 21:42:42 +00001050 if match: return map(unquote, match.group(1, 2))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001051 return None, host
Guido van Rossum7c395db1994-07-04 22:14:49 +00001052
Guido van Rossum332e1441997-09-29 23:23:46 +00001053_passwdprog = None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001054def splitpasswd(user):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001055 """splitpasswd('user:passwd') -> 'user', 'passwd'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001056 global _passwdprog
1057 if _passwdprog is None:
1058 import re
1059 _passwdprog = re.compile('^([^:]*):(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001060
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001061 match = _passwdprog.match(user)
1062 if match: return match.group(1, 2)
1063 return user, None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001064
Guido van Rossume7b146f2000-02-04 15:28:42 +00001065# splittag('/path#tag') --> '/path', 'tag'
Guido van Rossum332e1441997-09-29 23:23:46 +00001066_portprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001067def splitport(host):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001068 """splitport('host:port') --> 'host', 'port'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001069 global _portprog
1070 if _portprog is None:
1071 import re
1072 _portprog = re.compile('^(.*):([0-9]+)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001073
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001074 match = _portprog.match(host)
1075 if match: return match.group(1, 2)
1076 return host, None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001077
Guido van Rossum332e1441997-09-29 23:23:46 +00001078_nportprog = None
Guido van Rossum53725a21996-06-13 19:12:35 +00001079def splitnport(host, defport=-1):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001080 """Split host and port, returning numeric port.
1081 Return given default port if no ':' found; defaults to -1.
1082 Return numerical port if a valid number are found after ':'.
1083 Return None if ':' but not a valid number."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001084 global _nportprog
1085 if _nportprog is None:
1086 import re
1087 _nportprog = re.compile('^(.*):(.*)$')
Guido van Rossum7e7ca0b1998-03-26 21:01:39 +00001088
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001089 match = _nportprog.match(host)
1090 if match:
1091 host, port = match.group(1, 2)
1092 try:
Guido van Rossumb2493f82000-12-15 15:01:37 +00001093 if not port: raise ValueError, "no digits"
1094 nport = int(port)
1095 except ValueError:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001096 nport = None
1097 return host, nport
1098 return host, defport
Guido van Rossum53725a21996-06-13 19:12:35 +00001099
Guido van Rossum332e1441997-09-29 23:23:46 +00001100_queryprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001101def splitquery(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001102 """splitquery('/path?query') --> '/path', 'query'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001103 global _queryprog
1104 if _queryprog is None:
1105 import re
1106 _queryprog = re.compile('^(.*)\?([^?]*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001107
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001108 match = _queryprog.match(url)
1109 if match: return match.group(1, 2)
1110 return url, None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001111
Guido van Rossum332e1441997-09-29 23:23:46 +00001112_tagprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001113def splittag(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001114 """splittag('/path#tag') --> '/path', 'tag'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001115 global _tagprog
1116 if _tagprog is None:
1117 import re
1118 _tagprog = re.compile('^(.*)#([^#]*)$')
Guido van Rossum7e7ca0b1998-03-26 21:01:39 +00001119
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001120 match = _tagprog.match(url)
1121 if match: return match.group(1, 2)
1122 return url, None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001123
Guido van Rossum7c395db1994-07-04 22:14:49 +00001124def splitattr(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001125 """splitattr('/path;attr1=value1;attr2=value2;...') ->
1126 '/path', ['attr1=value1', 'attr2=value2', ...]."""
Guido van Rossumb2493f82000-12-15 15:01:37 +00001127 words = url.split(';')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001128 return words[0], words[1:]
Guido van Rossum7c395db1994-07-04 22:14:49 +00001129
Guido van Rossum332e1441997-09-29 23:23:46 +00001130_valueprog = None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001131def splitvalue(attr):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001132 """splitvalue('attr=value') --> 'attr', 'value'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001133 global _valueprog
1134 if _valueprog is None:
1135 import re
1136 _valueprog = re.compile('^([^=]*)=(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001137
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001138 match = _valueprog.match(attr)
1139 if match: return match.group(1, 2)
1140 return attr, None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001141
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001142def splitgophertype(selector):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001143 """splitgophertype('/Xselector') --> 'X', 'selector'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001144 if selector[:1] == '/' and selector[1:2]:
1145 return selector[1], selector[2:]
1146 return None, selector
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001147
Raymond Hettinger803ce802005-09-10 06:49:04 +00001148_hextochr = dict(('%02x' % i, chr(i)) for i in range(256))
1149_hextochr.update(('%02X' % i, chr(i)) for i in range(256))
1150
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001151def unquote(s):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001152 """unquote('abc%20def') -> 'abc def'."""
Raymond Hettinger803ce802005-09-10 06:49:04 +00001153 res = s.split('%')
1154 for i in xrange(1, len(res)):
1155 item = res[i]
1156 try:
1157 res[i] = _hextochr[item[:2]] + item[2:]
1158 except KeyError:
1159 res[i] = '%' + item
Raymond Hettinger4b0f20d2005-10-15 16:41:53 +00001160 except UnicodeDecodeError:
1161 res[i] = unichr(int(item[:2], 16)) + item[2:]
Guido van Rossumb2493f82000-12-15 15:01:37 +00001162 return "".join(res)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001163
Guido van Rossum0564e121996-12-13 14:47:36 +00001164def unquote_plus(s):
Skip Montanaro79f1c172000-08-22 03:00:52 +00001165 """unquote('%7e/abc+def') -> '~/abc def'"""
Brett Cannonaaeffaf2004-03-23 23:50:17 +00001166 s = s.replace('+', ' ')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001167 return unquote(s)
Guido van Rossum0564e121996-12-13 14:47:36 +00001168
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001169always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
Jeremy Hylton6102e292000-08-31 15:48:10 +00001170 'abcdefghijklmnopqrstuvwxyz'
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001171 '0123456789' '_.-')
Raymond Hettinger199d2f72005-09-09 22:27:13 +00001172_safemaps = {}
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001173
Guido van Rossum7c395db1994-07-04 22:14:49 +00001174def quote(s, safe = '/'):
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001175 """quote('abc def') -> 'abc%20def'
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001176
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001177 Each part of a URL, e.g. the path info, the query, etc., has a
1178 different set of reserved characters that must be quoted.
1179
1180 RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
1181 the following reserved characters.
1182
1183 reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
1184 "$" | ","
1185
1186 Each of these characters is reserved in some component of a URL,
1187 but not necessarily in all of them.
1188
1189 By default, the quote function is intended for quoting the path
1190 section of a URL. Thus, it will not encode '/'. This character
1191 is reserved, but in typical usage the quote function is being
1192 called on a path where the existing slash characters are used as
1193 reserved characters.
1194 """
Raymond Hettinger199d2f72005-09-09 22:27:13 +00001195 cachekey = (safe, always_safe)
1196 try:
1197 safe_map = _safemaps[cachekey]
1198 except KeyError:
1199 safe += always_safe
1200 safe_map = {}
1201 for i in range(256):
1202 c = chr(i)
1203 safe_map[c] = (c in safe) and c or ('%%%02X' % i)
1204 _safemaps[cachekey] = safe_map
1205 res = map(safe_map.__getitem__, s)
Guido van Rossumb2493f82000-12-15 15:01:37 +00001206 return ''.join(res)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001207
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001208def quote_plus(s, safe = ''):
1209 """Quote the query fragment of a URL; replacing ' ' with '+'"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001210 if ' ' in s:
Raymond Hettingercf6b6322005-09-10 18:17:54 +00001211 s = quote(s, safe + ' ')
1212 return s.replace(' ', '+')
1213 return quote(s, safe)
Guido van Rossum0564e121996-12-13 14:47:36 +00001214
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001215def urlencode(query,doseq=0):
1216 """Encode a sequence of two-element tuples or dictionary into a URL query string.
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001217
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001218 If any values in the query arg are sequences and doseq is true, each
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001219 sequence element is converted to a separate parameter.
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001220
1221 If the query arg is a sequence of two-element tuples, the order of the
1222 parameters in the output will match the order of parameters in the
1223 input.
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001224 """
Tim Peters658cba62001-02-09 20:06:00 +00001225
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001226 if hasattr(query,"items"):
1227 # mapping objects
1228 query = query.items()
1229 else:
1230 # it's a bother at times that strings and string-like objects are
1231 # sequences...
1232 try:
1233 # non-sequence items should not work with len()
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001234 # non-empty strings will fail this
Walter Dörwald65230a22002-06-03 15:58:32 +00001235 if len(query) and not isinstance(query[0], tuple):
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001236 raise TypeError
1237 # zero-length sequences of all types will get here and succeed,
1238 # but that's a minor nit - since the original implementation
1239 # allowed empty dicts that type of behavior probably should be
1240 # preserved for consistency
1241 except TypeError:
1242 ty,va,tb = sys.exc_info()
1243 raise TypeError, "not a valid non-string sequence or mapping object", tb
1244
Guido van Rossume7b146f2000-02-04 15:28:42 +00001245 l = []
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001246 if not doseq:
1247 # preserve old behavior
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001248 for k, v in query:
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001249 k = quote_plus(str(k))
1250 v = quote_plus(str(v))
1251 l.append(k + '=' + v)
1252 else:
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001253 for k, v in query:
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001254 k = quote_plus(str(k))
Walter Dörwald65230a22002-06-03 15:58:32 +00001255 if isinstance(v, str):
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001256 v = quote_plus(v)
1257 l.append(k + '=' + v)
Guido van Rossum4b46c0a2002-05-24 17:58:05 +00001258 elif _is_unicode(v):
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001259 # is there a reasonable way to convert to ASCII?
1260 # encode generates a string, but "replace" or "ignore"
1261 # lose information and "strict" can raise UnicodeError
1262 v = quote_plus(v.encode("ASCII","replace"))
1263 l.append(k + '=' + v)
1264 else:
1265 try:
1266 # is this a sufficient test for sequence-ness?
1267 x = len(v)
1268 except TypeError:
1269 # not a sequence
1270 v = quote_plus(str(v))
1271 l.append(k + '=' + v)
1272 else:
1273 # loop over the sequence
1274 for elt in v:
1275 l.append(k + '=' + quote_plus(str(elt)))
Guido van Rossumb2493f82000-12-15 15:01:37 +00001276 return '&'.join(l)
Guido van Rossum810a3391998-07-22 21:33:23 +00001277
Guido van Rossum442e7201996-03-20 15:33:11 +00001278# Proxy handling
Mark Hammond4f570b92000-07-26 07:04:38 +00001279def getproxies_environment():
1280 """Return a dictionary of scheme -> proxy server URL mappings.
1281
1282 Scan the environment for variables named <scheme>_proxy;
1283 this seems to be the standard convention. If you need a
1284 different way, you can pass a proxies dictionary to the
1285 [Fancy]URLopener constructor.
1286
1287 """
1288 proxies = {}
1289 for name, value in os.environ.items():
Guido van Rossumb2493f82000-12-15 15:01:37 +00001290 name = name.lower()
Mark Hammond4f570b92000-07-26 07:04:38 +00001291 if value and name[-6:] == '_proxy':
1292 proxies[name[:-6]] = value
1293 return proxies
1294
Jack Jansen11d9b062004-07-16 11:45:00 +00001295if sys.platform == 'darwin':
1296 def getproxies_internetconfig():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001297 """Return a dictionary of scheme -> proxy server URL mappings.
Guido van Rossum442e7201996-03-20 15:33:11 +00001298
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001299 By convention the mac uses Internet Config to store
1300 proxies. An HTTP proxy, for instance, is stored under
1301 the HttpProxy key.
Guido van Rossum442e7201996-03-20 15:33:11 +00001302
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001303 """
1304 try:
1305 import ic
1306 except ImportError:
1307 return {}
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001308
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001309 try:
1310 config = ic.IC()
1311 except ic.error:
1312 return {}
1313 proxies = {}
1314 # HTTP:
Raymond Hettinger54f02222002-06-01 14:18:47 +00001315 if 'UseHTTPProxy' in config and config['UseHTTPProxy']:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001316 try:
1317 value = config['HTTPProxyHost']
1318 except ic.error:
1319 pass
1320 else:
1321 proxies['http'] = 'http://%s' % value
1322 # FTP: XXXX To be done.
1323 # Gopher: XXXX To be done.
1324 return proxies
Mark Hammond4f570b92000-07-26 07:04:38 +00001325
Tim Peters55c12d42001-08-09 18:04:14 +00001326 def proxy_bypass(x):
1327 return 0
1328
Jack Jansen11d9b062004-07-16 11:45:00 +00001329 def getproxies():
1330 return getproxies_environment() or getproxies_internetconfig()
Tim Peters182b5ac2004-07-18 06:16:08 +00001331
Mark Hammond4f570b92000-07-26 07:04:38 +00001332elif os.name == 'nt':
1333 def getproxies_registry():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001334 """Return a dictionary of scheme -> proxy server URL mappings.
Mark Hammond4f570b92000-07-26 07:04:38 +00001335
1336 Win32 uses the registry to store proxies.
1337
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001338 """
1339 proxies = {}
Mark Hammond4f570b92000-07-26 07:04:38 +00001340 try:
1341 import _winreg
1342 except ImportError:
1343 # Std module, so should be around - but you never know!
1344 return proxies
1345 try:
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001346 internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1347 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
Mark Hammond4f570b92000-07-26 07:04:38 +00001348 proxyEnable = _winreg.QueryValueEx(internetSettings,
1349 'ProxyEnable')[0]
1350 if proxyEnable:
1351 # Returned as Unicode but problems if not converted to ASCII
1352 proxyServer = str(_winreg.QueryValueEx(internetSettings,
1353 'ProxyServer')[0])
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001354 if '=' in proxyServer:
1355 # Per-protocol settings
Mark Hammond4f570b92000-07-26 07:04:38 +00001356 for p in proxyServer.split(';'):
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001357 protocol, address = p.split('=', 1)
Guido van Rossumb955d6c2002-03-31 23:38:48 +00001358 # See if address has a type:// prefix
Guido van Rossum64e5aa92002-04-02 14:38:16 +00001359 import re
1360 if not re.match('^([^/:]+)://', address):
Guido van Rossumb955d6c2002-03-31 23:38:48 +00001361 address = '%s://%s' % (protocol, address)
1362 proxies[protocol] = address
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001363 else:
1364 # Use one setting for all protocols
1365 if proxyServer[:5] == 'http:':
1366 proxies['http'] = proxyServer
1367 else:
1368 proxies['http'] = 'http://%s' % proxyServer
1369 proxies['ftp'] = 'ftp://%s' % proxyServer
Mark Hammond4f570b92000-07-26 07:04:38 +00001370 internetSettings.Close()
1371 except (WindowsError, ValueError, TypeError):
1372 # Either registry key not found etc, or the value in an
1373 # unexpected format.
1374 # proxies already set up to be empty so nothing to do
1375 pass
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001376 return proxies
Guido van Rossum442e7201996-03-20 15:33:11 +00001377
Mark Hammond4f570b92000-07-26 07:04:38 +00001378 def getproxies():
1379 """Return a dictionary of scheme -> proxy server URL mappings.
1380
1381 Returns settings gathered from the environment, if specified,
1382 or the registry.
1383
1384 """
1385 return getproxies_environment() or getproxies_registry()
Tim Peters55c12d42001-08-09 18:04:14 +00001386
1387 def proxy_bypass(host):
1388 try:
1389 import _winreg
1390 import re
Tim Peters55c12d42001-08-09 18:04:14 +00001391 except ImportError:
1392 # Std modules, so should be around - but you never know!
1393 return 0
1394 try:
1395 internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1396 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
1397 proxyEnable = _winreg.QueryValueEx(internetSettings,
1398 'ProxyEnable')[0]
1399 proxyOverride = str(_winreg.QueryValueEx(internetSettings,
1400 'ProxyOverride')[0])
1401 # ^^^^ Returned as Unicode but problems if not converted to ASCII
1402 except WindowsError:
1403 return 0
1404 if not proxyEnable or not proxyOverride:
1405 return 0
1406 # try to make a host list from name and IP address.
Georg Brandl1f636702006-02-18 23:10:23 +00001407 rawHost, port = splitport(host)
1408 host = [rawHost]
Tim Peters55c12d42001-08-09 18:04:14 +00001409 try:
Georg Brandl1f636702006-02-18 23:10:23 +00001410 addr = socket.gethostbyname(rawHost)
1411 if addr != rawHost:
Tim Peters55c12d42001-08-09 18:04:14 +00001412 host.append(addr)
1413 except socket.error:
1414 pass
Georg Brandl1f636702006-02-18 23:10:23 +00001415 try:
1416 fqdn = socket.getfqdn(rawHost)
1417 if fqdn != rawHost:
1418 host.append(fqdn)
1419 except socket.error:
1420 pass
Tim Peters55c12d42001-08-09 18:04:14 +00001421 # make a check value list from the registry entry: replace the
1422 # '<local>' string by the localhost entry and the corresponding
1423 # canonical entry.
1424 proxyOverride = proxyOverride.split(';')
1425 i = 0
1426 while i < len(proxyOverride):
1427 if proxyOverride[i] == '<local>':
1428 proxyOverride[i:i+1] = ['localhost',
1429 '127.0.0.1',
1430 socket.gethostname(),
1431 socket.gethostbyname(
1432 socket.gethostname())]
1433 i += 1
1434 # print proxyOverride
1435 # now check if we match one of the registry values.
1436 for test in proxyOverride:
Tim Petersab9ba272001-08-09 21:40:30 +00001437 test = test.replace(".", r"\.") # mask dots
1438 test = test.replace("*", r".*") # change glob sequence
1439 test = test.replace("?", r".") # change glob char
Tim Peters55c12d42001-08-09 18:04:14 +00001440 for val in host:
1441 # print "%s <--> %s" %( test, val )
1442 if re.match(test, val, re.I):
1443 return 1
1444 return 0
1445
Mark Hammond4f570b92000-07-26 07:04:38 +00001446else:
1447 # By default use environment variables
1448 getproxies = getproxies_environment
1449
Tim Peters55c12d42001-08-09 18:04:14 +00001450 def proxy_bypass(host):
1451 return 0
Guido van Rossum442e7201996-03-20 15:33:11 +00001452
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001453# Test and time quote() and unquote()
1454def test1():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001455 s = ''
1456 for i in range(256): s = s + chr(i)
1457 s = s*4
1458 t0 = time.time()
1459 qs = quote(s)
1460 uqs = unquote(qs)
1461 t1 = time.time()
1462 if uqs != s:
1463 print 'Wrong!'
Walter Dörwald70a6b492004-02-12 17:35:32 +00001464 print repr(s)
1465 print repr(qs)
1466 print repr(uqs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001467 print round(t1 - t0, 3), 'sec'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001468
1469
Guido van Rossum9ab96d41998-09-28 14:07:00 +00001470def reporthook(blocknum, blocksize, totalsize):
1471 # Report during remote transfers
Guido van Rossumb2493f82000-12-15 15:01:37 +00001472 print "Block number: %d, Block size: %d, Total size: %d" % (
1473 blocknum, blocksize, totalsize)
Guido van Rossum9ab96d41998-09-28 14:07:00 +00001474
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001475# Test program
Guido van Rossum23490151998-06-25 02:39:00 +00001476def test(args=[]):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001477 if not args:
1478 args = [
1479 '/etc/passwd',
1480 'file:/etc/passwd',
1481 'file://localhost/etc/passwd',
Andrew M. Kuchling56a42352002-03-18 22:18:46 +00001482 'ftp://ftp.python.org/pub/python/README',
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001483## 'gopher://gopher.micro.umn.edu/1/',
1484 'http://www.python.org/index.html',
1485 ]
Guido van Rossum09c8b6c1999-12-07 21:37:17 +00001486 if hasattr(URLopener, "open_https"):
1487 args.append('https://synergy.as.cmu.edu/~geek/')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001488 try:
1489 for url in args:
1490 print '-'*10, url, '-'*10
1491 fn, h = urlretrieve(url, None, reporthook)
Guido van Rossumb2493f82000-12-15 15:01:37 +00001492 print fn
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001493 if h:
1494 print '======'
1495 for k in h.keys(): print k + ':', h[k]
1496 print '======'
1497 fp = open(fn, 'rb')
1498 data = fp.read()
1499 del fp
1500 if '\r' in data:
1501 table = string.maketrans("", "")
Guido van Rossumb2493f82000-12-15 15:01:37 +00001502 data = data.translate(table, "\r")
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001503 print data
1504 fn, h = None, None
1505 print '-'*40
1506 finally:
1507 urlcleanup()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001508
Guido van Rossum23490151998-06-25 02:39:00 +00001509def main():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001510 import getopt, sys
1511 try:
1512 opts, args = getopt.getopt(sys.argv[1:], "th")
1513 except getopt.error, msg:
1514 print msg
1515 print "Use -h for help"
1516 return
1517 t = 0
1518 for o, a in opts:
1519 if o == '-t':
1520 t = t + 1
1521 if o == '-h':
1522 print "Usage: python urllib.py [-t] [url ...]"
1523 print "-t runs self-test;",
1524 print "otherwise, contents of urls are printed"
1525 return
1526 if t:
1527 if t > 1:
1528 test1()
1529 test(args)
1530 else:
1531 if not args:
1532 print "Use -h for help"
1533 for url in args:
1534 print urlopen(url).read(),
Guido van Rossum23490151998-06-25 02:39:00 +00001535
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001536# Run test program when run as a script
1537if __name__ == '__main__':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001538 main()