blob: d367dd8677ed38ae9a2410bef023f3b81d1d31a3 [file] [log] [blame]
Guido van Rossume7b146f2000-02-04 15:28:42 +00001"""Open an arbitrary URL.
2
3See the following document for more info on URLs:
4"Names and Addresses, URIs, URLs, URNs, URCs", at
5http://www.w3.org/pub/WWW/Addressing/Overview.html
6
7See also the HTTP spec (from which the error codes are derived):
8"HTTP - Hypertext Transfer Protocol", at
9http://www.w3.org/pub/WWW/Protocols/
10
11Related standards and specs:
12- RFC1808: the "relative URL" spec. (authoritative status)
13- RFC1738 - the "URL standard". (authoritative status)
14- RFC1630 - the "URI spec". (informational status)
15
16The object returned by URLopener().open(file) will differ per
17protocol. All you know is that is has methods read(), readline(),
18readlines(), fileno(), close() and info(). The read*(), fileno()
Fredrik Lundhb49f88b2000-09-24 18:51:25 +000019and close() methods work like those of open files.
Guido van Rossume7b146f2000-02-04 15:28:42 +000020The info() method returns a mimetools.Message object which can be
21used to query various info about the object, if available.
22(mimetools.Message objects are queried with the getheader() method.)
23"""
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000024
Guido van Rossum7c395db1994-07-04 22:14:49 +000025import string
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000026import socket
Jack Jansendc3e3f61995-12-15 13:22:13 +000027import os
Guido van Rossumf0713d32001-08-09 17:43:35 +000028import time
Guido van Rossum3c8484e1996-11-20 22:02:24 +000029import sys
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000030
Skip Montanaro40fc1602001-03-01 04:27:19 +000031__all__ = ["urlopen", "URLopener", "FancyURLopener", "urlretrieve",
32 "urlcleanup", "quote", "quote_plus", "unquote", "unquote_plus",
Skip Montanaro44d5e0c2001-03-13 19:47:16 +000033 "urlencode", "url2pathname", "pathname2url", "splittag",
34 "localhost", "thishost", "ftperrors", "basejoin", "unwrap",
35 "splittype", "splithost", "splituser", "splitpasswd", "splitport",
36 "splitnport", "splitquery", "splitattr", "splitvalue",
37 "splitgophertype", "getproxies"]
Skip Montanaro40fc1602001-03-01 04:27:19 +000038
Guido van Rossumb2493f82000-12-15 15:01:37 +000039__version__ = '1.15' # XXX This version is not always updated :-(
Guido van Rossumf668d171997-06-06 21:11:11 +000040
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000041MAXFTPCACHE = 10 # Trim the ftp cache beyond this size
Guido van Rossum6cb15a01995-06-22 19:00:13 +000042
Jack Jansendc3e3f61995-12-15 13:22:13 +000043# Helper for non-unix systems
44if os.name == 'mac':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000045 from macurl2path import url2pathname, pathname2url
Guido van Rossum7e7ca0b1998-03-26 21:01:39 +000046elif os.name == 'nt':
Fredrik Lundhb49f88b2000-09-24 18:51:25 +000047 from nturl2path import url2pathname, pathname2url
Guido van Rossumd74fb6b2001-03-02 06:43:49 +000048elif os.name == 'riscos':
49 from rourl2path import url2pathname, pathname2url
Jack Jansendc3e3f61995-12-15 13:22:13 +000050else:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000051 def url2pathname(pathname):
Guido van Rossum367ac801999-03-12 14:31:10 +000052 return unquote(pathname)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000053 def pathname2url(pathname):
Guido van Rossum367ac801999-03-12 14:31:10 +000054 return quote(pathname)
Guido van Rossum33add0a1998-12-18 15:25:22 +000055
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000056# This really consists of two pieces:
57# (1) a class which handles opening of all sorts of URLs
58# (plus assorted utilities etc.)
59# (2) a set of functions for parsing URLs
60# XXX Should these be separated out into different modules?
61
62
63# Shortcut for basic usage
64_urlopener = None
Fred Drakedf6eca72002-04-04 20:41:34 +000065def urlopen(url, data=None, proxies=None):
Skip Montanaro79f1c172000-08-22 03:00:52 +000066 """urlopen(url [, data]) -> open file-like object"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000067 global _urlopener
Fred Drakedf6eca72002-04-04 20:41:34 +000068 if proxies is not None:
69 opener = FancyURLopener(proxies=proxies)
70 elif not _urlopener:
71 opener = FancyURLopener()
72 _urlopener = opener
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000073 else:
Fred Drakedf6eca72002-04-04 20:41:34 +000074 opener = _urlopener
75 if data is None:
76 return opener.open(url)
77 else:
78 return opener.open(url, data)
Fred Drake316a7932000-08-24 01:01:26 +000079def urlretrieve(url, filename=None, reporthook=None, data=None):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000080 global _urlopener
81 if not _urlopener:
82 _urlopener = FancyURLopener()
Fred Drake316a7932000-08-24 01:01:26 +000083 return _urlopener.retrieve(url, filename, reporthook, data)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000084def urlcleanup():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000085 if _urlopener:
86 _urlopener.cleanup()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000087
88
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000089ftpcache = {}
90class URLopener:
Guido van Rossume7b146f2000-02-04 15:28:42 +000091 """Class to open URLs.
92 This is a class rather than just a subroutine because we may need
93 more than one set of global protocol-specific options.
94 Note -- this is a base class for those who don't want the
95 automatic handling of errors type 302 (relocated) and 401
96 (authorization needed)."""
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000097
Jeremy Hyltonf90b0021999-02-25 16:12:12 +000098 __tempfiles = None
Guido van Rossum29e77811996-11-27 19:39:58 +000099
Guido van Rossumba311382000-08-24 16:18:04 +0000100 version = "Python-urllib/%s" % __version__
101
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000102 # Constructor
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000103 def __init__(self, proxies=None, **x509):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000104 if proxies is None:
105 proxies = getproxies()
106 assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
107 self.proxies = proxies
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000108 self.key_file = x509.get('key_file')
109 self.cert_file = x509.get('cert_file')
Guido van Rossumba311382000-08-24 16:18:04 +0000110 self.addheaders = [('User-agent', self.version)]
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000111 self.__tempfiles = []
112 self.__unlink = os.unlink # See cleanup()
113 self.tempcache = None
114 # Undocumented feature: if you assign {} to tempcache,
115 # it is used to cache files retrieved with
116 # self.retrieve(). This is not enabled by default
117 # since it does not work for changing documents (and I
118 # haven't got the logic to check expiration headers
119 # yet).
120 self.ftpcache = ftpcache
121 # Undocumented feature: you can use a different
122 # ftp cache by assigning to the .ftpcache member;
123 # in case you want logically independent URL openers
124 # XXX This is not threadsafe. Bah.
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000125
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000126 def __del__(self):
127 self.close()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000128
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000129 def close(self):
130 self.cleanup()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000131
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000132 def cleanup(self):
133 # This code sometimes runs when the rest of this module
134 # has already been deleted, so it can't use any globals
135 # or import anything.
136 if self.__tempfiles:
137 for file in self.__tempfiles:
138 try:
139 self.__unlink(file)
Martin v. Löwis58682b72001-08-11 15:02:57 +0000140 except OSError:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000141 pass
142 del self.__tempfiles[:]
143 if self.tempcache:
144 self.tempcache.clear()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000145
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000146 def addheader(self, *args):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000147 """Add a header to be used by the HTTP interface only
148 e.g. u.addheader('Accept', 'sound/basic')"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000149 self.addheaders.append(args)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000150
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000151 # External interface
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000152 def open(self, fullurl, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000153 """Use URLopener().open(file) instead of open(file, 'r')."""
Martin v. Löwis1d994332000-12-03 18:30:10 +0000154 fullurl = unwrap(toBytes(fullurl))
Raymond Hettinger54f02222002-06-01 14:18:47 +0000155 if self.tempcache and fullurl in self.tempcache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000156 filename, headers = self.tempcache[fullurl]
157 fp = open(filename, 'rb')
158 return addinfourl(fp, headers, fullurl)
Martin v. Löwis1d994332000-12-03 18:30:10 +0000159 urltype, url = splittype(fullurl)
160 if not urltype:
161 urltype = 'file'
Raymond Hettinger54f02222002-06-01 14:18:47 +0000162 if urltype in self.proxies:
Martin v. Löwis1d994332000-12-03 18:30:10 +0000163 proxy = self.proxies[urltype]
164 urltype, proxyhost = splittype(proxy)
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000165 host, selector = splithost(proxyhost)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000166 url = (host, fullurl) # Signal special case to open_*()
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000167 else:
168 proxy = None
Martin v. Löwis1d994332000-12-03 18:30:10 +0000169 name = 'open_' + urltype
170 self.type = urltype
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000171 if '-' in name:
172 # replace - with _
Guido van Rossumb2493f82000-12-15 15:01:37 +0000173 name = '_'.join(name.split('-'))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000174 if not hasattr(self, name):
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000175 if proxy:
176 return self.open_unknown_proxy(proxy, fullurl, data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000177 else:
178 return self.open_unknown(fullurl, data)
179 try:
180 if data is None:
181 return getattr(self, name)(url)
182 else:
183 return getattr(self, name)(url, data)
184 except socket.error, msg:
185 raise IOError, ('socket error', msg), sys.exc_info()[2]
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000186
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000187 def open_unknown(self, fullurl, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000188 """Overridable interface to open unknown URL type."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000189 type, url = splittype(fullurl)
190 raise IOError, ('url error', 'unknown url type', type)
Guido van Rossumca445401995-08-29 19:19:12 +0000191
Jeremy Hyltond52755f2000-10-02 23:04:02 +0000192 def open_unknown_proxy(self, proxy, fullurl, data=None):
193 """Overridable interface to open unknown URL type."""
194 type, url = splittype(fullurl)
195 raise IOError, ('url error', 'invalid proxy for %s' % type, proxy)
196
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000197 # External interface
Sjoerd Mullenderd7b86f02000-08-25 11:23:36 +0000198 def retrieve(self, url, filename=None, reporthook=None, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000199 """retrieve(url) returns (filename, None) for a local object
200 or (tempfilename, headers) for a remote object."""
Martin v. Löwis1d994332000-12-03 18:30:10 +0000201 url = unwrap(toBytes(url))
Raymond Hettinger54f02222002-06-01 14:18:47 +0000202 if self.tempcache and url in self.tempcache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000203 return self.tempcache[url]
204 type, url1 = splittype(url)
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000205 if filename is None and (not type or type == 'file'):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000206 try:
207 fp = self.open_local_file(url1)
208 hdrs = fp.info()
209 del fp
210 return url2pathname(splithost(url1)[1]), hdrs
211 except IOError, msg:
212 pass
Fred Drake316a7932000-08-24 01:01:26 +0000213 fp = self.open(url, data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000214 headers = fp.info()
Guido van Rossum3b0a3292002-08-09 16:38:32 +0000215 if filename:
216 tfp = open(filename, 'wb')
217 else:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000218 import tempfile
219 garbage, path = splittype(url)
220 garbage, path = splithost(path or "")
221 path, garbage = splitquery(path or "")
222 path, garbage = splitattr(path or "")
223 suffix = os.path.splitext(path)[1]
Guido van Rossum3b0a3292002-08-09 16:38:32 +0000224 (fd, filename) = tempfile.mkstemp(suffix)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000225 self.__tempfiles.append(filename)
Guido van Rossum3b0a3292002-08-09 16:38:32 +0000226 tfp = os.open(fd, 'wb')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000227 result = filename, headers
228 if self.tempcache is not None:
229 self.tempcache[url] = result
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000230 bs = 1024*8
231 size = -1
232 blocknum = 1
233 if reporthook:
Raymond Hettinger54f02222002-06-01 14:18:47 +0000234 if "content-length" in headers:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000235 size = int(headers["Content-Length"])
236 reporthook(0, bs, size)
237 block = fp.read(bs)
238 if reporthook:
239 reporthook(1, bs, size)
240 while block:
241 tfp.write(block)
242 block = fp.read(bs)
243 blocknum = blocknum + 1
244 if reporthook:
245 reporthook(blocknum, bs, size)
246 fp.close()
247 tfp.close()
248 del fp
249 del tfp
250 return result
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000251
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000252 # Each method named open_<type> knows how to open that type of URL
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000253
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000254 def open_http(self, url, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000255 """Use HTTP protocol."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000256 import httplib
257 user_passwd = None
Walter Dörwald65230a22002-06-03 15:58:32 +0000258 if isinstance(url, str):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000259 host, selector = splithost(url)
260 if host:
261 user_passwd, host = splituser(host)
262 host = unquote(host)
263 realhost = host
264 else:
265 host, selector = url
266 urltype, rest = splittype(selector)
267 url = rest
268 user_passwd = None
Guido van Rossumb2493f82000-12-15 15:01:37 +0000269 if urltype.lower() != 'http':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000270 realhost = None
271 else:
272 realhost, rest = splithost(rest)
273 if realhost:
274 user_passwd, realhost = splituser(realhost)
275 if user_passwd:
276 selector = "%s://%s%s" % (urltype, realhost, rest)
Tim Peters55c12d42001-08-09 18:04:14 +0000277 if proxy_bypass(realhost):
278 host = realhost
279
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000280 #print "proxy via http:", host, selector
281 if not host: raise IOError, ('http error', 'no host given')
282 if user_passwd:
283 import base64
Guido van Rossumb2493f82000-12-15 15:01:37 +0000284 auth = base64.encodestring(user_passwd).strip()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000285 else:
286 auth = None
287 h = httplib.HTTP(host)
288 if data is not None:
289 h.putrequest('POST', selector)
290 h.putheader('Content-type', 'application/x-www-form-urlencoded')
291 h.putheader('Content-length', '%d' % len(data))
292 else:
293 h.putrequest('GET', selector)
294 if auth: h.putheader('Authorization', 'Basic %s' % auth)
295 if realhost: h.putheader('Host', realhost)
296 for args in self.addheaders: apply(h.putheader, args)
297 h.endheaders()
298 if data is not None:
Fred Drakeec3dfde2001-07-04 05:18:29 +0000299 h.send(data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000300 errcode, errmsg, headers = h.getreply()
301 fp = h.getfile()
302 if errcode == 200:
303 return addinfourl(fp, headers, "http:" + url)
304 else:
305 if data is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000306 return self.http_error(url, fp, errcode, errmsg, headers)
Guido van Rossum29aab751999-03-09 19:31:21 +0000307 else:
308 return self.http_error(url, fp, errcode, errmsg, headers, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000309
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000310 def http_error(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000311 """Handle http errors.
312 Derived class can override this, or provide specific handlers
313 named http_error_DDD where DDD is the 3-digit error code."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000314 # First check if there's a specific handler for this error
315 name = 'http_error_%d' % errcode
316 if hasattr(self, name):
317 method = getattr(self, name)
318 if data is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000319 result = method(url, fp, errcode, errmsg, headers)
Jeremy Hyltonb30f52a1999-02-25 16:14:58 +0000320 else:
321 result = method(url, fp, errcode, errmsg, headers, data)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000322 if result: return result
Jeremy Hyltonb30f52a1999-02-25 16:14:58 +0000323 return self.http_error_default(url, fp, errcode, errmsg, headers)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000324
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000325 def http_error_default(self, url, fp, errcode, errmsg, headers):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000326 """Default error handler: close the connection and raise IOError."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000327 void = fp.read()
328 fp.close()
329 raise IOError, ('http error', errcode, errmsg, headers)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000330
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000331 if hasattr(socket, "ssl"):
Andrew M. Kuchling141e9892000-04-23 02:53:11 +0000332 def open_https(self, url, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000333 """Use HTTPS protocol."""
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000334 import httplib
Fred Drake567ca8e2000-08-21 21:42:42 +0000335 user_passwd = None
Walter Dörwald65230a22002-06-03 15:58:32 +0000336 if isinstance(url, str):
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000337 host, selector = splithost(url)
Fred Drake567ca8e2000-08-21 21:42:42 +0000338 if host:
339 user_passwd, host = splituser(host)
340 host = unquote(host)
341 realhost = host
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000342 else:
343 host, selector = url
344 urltype, rest = splittype(selector)
Fred Drake567ca8e2000-08-21 21:42:42 +0000345 url = rest
346 user_passwd = None
Guido van Rossumb2493f82000-12-15 15:01:37 +0000347 if urltype.lower() != 'https':
Fred Drake567ca8e2000-08-21 21:42:42 +0000348 realhost = None
349 else:
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000350 realhost, rest = splithost(rest)
Fred Drake567ca8e2000-08-21 21:42:42 +0000351 if realhost:
352 user_passwd, realhost = splituser(realhost)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000353 if user_passwd:
354 selector = "%s://%s%s" % (urltype, realhost, rest)
Andrew M. Kuchling7ad47922000-06-10 01:41:48 +0000355 #print "proxy via https:", host, selector
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000356 if not host: raise IOError, ('https error', 'no host given')
357 if user_passwd:
358 import base64
Guido van Rossumb2493f82000-12-15 15:01:37 +0000359 auth = base64.encodestring(user_passwd).strip()
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000360 else:
361 auth = None
362 h = httplib.HTTPS(host, 0,
363 key_file=self.key_file,
364 cert_file=self.cert_file)
Andrew M. Kuchling141e9892000-04-23 02:53:11 +0000365 if data is not None:
366 h.putrequest('POST', selector)
367 h.putheader('Content-type',
368 'application/x-www-form-urlencoded')
369 h.putheader('Content-length', '%d' % len(data))
370 else:
371 h.putrequest('GET', selector)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000372 if auth: h.putheader('Authorization: Basic %s' % auth)
Fred Drake567ca8e2000-08-21 21:42:42 +0000373 if realhost: h.putheader('Host', realhost)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000374 for args in self.addheaders: apply(h.putheader, args)
375 h.endheaders()
Andrew M. Kuchling43c5af02000-04-24 14:17:06 +0000376 if data is not None:
Fred Drakeec3dfde2001-07-04 05:18:29 +0000377 h.send(data)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000378 errcode, errmsg, headers = h.getreply()
379 fp = h.getfile()
380 if errcode == 200:
Guido van Rossumb931bf32001-12-08 17:09:07 +0000381 return addinfourl(fp, headers, "https:" + url)
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000382 else:
Fred Drake567ca8e2000-08-21 21:42:42 +0000383 if data is None:
384 return self.http_error(url, fp, errcode, errmsg, headers)
385 else:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000386 return self.http_error(url, fp, errcode, errmsg, headers,
387 data)
Fred Drake567ca8e2000-08-21 21:42:42 +0000388
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000389 def open_gopher(self, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000390 """Use Gopher protocol."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000391 import gopherlib
392 host, selector = splithost(url)
393 if not host: raise IOError, ('gopher error', 'no host given')
394 host = unquote(host)
395 type, selector = splitgophertype(selector)
396 selector, query = splitquery(selector)
397 selector = unquote(selector)
398 if query:
399 query = unquote(query)
400 fp = gopherlib.send_query(selector, query, host)
401 else:
402 fp = gopherlib.send_selector(selector, host)
403 return addinfourl(fp, noheaders(), "gopher:" + url)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000404
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000405 def open_file(self, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000406 """Use local file or FTP depending on form of URL."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000407 if url[:2] == '//' and url[2:3] != '/':
408 return self.open_ftp(url)
409 else:
410 return self.open_local_file(url)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000411
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000412 def open_local_file(self, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000413 """Use local file."""
Jeremy Hylton6d8c1aa2001-08-27 20:16:53 +0000414 import mimetypes, mimetools, rfc822, StringIO
Guido van Rossumf0713d32001-08-09 17:43:35 +0000415 host, file = splithost(url)
416 localname = url2pathname(file)
Guido van Rossuma2da3052002-04-15 00:25:01 +0000417 try:
418 stats = os.stat(localname)
419 except OSError, e:
420 raise IOError(e.errno, e.strerror, e.filename)
Walter Dörwald92b48b72002-03-22 17:30:38 +0000421 size = stats.st_size
422 modified = rfc822.formatdate(stats.st_mtime)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000423 mtype = mimetypes.guess_type(url)[0]
424 headers = mimetools.Message(StringIO.StringIO(
Guido van Rossumf0713d32001-08-09 17:43:35 +0000425 'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
426 (mtype or 'text/plain', size, modified)))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000427 if not host:
Guido van Rossum336a2011999-06-24 15:27:36 +0000428 urlfile = file
429 if file[:1] == '/':
430 urlfile = 'file://' + file
Guido van Rossumf0713d32001-08-09 17:43:35 +0000431 return addinfourl(open(localname, 'rb'),
Guido van Rossum336a2011999-06-24 15:27:36 +0000432 headers, urlfile)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000433 host, port = splitport(host)
434 if not port \
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000435 and socket.gethostbyname(host) in (localhost(), thishost()):
Guido van Rossum336a2011999-06-24 15:27:36 +0000436 urlfile = file
437 if file[:1] == '/':
438 urlfile = 'file://' + file
Guido van Rossumf0713d32001-08-09 17:43:35 +0000439 return addinfourl(open(localname, 'rb'),
Guido van Rossum336a2011999-06-24 15:27:36 +0000440 headers, urlfile)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000441 raise IOError, ('local file error', 'not on local host')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000442
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000443 def open_ftp(self, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000444 """Use FTP protocol."""
Guido van Rossum88e0b5b2001-08-23 13:38:15 +0000445 import mimetypes, mimetools, StringIO
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000446 host, path = splithost(url)
447 if not host: raise IOError, ('ftp error', 'no host given')
448 host, port = splitport(host)
449 user, host = splituser(host)
450 if user: user, passwd = splitpasswd(user)
451 else: passwd = None
452 host = unquote(host)
453 user = unquote(user or '')
454 passwd = unquote(passwd or '')
455 host = socket.gethostbyname(host)
456 if not port:
457 import ftplib
458 port = ftplib.FTP_PORT
459 else:
460 port = int(port)
461 path, attrs = splitattr(path)
462 path = unquote(path)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000463 dirs = path.split('/')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000464 dirs, file = dirs[:-1], dirs[-1]
465 if dirs and not dirs[0]: dirs = dirs[1:]
Guido van Rossum5e006a31999-08-18 17:40:33 +0000466 if dirs and not dirs[0]: dirs[0] = '/'
Guido van Rossumb2493f82000-12-15 15:01:37 +0000467 key = user, host, port, '/'.join(dirs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000468 # XXX thread unsafe!
469 if len(self.ftpcache) > MAXFTPCACHE:
470 # Prune the cache, rather arbitrarily
471 for k in self.ftpcache.keys():
472 if k != key:
473 v = self.ftpcache[k]
474 del self.ftpcache[k]
475 v.close()
476 try:
Raymond Hettinger54f02222002-06-01 14:18:47 +0000477 if not key in self.ftpcache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000478 self.ftpcache[key] = \
479 ftpwrapper(user, passwd, host, port, dirs)
480 if not file: type = 'D'
481 else: type = 'I'
482 for attr in attrs:
483 attr, value = splitvalue(attr)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000484 if attr.lower() == 'type' and \
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000485 value in ('a', 'A', 'i', 'I', 'd', 'D'):
Guido van Rossumb2493f82000-12-15 15:01:37 +0000486 type = value.upper()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000487 (fp, retrlen) = self.ftpcache[key].retrfile(file, type)
Guido van Rossum88e0b5b2001-08-23 13:38:15 +0000488 mtype = mimetypes.guess_type("ftp:" + url)[0]
489 headers = ""
490 if mtype:
491 headers += "Content-Type: %s\n" % mtype
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000492 if retrlen is not None and retrlen >= 0:
Guido van Rossum88e0b5b2001-08-23 13:38:15 +0000493 headers += "Content-Length: %d\n" % retrlen
494 headers = mimetools.Message(StringIO.StringIO(headers))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000495 return addinfourl(fp, headers, "ftp:" + url)
496 except ftperrors(), msg:
497 raise IOError, ('ftp error', msg), sys.exc_info()[2]
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000498
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000499 def open_data(self, url, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000500 """Use "data" URL."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000501 # ignore POSTed data
502 #
503 # syntax of data URLs:
504 # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
505 # mediatype := [ type "/" subtype ] *( ";" parameter )
506 # data := *urlchar
507 # parameter := attribute "=" value
Neal Norwitzaad18492002-03-26 16:25:01 +0000508 import StringIO, mimetools
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000509 try:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000510 [type, data] = url.split(',', 1)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000511 except ValueError:
512 raise IOError, ('data error', 'bad data URL')
513 if not type:
514 type = 'text/plain;charset=US-ASCII'
Guido van Rossumb2493f82000-12-15 15:01:37 +0000515 semi = type.rfind(';')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000516 if semi >= 0 and '=' not in type[semi:]:
517 encoding = type[semi+1:]
518 type = type[:semi]
519 else:
520 encoding = ''
521 msg = []
522 msg.append('Date: %s'%time.strftime('%a, %d %b %Y %T GMT',
523 time.gmtime(time.time())))
524 msg.append('Content-type: %s' % type)
525 if encoding == 'base64':
526 import base64
527 data = base64.decodestring(data)
528 else:
529 data = unquote(data)
530 msg.append('Content-length: %d' % len(data))
531 msg.append('')
532 msg.append(data)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000533 msg = '\n'.join(msg)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000534 f = StringIO.StringIO(msg)
535 headers = mimetools.Message(f, 0)
536 f.fileno = None # needed for addinfourl
537 return addinfourl(f, headers, url)
Guido van Rossum6d4d1c21998-03-12 14:32:55 +0000538
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000539
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000540class FancyURLopener(URLopener):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000541 """Derived class with handlers for errors we can handle (perhaps)."""
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000542
Neal Norwitz60e04cd2002-06-11 13:38:51 +0000543 def __init__(self, *args, **kwargs):
544 apply(URLopener.__init__, (self,) + args, kwargs)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000545 self.auth_cache = {}
Skip Montanaroc3e11d62001-02-15 16:56:36 +0000546 self.tries = 0
547 self.maxtries = 10
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000548
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000549 def http_error_default(self, url, fp, errcode, errmsg, headers):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000550 """Default error handling -- don't raise an exception."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000551 return addinfourl(fp, headers, "http:" + url)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000552
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000553 def http_error_302(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000554 """Error 302 -- relocated (temporarily)."""
Skip Montanaroc3e11d62001-02-15 16:56:36 +0000555 self.tries += 1
556 if self.maxtries and self.tries >= self.maxtries:
557 if hasattr(self, "http_error_500"):
558 meth = self.http_error_500
559 else:
560 meth = self.http_error_default
561 self.tries = 0
562 return meth(url, fp, 500,
563 "Internal Server Error: Redirect Recursion", headers)
564 result = self.redirect_internal(url, fp, errcode, errmsg, headers,
565 data)
566 self.tries = 0
567 return result
568
569 def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
Raymond Hettinger54f02222002-06-01 14:18:47 +0000570 if 'location' in headers:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000571 newurl = headers['location']
Raymond Hettinger54f02222002-06-01 14:18:47 +0000572 elif 'uri' in headers:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000573 newurl = headers['uri']
574 else:
575 return
576 void = fp.read()
577 fp.close()
Guido van Rossum3527f591999-03-29 20:23:41 +0000578 # In case the server sent a relative URL, join with original:
Moshe Zadka5d87d472001-04-09 14:54:21 +0000579 newurl = basejoin(self.type + ":" + url, newurl)
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000580 if data is None:
581 return self.open(newurl)
582 else:
583 return self.open(newurl, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000584
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000585 def http_error_301(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000586 """Error 301 -- also relocated (permanently)."""
587 return self.http_error_302(url, fp, errcode, errmsg, headers, data)
Guido van Rossume6ad8911996-09-10 17:02:56 +0000588
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000589 def http_error_401(self, url, fp, errcode, errmsg, headers, data=None):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000590 """Error 401 -- authentication required.
591 See this URL for a description of the basic authentication scheme:
592 http://www.ics.uci.edu/pub/ietf/http/draft-ietf-http-v10-spec-00.txt"""
Raymond Hettinger54f02222002-06-01 14:18:47 +0000593 if not 'www-authenticate' in headers:
Tim Peters85ba6732001-02-28 08:26:44 +0000594 URLopener.http_error_default(self, url, fp,
Fred Drakec680ae82001-10-13 18:37:07 +0000595 errcode, errmsg, headers)
Moshe Zadkae99bd172001-02-27 06:27:04 +0000596 stuff = headers['www-authenticate']
597 import re
598 match = re.match('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"', stuff)
599 if not match:
Tim Peters85ba6732001-02-28 08:26:44 +0000600 URLopener.http_error_default(self, url, fp,
Moshe Zadkae99bd172001-02-27 06:27:04 +0000601 errcode, errmsg, headers)
602 scheme, realm = match.groups()
603 if scheme.lower() != 'basic':
Tim Peters85ba6732001-02-28 08:26:44 +0000604 URLopener.http_error_default(self, url, fp,
Moshe Zadkae99bd172001-02-27 06:27:04 +0000605 errcode, errmsg, headers)
606 name = 'retry_' + self.type + '_basic_auth'
607 if data is None:
608 return getattr(self,name)(url, realm)
609 else:
610 return getattr(self,name)(url, realm, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000611
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000612 def retry_http_basic_auth(self, url, realm, data=None):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000613 host, selector = splithost(url)
Guido van Rossumb2493f82000-12-15 15:01:37 +0000614 i = host.find('@') + 1
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000615 host = host[i:]
616 user, passwd = self.get_user_passwd(host, realm, i)
617 if not (user or passwd): return None
Guido van Rossumafc4f042001-01-15 18:31:13 +0000618 host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000619 newurl = 'http://' + host + selector
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000620 if data is None:
621 return self.open(newurl)
622 else:
623 return self.open(newurl, data)
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000624
Guido van Rossum3c8baed2000-02-01 23:36:55 +0000625 def retry_https_basic_auth(self, url, realm, data=None):
Tim Peterse1190062001-01-15 03:34:38 +0000626 host, selector = splithost(url)
627 i = host.find('@') + 1
628 host = host[i:]
629 user, passwd = self.get_user_passwd(host, realm, i)
630 if not (user or passwd): return None
Guido van Rossumafc4f042001-01-15 18:31:13 +0000631 host = quote(user, safe='') + ':' + quote(passwd, safe='') + '@' + host
Tim Peterse1190062001-01-15 03:34:38 +0000632 newurl = '//' + host + selector
Guido van Rossumafc4f042001-01-15 18:31:13 +0000633 return self.open_https(newurl, data)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000634
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000635 def get_user_passwd(self, host, realm, clear_cache = 0):
Guido van Rossumb2493f82000-12-15 15:01:37 +0000636 key = realm + '@' + host.lower()
Raymond Hettinger54f02222002-06-01 14:18:47 +0000637 if key in self.auth_cache:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000638 if clear_cache:
639 del self.auth_cache[key]
640 else:
641 return self.auth_cache[key]
642 user, passwd = self.prompt_user_passwd(host, realm)
643 if user or passwd: self.auth_cache[key] = (user, passwd)
644 return user, passwd
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000645
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000646 def prompt_user_passwd(self, host, realm):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000647 """Override this in a GUI environment!"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000648 import getpass
649 try:
650 user = raw_input("Enter username for %s at %s: " % (realm,
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000651 host))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000652 passwd = getpass.getpass("Enter password for %s in %s at %s: " %
653 (user, realm, host))
654 return user, passwd
655 except KeyboardInterrupt:
656 print
657 return None, None
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000658
659
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000660# Utility functions
661
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000662_localhost = None
663def localhost():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000664 """Return the IP address of the magic hostname 'localhost'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000665 global _localhost
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000666 if _localhost is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000667 _localhost = socket.gethostbyname('localhost')
668 return _localhost
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000669
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000670_thishost = None
671def thishost():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000672 """Return the IP address of the current host."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000673 global _thishost
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000674 if _thishost is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000675 _thishost = socket.gethostbyname(socket.gethostname())
676 return _thishost
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000677
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000678_ftperrors = None
679def ftperrors():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000680 """Return the set of errors raised by the FTP class."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000681 global _ftperrors
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000682 if _ftperrors is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000683 import ftplib
684 _ftperrors = ftplib.all_errors
685 return _ftperrors
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000686
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000687_noheaders = None
688def noheaders():
Guido van Rossume7b146f2000-02-04 15:28:42 +0000689 """Return an empty mimetools.Message object."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000690 global _noheaders
Raymond Hettinger10ff7062002-06-02 03:04:52 +0000691 if _noheaders is None:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000692 import mimetools
693 import StringIO
694 _noheaders = mimetools.Message(StringIO.StringIO(), 0)
695 _noheaders.fp.close() # Recycle file descriptor
696 return _noheaders
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000697
698
699# Utility classes
700
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000701class ftpwrapper:
Guido van Rossume7b146f2000-02-04 15:28:42 +0000702 """Class used by open_ftp() for cache of open FTP connections."""
703
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000704 def __init__(self, user, passwd, host, port, dirs):
705 self.user = user
706 self.passwd = passwd
707 self.host = host
708 self.port = port
709 self.dirs = dirs
710 self.init()
Guido van Rossume7b146f2000-02-04 15:28:42 +0000711
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000712 def init(self):
713 import ftplib
714 self.busy = 0
715 self.ftp = ftplib.FTP()
716 self.ftp.connect(self.host, self.port)
717 self.ftp.login(self.user, self.passwd)
718 for dir in self.dirs:
719 self.ftp.cwd(dir)
Guido van Rossume7b146f2000-02-04 15:28:42 +0000720
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000721 def retrfile(self, file, type):
722 import ftplib
723 self.endtransfer()
724 if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
725 else: cmd = 'TYPE ' + type; isdir = 0
726 try:
727 self.ftp.voidcmd(cmd)
728 except ftplib.all_errors:
729 self.init()
730 self.ftp.voidcmd(cmd)
731 conn = None
732 if file and not isdir:
733 # Use nlst to see if the file exists at all
734 try:
735 self.ftp.nlst(file)
736 except ftplib.error_perm, reason:
737 raise IOError, ('ftp error', reason), sys.exc_info()[2]
738 # Restore the transfer mode!
739 self.ftp.voidcmd(cmd)
740 # Try to retrieve as a file
741 try:
742 cmd = 'RETR ' + file
743 conn = self.ftp.ntransfercmd(cmd)
744 except ftplib.error_perm, reason:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000745 if str(reason)[:3] != '550':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000746 raise IOError, ('ftp error', reason), sys.exc_info()[2]
747 if not conn:
748 # Set transfer mode to ASCII!
749 self.ftp.voidcmd('TYPE A')
750 # Try a directory listing
751 if file: cmd = 'LIST ' + file
752 else: cmd = 'LIST'
753 conn = self.ftp.ntransfercmd(cmd)
754 self.busy = 1
755 # Pass back both a suitably decorated object and a retrieval length
756 return (addclosehook(conn[0].makefile('rb'),
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000757 self.endtransfer), conn[1])
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000758 def endtransfer(self):
759 if not self.busy:
760 return
761 self.busy = 0
762 try:
763 self.ftp.voidresp()
764 except ftperrors():
765 pass
Guido van Rossume7b146f2000-02-04 15:28:42 +0000766
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000767 def close(self):
768 self.endtransfer()
769 try:
770 self.ftp.close()
771 except ftperrors():
772 pass
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000773
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000774class addbase:
Guido van Rossume7b146f2000-02-04 15:28:42 +0000775 """Base class for addinfo and addclosehook."""
776
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000777 def __init__(self, fp):
778 self.fp = fp
779 self.read = self.fp.read
780 self.readline = self.fp.readline
Guido van Rossum09c8b6c1999-12-07 21:37:17 +0000781 if hasattr(self.fp, "readlines"): self.readlines = self.fp.readlines
782 if hasattr(self.fp, "fileno"): self.fileno = self.fp.fileno
Guido van Rossume7b146f2000-02-04 15:28:42 +0000783
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000784 def __repr__(self):
785 return '<%s at %s whose fp = %s>' % (self.__class__.__name__,
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000786 `id(self)`, `self.fp`)
Guido van Rossume7b146f2000-02-04 15:28:42 +0000787
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000788 def close(self):
789 self.read = None
790 self.readline = None
791 self.readlines = None
792 self.fileno = None
793 if self.fp: self.fp.close()
794 self.fp = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000795
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000796class addclosehook(addbase):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000797 """Class to add a close hook to an open file."""
798
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000799 def __init__(self, fp, closehook, *hookargs):
800 addbase.__init__(self, fp)
801 self.closehook = closehook
802 self.hookargs = hookargs
Guido van Rossume7b146f2000-02-04 15:28:42 +0000803
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000804 def close(self):
Guido van Rossumc580dae2000-05-24 13:21:46 +0000805 addbase.close(self)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000806 if self.closehook:
807 apply(self.closehook, self.hookargs)
808 self.closehook = None
809 self.hookargs = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000810
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000811class addinfo(addbase):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000812 """class to add an info() method to an open file."""
813
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000814 def __init__(self, fp, headers):
815 addbase.__init__(self, fp)
816 self.headers = headers
Guido van Rossume7b146f2000-02-04 15:28:42 +0000817
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000818 def info(self):
819 return self.headers
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000820
Guido van Rossume6ad8911996-09-10 17:02:56 +0000821class addinfourl(addbase):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000822 """class to add info() and geturl() methods to an open file."""
823
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000824 def __init__(self, fp, headers, url):
825 addbase.__init__(self, fp)
826 self.headers = headers
827 self.url = url
Guido van Rossume7b146f2000-02-04 15:28:42 +0000828
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000829 def info(self):
830 return self.headers
Guido van Rossume7b146f2000-02-04 15:28:42 +0000831
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000832 def geturl(self):
833 return self.url
Guido van Rossume6ad8911996-09-10 17:02:56 +0000834
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000835
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000836def basejoin(base, url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000837 """Utility to combine a URL with a base URL to form a new URL."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000838 type, path = splittype(url)
839 if type:
840 # if url is complete (i.e., it contains a type), return it
841 return url
842 host, path = splithost(path)
843 type, basepath = splittype(base) # inherit type from base
844 if host:
845 # if url contains host, just inherit type
846 if type: return type + '://' + host + path
847 else:
848 # no type inherited, so url must have started with //
849 # just return it
850 return url
851 host, basepath = splithost(basepath) # inherit host
Thomas Wouters7e474022000-07-16 12:04:32 +0000852 basepath, basetag = splittag(basepath) # remove extraneous cruft
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000853 basepath, basequery = splitquery(basepath) # idem
854 if path[:1] != '/':
855 # non-absolute path name
856 if path[:1] in ('#', '?'):
857 # path is just a tag or query, attach to basepath
858 i = len(basepath)
859 else:
860 # else replace last component
Guido van Rossumb2493f82000-12-15 15:01:37 +0000861 i = basepath.rfind('/')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000862 if i < 0:
863 # basepath not absolute
864 if host:
865 # host present, make absolute
866 basepath = '/'
867 else:
868 # else keep non-absolute
869 basepath = ''
870 else:
871 # remove last file component
872 basepath = basepath[:i+1]
873 # Interpret ../ (important because of symlinks)
874 while basepath and path[:3] == '../':
875 path = path[3:]
Guido van Rossumb2493f82000-12-15 15:01:37 +0000876 i = basepath[:-1].rfind('/')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000877 if i > 0:
878 basepath = basepath[:i+1]
879 elif i == 0:
880 basepath = '/'
881 break
882 else:
883 basepath = ''
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000884
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000885 path = basepath + path
Guido van Rossumb8bf3be2001-04-15 20:47:33 +0000886 if host and path and path[0] != '/':
887 path = '/' + path
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000888 if type and host: return type + '://' + host + path
889 elif type: return type + ':' + path
890 elif host: return '//' + host + path # don't know what this means
891 else: return path
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000892
893
Guido van Rossum7c395db1994-07-04 22:14:49 +0000894# Utilities to parse URLs (most of these return None for missing parts):
Sjoerd Mullendere0371b81995-11-10 10:36:07 +0000895# unwrap('<URL:type://host/path>') --> 'type://host/path'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000896# splittype('type:opaquestring') --> 'type', 'opaquestring'
897# splithost('//host[:port]/path') --> 'host[:port]', '/path'
Guido van Rossum7c395db1994-07-04 22:14:49 +0000898# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
899# splitpasswd('user:passwd') -> 'user', 'passwd'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000900# splitport('host:port') --> 'host', 'port'
901# splitquery('/path?query') --> '/path', 'query'
902# splittag('/path#tag') --> '/path', 'tag'
Guido van Rossum7c395db1994-07-04 22:14:49 +0000903# splitattr('/path;attr1=value1;attr2=value2;...') ->
904# '/path', ['attr1=value1', 'attr2=value2', ...]
905# splitvalue('attr=value') --> 'attr', 'value'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000906# splitgophertype('/Xselector') --> 'X', 'selector'
907# unquote('abc%20def') -> 'abc def'
908# quote('abc def') -> 'abc%20def')
909
Walter Dörwald65230a22002-06-03 15:58:32 +0000910try:
911 unicode
912except NameError:
Guido van Rossum4b46c0a2002-05-24 17:58:05 +0000913 def _is_unicode(x):
914 return 0
Walter Dörwald65230a22002-06-03 15:58:32 +0000915else:
916 def _is_unicode(x):
917 return isinstance(x, unicode)
Guido van Rossum4b46c0a2002-05-24 17:58:05 +0000918
Martin v. Löwis1d994332000-12-03 18:30:10 +0000919def toBytes(url):
920 """toBytes(u"URL") --> 'URL'."""
921 # Most URL schemes require ASCII. If that changes, the conversion
922 # can be relaxed
Guido van Rossum4b46c0a2002-05-24 17:58:05 +0000923 if _is_unicode(url):
Martin v. Löwis1d994332000-12-03 18:30:10 +0000924 try:
925 url = url.encode("ASCII")
926 except UnicodeError:
Guido van Rossumb2493f82000-12-15 15:01:37 +0000927 raise UnicodeError("URL " + repr(url) +
928 " contains non-ASCII characters")
Martin v. Löwis1d994332000-12-03 18:30:10 +0000929 return url
930
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000931def unwrap(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000932 """unwrap('<URL:type://host/path>') --> 'type://host/path'."""
Guido van Rossumb2493f82000-12-15 15:01:37 +0000933 url = url.strip()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000934 if url[:1] == '<' and url[-1:] == '>':
Guido van Rossumb2493f82000-12-15 15:01:37 +0000935 url = url[1:-1].strip()
936 if url[:4] == 'URL:': url = url[4:].strip()
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000937 return url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000938
Guido van Rossum332e1441997-09-29 23:23:46 +0000939_typeprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000940def splittype(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000941 """splittype('type:opaquestring') --> 'type', 'opaquestring'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000942 global _typeprog
943 if _typeprog is None:
944 import re
945 _typeprog = re.compile('^([^/:]+):')
Guido van Rossum332e1441997-09-29 23:23:46 +0000946
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000947 match = _typeprog.match(url)
948 if match:
949 scheme = match.group(1)
Fred Drake9e94afd2000-07-01 07:03:30 +0000950 return scheme.lower(), url[len(scheme) + 1:]
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000951 return None, url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000952
Guido van Rossum332e1441997-09-29 23:23:46 +0000953_hostprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000954def splithost(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000955 """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000956 global _hostprog
957 if _hostprog is None:
958 import re
Guido van Rossum3427c1f1999-07-01 23:20:56 +0000959 _hostprog = re.compile('^//([^/]*)(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +0000960
Fredrik Lundhb49f88b2000-09-24 18:51:25 +0000961 match = _hostprog.match(url)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000962 if match: return match.group(1, 2)
963 return None, url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000964
Guido van Rossum332e1441997-09-29 23:23:46 +0000965_userprog = None
Guido van Rossum7c395db1994-07-04 22:14:49 +0000966def splituser(host):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000967 """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000968 global _userprog
969 if _userprog is None:
970 import re
971 _userprog = re.compile('^([^@]*)@(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +0000972
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000973 match = _userprog.match(host)
Fred Drake567ca8e2000-08-21 21:42:42 +0000974 if match: return map(unquote, match.group(1, 2))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000975 return None, host
Guido van Rossum7c395db1994-07-04 22:14:49 +0000976
Guido van Rossum332e1441997-09-29 23:23:46 +0000977_passwdprog = None
Guido van Rossum7c395db1994-07-04 22:14:49 +0000978def splitpasswd(user):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000979 """splitpasswd('user:passwd') -> 'user', 'passwd'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000980 global _passwdprog
981 if _passwdprog is None:
982 import re
983 _passwdprog = re.compile('^([^:]*):(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +0000984
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000985 match = _passwdprog.match(user)
986 if match: return match.group(1, 2)
987 return user, None
Guido van Rossum7c395db1994-07-04 22:14:49 +0000988
Guido van Rossume7b146f2000-02-04 15:28:42 +0000989# splittag('/path#tag') --> '/path', 'tag'
Guido van Rossum332e1441997-09-29 23:23:46 +0000990_portprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000991def splitport(host):
Guido van Rossume7b146f2000-02-04 15:28:42 +0000992 """splitport('host:port') --> 'host', 'port'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000993 global _portprog
994 if _portprog is None:
995 import re
996 _portprog = re.compile('^(.*):([0-9]+)$')
Guido van Rossum332e1441997-09-29 23:23:46 +0000997
Jeremy Hyltonf90b0021999-02-25 16:12:12 +0000998 match = _portprog.match(host)
999 if match: return match.group(1, 2)
1000 return host, None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001001
Guido van Rossum332e1441997-09-29 23:23:46 +00001002_nportprog = None
Guido van Rossum53725a21996-06-13 19:12:35 +00001003def splitnport(host, defport=-1):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001004 """Split host and port, returning numeric port.
1005 Return given default port if no ':' found; defaults to -1.
1006 Return numerical port if a valid number are found after ':'.
1007 Return None if ':' but not a valid number."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001008 global _nportprog
1009 if _nportprog is None:
1010 import re
1011 _nportprog = re.compile('^(.*):(.*)$')
Guido van Rossum7e7ca0b1998-03-26 21:01:39 +00001012
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001013 match = _nportprog.match(host)
1014 if match:
1015 host, port = match.group(1, 2)
1016 try:
Guido van Rossumb2493f82000-12-15 15:01:37 +00001017 if not port: raise ValueError, "no digits"
1018 nport = int(port)
1019 except ValueError:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001020 nport = None
1021 return host, nport
1022 return host, defport
Guido van Rossum53725a21996-06-13 19:12:35 +00001023
Guido van Rossum332e1441997-09-29 23:23:46 +00001024_queryprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001025def splitquery(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001026 """splitquery('/path?query') --> '/path', 'query'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001027 global _queryprog
1028 if _queryprog is None:
1029 import re
1030 _queryprog = re.compile('^(.*)\?([^?]*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001031
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001032 match = _queryprog.match(url)
1033 if match: return match.group(1, 2)
1034 return url, None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001035
Guido van Rossum332e1441997-09-29 23:23:46 +00001036_tagprog = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001037def splittag(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001038 """splittag('/path#tag') --> '/path', 'tag'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001039 global _tagprog
1040 if _tagprog is None:
1041 import re
1042 _tagprog = re.compile('^(.*)#([^#]*)$')
Guido van Rossum7e7ca0b1998-03-26 21:01:39 +00001043
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001044 match = _tagprog.match(url)
1045 if match: return match.group(1, 2)
1046 return url, None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001047
Guido van Rossum7c395db1994-07-04 22:14:49 +00001048def splitattr(url):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001049 """splitattr('/path;attr1=value1;attr2=value2;...') ->
1050 '/path', ['attr1=value1', 'attr2=value2', ...]."""
Guido van Rossumb2493f82000-12-15 15:01:37 +00001051 words = url.split(';')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001052 return words[0], words[1:]
Guido van Rossum7c395db1994-07-04 22:14:49 +00001053
Guido van Rossum332e1441997-09-29 23:23:46 +00001054_valueprog = None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001055def splitvalue(attr):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001056 """splitvalue('attr=value') --> 'attr', 'value'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001057 global _valueprog
1058 if _valueprog is None:
1059 import re
1060 _valueprog = re.compile('^([^=]*)=(.*)$')
Guido van Rossum332e1441997-09-29 23:23:46 +00001061
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001062 match = _valueprog.match(attr)
1063 if match: return match.group(1, 2)
1064 return attr, None
Guido van Rossum7c395db1994-07-04 22:14:49 +00001065
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001066def splitgophertype(selector):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001067 """splitgophertype('/Xselector') --> 'X', 'selector'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001068 if selector[:1] == '/' and selector[1:2]:
1069 return selector[1], selector[2:]
1070 return None, selector
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001071
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001072def unquote(s):
Guido van Rossume7b146f2000-02-04 15:28:42 +00001073 """unquote('abc%20def') -> 'abc def'."""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001074 mychr = chr
Guido van Rossumb2493f82000-12-15 15:01:37 +00001075 myatoi = int
1076 list = s.split('%')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001077 res = [list[0]]
1078 myappend = res.append
1079 del list[0]
1080 for item in list:
1081 if item[1:2]:
1082 try:
1083 myappend(mychr(myatoi(item[:2], 16))
1084 + item[2:])
Martin v. Löwis58682b72001-08-11 15:02:57 +00001085 except ValueError:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001086 myappend('%' + item)
1087 else:
1088 myappend('%' + item)
Guido van Rossumb2493f82000-12-15 15:01:37 +00001089 return "".join(res)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001090
Guido van Rossum0564e121996-12-13 14:47:36 +00001091def unquote_plus(s):
Skip Montanaro79f1c172000-08-22 03:00:52 +00001092 """unquote('%7e/abc+def') -> '~/abc def'"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001093 if '+' in s:
1094 # replace '+' with ' '
Guido van Rossumb2493f82000-12-15 15:01:37 +00001095 s = ' '.join(s.split('+'))
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001096 return unquote(s)
Guido van Rossum0564e121996-12-13 14:47:36 +00001097
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001098always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
Jeremy Hylton6102e292000-08-31 15:48:10 +00001099 'abcdefghijklmnopqrstuvwxyz'
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001100 '0123456789' '_.-')
1101
1102_fast_safe_test = always_safe + '/'
1103_fast_safe = None
1104
1105def _fast_quote(s):
1106 global _fast_safe
1107 if _fast_safe is None:
1108 _fast_safe = {}
1109 for c in _fast_safe_test:
1110 _fast_safe[c] = c
1111 res = list(s)
1112 for i in range(len(res)):
1113 c = res[i]
Raymond Hettinger54f02222002-06-01 14:18:47 +00001114 if not c in _fast_safe:
Guido van Rossume27a7b82001-01-19 03:28:15 +00001115 res[i] = '%%%02X' % ord(c)
Guido van Rossumb2493f82000-12-15 15:01:37 +00001116 return ''.join(res)
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001117
Guido van Rossum7c395db1994-07-04 22:14:49 +00001118def quote(s, safe = '/'):
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001119 """quote('abc def') -> 'abc%20def'
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001120
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001121 Each part of a URL, e.g. the path info, the query, etc., has a
1122 different set of reserved characters that must be quoted.
1123
1124 RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists
1125 the following reserved characters.
1126
1127 reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
1128 "$" | ","
1129
1130 Each of these characters is reserved in some component of a URL,
1131 but not necessarily in all of them.
1132
1133 By default, the quote function is intended for quoting the path
1134 section of a URL. Thus, it will not encode '/'. This character
1135 is reserved, but in typical usage the quote function is being
1136 called on a path where the existing slash characters are used as
1137 reserved characters.
1138 """
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001139 safe = always_safe + safe
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001140 if _fast_safe_test == safe:
1141 return _fast_quote(s)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001142 res = list(s)
1143 for i in range(len(res)):
1144 c = res[i]
1145 if c not in safe:
Guido van Rossume27a7b82001-01-19 03:28:15 +00001146 res[i] = '%%%02X' % ord(c)
Guido van Rossumb2493f82000-12-15 15:01:37 +00001147 return ''.join(res)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001148
Jeremy Hylton7ae51bf2000-09-14 16:59:07 +00001149def quote_plus(s, safe = ''):
1150 """Quote the query fragment of a URL; replacing ' ' with '+'"""
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001151 if ' ' in s:
Guido van Rossumb2493f82000-12-15 15:01:37 +00001152 l = s.split(' ')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001153 for i in range(len(l)):
1154 l[i] = quote(l[i], safe)
Guido van Rossumb2493f82000-12-15 15:01:37 +00001155 return '+'.join(l)
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001156 else:
1157 return quote(s, safe)
Guido van Rossum0564e121996-12-13 14:47:36 +00001158
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001159def urlencode(query,doseq=0):
1160 """Encode a sequence of two-element tuples or dictionary into a URL query string.
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001161
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001162 If any values in the query arg are sequences and doseq is true, each
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001163 sequence element is converted to a separate parameter.
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001164
1165 If the query arg is a sequence of two-element tuples, the order of the
1166 parameters in the output will match the order of parameters in the
1167 input.
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001168 """
Tim Peters658cba62001-02-09 20:06:00 +00001169
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001170 if hasattr(query,"items"):
1171 # mapping objects
1172 query = query.items()
1173 else:
1174 # it's a bother at times that strings and string-like objects are
1175 # sequences...
1176 try:
1177 # non-sequence items should not work with len()
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001178 # non-empty strings will fail this
Walter Dörwald65230a22002-06-03 15:58:32 +00001179 if len(query) and not isinstance(query[0], tuple):
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001180 raise TypeError
1181 # zero-length sequences of all types will get here and succeed,
1182 # but that's a minor nit - since the original implementation
1183 # allowed empty dicts that type of behavior probably should be
1184 # preserved for consistency
1185 except TypeError:
1186 ty,va,tb = sys.exc_info()
1187 raise TypeError, "not a valid non-string sequence or mapping object", tb
1188
Guido van Rossume7b146f2000-02-04 15:28:42 +00001189 l = []
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001190 if not doseq:
1191 # preserve old behavior
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001192 for k, v in query:
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001193 k = quote_plus(str(k))
1194 v = quote_plus(str(v))
1195 l.append(k + '=' + v)
1196 else:
Skip Montanaro14f1ad42001-01-28 21:11:12 +00001197 for k, v in query:
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001198 k = quote_plus(str(k))
Walter Dörwald65230a22002-06-03 15:58:32 +00001199 if isinstance(v, str):
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001200 v = quote_plus(v)
1201 l.append(k + '=' + v)
Guido van Rossum4b46c0a2002-05-24 17:58:05 +00001202 elif _is_unicode(v):
Skip Montanaroa5d23a12001-01-20 15:56:39 +00001203 # is there a reasonable way to convert to ASCII?
1204 # encode generates a string, but "replace" or "ignore"
1205 # lose information and "strict" can raise UnicodeError
1206 v = quote_plus(v.encode("ASCII","replace"))
1207 l.append(k + '=' + v)
1208 else:
1209 try:
1210 # is this a sufficient test for sequence-ness?
1211 x = len(v)
1212 except TypeError:
1213 # not a sequence
1214 v = quote_plus(str(v))
1215 l.append(k + '=' + v)
1216 else:
1217 # loop over the sequence
1218 for elt in v:
1219 l.append(k + '=' + quote_plus(str(elt)))
Guido van Rossumb2493f82000-12-15 15:01:37 +00001220 return '&'.join(l)
Guido van Rossum810a3391998-07-22 21:33:23 +00001221
Guido van Rossum442e7201996-03-20 15:33:11 +00001222# Proxy handling
Mark Hammond4f570b92000-07-26 07:04:38 +00001223def getproxies_environment():
1224 """Return a dictionary of scheme -> proxy server URL mappings.
1225
1226 Scan the environment for variables named <scheme>_proxy;
1227 this seems to be the standard convention. If you need a
1228 different way, you can pass a proxies dictionary to the
1229 [Fancy]URLopener constructor.
1230
1231 """
1232 proxies = {}
1233 for name, value in os.environ.items():
Guido van Rossumb2493f82000-12-15 15:01:37 +00001234 name = name.lower()
Mark Hammond4f570b92000-07-26 07:04:38 +00001235 if value and name[-6:] == '_proxy':
1236 proxies[name[:-6]] = value
1237 return proxies
1238
Guido van Rossum4163e701998-08-06 13:39:09 +00001239if os.name == 'mac':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001240 def getproxies():
1241 """Return a dictionary of scheme -> proxy server URL mappings.
Guido van Rossum442e7201996-03-20 15:33:11 +00001242
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001243 By convention the mac uses Internet Config to store
1244 proxies. An HTTP proxy, for instance, is stored under
1245 the HttpProxy key.
Guido van Rossum442e7201996-03-20 15:33:11 +00001246
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001247 """
1248 try:
1249 import ic
1250 except ImportError:
1251 return {}
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001252
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001253 try:
1254 config = ic.IC()
1255 except ic.error:
1256 return {}
1257 proxies = {}
1258 # HTTP:
Raymond Hettinger54f02222002-06-01 14:18:47 +00001259 if 'UseHTTPProxy' in config and config['UseHTTPProxy']:
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001260 try:
1261 value = config['HTTPProxyHost']
1262 except ic.error:
1263 pass
1264 else:
1265 proxies['http'] = 'http://%s' % value
1266 # FTP: XXXX To be done.
1267 # Gopher: XXXX To be done.
1268 return proxies
Mark Hammond4f570b92000-07-26 07:04:38 +00001269
Tim Peters55c12d42001-08-09 18:04:14 +00001270 def proxy_bypass(x):
1271 return 0
1272
Mark Hammond4f570b92000-07-26 07:04:38 +00001273elif os.name == 'nt':
1274 def getproxies_registry():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001275 """Return a dictionary of scheme -> proxy server URL mappings.
Mark Hammond4f570b92000-07-26 07:04:38 +00001276
1277 Win32 uses the registry to store proxies.
1278
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001279 """
1280 proxies = {}
Mark Hammond4f570b92000-07-26 07:04:38 +00001281 try:
1282 import _winreg
1283 except ImportError:
1284 # Std module, so should be around - but you never know!
1285 return proxies
1286 try:
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001287 internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1288 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
Mark Hammond4f570b92000-07-26 07:04:38 +00001289 proxyEnable = _winreg.QueryValueEx(internetSettings,
1290 'ProxyEnable')[0]
1291 if proxyEnable:
1292 # Returned as Unicode but problems if not converted to ASCII
1293 proxyServer = str(_winreg.QueryValueEx(internetSettings,
1294 'ProxyServer')[0])
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001295 if '=' in proxyServer:
1296 # Per-protocol settings
Mark Hammond4f570b92000-07-26 07:04:38 +00001297 for p in proxyServer.split(';'):
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001298 protocol, address = p.split('=', 1)
Guido van Rossumb955d6c2002-03-31 23:38:48 +00001299 # See if address has a type:// prefix
Guido van Rossum64e5aa92002-04-02 14:38:16 +00001300 import re
1301 if not re.match('^([^/:]+)://', address):
Guido van Rossumb955d6c2002-03-31 23:38:48 +00001302 address = '%s://%s' % (protocol, address)
1303 proxies[protocol] = address
Fredrik Lundhb49f88b2000-09-24 18:51:25 +00001304 else:
1305 # Use one setting for all protocols
1306 if proxyServer[:5] == 'http:':
1307 proxies['http'] = proxyServer
1308 else:
1309 proxies['http'] = 'http://%s' % proxyServer
1310 proxies['ftp'] = 'ftp://%s' % proxyServer
Mark Hammond4f570b92000-07-26 07:04:38 +00001311 internetSettings.Close()
1312 except (WindowsError, ValueError, TypeError):
1313 # Either registry key not found etc, or the value in an
1314 # unexpected format.
1315 # proxies already set up to be empty so nothing to do
1316 pass
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001317 return proxies
Guido van Rossum442e7201996-03-20 15:33:11 +00001318
Mark Hammond4f570b92000-07-26 07:04:38 +00001319 def getproxies():
1320 """Return a dictionary of scheme -> proxy server URL mappings.
1321
1322 Returns settings gathered from the environment, if specified,
1323 or the registry.
1324
1325 """
1326 return getproxies_environment() or getproxies_registry()
Tim Peters55c12d42001-08-09 18:04:14 +00001327
1328 def proxy_bypass(host):
1329 try:
1330 import _winreg
1331 import re
Tim Peters55c12d42001-08-09 18:04:14 +00001332 except ImportError:
1333 # Std modules, so should be around - but you never know!
1334 return 0
1335 try:
1336 internetSettings = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
1337 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
1338 proxyEnable = _winreg.QueryValueEx(internetSettings,
1339 'ProxyEnable')[0]
1340 proxyOverride = str(_winreg.QueryValueEx(internetSettings,
1341 'ProxyOverride')[0])
1342 # ^^^^ Returned as Unicode but problems if not converted to ASCII
1343 except WindowsError:
1344 return 0
1345 if not proxyEnable or not proxyOverride:
1346 return 0
1347 # try to make a host list from name and IP address.
1348 host = [host]
1349 try:
1350 addr = socket.gethostbyname(host[0])
1351 if addr != host:
1352 host.append(addr)
1353 except socket.error:
1354 pass
1355 # make a check value list from the registry entry: replace the
1356 # '<local>' string by the localhost entry and the corresponding
1357 # canonical entry.
1358 proxyOverride = proxyOverride.split(';')
1359 i = 0
1360 while i < len(proxyOverride):
1361 if proxyOverride[i] == '<local>':
1362 proxyOverride[i:i+1] = ['localhost',
1363 '127.0.0.1',
1364 socket.gethostname(),
1365 socket.gethostbyname(
1366 socket.gethostname())]
1367 i += 1
1368 # print proxyOverride
1369 # now check if we match one of the registry values.
1370 for test in proxyOverride:
Tim Petersab9ba272001-08-09 21:40:30 +00001371 test = test.replace(".", r"\.") # mask dots
1372 test = test.replace("*", r".*") # change glob sequence
1373 test = test.replace("?", r".") # change glob char
Tim Peters55c12d42001-08-09 18:04:14 +00001374 for val in host:
1375 # print "%s <--> %s" %( test, val )
1376 if re.match(test, val, re.I):
1377 return 1
1378 return 0
1379
Mark Hammond4f570b92000-07-26 07:04:38 +00001380else:
1381 # By default use environment variables
1382 getproxies = getproxies_environment
1383
Tim Peters55c12d42001-08-09 18:04:14 +00001384 def proxy_bypass(host):
1385 return 0
Guido van Rossum442e7201996-03-20 15:33:11 +00001386
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001387# Test and time quote() and unquote()
1388def test1():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001389 s = ''
1390 for i in range(256): s = s + chr(i)
1391 s = s*4
1392 t0 = time.time()
1393 qs = quote(s)
1394 uqs = unquote(qs)
1395 t1 = time.time()
1396 if uqs != s:
1397 print 'Wrong!'
1398 print `s`
1399 print `qs`
1400 print `uqs`
1401 print round(t1 - t0, 3), 'sec'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001402
1403
Guido van Rossum9ab96d41998-09-28 14:07:00 +00001404def reporthook(blocknum, blocksize, totalsize):
1405 # Report during remote transfers
Guido van Rossumb2493f82000-12-15 15:01:37 +00001406 print "Block number: %d, Block size: %d, Total size: %d" % (
1407 blocknum, blocksize, totalsize)
Guido van Rossum9ab96d41998-09-28 14:07:00 +00001408
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001409# Test program
Guido van Rossum23490151998-06-25 02:39:00 +00001410def test(args=[]):
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001411 if not args:
1412 args = [
1413 '/etc/passwd',
1414 'file:/etc/passwd',
1415 'file://localhost/etc/passwd',
Andrew M. Kuchling56a42352002-03-18 22:18:46 +00001416 'ftp://ftp.python.org/pub/python/README',
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001417## 'gopher://gopher.micro.umn.edu/1/',
1418 'http://www.python.org/index.html',
1419 ]
Guido van Rossum09c8b6c1999-12-07 21:37:17 +00001420 if hasattr(URLopener, "open_https"):
1421 args.append('https://synergy.as.cmu.edu/~geek/')
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001422 try:
1423 for url in args:
1424 print '-'*10, url, '-'*10
1425 fn, h = urlretrieve(url, None, reporthook)
Guido van Rossumb2493f82000-12-15 15:01:37 +00001426 print fn
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001427 if h:
1428 print '======'
1429 for k in h.keys(): print k + ':', h[k]
1430 print '======'
1431 fp = open(fn, 'rb')
1432 data = fp.read()
1433 del fp
1434 if '\r' in data:
1435 table = string.maketrans("", "")
Guido van Rossumb2493f82000-12-15 15:01:37 +00001436 data = data.translate(table, "\r")
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001437 print data
1438 fn, h = None, None
1439 print '-'*40
1440 finally:
1441 urlcleanup()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001442
Guido van Rossum23490151998-06-25 02:39:00 +00001443def main():
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001444 import getopt, sys
1445 try:
1446 opts, args = getopt.getopt(sys.argv[1:], "th")
1447 except getopt.error, msg:
1448 print msg
1449 print "Use -h for help"
1450 return
1451 t = 0
1452 for o, a in opts:
1453 if o == '-t':
1454 t = t + 1
1455 if o == '-h':
1456 print "Usage: python urllib.py [-t] [url ...]"
1457 print "-t runs self-test;",
1458 print "otherwise, contents of urls are printed"
1459 return
1460 if t:
1461 if t > 1:
1462 test1()
1463 test(args)
1464 else:
1465 if not args:
1466 print "Use -h for help"
1467 for url in args:
1468 print urlopen(url).read(),
Guido van Rossum23490151998-06-25 02:39:00 +00001469
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001470# Run test program when run as a script
1471if __name__ == '__main__':
Jeremy Hyltonf90b0021999-02-25 16:12:12 +00001472 main()