blob: dfc809c7fa6ff0e7fd03351cea9935a7ae3c1be6 [file] [log] [blame]
Guido van Rossum7c6ebb51994-03-22 12:05:32 +00001# Open an arbitrary URL
2#
Guido van Rossum838cb281997-02-10 17:51:56 +00003# See the following document for more info on URLs:
4# "Names and Addresses, URIs, URLs, URNs, URCs", at
5# http://www.w3.org/pub/WWW/Addressing/Overview.html
6#
7# See also the HTTP spec (from which the error codes are derived):
8# "HTTP - Hypertext Transfer Protocol", at
9# http://www.w3.org/pub/WWW/Protocols/
10#
11# Related standards and specs:
12# - RFC1808: the "relative URL" spec. (authoritative status)
13# - RFC1738 - the "URL standard". (authoritative status)
14# - RFC1630 - the "URI spec". (informational status)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000015#
16# The object returned by URLopener().open(file) will differ per
17# protocol. All you know is that is has methods read(), readline(),
18# readlines(), fileno(), close() and info(). The read*(), fileno()
19# and close() methods work like those of open files.
Guido van Rossum838cb281997-02-10 17:51:56 +000020# The info() method returns a mimetools.Message object which can be
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000021# used to query various info about the object, if available.
Guido van Rossumbbb0a051995-08-04 04:29:05 +000022# (mimetools.Message objects are queried with the getheader() method.)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000023
Guido van Rossum7c395db1994-07-04 22:14:49 +000024import string
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000025import socket
26import regex
Jack Jansendc3e3f61995-12-15 13:22:13 +000027import os
Guido van Rossum3c8484e1996-11-20 22:02:24 +000028import sys
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000029
30
Guido van Rossumf668d171997-06-06 21:11:11 +000031__version__ = '1.7'
32
33MAXFTPCACHE = 10 # Trim the ftp cache beyond this size
Guido van Rossum6cb15a01995-06-22 19:00:13 +000034
Jack Jansendc3e3f61995-12-15 13:22:13 +000035# Helper for non-unix systems
36if os.name == 'mac':
Guido van Rossum71ac9451996-03-21 16:31:41 +000037 from macurl2path import url2pathname, pathname2url
Guido van Rossum2281d351996-06-26 19:47:37 +000038elif os.name == 'nt':
39 from nturl2path import url2pathname, pathname2url
Jack Jansendc3e3f61995-12-15 13:22:13 +000040else:
Jack Jansene8ea21b1995-12-21 15:43:53 +000041 def url2pathname(pathname):
Jack Jansendc3e3f61995-12-15 13:22:13 +000042 return pathname
Jack Jansene8ea21b1995-12-21 15:43:53 +000043 def pathname2url(pathname):
44 return pathname
Guido van Rossum6cb15a01995-06-22 19:00:13 +000045
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000046# This really consists of two pieces:
47# (1) a class which handles opening of all sorts of URLs
48# (plus assorted utilities etc.)
49# (2) a set of functions for parsing URLs
50# XXX Should these be separated out into different modules?
51
52
53# Shortcut for basic usage
54_urlopener = None
Guido van Rossumbd013741996-12-10 16:00:28 +000055def urlopen(url, data=None):
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000056 global _urlopener
57 if not _urlopener:
Guido van Rossumbbb0a051995-08-04 04:29:05 +000058 _urlopener = FancyURLopener()
Guido van Rossumbd013741996-12-10 16:00:28 +000059 if data is None:
60 return _urlopener.open(url)
61 else:
62 return _urlopener.open(url, data)
Guido van Rossuma7e4b281996-06-11 00:16:27 +000063def urlretrieve(url, filename=None):
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000064 global _urlopener
65 if not _urlopener:
Guido van Rossumbbb0a051995-08-04 04:29:05 +000066 _urlopener = FancyURLopener()
Guido van Rossuma7e4b281996-06-11 00:16:27 +000067 if filename:
68 return _urlopener.retrieve(url, filename)
69 else:
70 return _urlopener.retrieve(url)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000071def urlcleanup():
72 if _urlopener:
73 _urlopener.cleanup()
74
75
76# Class to open URLs.
77# This is a class rather than just a subroutine because we may need
78# more than one set of global protocol-specific options.
Guido van Rossumbbb0a051995-08-04 04:29:05 +000079# Note -- this is a base class for those who don't want the
80# automatic handling of errors type 302 (relocated) and 401
81# (authorization needed).
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000082ftpcache = {}
83class URLopener:
84
Guido van Rossum2b3fd761997-09-03 22:36:15 +000085 __tempfiles = []
Guido van Rossum29e77811996-11-27 19:39:58 +000086
Guido van Rossum7c6ebb51994-03-22 12:05:32 +000087 # Constructor
Guido van Rossum442e7201996-03-20 15:33:11 +000088 def __init__(self, proxies=None):
89 if proxies is None:
90 proxies = getproxies()
91 self.proxies = proxies
Guido van Rossum6cb15a01995-06-22 19:00:13 +000092 server_version = "Python-urllib/%s" % __version__
93 self.addheaders = [('User-agent', server_version)]
Guido van Rossum10499321997-09-08 02:16:33 +000094 self.__tempfiles = []
Guido van Rossum7aeb4b91994-08-23 13:32:20 +000095 self.tempcache = None
96 # Undocumented feature: if you assign {} to tempcache,
97 # it is used to cache files retrieved with
98 # self.retrieve(). This is not enabled by default
99 # since it does not work for changing documents (and I
100 # haven't got the logic to check expiration headers
101 # yet).
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000102 self.ftpcache = ftpcache
103 # Undocumented feature: you can use a different
104 # ftp cache by assigning to the .ftpcache member;
105 # in case you want logically independent URL openers
106
107 def __del__(self):
108 self.close()
109
110 def close(self):
111 self.cleanup()
112
113 def cleanup(self):
Guido van Rossum2b3fd761997-09-03 22:36:15 +0000114 if self.__tempfiles:
Guido van Rossumd23d9401997-01-30 15:54:58 +0000115 import os
Guido van Rossum2b3fd761997-09-03 22:36:15 +0000116 for file in self.__tempfiles:
Guido van Rossum7aeb4b91994-08-23 13:32:20 +0000117 try:
Guido van Rossum2b3fd761997-09-03 22:36:15 +0000118 os.unlink(file)
Guido van Rossum7aeb4b91994-08-23 13:32:20 +0000119 except os.error:
120 pass
Guido van Rossum2b3fd761997-09-03 22:36:15 +0000121 URLopener.__tempfiles = []
122 self.tempcache = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000123
124 # Add a header to be used by the HTTP interface only
125 # e.g. u.addheader('Accept', 'sound/basic')
126 def addheader(self, *args):
127 self.addheaders.append(args)
128
129 # External interface
130 # Use URLopener().open(file) instead of open(file, 'r')
Guido van Rossumbd013741996-12-10 16:00:28 +0000131 def open(self, fullurl, data=None):
Guido van Rossumca445401995-08-29 19:19:12 +0000132 fullurl = unwrap(fullurl)
133 type, url = splittype(fullurl)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000134 if not type: type = 'file'
Guido van Rossume6ad8911996-09-10 17:02:56 +0000135 self.openedurl = '%s:%s' % (type, url)
Guido van Rossum442e7201996-03-20 15:33:11 +0000136 if self.proxies.has_key(type):
137 proxy = self.proxies[type]
138 type, proxy = splittype(proxy)
139 host, selector = splithost(proxy)
140 url = (host, fullurl) # Signal special case to open_*()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000141 name = 'open_' + type
142 if '-' in name:
143 import regsub
144 name = regsub.gsub('-', '_', name)
145 if not hasattr(self, name):
Guido van Rossumbd013741996-12-10 16:00:28 +0000146 if data is None:
147 return self.open_unknown(fullurl)
148 else:
149 return self.open_unknown(fullurl, data)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000150 try:
Guido van Rossumbd013741996-12-10 16:00:28 +0000151 if data is None:
152 return getattr(self, name)(url)
153 else:
154 return getattr(self, name)(url, data)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000155 except socket.error, msg:
Guido van Rossum3c8484e1996-11-20 22:02:24 +0000156 raise IOError, ('socket error', msg), sys.exc_traceback
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000157
Guido van Rossumca445401995-08-29 19:19:12 +0000158 # Overridable interface to open unknown URL type
Guido van Rossumbd013741996-12-10 16:00:28 +0000159 def open_unknown(self, fullurl, data=None):
Guido van Rossumca445401995-08-29 19:19:12 +0000160 type, url = splittype(fullurl)
161 raise IOError, ('url error', 'unknown url type', type)
162
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000163 # External interface
164 # retrieve(url) returns (filename, None) for a local object
165 # or (tempfilename, headers) for a remote object
Guido van Rossuma7e4b281996-06-11 00:16:27 +0000166 def retrieve(self, url, filename=None):
Guido van Rossum7aeb4b91994-08-23 13:32:20 +0000167 if self.tempcache and self.tempcache.has_key(url):
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000168 return self.tempcache[url]
169 url1 = unwrap(url)
Guido van Rossum5b1b33c1996-10-22 13:28:37 +0000170 self.openedurl = url1
Guido van Rossum7aeb4b91994-08-23 13:32:20 +0000171 if self.tempcache and self.tempcache.has_key(url1):
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000172 self.tempcache[url] = self.tempcache[url1]
173 return self.tempcache[url1]
174 type, url1 = splittype(url1)
Guido van Rossuma7e4b281996-06-11 00:16:27 +0000175 if not filename and (not type or type == 'file'):
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000176 try:
177 fp = self.open_local_file(url1)
178 del fp
Jack Jansene8ea21b1995-12-21 15:43:53 +0000179 return url2pathname(splithost(url1)[1]), None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000180 except IOError, msg:
181 pass
182 fp = self.open(url)
183 headers = fp.info()
Guido van Rossuma7e4b281996-06-11 00:16:27 +0000184 if not filename:
185 import tempfile
186 filename = tempfile.mktemp()
Guido van Rossum2b3fd761997-09-03 22:36:15 +0000187 self.__tempfiles.append(filename)
Guido van Rossuma7e4b281996-06-11 00:16:27 +0000188 result = filename, headers
Guido van Rossum7aeb4b91994-08-23 13:32:20 +0000189 if self.tempcache is not None:
Guido van Rossumfa59e831994-09-21 11:36:19 +0000190 self.tempcache[url] = result
Guido van Rossumc511aee1997-04-11 19:01:48 +0000191 tfp = open(filename, 'wb')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000192 bs = 1024*8
193 block = fp.read(bs)
194 while block:
195 tfp.write(block)
196 block = fp.read(bs)
Guido van Rossumab0abdc1997-08-26 19:06:40 +0000197 fp.close()
198 tfp.close()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000199 del fp
200 del tfp
201 return result
202
203 # Each method named open_<type> knows how to open that type of URL
204
205 # Use HTTP protocol
Guido van Rossumbd013741996-12-10 16:00:28 +0000206 def open_http(self, url, data=None):
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000207 import httplib
Guido van Rossum442e7201996-03-20 15:33:11 +0000208 if type(url) is type(""):
209 host, selector = splithost(url)
Guido van Rossum78c96371996-08-26 18:09:59 +0000210 user_passwd, host = splituser(host)
Guido van Rossumc24751b1997-06-03 14:34:19 +0000211 realhost = host
Guido van Rossum442e7201996-03-20 15:33:11 +0000212 else:
213 host, selector = url
Guido van Rossum78c96371996-08-26 18:09:59 +0000214 urltype, rest = splittype(selector)
Guido van Rossumfd795661997-04-02 05:46:35 +0000215 user_passwd = None
Guido van Rossumc24751b1997-06-03 14:34:19 +0000216 if string.lower(urltype) != 'http':
217 realhost = None
218 else:
Guido van Rossum78c96371996-08-26 18:09:59 +0000219 realhost, rest = splithost(rest)
220 user_passwd, realhost = splituser(realhost)
221 if user_passwd:
222 selector = "%s://%s%s" % (urltype,
223 realhost, rest)
Guido van Rossumfd795661997-04-02 05:46:35 +0000224 #print "proxy via http:", host, selector
Guido van Rossum590b2891994-04-18 09:39:56 +0000225 if not host: raise IOError, ('http error', 'no host given')
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000226 if user_passwd:
227 import base64
228 auth = string.strip(base64.encodestring(user_passwd))
229 else:
230 auth = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000231 h = httplib.HTTP(host)
Guido van Rossumbd013741996-12-10 16:00:28 +0000232 if data is not None:
233 h.putrequest('POST', selector)
234 h.putheader('Content-type',
235 'application/x-www-form-urlencoded')
236 h.putheader('Content-length', '%d' % len(data))
237 else:
238 h.putrequest('GET', selector)
Guido van Rossumc5d7e801996-11-11 19:01:17 +0000239 if auth: h.putheader('Authorization', 'Basic %s' % auth)
Guido van Rossumc24751b1997-06-03 14:34:19 +0000240 if realhost: h.putheader('Host', realhost)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000241 for args in self.addheaders: apply(h.putheader, args)
Guido van Rossum6cb15a01995-06-22 19:00:13 +0000242 h.endheaders()
Guido van Rossumbd013741996-12-10 16:00:28 +0000243 if data is not None:
244 h.send(data + '\r\n')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000245 errcode, errmsg, headers = h.getreply()
Guido van Rossum6cb15a01995-06-22 19:00:13 +0000246 fp = h.getfile()
247 if errcode == 200:
Guido van Rossume6ad8911996-09-10 17:02:56 +0000248 return addinfourl(fp, headers, self.openedurl)
Guido van Rossum6cb15a01995-06-22 19:00:13 +0000249 else:
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000250 return self.http_error(url,
251 fp, errcode, errmsg, headers)
252
253 # Handle http errors.
254 # Derived class can override this, or provide specific handlers
255 # named http_error_DDD where DDD is the 3-digit error code
256 def http_error(self, url, fp, errcode, errmsg, headers):
257 # First check if there's a specific handler for this error
258 name = 'http_error_%d' % errcode
259 if hasattr(self, name):
260 method = getattr(self, name)
261 result = method(url, fp, errcode, errmsg, headers)
262 if result: return result
263 return self.http_error_default(
264 url, fp, errcode, errmsg, headers)
265
266 # Default http error handler: close the connection and raises IOError
267 def http_error_default(self, url, fp, errcode, errmsg, headers):
268 void = fp.read()
269 fp.close()
270 raise IOError, ('http error', errcode, errmsg, headers)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000271
272 # Use Gopher protocol
273 def open_gopher(self, url):
274 import gopherlib
275 host, selector = splithost(url)
Guido van Rossum590b2891994-04-18 09:39:56 +0000276 if not host: raise IOError, ('gopher error', 'no host given')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000277 type, selector = splitgophertype(selector)
278 selector, query = splitquery(selector)
Guido van Rossum7c395db1994-07-04 22:14:49 +0000279 selector = unquote(selector)
280 if query:
281 query = unquote(query)
282 fp = gopherlib.send_query(selector, query, host)
283 else:
284 fp = gopherlib.send_selector(selector, host)
Guido van Rossume6ad8911996-09-10 17:02:56 +0000285 return addinfourl(fp, noheaders(), self.openedurl)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000286
287 # Use local file or FTP depending on form of URL
288 def open_file(self, url):
Guido van Rossumb6784dc1997-08-20 23:34:01 +0000289 if url[:2] == '//' and url[2:3] != '/':
290 return self.open_ftp(url)
291 else:
292 return self.open_local_file(url)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000293
294 # Use local file
295 def open_local_file(self, url):
296 host, file = splithost(url)
Guido van Rossumb030bc01996-10-10 16:01:16 +0000297 if not host:
Guido van Rossum2966b321997-06-06 17:44:07 +0000298 return addinfourl(
299 open(url2pathname(file), 'rb'),
300 noheaders(), 'file:'+file)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000301 host, port = splitport(host)
302 if not port and socket.gethostbyname(host) in (
303 localhost(), thishost()):
Guido van Rossum7c395db1994-07-04 22:14:49 +0000304 file = unquote(file)
Guido van Rossum2966b321997-06-06 17:44:07 +0000305 return addinfourl(
306 open(url2pathname(file), 'rb'),
307 noheaders(), 'file:'+file)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000308 raise IOError, ('local file error', 'not on local host')
309
310 # Use FTP protocol
311 def open_ftp(self, url):
Guido van Rossum7c395db1994-07-04 22:14:49 +0000312 host, path = splithost(url)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000313 if not host: raise IOError, ('ftp error', 'no host given')
314 host, port = splitport(host)
Guido van Rossum7c395db1994-07-04 22:14:49 +0000315 user, host = splituser(host)
316 if user: user, passwd = splitpasswd(user)
317 else: passwd = None
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000318 host = socket.gethostbyname(host)
319 if not port:
320 import ftplib
321 port = ftplib.FTP_PORT
Guido van Rossum7c395db1994-07-04 22:14:49 +0000322 path, attrs = splitattr(path)
323 dirs = string.splitfields(path, '/')
324 dirs, file = dirs[:-1], dirs[-1]
325 if dirs and not dirs[0]: dirs = dirs[1:]
326 key = (user, host, port, string.joinfields(dirs, '/'))
Guido van Rossumf668d171997-06-06 21:11:11 +0000327 if len(self.ftpcache) > MAXFTPCACHE:
328 # Prune the cache, rather arbitrarily
329 for k in self.ftpcache.keys():
330 if k != key:
331 v = self.ftpcache[k]
332 del self.ftpcache[k]
333 v.close()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000334 try:
335 if not self.ftpcache.has_key(key):
Guido van Rossum7c395db1994-07-04 22:14:49 +0000336 self.ftpcache[key] = \
337 ftpwrapper(user, passwd,
338 host, port, dirs)
339 if not file: type = 'D'
340 else: type = 'I'
341 for attr in attrs:
342 attr, value = splitvalue(attr)
343 if string.lower(attr) == 'type' and \
344 value in ('a', 'A', 'i', 'I', 'd', 'D'):
345 type = string.upper(value)
Guido van Rossum2966b321997-06-06 17:44:07 +0000346 return addinfourl(
347 self.ftpcache[key].retrfile(file, type),
348 noheaders(), self.openedurl)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000349 except ftperrors(), msg:
Guido van Rossum3c8484e1996-11-20 22:02:24 +0000350 raise IOError, ('ftp error', msg), sys.exc_traceback
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000351
352
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000353# Derived class with handlers for errors we can handle (perhaps)
354class FancyURLopener(URLopener):
355
356 def __init__(self, *args):
357 apply(URLopener.__init__, (self,) + args)
358 self.auth_cache = {}
359
360 # Default error handling -- don't raise an exception
361 def http_error_default(self, url, fp, errcode, errmsg, headers):
Guido van Rossume6ad8911996-09-10 17:02:56 +0000362 return addinfourl(fp, headers, self.openedurl)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000363
Guido van Rossume6ad8911996-09-10 17:02:56 +0000364 # Error 302 -- relocated (temporarily)
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000365 def http_error_302(self, url, fp, errcode, errmsg, headers):
366 # XXX The server can force infinite recursion here!
367 if headers.has_key('location'):
368 newurl = headers['location']
369 elif headers.has_key('uri'):
370 newurl = headers['uri']
371 else:
372 return
373 void = fp.read()
374 fp.close()
375 return self.open(newurl)
376
Guido van Rossume6ad8911996-09-10 17:02:56 +0000377 # Error 301 -- also relocated (permanently)
378 http_error_301 = http_error_302
379
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000380 # Error 401 -- authentication required
381 # See this URL for a description of the basic authentication scheme:
382 # http://www.ics.uci.edu/pub/ietf/http/draft-ietf-http-v10-spec-00.txt
383 def http_error_401(self, url, fp, errcode, errmsg, headers):
384 if headers.has_key('www-authenticate'):
385 stuff = headers['www-authenticate']
386 p = regex.compile(
387 '[ \t]*\([^ \t]+\)[ \t]+realm="\([^"]*\)"')
388 if p.match(stuff) >= 0:
389 scheme, realm = p.group(1, 2)
390 if string.lower(scheme) == 'basic':
391 return self.retry_http_basic_auth(
392 url, realm)
393
394 def retry_http_basic_auth(self, url, realm):
395 host, selector = splithost(url)
396 i = string.find(host, '@') + 1
397 host = host[i:]
398 user, passwd = self.get_user_passwd(host, realm, i)
399 if not (user or passwd): return None
400 host = user + ':' + passwd + '@' + host
401 newurl = '//' + host + selector
402 return self.open_http(newurl)
403
404 def get_user_passwd(self, host, realm, clear_cache = 0):
405 key = realm + '@' + string.lower(host)
406 if self.auth_cache.has_key(key):
407 if clear_cache:
408 del self.auth_cache[key]
409 else:
410 return self.auth_cache[key]
411 user, passwd = self.prompt_user_passwd(host, realm)
412 if user or passwd: self.auth_cache[key] = (user, passwd)
413 return user, passwd
414
415 def prompt_user_passwd(self, host, realm):
416 # Override this in a GUI environment!
417 try:
418 user = raw_input("Enter username for %s at %s: " %
419 (realm, host))
420 self.echo_off()
421 try:
422 passwd = raw_input(
423 "Enter password for %s in %s at %s: " %
424 (user, realm, host))
425 finally:
426 self.echo_on()
427 return user, passwd
428 except KeyboardInterrupt:
429 return None, None
430
431 def echo_off(self):
432 import os
433 os.system("stty -echo")
434
435 def echo_on(self):
436 import os
437 print
438 os.system("stty echo")
439
440
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000441# Utility functions
442
443# Return the IP address of the magic hostname 'localhost'
444_localhost = None
445def localhost():
446 global _localhost
447 if not _localhost:
448 _localhost = socket.gethostbyname('localhost')
449 return _localhost
450
451# Return the IP address of the current host
452_thishost = None
453def thishost():
454 global _thishost
455 if not _thishost:
456 _thishost = socket.gethostbyname(socket.gethostname())
457 return _thishost
458
459# Return the set of errors raised by the FTP class
460_ftperrors = None
461def ftperrors():
462 global _ftperrors
463 if not _ftperrors:
464 import ftplib
Guido van Rossum2966b321997-06-06 17:44:07 +0000465 _ftperrors = ftplib.all_errors
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000466 return _ftperrors
467
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000468# Return an empty mimetools.Message object
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000469_noheaders = None
470def noheaders():
471 global _noheaders
472 if not _noheaders:
Guido van Rossumbbb0a051995-08-04 04:29:05 +0000473 import mimetools
474 import StringIO
475 _noheaders = mimetools.Message(StringIO.StringIO(), 0)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000476 _noheaders.fp.close() # Recycle file descriptor
477 return _noheaders
478
479
480# Utility classes
481
482# Class used by open_ftp() for cache of open FTP connections
483class ftpwrapper:
Guido van Rossum7c395db1994-07-04 22:14:49 +0000484 def __init__(self, user, passwd, host, port, dirs):
485 self.user = unquote(user or '')
486 self.passwd = unquote(passwd or '')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000487 self.host = host
488 self.port = port
Guido van Rossum7c395db1994-07-04 22:14:49 +0000489 self.dirs = []
490 for dir in dirs:
491 self.dirs.append(unquote(dir))
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000492 self.init()
493 def init(self):
494 import ftplib
495 self.ftp = ftplib.FTP()
496 self.ftp.connect(self.host, self.port)
Guido van Rossum7c395db1994-07-04 22:14:49 +0000497 self.ftp.login(self.user, self.passwd)
498 for dir in self.dirs:
499 self.ftp.cwd(dir)
500 def retrfile(self, file, type):
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000501 import ftplib
Guido van Rossum7c395db1994-07-04 22:14:49 +0000502 if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
503 else: cmd = 'TYPE ' + type; isdir = 0
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000504 try:
Guido van Rossum7c395db1994-07-04 22:14:49 +0000505 self.ftp.voidcmd(cmd)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000506 except ftplib.all_errors:
507 self.init()
Guido van Rossum7c395db1994-07-04 22:14:49 +0000508 self.ftp.voidcmd(cmd)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000509 conn = None
Guido van Rossum7c395db1994-07-04 22:14:49 +0000510 if file and not isdir:
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000511 try:
512 cmd = 'RETR ' + file
513 conn = self.ftp.transfercmd(cmd)
514 except ftplib.error_perm, reason:
515 if reason[:3] != '550':
Guido van Rossum3c8484e1996-11-20 22:02:24 +0000516 raise IOError, ('ftp error', reason), \
517 sys.exc_traceback
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000518 if not conn:
519 # Try a directory listing
520 if file: cmd = 'LIST ' + file
521 else: cmd = 'LIST'
522 conn = self.ftp.transfercmd(cmd)
Guido van Rossumf668d171997-06-06 21:11:11 +0000523 return addclosehook(conn.makefile('rb'), self.endtransfer)
524 def endtransfer(self):
525 try:
526 self.ftp.voidresp()
527 except ftperrors():
528 pass
529 def close(self):
530 try:
531 self.ftp.close()
532 except ftperrors():
533 pass
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000534
535# Base class for addinfo and addclosehook
536class addbase:
537 def __init__(self, fp):
538 self.fp = fp
539 self.read = self.fp.read
540 self.readline = self.fp.readline
541 self.readlines = self.fp.readlines
542 self.fileno = self.fp.fileno
543 def __repr__(self):
544 return '<%s at %s whose fp = %s>' % (
545 self.__class__.__name__, `id(self)`, `self.fp`)
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000546 def close(self):
547 self.read = None
548 self.readline = None
549 self.readlines = None
550 self.fileno = None
Guido van Rossum6cb15a01995-06-22 19:00:13 +0000551 if self.fp: self.fp.close()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000552 self.fp = None
553
554# Class to add a close hook to an open file
555class addclosehook(addbase):
556 def __init__(self, fp, closehook, *hookargs):
557 addbase.__init__(self, fp)
558 self.closehook = closehook
559 self.hookargs = hookargs
560 def close(self):
561 if self.closehook:
562 apply(self.closehook, self.hookargs)
563 self.closehook = None
564 self.hookargs = None
565 addbase.close(self)
566
567# class to add an info() method to an open file
568class addinfo(addbase):
569 def __init__(self, fp, headers):
570 addbase.__init__(self, fp)
571 self.headers = headers
572 def info(self):
573 return self.headers
574
Guido van Rossume6ad8911996-09-10 17:02:56 +0000575# class to add info() and geturl() methods to an open file
576class addinfourl(addbase):
577 def __init__(self, fp, headers, url):
578 addbase.__init__(self, fp)
579 self.headers = headers
580 self.url = url
581 def info(self):
582 return self.headers
583 def geturl(self):
584 return self.url
585
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000586
587# Utility to combine a URL with a base URL to form a new URL
588
589def basejoin(base, url):
590 type, path = splittype(url)
Sjoerd Mullendere0371b81995-11-10 10:36:07 +0000591 if type:
592 # if url is complete (i.e., it contains a type), return it
593 return url
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000594 host, path = splithost(path)
Sjoerd Mullendere0371b81995-11-10 10:36:07 +0000595 type, basepath = splittype(base) # inherit type from base
596 if host:
597 # if url contains host, just inherit type
598 if type: return type + '://' + host + path
599 else:
600 # no type inherited, so url must have started with //
601 # just return it
602 return url
603 host, basepath = splithost(basepath) # inherit host
604 basepath, basetag = splittag(basepath) # remove extraneuous cruft
605 basepath, basequery = splitquery(basepath) # idem
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000606 if path[:1] != '/':
Sjoerd Mullendere0371b81995-11-10 10:36:07 +0000607 # non-absolute path name
608 if path[:1] in ('#', '?'):
609 # path is just a tag or query, attach to basepath
610 i = len(basepath)
611 else:
612 # else replace last component
613 i = string.rfind(basepath, '/')
614 if i < 0:
615 # basepath not absolute
616 if host:
617 # host present, make absolute
618 basepath = '/'
619 else:
620 # else keep non-absolute
621 basepath = ''
622 else:
623 # remove last file component
624 basepath = basepath[:i+1]
Guido van Rossum54a1d0b1997-04-11 19:09:51 +0000625 # Interpret ../ (important because of symlinks)
626 while basepath and path[:3] == '../':
627 path = path[3:]
Guido van Rossum2b3fd761997-09-03 22:36:15 +0000628 i = string.rfind(basepath[:-1], '/')
Guido van Rossum54a1d0b1997-04-11 19:09:51 +0000629 if i > 0:
Guido van Rossum2b3fd761997-09-03 22:36:15 +0000630 basepath = basepath[:i+1]
631 elif i == 0:
632 basepath = '/'
633 break
634 else:
635 basepath = ''
Guido van Rossum54a1d0b1997-04-11 19:09:51 +0000636
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000637 path = basepath + path
Sjoerd Mullendere0371b81995-11-10 10:36:07 +0000638 if type and host: return type + '://' + host + path
639 elif type: return type + ':' + path
640 elif host: return '//' + host + path # don't know what this means
641 else: return path
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000642
643
Guido van Rossum7c395db1994-07-04 22:14:49 +0000644# Utilities to parse URLs (most of these return None for missing parts):
Sjoerd Mullendere0371b81995-11-10 10:36:07 +0000645# unwrap('<URL:type://host/path>') --> 'type://host/path'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000646# splittype('type:opaquestring') --> 'type', 'opaquestring'
647# splithost('//host[:port]/path') --> 'host[:port]', '/path'
Guido van Rossum7c395db1994-07-04 22:14:49 +0000648# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
649# splitpasswd('user:passwd') -> 'user', 'passwd'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000650# splitport('host:port') --> 'host', 'port'
651# splitquery('/path?query') --> '/path', 'query'
652# splittag('/path#tag') --> '/path', 'tag'
Guido van Rossum7c395db1994-07-04 22:14:49 +0000653# splitattr('/path;attr1=value1;attr2=value2;...') ->
654# '/path', ['attr1=value1', 'attr2=value2', ...]
655# splitvalue('attr=value') --> 'attr', 'value'
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000656# splitgophertype('/Xselector') --> 'X', 'selector'
657# unquote('abc%20def') -> 'abc def'
658# quote('abc def') -> 'abc%20def')
659
660def unwrap(url):
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000661 url = string.strip(url)
662 if url[:1] == '<' and url[-1:] == '>':
663 url = string.strip(url[1:-1])
664 if url[:4] == 'URL:': url = string.strip(url[4:])
665 return url
666
Guido van Rossumab0d1af1997-04-16 15:17:06 +0000667_typeprog = regex.compile('^\([^/:]+\):')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000668def splittype(url):
Guido van Rossumab0d1af1997-04-16 15:17:06 +0000669 if _typeprog.match(url) >= 0:
670 scheme = _typeprog.group(1)
671 return scheme, url[len(scheme) + 1:]
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000672 return None, url
673
674_hostprog = regex.compile('^//\([^/]+\)\(.*\)$')
675def splithost(url):
676 if _hostprog.match(url) >= 0: return _hostprog.group(1, 2)
677 return None, url
678
Guido van Rossum7c395db1994-07-04 22:14:49 +0000679_userprog = regex.compile('^\([^@]*\)@\(.*\)$')
680def splituser(host):
681 if _userprog.match(host) >= 0: return _userprog.group(1, 2)
682 return None, host
683
684_passwdprog = regex.compile('^\([^:]*\):\(.*\)$')
685def splitpasswd(user):
686 if _passwdprog.match(user) >= 0: return _passwdprog.group(1, 2)
687 return user, None
688
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000689_portprog = regex.compile('^\(.*\):\([0-9]+\)$')
690def splitport(host):
691 if _portprog.match(host) >= 0: return _portprog.group(1, 2)
692 return host, None
693
Guido van Rossum53725a21996-06-13 19:12:35 +0000694# Split host and port, returning numeric port.
695# Return given default port if no ':' found; defaults to -1.
Guido van Rossum84a00a81996-06-17 17:11:40 +0000696# Return numerical port if a valid number are found after ':'.
697# Return None if ':' but not a valid number.
698_nportprog = regex.compile('^\(.*\):\(.*\)$')
Guido van Rossum53725a21996-06-13 19:12:35 +0000699def splitnport(host, defport=-1):
700 if _nportprog.match(host) >= 0:
Guido van Rossum84a00a81996-06-17 17:11:40 +0000701 host, port = _nportprog.group(1, 2)
702 try:
703 if not port: raise string.atoi_error, "no digits"
704 nport = string.atoi(port)
705 except string.atoi_error:
706 nport = None
Guido van Rossum53725a21996-06-13 19:12:35 +0000707 return host, nport
708 return host, defport
709
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000710_queryprog = regex.compile('^\(.*\)\?\([^?]*\)$')
711def splitquery(url):
712 if _queryprog.match(url) >= 0: return _queryprog.group(1, 2)
713 return url, None
714
715_tagprog = regex.compile('^\(.*\)#\([^#]*\)$')
716def splittag(url):
717 if _tagprog.match(url) >= 0: return _tagprog.group(1, 2)
718 return url, None
719
Guido van Rossum7c395db1994-07-04 22:14:49 +0000720def splitattr(url):
721 words = string.splitfields(url, ';')
722 return words[0], words[1:]
723
724_valueprog = regex.compile('^\([^=]*\)=\(.*\)$')
725def splitvalue(attr):
726 if _valueprog.match(attr) >= 0: return _valueprog.group(1, 2)
727 return attr, None
728
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000729def splitgophertype(selector):
730 if selector[:1] == '/' and selector[1:2]:
731 return selector[1], selector[2:]
732 return None, selector
733
734_quoteprog = regex.compile('%[0-9a-fA-F][0-9a-fA-F]')
735def unquote(s):
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000736 i = 0
737 n = len(s)
Guido van Rossumf8abb381996-08-26 15:56:12 +0000738 res = []
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000739 while 0 <= i < n:
740 j = _quoteprog.search(s, i)
741 if j < 0:
Guido van Rossumf8abb381996-08-26 15:56:12 +0000742 res.append(s[i:])
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000743 break
Guido van Rossumf8abb381996-08-26 15:56:12 +0000744 res.append(s[i:j] + chr(string.atoi(s[j+1:j+3], 16)))
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000745 i = j+3
Guido van Rossumf8abb381996-08-26 15:56:12 +0000746 return string.joinfields(res, '')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000747
Guido van Rossum0564e121996-12-13 14:47:36 +0000748def unquote_plus(s):
749 if '+' in s:
750 import regsub
751 s = regsub.gsub('+', ' ', s)
752 return unquote(s)
753
Guido van Rossum3bb54481994-08-29 10:52:58 +0000754always_safe = string.letters + string.digits + '_,.-'
Guido van Rossum7c395db1994-07-04 22:14:49 +0000755def quote(s, safe = '/'):
756 safe = always_safe + safe
Guido van Rossumf8abb381996-08-26 15:56:12 +0000757 res = []
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000758 for c in s:
Guido van Rossum7c395db1994-07-04 22:14:49 +0000759 if c in safe:
Guido van Rossumf8abb381996-08-26 15:56:12 +0000760 res.append(c)
Guido van Rossum7c395db1994-07-04 22:14:49 +0000761 else:
Guido van Rossumf8abb381996-08-26 15:56:12 +0000762 res.append('%%%02x' % ord(c))
763 return string.joinfields(res, '')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000764
Guido van Rossum0564e121996-12-13 14:47:36 +0000765def quote_plus(s, safe = '/'):
766 if ' ' in s:
767 import regsub
768 s = regsub.gsub(' ', '+', s)
769 return quote(s, safe + '+')
770 else:
771 return quote(s, safe)
772
Guido van Rossum442e7201996-03-20 15:33:11 +0000773
774# Proxy handling
775def getproxies():
776 """Return a dictionary of protocol scheme -> proxy server URL mappings.
777
778 Scan the environment for variables named <scheme>_proxy;
779 this seems to be the standard convention. If you need a
780 different way, you can pass a proxies dictionary to the
781 [Fancy]URLopener constructor.
782
783 """
784 proxies = {}
785 for name, value in os.environ.items():
Guido van Rossum1aec3f01997-05-28 15:37:19 +0000786 name = string.lower(name)
Guido van Rossum442e7201996-03-20 15:33:11 +0000787 if value and name[-6:] == '_proxy':
788 proxies[name[:-6]] = value
789 return proxies
790
791
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000792# Test and time quote() and unquote()
793def test1():
794 import time
795 s = ''
796 for i in range(256): s = s + chr(i)
797 s = s*4
798 t0 = time.time()
799 qs = quote(s)
800 uqs = unquote(qs)
801 t1 = time.time()
802 if uqs != s:
803 print 'Wrong!'
804 print `s`
805 print `qs`
806 print `uqs`
807 print round(t1 - t0, 3), 'sec'
808
809
810# Test program
811def test():
812 import sys
813 import regsub
814 args = sys.argv[1:]
815 if not args:
816 args = [
817 '/etc/passwd',
818 'file:/etc/passwd',
819 'file://localhost/etc/passwd',
820 'ftp://ftp.cwi.nl/etc/passwd',
821 'gopher://gopher.cwi.nl/11/',
822 'http://www.cwi.nl/index.html',
823 ]
824 try:
825 for url in args:
826 print '-'*10, url, '-'*10
827 fn, h = urlretrieve(url)
828 print fn, h
829 if h:
830 print '======'
831 for k in h.keys(): print k + ':', h[k]
832 print '======'
Guido van Rossumc511aee1997-04-11 19:01:48 +0000833 fp = open(fn, 'rb')
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000834 data = fp.read()
835 del fp
836 print regsub.gsub('\r', '', data)
837 fn, h = None, None
838 print '-'*40
839 finally:
840 urlcleanup()
841
842# Run test program when run as a script
843if __name__ == '__main__':
Guido van Rossum7c395db1994-07-04 22:14:49 +0000844## test1()
Guido van Rossum7c6ebb51994-03-22 12:05:32 +0000845 test()