EXPERIMENTAL
An extensible library for opening URLs using a variety protocols.
Intended as a replacement for urllib.
diff --git a/Lib/urllib2.py b/Lib/urllib2.py
new file mode 100644
index 0000000..40a6715
--- /dev/null
+++ b/Lib/urllib2.py
@@ -0,0 +1,1063 @@
+"""An extensible library for opening URLs using a variety protocols
+
+The simplest way to use this module is to call the urlopen function,
+which accepts a string containing a URL or a Request object (described
+below). It opens the URL and returns the results as file-like
+object; the returned object has some extra methods described below.
+
+The OpenerDirectory manages a collection of Handler objects that do
+all the actual work. Each Handler implements a particular protocol or
+option. The OpenerDirector is a composite object that invokes the
+Handlers needed to open the requested URL. For example, the
+HTTPHandler performs HTTP GET and POST requests and deals with
+non-error returns. The HTTPRedirectHandler automatically deals with
+HTTP 301 & 302 redirect errors, and the HTTPDigestAuthHandler deals
+with digest authentication.
+
+urlopen(url, data=None) -- basic usage is that same as original
+urllib. pass the url and optionally data to post to an HTTP URL, and
+get a file-like object back. One difference is that you can also pass
+a Request instance instead of URL. Raises a URLError (subclass of
+IOError); for HTTP errors, raises an HTTPError, which can also be
+treated as a valid response.
+
+build_opener -- function that creates a new OpenerDirector instance.
+will install the default handlers. accepts one or more Handlers as
+arguments, either instances or Handler classes that it will
+instantiate. if one of the argument is a subclass of the default
+handler, the argument will be installed instead of the default.
+
+install_opener -- installs a new opener as the default opener.
+
+objects of interest:
+OpenerDirector --
+
+Request -- an object that encapsulates the state of a request. the
+state can be a simple as the URL. it can also include extra HTTP
+headers, e.g. a User-Agent.
+
+BaseHandler --
+
+exceptions:
+URLError-- a subclass of IOError, individual protocols have their own
+specific subclass
+
+HTTPError-- also a valid HTTP response, so you can treat an HTTP error
+as an exceptional event or valid response
+
+internals:
+BaseHandler and parent
+_call_chain conventions
+
+Example usage:
+
+import urllib2
+
+# set up authentication info
+authinfo = urllib2.HTTPBasicAuthHandler()
+authinfo.add_password('realm', 'host', 'username', 'password')
+
+# build a new opener that adds authentication and caching FTP handlers
+opener = urllib2.build_opener(authinfo, urllib2.CacheFTPHandler)
+
+# install it
+urllib2.install_opener(opener)
+
+f = urllib2.urlopen('http://www.python.org/')
+
+
+"""
+
+# XXX issues:
+# If an authentication error handler that tries to perform
+ # authentication for some reason but fails, how should the error be
+ # signalled? The client needs to know the HTTP error code. But if
+ # the handler knows that the problem was, e.g., that it didn't know
+ # that hash algo that requested in the challenge, it would be good to
+ # pass that information along to the client, too.
+
+# XXX to do:
+# name!
+# documentation (getting there)
+# complex proxies
+# abstract factory for opener
+# ftp errors aren't handled cleanly
+# gopher can return a socket.error
+# check digest against correct (i.e. non-apache) implementation
+
+import string
+import socket
+import UserDict
+import httplib
+import re
+import base64
+import types
+import urlparse
+import os
+import md5
+import mimetypes
+import mimetools
+import ftplib
+import sys
+import time
+import gopherlib
+
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
+try:
+ import sha
+except ImportError:
+ # need 1.5.2 final
+ sha = None
+
+# not sure how many of these need to be gotten rid of
+from urllib import unwrap, unquote, splittype, splithost, \
+ addinfourl, splitport, splitgophertype, splitquery, \
+ splitattr, ftpwrapper, noheaders
+
+# support for proxies via environment variables
+from urllib import getproxies
+
+# support for FileHandler
+from urllib import localhost, thishost, url2pathname, pathname2url
+
+# support for GopherHandler
+from urllib import splitgophertype, splitquery
+
+__version__ = "2.0a1"
+
+_opener = None
+def urlopen(url, data=None):
+ global _opener
+ if _opener is None:
+ _opener = build_opener()
+ return _opener.open(url, data)
+
+def install_opener(opener):
+ global _opener
+ _opener = opener
+
+# do these error classes make sense?
+# make sure all of the IOError stuff is overriden. we just want to be
+ # subtypes.
+
+class URLError(IOError):
+ # URLError is a sub-type of IOError, but it doesn't share any of
+ # the implementation. need to override __init__ and __str__
+ def __init__(self, reason):
+ self.reason = reason
+
+ def __str__(self):
+ return '<urlopen error %s>' % self.reason
+
+class HTTPError(URLError, addinfourl):
+ """Raised when HTTP error occurs, but also acts like non-error return"""
+
+ def __init__(self, url, code, msg, hdrs, fp):
+ addinfourl.__init__(self, fp, hdrs, url)
+ self.code = code
+ self.msg = msg
+ self.hdrs = hdrs
+ self.fp = fp
+ # XXX
+ self.filename = url
+
+ def __str__(self):
+ return 'HTTP Error %s: %s' % (self.code, self.msg)
+
+ def __del__(self):
+ # XXX is this safe? what if user catches exception, then
+ # extracts fp and discards exception?
+ self.fp.close()
+
+class GopherError(URLError):
+ pass
+
+class Request:
+ def __init__(self, url, data=None, headers={}):
+ # unwrap('<URL:type://host/path>') --> 'type://host/path'
+ self.__original = unwrap(url)
+ self.type = None
+ # self.__r_type is what's left after doing the splittype
+ self.host = None
+ self.port = None
+ self.data = data
+ self.headers = {}
+ self.headers.update(headers)
+
+ def __getattr__(self, attr):
+ # XXX this is a fallback mechanism to guard against these
+ # methods getting called in a non-standard order. this may be
+ # too complicated and/or unnecessary.
+ # XXX should the __r_XXX attributes be public?
+ if attr[:12] == '_Request__r_':
+ name = attr[12:]
+ if hasattr(Request, 'get_' + name):
+ getattr(self, 'get_' + name)()
+ return getattr(self, attr)
+ raise AttributeError, attr
+
+ def add_data(self, data):
+ self.data = data
+
+ def has_data(self):
+ return self.data is not None
+
+ def get_data(self):
+ return self.data
+
+ def get_full_url(self):
+ return self.__original
+
+ def get_type(self):
+ if self.type is None:
+ self.type, self.__r_type = splittype(self.__original)
+ return self.type
+
+ def get_host(self):
+ if self.host is None:
+ self.host, self.__r_host = splithost(self.__r_type)
+ if self.host:
+ self.host = unquote(self.host)
+ return self.host
+
+ def get_selector(self):
+ return self.__r_host
+
+ def set_proxy(self, proxy):
+ self.__proxy = proxy
+ # XXX this code is based on urllib, but it doesn't seem
+ # correct. specifically, if the proxy has a port number then
+ # splittype will return the hostname as the type and the port
+ # will be include with everything else
+ self.type, self.__r_type = splittype(self.__proxy)
+ self.host, XXX = splithost(self.__r_type)
+ self.host = unquote(self.host)
+ self.__r_host = self.__original
+
+ def add_header(self, key, val):
+ # useful for something like authentication
+ self.headers[key] = val
+
+class OpenerDirector:
+ def __init__(self):
+ server_version = "Python-urllib/%s" % __version__
+ self.addheaders = [('User-agent', server_version)]
+ # manage the individual handlers
+ self.handlers = []
+ self.handle_open = {}
+ self.handle_error = {}
+
+ def add_handler(self, handler):
+ added = 0
+ for meth in get_methods(handler):
+ if meth[-5:] == '_open':
+ protocol = meth[:-5]
+ if self.handle_open.has_key(protocol):
+ self.handle_open[protocol].append(handler)
+ else:
+ self.handle_open[protocol] = [handler]
+ added = 1
+ continue
+ i = string.find(meth, '_')
+ j = string.find(meth[i+1:], '_') + i + 1
+ if j != -1 and meth[i+1:j] == 'error':
+ proto = meth[:i]
+ kind = meth[j+1:]
+ try:
+ kind = string.atoi(kind)
+ except ValueError:
+ pass
+ dict = self.handle_error.get(proto, {})
+ if dict.has_key(kind):
+ dict[kind].append(handler)
+ else:
+ dict[kind] = [handler]
+ self.handle_error[proto] = dict
+ added = 1
+ continue
+ if added:
+ self.handlers.append(handler)
+ handler.add_parent(self)
+
+ def __del__(self):
+ self.close()
+
+ def close(self):
+ for handler in self.handlers:
+ handler.close()
+ self.handlers = []
+
+ def _call_chain(self, chain, kind, meth_name, *args):
+ # XXX raise an exception if no one else should try to handle
+ # this url. return None if you can't but someone else could.
+ handlers = chain.get(kind, ())
+ for handler in handlers:
+ func = getattr(handler, meth_name)
+ result = apply(func, args)
+ if result is not None:
+ return result
+
+ def open(self, fullurl, data=None):
+ # accept a URL or a Request object
+ if type(fullurl) == types.StringType:
+ req = Request(fullurl, data)
+ else:
+ req = fullurl
+ if data is not None:
+ req.add_data(data)
+ assert isinstance(req, Request) # really only care about interface
+
+ result = self._call_chain(self.handle_open, 'default',
+ 'default_open', req)
+ if result:
+ return result
+
+ type_ = req.get_type()
+ result = self._call_chain(self.handle_open, type_, type_ + \
+ '_open', req)
+ if result:
+ return result
+
+ return self._call_chain(self.handle_open, 'unknown',
+ 'unknown_open', req)
+
+ def error(self, proto, *args):
+ if proto == 'http':
+ # XXX http protocol is special cased
+ dict = self.handle_error[proto]
+ proto = args[2] # YUCK!
+ meth_name = 'http_error_%d' % proto
+ http_err = 1
+ orig_args = args
+ else:
+ dict = self.handle_error
+ meth_name = proto + '_error'
+ http_err = 0
+ args = (dict, proto, meth_name) + args
+ result = apply(self._call_chain, args)
+ if result:
+ return result
+
+ if http_err:
+ args = (dict, 'default', 'http_error_default') + orig_args
+ return apply(self._call_chain, args)
+
+def is_callable(obj):
+ # not quite like builtin callable (which I didn't know existed),
+ # not entirely sure it needs to be different
+ if type(obj) in (types.BuiltinFunctionType,
+ types.BuiltinMethodType, types.LambdaType,
+ types.MethodType):
+ return 1
+ if type(obj) == types.InstanceType:
+ return hasattr(obj, '__call__')
+ return 0
+
+def get_methods(inst):
+ methods = {}
+ classes = []
+ classes.append(inst.__class__)
+ while classes:
+ klass = classes[0]
+ del classes[0]
+ classes = classes + list(klass.__bases__)
+ for name in dir(klass):
+ attr = getattr(klass, name)
+ if type(attr) == types.UnboundMethodType:
+ methods[name] = 1
+ for name in dir(inst):
+ if is_callable(getattr(inst, name)):
+ methods[name] = 1
+ return methods.keys()
+
+# XXX probably also want an abstract factory that knows things like
+ # the fact that a ProxyHandler needs to get inserted first.
+# would also know when it makes sense to skip a superclass in favor of
+ # a subclass and when it might make sense to include both
+
+def build_opener(*handlers):
+ """Create an opener object from a list of handlers.
+
+ The opener will use several default handlers, including support
+ for HTTP and FTP. If there is a ProxyHandler, it must be at the
+ front of the list of handlers. (Yuck.)
+
+ If any of the handlers passed as arguments are subclasses of the
+ default handlers, the default handlers will not be used.
+ """
+
+ opener = OpenerDirector()
+ default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
+ HTTPDefaultErrorHandler, HTTPRedirectHandler,
+ FTPHandler, FileHandler]
+ skip = []
+ for klass in default_classes:
+ for check in handlers:
+ if type(check) == types.ClassType:
+ if issubclass(check, klass):
+ skip.append(klass)
+ elif type(check) == types.InstanceType:
+ if isinstance(check, klass):
+ skip.append(klass)
+ for klass in skip:
+ default_classes.remove(klass)
+
+ for klass in default_classes:
+ opener.add_handler(klass())
+
+ for h in handlers:
+ if type(h) == types.ClassType:
+ h = h()
+ opener.add_handler(h)
+ return opener
+
+class BaseHandler:
+ def add_parent(self, parent):
+ self.parent = parent
+ def close(self):
+ self.parent = None
+
+class HTTPDefaultErrorHandler(BaseHandler):
+ def http_error_default(self, req, fp, code, msg, hdrs):
+ raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
+
+class HTTPRedirectHandler(BaseHandler):
+ # Implementation note: To avoid the server sending us into an
+ # infinite loop, the request object needs to track what URLs we
+ # have already seen. Do this by adding a handler-specific
+ # attribute to the Request object.
+ def http_error_302(self, req, fp, code, msg, headers):
+ if headers.has_key('location'):
+ newurl = headers['location']
+ elif headers.has_key('uri'):
+ newurl = headers['uri']
+ else:
+ return
+ nil = fp.read()
+ fp.close()
+
+ # XXX Probably want to forget about the state of the current
+ # request, although that might interact poorly with other
+ # handlers that also use handler-specific request attributes
+ new = Request(newurl, req.get_data())
+ new.error_302_dict = {}
+ if hasattr(req, 'error_302_dict'):
+ if req.error_302_dict.has_key(newurl):
+ raise HTTPError(req.get_full_url(), code,
+ self.inf_msg + msg, headers)
+ new.error_302_dict.update(req.error_302_dict)
+ new.error_302_dict[newurl] = newurl
+ return self.parent.open(new)
+
+ http_error_301 = http_error_302
+
+ inf_msg = "The HTTP server returned a redirect error that would" \
+ "lead to an inifinte loop.\n" \
+ "The last 302 error message was:\n"
+
+class ProxyHandler(BaseHandler):
+ def __init__(self, proxies=None):
+ if proxies is None:
+ proxies = getproxies()
+ assert hasattr(proxies, 'has_key'), "proxies must be a mapping"
+ self.proxies = proxies
+ for type, url in proxies.items():
+ setattr(self, '%s_open' % type,
+ lambda r, proxy=url, type=type, meth=self.proxy_open: \
+ meth(r, proxy, type))
+
+ def proxy_open(self, req, proxy, type):
+ orig_type = req.get_type()
+ req.set_proxy(proxy)
+ if orig_type == type:
+ # let other handlers take care of it
+ # XXX this only makes sense if the proxy is before the
+ # other handlers
+ return None
+ else:
+ # need to start over, because the other handlers don't
+ # grok the proxy's URL type
+ return self.parent.open(req)
+
+# feature suggested by Duncan Booth
+# XXX custom is not a good name
+class CustomProxy:
+ # either pass a function to the constructor or override handle
+ def __init__(self, proto, func=None, proxy_addr=None):
+ self.proto = proto
+ self.func = func
+ self.addr = proxy_addr
+
+ def handle(self, req):
+ if self.func and self.func(req):
+ return 1
+
+ def get_proxy(self):
+ return self.addr
+
+class CustomProxyHandler(BaseHandler):
+ def __init__(self, *proxies):
+ self.proxies = {}
+
+ def proxy_open(self, req):
+ proto = req.get_type()
+ try:
+ proxies = self.proxies[proto]
+ except KeyError:
+ return None
+ for p in proxies:
+ if p.handle(req):
+ req.set_proxy(p.get_proxy())
+ return self.parent.open(req)
+ return None
+
+ def do_proxy(self, p, req):
+ p
+ return self.parent.open(req)
+
+ def add_proxy(self, cpo):
+ if self.proxies.has_key(cpo.proto):
+ self.proxies[cpo.proto].append(cpo)
+ else:
+ self.proxies[cpo.proto] = [cpo]
+
+class HTTPPasswordMgr:
+ def __init__(self):
+ self.passwd = {}
+
+ def add_password(self, realm, uri, user, passwd):
+ # uri could be a single URI or a sequence
+ if type(uri) == types.StringType:
+ uri = [uri]
+ uri = tuple(map(self.reduce_uri, uri))
+ if not self.passwd.has_key(realm):
+ self.passwd[realm] = {}
+ self.passwd[realm][uri] = (user, passwd)
+
+ def find_user_password(self, realm, authuri):
+ domains = self.passwd.get(realm, {})
+ authuri = self.reduce_uri(authuri)
+ for uris, authinfo in domains.items():
+ for uri in uris:
+ if self.is_suburi(uri, authuri):
+ return authinfo
+ return None, None
+
+ def reduce_uri(self, uri):
+ """Accept netloc or URI and extract only the netloc and path"""
+ parts = urlparse.urlparse(uri)
+ if parts[1]:
+ return parts[1], parts[2] or '/'
+ else:
+ return parts[2], '/'
+
+ def is_suburi(self, base, test):
+ """Check if test is below base in a URI tree
+
+ Both args must be URIs in reduced form.
+ """
+ if base == test:
+ return 1
+ if base[0] != test[0]:
+ return 0
+ common = os.path.commonprefix((base[1], test[1]))
+ if len(common) == len(base[1]):
+ return 1
+ return 0
+
+
+class HTTPBasicAuthHandler(BaseHandler):
+ rx = re.compile('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"')
+
+ # XXX there can actually be multiple auth-schemes in a
+ # www-authenticate header. should probably be a lot more careful
+ # in parsing them to extract multiple alternatives
+
+ def __init__(self):
+ self.passwd = HTTPPasswordMgr()
+ self.add_password = self.passwd.add_password
+ self.__current_realm = None
+ # if __current_realm is not None, then the server must have
+ # refused our name/password and is asking for authorization
+ # again. must be careful to set it to None on successful
+ # return.
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ # XXX could be mult. headers
+ authreq = headers.get('www-authenticate', None)
+ if authreq:
+ mo = HTTPBasicAuthHandler.rx.match(authreq)
+ if mo:
+ scheme, realm = mo.groups()
+ if string.lower(scheme) == 'basic':
+ return self.retry_http_basic_auth(req, realm)
+
+ def retry_http_basic_auth(self, req, realm):
+ if self.__current_realm is None:
+ self.__current_realm = realm
+ else:
+ self.__current_realm = realm
+ return None
+ # XXX host isn't really the correct URI?
+ host = req.get_host()
+ user,pw = self.passwd.find_user_password(realm, host)
+ if pw:
+ raw = "%s:%s" % (user, pw)
+ auth = string.strip(base64.encodestring(raw))
+ req.add_header('Authorization', 'Basic %s' % auth)
+ resp = self.parent.open(req)
+ self.__current_realm = None
+ return resp
+ else:
+ self.__current_realm = None
+ return None
+
+class HTTPDigestAuthHandler(BaseHandler):
+ """An authentication protocol defined by RFC 2069
+
+ Digest authentication improves on basic authentication because it
+ does not transmit passwords in the clear.
+ """
+
+ def __init__(self):
+ self.passwd = HTTPPasswordMgr()
+ self.add_password = self.passwd.add_password
+ self.__current_realm = None
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ # XXX could be mult. headers
+ authreq = headers.get('www-authenticate', None)
+ if authreq:
+ kind = string.split(authreq)[0]
+ if kind == 'Digest':
+ return self.retry_http_digest_auth(req, authreq)
+
+ def retry_http_digest_auth(self, req, auth):
+ token, challenge = string.split(auth, ' ', 1)
+ chal = parse_keqv_list(parse_http_list(challenge))
+ auth = self.get_authorization(req, chal)
+ if auth:
+ req.add_header('Authorization', 'Digest %s' % auth)
+ resp = self.parent.open(req)
+ self.__current_realm = None
+ return resp
+
+ def get_authorization(self, req, chal):
+ try:
+ realm = chal['realm']
+ nonce = chal['nonce']
+ algorithm = chal.get('algorithm', 'MD5')
+ # mod_digest doesn't send an opaque, even though it isn't
+ # supposed to be optional
+ opaque = chal.get('opaque', None)
+ except KeyError:
+ return None
+
+ if self.__current_realm is None:
+ self.__current_realm = realm
+ else:
+ self.__current_realm = realm
+ return None
+
+ H, KD = self.get_algorithm_impls(algorithm)
+ if H is None:
+ return None
+
+ user, pw = self.passwd.find_user_password(realm,
+ req.get_full_url())
+ if user is None:
+ return None
+
+ # XXX not implemented yet
+ if req.has_data():
+ entdig = self.get_entity_digest(req.get_data(), chal)
+ else:
+ entdig = None
+
+ A1 = "%s:%s:%s" % (user, realm, pw)
+ A2 = "%s:%s" % (req.has_data() and 'POST' or 'GET',
+ # XXX selector: what about proxies and full urls
+ req.get_selector())
+ respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
+ # XXX should the partial digests be encoded too?
+
+ base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
+ 'response="%s"' % (user, realm, nonce, req.get_selector(),
+ respdig)
+ if opaque:
+ base = base + ', opaque="%s"' % opaque
+ if entdig:
+ base = base + ', digest="%s"' % entdig
+ if algorithm != 'MD5':
+ base = base + ', algorithm="%s"' % algorithm
+ return base
+
+ def get_algorithm_impls(self, algorithm):
+ # lambdas assume digest modules are imported at the top level
+ if algorithm == 'MD5':
+ H = lambda x, e=encode_digest:e(md5.new(x).digest())
+ elif algorithm == 'SHA':
+ H = lambda x, e=encode_digest:e(sha.new(x).digest())
+ # XXX MD5-sess
+ KD = lambda s, d, H=H: H("%s:%s" % (s, d))
+ return H, KD
+
+ def get_entity_digest(self, data, chal):
+ # XXX not implemented yet
+ return None
+
+def encode_digest(digest):
+ hexrep = []
+ for c in digest:
+ n = (ord(c) >> 4) & 0xf
+ hexrep.append(hex(n)[-1])
+ n = ord(c) & 0xf
+ hexrep.append(hex(n)[-1])
+ return string.join(hexrep, '')
+
+
+class HTTPHandler(BaseHandler):
+ def http_open(self, req):
+ # XXX devise a new mechanism to specify user/password
+ host = req.get_host()
+ if not host:
+ raise URLError('no host given')
+
+ h = httplib.HTTP(host) # will parse host:port
+## h.set_debuglevel(1)
+ if req.has_data():
+ data = req.get_data()
+ h.putrequest('POST', req.get_selector())
+ h.putheader('Content-type', 'application/x-www-form-urlencoded')
+ h.putheader('Content-length', '%d' % len(data))
+ else:
+ h.putrequest('GET', req.get_selector())
+ # XXX proxies would have different host here
+ h.putheader('Host', host)
+ for args in self.parent.addheaders:
+ apply(h.putheader, args)
+ for k, v in req.headers.items():
+ h.putheader(k, v)
+ h.endheaders()
+ if req.has_data():
+ h.send(data + '\r\n')
+
+ code, msg, hdrs = h.getreply()
+ fp = h.getfile()
+ if code == 200:
+ return addinfourl(fp, hdrs, req.get_full_url())
+ else:
+ # want to make sure the socket is closed, even if error
+ # handling doesn't return immediately. the socket won't
+ # actually be closed until fp is also closed.
+ if h.sock:
+ h.sock.close()
+ h.sock = None
+ return self.parent.error('http', req, fp, code, msg, hdrs)
+
+class UnknownHandler(BaseHandler):
+ def unknown_open(self, req):
+ type = req.get_type()
+ raise URLError('unknown url type: %s' % type)
+
+def parse_keqv_list(l):
+ """Parse list of key=value strings where keys are not duplicated."""
+ parsed = {}
+ for elt in l:
+ k, v = string.split(elt, '=', 1)
+ if v[0] == '"' and v[-1] == '"':
+ v = v[1:-1]
+ parsed[k] = v
+ return parsed
+
+def parse_http_list(s):
+ """Parse lists as described by RFC 2068 Section 2.
+
+ In particular, parse comman-separated lists where the elements of
+ the list may include quoted-strings. A quoted-string could
+ contain a comma.
+ """
+ # XXX this function could probably use more testing
+
+ list = []
+ end = len(s)
+ i = 0
+ inquote = 0
+ start = 0
+ while i < end:
+ cur = s[i:]
+ c = string.find(cur, ',')
+ q = string.find(cur, '"')
+ if c == -1:
+ list.append(s[start:])
+ break
+ if q == -1:
+ if inquote:
+ raise ValueError, "unbalanced quotes"
+ else:
+ list.append(s[start:i+c])
+ i = i + c + 1
+ continue
+ if inquote:
+ if q < c:
+ list.append(s[start:i+c])
+ i = i + c + 1
+ start = i
+ inquote = 0
+ else:
+ i = i + q
+ else:
+ if c < q:
+ list.append(s[start:i+c])
+ i = i + c + 1
+ start = i
+ else:
+ inquote = 1
+ i = i + q + 1
+ return map(string.strip, list)
+
+class FileHandler(BaseHandler):
+ # Use local file or FTP depending on form of URL
+ def file_open(self, req):
+ url = req.get_selector()
+ if url[:2] == '//' and url[2:3] != '/':
+ req.type = 'ftp'
+ return self.parent.open(req)
+ else:
+ return self.open_local_file(req)
+
+ # names for the localhost
+ names = None
+ def get_names(self):
+ if FileHandler.names is None:
+ FileHandler.names = (socket.gethostbyname('localhost'),
+ socket.gethostbyname(socket.gethostname()))
+ return FileHandler.names
+
+ # not entirely sure what the rules are here
+ def open_local_file(self, req):
+ mtype = mimetypes.guess_type(req.get_selector())[0]
+ headers = mimetools.Message(StringIO('Content-Type: %s\n' \
+ % (mtype or 'text/plain')))
+ host = req.get_host()
+ file = req.get_selector()
+ if host:
+ host, port = splitport(host)
+ if not host or \
+ (not port and socket.gethostbyname(host) in self.get_names()):
+ return addinfourl(open(url2pathname(file), 'rb'),
+ headers, 'file:'+file)
+ raise URLError('file not on local host')
+
+class FTPHandler(BaseHandler):
+ def ftp_open(self, req):
+ host = req.get_host()
+ if not host:
+ raise IOError, ('ftp error', 'no host given')
+ # XXX handle custom username & password
+ host = socket.gethostbyname(host)
+ host, port = splitport(host)
+ if port is None:
+ port = ftplib.FTP_PORT
+ path, attrs = splitattr(req.get_selector())
+ path = unquote(path)
+ dirs = string.splitfields(path, '/')
+ dirs, file = dirs[:-1], dirs[-1]
+ if dirs and not dirs[0]:
+ dirs = dirs[1:]
+ user = passwd = '' # XXX
+ try:
+ fw = self.connect_ftp(user, passwd, host, port, dirs)
+ type = file and 'I' or 'D'
+ for attr in attrs:
+ attr, value = splitattr(attr)
+ if string.lower(attr) == 'type' and \
+ value in ('a', 'A', 'i', 'I', 'd', 'D'):
+ type = string.upper(value)
+ fp, retrlen = fw.retrfile(file, type)
+ if retrlen is not None and retrlen >= 0:
+ sf = StringIO('Content-Length: %d\n' % retrlen)
+ headers = mimetools.Message(sf)
+ else:
+ headers = noheaders()
+ return addinfourl(fp, headers, req.get_full_url())
+ except ftplib.all_errors, msg:
+ raise IOError, ('ftp error', msg), sys.exc_info()[2]
+
+ def connect_ftp(self, user, passwd, host, port, dirs):
+ fw = ftpwrapper(user, passwd, host, port, dirs)
+## fw.ftp.set_debuglevel(1)
+ return fw
+
+class CacheFTPHandler(FTPHandler):
+ # XXX would be nice to have pluggable cache strategies
+ # XXX this stuff is definitely not thread safe
+ def __init__(self):
+ self.cache = {}
+ self.timeout = {}
+ self.soonest = 0
+ self.delay = 60
+ self.max_conns = 16
+
+ def setTimeout(self, t):
+ self.delay = t
+
+ def setMaxConns(self, m):
+ self.max_conns = m
+
+ def connect_ftp(self, user, passwd, host, port, dirs):
+ key = user, passwd, host, port
+ if self.cache.has_key(key):
+ self.timeout[key] = time.time() + self.delay
+ else:
+ self.cache[key] = ftpwrapper(user, passwd, host, port, dirs)
+ self.timeout[key] = time.time() + self.delay
+ self.check_cache()
+ return self.cache[key]
+
+ def check_cache(self):
+ # first check for old ones
+ t = time.time()
+ if self.soonest <= t:
+ for k, v in self.timeout.items():
+ if v < t:
+ self.cache[k].close()
+ del self.cache[k]
+ del self.timeout[k]
+ self.soonest = min(self.timeout.values())
+
+ # then check the size
+ if len(self.cache) == self.max_conns:
+ for k, v in self.timeout.items():
+ if v == self.soonest:
+ del self.cache[k]
+ del self.timeout[k]
+ break
+ self.soonest = min(self.timeout.values())
+
+class GopherHandler(BaseHandler):
+ def gopher_open(self, req):
+ host = req.get_host()
+ if not host:
+ raise GopherError('no host given')
+ host = unquote(host)
+ selector = req.get_selector()
+ type, selector = splitgophertype(selector)
+ selector, query = splitquery(selector)
+ selector = unquote(selector)
+ if query:
+ query = unquote(query)
+ fp = gopherlib.send_query(selector, query, host)
+ else:
+ fp = gopherlib.send_selector(selector, host)
+ return addinfourl(fp, noheaders(), req.get_full_url())
+
+#bleck! don't use this yet
+class OpenerFactory:
+
+ default_handlers = [UnknownHandler, HTTPHandler,
+ HTTPDefaultErrorHandler, HTTPRedirectHandler,
+ FTPHandler, FileHandler]
+ proxy_handlers = [ProxyHandler]
+ handlers = []
+ replacement_handlers = []
+
+ def add_proxy_handler(self, ph):
+ self.proxy_handlers = self.proxy_handlers + [ph]
+
+ def add_handler(self, h):
+ self.handlers = self.handlers + [h]
+
+ def replace_handler(self, h):
+ pass
+
+ def build_opener(self):
+ opener = OpenerDirectory()
+ for ph in self.proxy_handlers:
+ if type(ph) == types.ClassType:
+ ph = ph()
+ opener.add_handler(ph)
+
+if __name__ == "__main__":
+ # XXX some of the test code depends on machine configurations that
+ # are internal to CNRI. Need to set up a public server with the
+ # right authentication configuration for test purposes.
+ if socket.gethostname() == 'bitdiddle':
+ localhost = 'bitdiddle.cnri.reston.va.us'
+ elif socket.gethostname() == 'walden':
+ localhost = 'localhost'
+ else:
+ localhost = None
+ urls = [
+ # Thanks to Fred for finding these!
+ 'gopher://gopher.lib.ncsu.edu/11/library/stacks/Alex',
+ 'gopher://gopher.vt.edu:10010/10/33',
+
+ 'file:/etc/passwd',
+ 'file://nonsensename/etc/passwd',
+ 'ftp://www.python.org/pub/tmp/httplib.py',
+ 'ftp://www.python.org/pub/tmp/imageop.c',
+ 'ftp://www.python.org/pub/tmp/blat',
+ 'http://www.espn.com/', # redirect
+ 'http://www.python.org/Spanish/Inquistion/',
+ ('http://grail.cnri.reston.va.us/cgi-bin/faqw.py',
+ 'query=pythonistas&querytype=simple&casefold=yes&req=search'),
+ 'http://www.python.org/',
+ 'ftp://prep.ai.mit.edu/welcome.msg',
+ 'ftp://www.python.org/pub/tmp/figure.prn',
+ 'ftp://www.python.org/pub/tmp/interp.pl',
+ 'http://checkproxy.cnri.reston.va.us/test/test.html',
+ ]
+
+ if localhost is not None:
+ urls = urls + [
+ 'file://%s/etc/passwd' % localhost,
+ 'http://%s/simple/' % localhost,
+ 'http://%s/digest/' % localhost,
+ 'http://%s/not/found.h' % localhost,
+ ]
+
+ bauth = HTTPBasicAuthHandler()
+ bauth.add_password('basic_test_realm', localhost, 'jhylton',
+ 'password')
+ dauth = HTTPDigestAuthHandler()
+ dauth.add_password('digest_test_realm', localhost, 'jhylton',
+ 'password')
+
+
+ cfh = CacheFTPHandler()
+ cfh.setTimeout(1)
+
+ # XXX try out some custom proxy objects too!
+ def at_cnri(req):
+ host = req.get_host()
+ print host
+ if host[-18:] == '.cnri.reston.va.us':
+ return 1
+ p = CustomProxy('http', at_cnri, 'proxy.cnri.reston.va.us')
+ ph = CustomProxyHandler(p)
+
+ install_opener(build_opener(dauth, bauth, cfh, GopherHandler, ph))
+
+ for url in urls:
+ if type(url) == types.TupleType:
+ url, req = url
+ else:
+ req = None
+ print url
+ try:
+ f = urlopen(url, req)
+ except IOError, err:
+ print "IOError:", err
+ except socket.error, err:
+ print "socket.error:", err
+ else:
+ buf = f.read()
+ f.close()
+ print "read %d bytes" % len(buf)
+ print
+ time.sleep(0.1)