Checking in patch 404826 -- urllib2 enhancements and documentations.
(please not that the library reference does *not* include the
urllib2 documnetation -- that will wiat for Fred)
diff --git a/Lib/urllib2.py b/Lib/urllib2.py
index a3ff482..a1e157b 100644
--- a/Lib/urllib2.py
+++ b/Lib/urllib2.py
@@ -57,8 +57,10 @@
authinfo = urllib2.HTTPBasicAuthHandler()
authinfo.add_password('realm', 'host', 'username', 'password')
+proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"})
+
# build a new opener that adds authentication and caching FTP handlers
-opener = urllib2.build_opener(authinfo, urllib2.CacheFTPHandler)
+opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler)
# install it
urllib2.install_opener(opener)
@@ -92,7 +94,6 @@
import base64
import types
import urlparse
-import os
import md5
import mimetypes
import mimetools
@@ -100,6 +101,7 @@
import sys
import time
import gopherlib
+import posixpath
try:
from cStringIO import StringIO
@@ -121,10 +123,7 @@
from urllib import getproxies
# support for FileHandler
-from urllib import localhost, thishost, url2pathname, pathname2url
-
-# support for GopherHandler
-from urllib import splitgophertype, splitquery
+from urllib import localhost, url2pathname
__version__ = "2.0a1"
@@ -177,7 +176,9 @@
class GopherError(URLError):
pass
+
class Request:
+
def __init__(self, url, data=None, headers={}):
# unwrap('<URL:type://host/path>') --> 'type://host/path'
self.__original = unwrap(url)
@@ -229,15 +230,8 @@
def get_selector(self):
return self.__r_host
- def set_proxy(self, proxy):
- self.__proxy = proxy
- # XXX this code is based on urllib, but it doesn't seem
- # correct. specifically, if the proxy has a port number then
- # splittype will return the hostname as the type and the port
- # will be include with everything else
- self.type, self.__r_type = splittype(self.__proxy)
- self.host, XXX = splithost(self.__r_type)
- self.host = unquote(self.host)
+ def set_proxy(self, host, type):
+ self.host, self.type = host, type
self.__r_host = self.__original
def add_header(self, key, val):
@@ -329,9 +323,9 @@
'unknown_open', req)
def error(self, proto, *args):
- if proto == 'http':
- # XXX http protocol is special cased
- dict = self.handle_error[proto]
+ if proto in ['http', 'https']:
+ # XXX http[s] protocols are special cased
+ dict = self.handle_error['http'] # https is not different then http
proto = args[2] # YUCK!
meth_name = 'http_error_%d' % proto
http_err = 1
@@ -397,6 +391,8 @@
default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
HTTPDefaultErrorHandler, HTTPRedirectHandler,
FTPHandler, FileHandler]
+ if hasattr(httplib, 'HTTPS'):
+ default_classes.append(HTTPSHandler)
skip = []
for klass in default_classes:
for check in handlers:
@@ -451,7 +447,7 @@
new = Request(newurl, req.get_data())
new.error_302_dict = {}
if hasattr(req, 'error_302_dict'):
- if req.error_302_dict.has_key(newurl):
+ if len(error_302_dict)>10 or req.error_302_dict.has_key(newurl):
raise HTTPError(req.get_full_url(), code,
self.inf_msg + msg, headers)
new.error_302_dict.update(req.error_302_dict)
@@ -477,7 +473,14 @@
def proxy_open(self, req, proxy, type):
orig_type = req.get_type()
- req.set_proxy(proxy)
+ type, r_type = splittype(proxy)
+ host, XXX = splithost(r_type)
+ if '@' in host:
+ user_pass, host = host.split('@', 1)
+ user_pass = base64.encode_string(unquote(user_passw)).strip()
+ req.addheader('Proxy-Authorization', user_pass)
+ host = unquote(host)
+ req.set_proxy(host, type)
if orig_type == type:
# let other handlers take care of it
# XXX this only makes sense if the proxy is before the
@@ -569,21 +572,33 @@
return 1
if base[0] != test[0]:
return 0
- common = os.path.commonprefix((base[1], test[1]))
+ common = posixpath.commonprefix((base[1], test[1]))
if len(common) == len(base[1]):
return 1
return 0
-class HTTPBasicAuthHandler(BaseHandler):
+class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
+
+ def find_user_password(self, realm, authuri):
+ user, password = HTTPPasswordMgr.find_user_password(self,realm,authuri)
+ if user is not None:
+ return user, password
+ return HTTPPasswordMgr.find_user_password(self, None, authuri)
+
+
+class AbstractBasicAuthHandler:
+
rx = re.compile('[ \t]*([^ \t]+)[ \t]+realm="([^"]*)"')
# XXX there can actually be multiple auth-schemes in a
# www-authenticate header. should probably be a lot more careful
# in parsing them to extract multiple alternatives
- def __init__(self):
- self.passwd = HTTPPasswordMgr()
+ def __init__(self, password_mgr=None):
+ if password_mgr is None:
+ password_mgr = HTTPPasswordMgr()
+ self.passwd = password_mgr
self.add_password = self.passwd.add_password
self.__current_realm = None
# if __current_realm is not None, then the server must have
@@ -591,29 +606,27 @@
# again. must be careful to set it to None on successful
# return.
- def http_error_401(self, req, fp, code, msg, headers):
- # XXX could be mult. headers
- authreq = headers.get('www-authenticate', None)
+ def http_error_auth_reqed(self, authreq, host, req, headers):
+ # XXX could be multiple headers
+ authreq = headers.get(authreq, None)
if authreq:
- mo = HTTPBasicAuthHandler.rx.match(authreq)
+ mo = AbstractBasicAuthHandler.rx.match(authreq)
if mo:
scheme, realm = mo.groups()
if scheme.lower() == 'basic':
- return self.retry_http_basic_auth(req, realm)
+ return self.retry_http_basic_auth(host, req, realm)
- def retry_http_basic_auth(self, req, realm):
+ def retry_http_basic_auth(self, host, req, realm):
if self.__current_realm is None:
self.__current_realm = realm
else:
self.__current_realm = realm
return None
- # XXX host isn't really the correct URI?
- host = req.get_host()
user,pw = self.passwd.find_user_password(realm, host)
if pw:
raw = "%s:%s" % (user, pw)
auth = base64.encodestring(raw).strip()
- req.add_header('Authorization', 'Basic %s' % auth)
+ req.add_header(self.header, 'Basic %s' % auth)
resp = self.parent.open(req)
self.__current_realm = None
return resp
@@ -621,21 +634,37 @@
self.__current_realm = None
return None
-class HTTPDigestAuthHandler(BaseHandler):
- """An authentication protocol defined by RFC 2069
+class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
- Digest authentication improves on basic authentication because it
- does not transmit passwords in the clear.
- """
+ header = 'Authorization'
- def __init__(self):
- self.passwd = HTTPPasswordMgr()
+ def http_error_401(self, req, fp, code, msg, headers):
+ host = urlparse.urlparse(req.get_full_url())[1]
+ return self.http_error_auth_reqed('www-authenticate',
+ host, req, headers)
+
+
+class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
+
+ header = 'Proxy-Authorization'
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ host = req.get_host()
+ return self.http_error_auth_reqed('proxy-authenticate',
+ host, req, headers)
+
+
+class AbstractDigestAuthHandler:
+
+ def __init__(self, passwd=None):
+ if passwd is None:
+ passwd = HTTPPassowrdMgr()
+ self.passwd = passwd
self.add_password = self.passwd.add_password
self.__current_realm = None
- def http_error_401(self, req, fp, code, msg, headers):
- # XXX could be mult. headers
- authreq = headers.get('www-authenticate', None)
+ def http_error_auth_reqed(self, authreq, host, req, headers):
+ authreq = headers.get(self.header, None)
if authreq:
kind = authreq.split()[0]
if kind == 'Digest':
@@ -646,7 +675,7 @@
chal = parse_keqv_list(parse_http_list(challenge))
auth = self.get_authorization(req, chal)
if auth:
- req.add_header('Authorization', 'Digest %s' % auth)
+ req.add_header(self.header, 'Digest %s' % auth)
resp = self.parent.open(req)
self.__current_realm = None
return resp
@@ -715,6 +744,30 @@
# XXX not implemented yet
return None
+
+class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+ """An authentication protocol defined by RFC 2069
+
+ Digest authentication improves on basic authentication because it
+ does not transmit passwords in the clear.
+ """
+
+ header = 'Authorization'
+
+ def http_error_401(self, req, fp, code, msg, headers):
+ host = urlparse.urlparse(req.get_full_url())[1]
+ self.http_error_auth_reqed('www-authenticate', host, req, headers)
+
+
+class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
+
+ header = 'Proxy-Authorization'
+
+ def http_error_407(self, req, fp, code, msg, headers):
+ host = req.get_host()
+ self.http_error_auth_reqed('proxy-authenticate', host, req, headers)
+
+
def encode_digest(digest):
hexrep = []
for c in digest:
@@ -725,15 +778,15 @@
return ''.join(hexrep)
-class HTTPHandler(BaseHandler):
- def http_open(self, req):
- # XXX devise a new mechanism to specify user/password
+class AbstractHTTPHandler(BaseHandler):
+
+ def do_open(self, http_class, req):
host = req.get_host()
if not host:
raise URLError('no host given')
try:
- h = httplib.HTTP(host) # will parse host:port
+ h = http_class(host) # will parse host:port
if req.has_data():
data = req.get_data()
h.putrequest('POST', req.get_selector())
@@ -762,6 +815,20 @@
else:
return self.parent.error('http', req, fp, code, msg, hdrs)
+
+class HTTPHandler(AbstractHTTPHandler):
+
+ def http_open(self, req):
+ return self.do_open(httplib.HTTP, req)
+
+
+if hasattr(httplib, 'HTTPS'):
+ class HTTPSHandler(AbstractHTTPHandler):
+
+ def https_open(self, req):
+ return self.do_open(httplib.HTTPS, req)
+
+
class UnknownHandler(BaseHandler):
def unknown_open(self, req):
type = req.get_type()