* Lib/urllib.py: implemented new quoting rules; added splituser,
splitpasswd, splitattr, splitvalue; new ftp syntax (user:passwd,
cwd to each subdir, type={a,i,d})
diff --git a/Lib/urllib.py b/Lib/urllib.py
index beabfc7..14fcefa 100644
--- a/Lib/urllib.py
+++ b/Lib/urllib.py
@@ -14,6 +14,7 @@
# used to query various info about the object, if available.
# (rfc822.Message objects are queried with the getheader() method.)
+import string
import socket
import regex
@@ -147,8 +148,12 @@
if not host: raise IOError, ('gopher error', 'no host given')
type, selector = splitgophertype(selector)
selector, query = splitquery(selector)
- if query: fp = gopherlib.send_query(selector, query, host)
- else: fp = gopherlib.send_selector(selector, host)
+ selector = unquote(selector)
+ if query:
+ query = unquote(query)
+ fp = gopherlib.send_query(selector, query, host)
+ else:
+ fp = gopherlib.send_selector(selector, host)
return addinfo(fp, noheaders())
# Use local file or FTP depending on form of URL
@@ -165,23 +170,41 @@
host, port = splitport(host)
if not port and socket.gethostbyname(host) in (
localhost(), thishost()):
+ file = unquote(file)
return addinfo(open(file, 'r'), noheaders())
raise IOError, ('local file error', 'not on local host')
# Use FTP protocol
def open_ftp(self, url):
- host, file = splithost(url)
+ host, path = splithost(url)
if not host: raise IOError, ('ftp error', 'no host given')
host, port = splitport(host)
+ user, host = splituser(host)
+ if user: user, passwd = splitpasswd(user)
+ else: passwd = None
host = socket.gethostbyname(host)
if not port:
import ftplib
port = ftplib.FTP_PORT
- key = (host, port)
+ path, attrs = splitattr(path)
+ dirs = string.splitfields(path, '/')
+ dirs, file = dirs[:-1], dirs[-1]
+ if dirs and not dirs[0]: dirs = dirs[1:]
+ key = (user, host, port, string.joinfields(dirs, '/'))
+ print 'key =', key
try:
if not self.ftpcache.has_key(key):
- self.ftpcache[key] = ftpwrapper(host, port)
- return addinfo(self.ftpcache[key].retrfile(file),
+ self.ftpcache[key] = \
+ ftpwrapper(user, passwd,
+ host, port, dirs)
+ if not file: type = 'D'
+ else: type = 'I'
+ for attr in attrs:
+ attr, value = splitvalue(attr)
+ if string.lower(attr) == 'type' and \
+ value in ('a', 'A', 'i', 'I', 'd', 'D'):
+ type = string.upper(value)
+ return addinfo(self.ftpcache[key].retrfile(file, type),
noheaders())
except ftperrors(), msg:
raise IOError, ('ftp error', msg)
@@ -232,24 +255,33 @@
# Class used by open_ftp() for cache of open FTP connections
class ftpwrapper:
- def __init__(self, host, port):
+ def __init__(self, user, passwd, host, port, dirs):
+ self.user = unquote(user or '')
+ self.passwd = unquote(passwd or '')
self.host = host
self.port = port
+ self.dirs = []
+ for dir in dirs:
+ self.dirs.append(unquote(dir))
self.init()
def init(self):
import ftplib
self.ftp = ftplib.FTP()
self.ftp.connect(self.host, self.port)
- self.ftp.login()
- def retrfile(self, file):
+ self.ftp.login(self.user, self.passwd)
+ for dir in self.dirs:
+ self.ftp.cwd(dir)
+ def retrfile(self, file, type):
import ftplib
+ if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
+ else: cmd = 'TYPE ' + type; isdir = 0
try:
- self.ftp.voidcmd('TYPE I')
+ self.ftp.voidcmd(cmd)
except ftplib.all_errors:
self.init()
- self.ftp.voidcmd('TYPE I')
+ self.ftp.voidcmd(cmd)
conn = None
- if file:
+ if file and not isdir:
try:
cmd = 'RETR ' + file
conn = self.ftp.transfercmd(cmd)
@@ -317,7 +349,6 @@
basepath, basequery = splitquery(basepath)
type = basetype or 'file'
if path[:1] != '/':
- import string
i = string.rfind(basepath, '/')
if i < 0: basepath = '/'
else: basepath = basepath[:i+1]
@@ -327,19 +358,23 @@
else: return type + ':' + path
-# Utilities to parse URLs:
+# Utilities to parse URLs (most of these return None for missing parts):
# unwrap('<URL:type//host/path>') --> 'type//host/path'
# splittype('type:opaquestring') --> 'type', 'opaquestring'
# splithost('//host[:port]/path') --> 'host[:port]', '/path'
+# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'
+# splitpasswd('user:passwd') -> 'user', 'passwd'
# splitport('host:port') --> 'host', 'port'
# splitquery('/path?query') --> '/path', 'query'
# splittag('/path#tag') --> '/path', 'tag'
+# splitattr('/path;attr1=value1;attr2=value2;...') ->
+# '/path', ['attr1=value1', 'attr2=value2', ...]
+# splitvalue('attr=value') --> 'attr', 'value'
# splitgophertype('/Xselector') --> 'X', 'selector'
# unquote('abc%20def') -> 'abc def'
# quote('abc def') -> 'abc%20def')
def unwrap(url):
- import string
url = string.strip(url)
if url[:1] == '<' and url[-1:] == '>':
url = string.strip(url[1:-1])
@@ -356,6 +391,16 @@
if _hostprog.match(url) >= 0: return _hostprog.group(1, 2)
return None, url
+_userprog = regex.compile('^\([^@]*\)@\(.*\)$')
+def splituser(host):
+ if _userprog.match(host) >= 0: return _userprog.group(1, 2)
+ return None, host
+
+_passwdprog = regex.compile('^\([^:]*\):\(.*\)$')
+def splitpasswd(user):
+ if _passwdprog.match(user) >= 0: return _passwdprog.group(1, 2)
+ return user, None
+
_portprog = regex.compile('^\(.*\):\([0-9]+\)$')
def splitport(host):
if _portprog.match(host) >= 0: return _portprog.group(1, 2)
@@ -371,6 +416,15 @@
if _tagprog.match(url) >= 0: return _tagprog.group(1, 2)
return url, None
+def splitattr(url):
+ words = string.splitfields(url, ';')
+ return words[0], words[1:]
+
+_valueprog = regex.compile('^\([^=]*\)=\(.*\)$')
+def splitvalue(attr):
+ if _valueprog.match(attr) >= 0: return _valueprog.group(1, 2)
+ return attr, None
+
def splitgophertype(selector):
if selector[:1] == '/' and selector[1:2]:
return selector[1], selector[2:]
@@ -378,7 +432,6 @@
_quoteprog = regex.compile('%[0-9a-fA-F][0-9a-fA-F]')
def unquote(s):
- import string
i = 0
n = len(s)
res = ''
@@ -391,13 +444,15 @@
i = j+3
return res
-_acceptable = \
- 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._@'
-def quote(s):
+always_safe = string.letters + string.digits + '_,.+-'
+def quote(s, safe = '/'):
+ safe = always_safe + safe
res = ''
for c in s:
- if c in _acceptable: res = res + c
- else: res = res + '%%%02x' % ord(c)
+ if c in safe:
+ res = res + c
+ else:
+ res = res + '%%%02x' % ord(c)
return res
# Test and time quote() and unquote()
@@ -452,5 +507,5 @@
# Run test program when run as a script
if __name__ == '__main__':
- test1()
+## test1()
test()