Make a new urllib package .
It consists of code from urllib, urllib2, urlparse, and robotparser.
The old modules have all been removed. The new package has five
submodules: urllib.parse, urllib.request, urllib.response,
urllib.error, and urllib.robotparser. The urllib.request.urlopen()
function uses the url opener from urllib2.
Note that the unittests have not been renamed for the
beta, but they will be renamed in the future.
Joint work with Senthil Kumaran.
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index 2b41cad..f5a9d5d 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -1,6 +1,7 @@
"""Regresssion tests for urllib"""
-import urllib
+import urllib.parse
+import urllib.request
import http.client
import email.message
import io
@@ -16,6 +17,23 @@
hex_repr = "0%s" % hex_repr
return "%" + hex_repr
+# Shortcut for testing FancyURLopener
+_urlopener = None
+def urlopen(url, data=None, proxies=None):
+ """urlopen(url [, data]) -> open file-like object"""
+ global _urlopener
+ if proxies is not None:
+ opener = urllib.request.FancyURLopener(proxies=proxies)
+ elif not _urlopener:
+ opener = urllib.request.FancyURLopener()
+ _urlopener = opener
+ else:
+ opener = _urlopener
+ if data is None:
+ return opener.open(url)
+ else:
+ return opener.open(url, data)
+
class urlopen_FileTests(unittest.TestCase):
"""Test urlopen() opening a temporary file.
@@ -25,15 +43,16 @@
"""
def setUp(self):
- """Setup of a temp file to use for testing"""
- self.text = bytes("test_urllib: %s\n" % self.__class__.__name__, "ascii")
- FILE = open(support.TESTFN, 'wb')
+ # Create a temp file to use for testing
+ self.text = bytes("test_urllib: %s\n" % self.__class__.__name__,
+ "ascii")
+ f = open(support.TESTFN, 'wb')
try:
- FILE.write(self.text)
+ f.write(self.text)
finally:
- FILE.close()
+ f.close()
self.pathname = support.TESTFN
- self.returned_obj = urllib.urlopen("file:%s" % self.pathname)
+ self.returned_obj = urlopen("file:%s" % self.pathname)
def tearDown(self):
"""Shut down the open object"""
@@ -119,7 +138,7 @@
def test_read(self):
self.fakehttp(b"Hello!")
try:
- fp = urllib.urlopen("http://python.org/")
+ fp = urlopen("http://python.org/")
self.assertEqual(fp.readline(), b"Hello!")
self.assertEqual(fp.readline(), b"")
self.assertEqual(fp.geturl(), 'http://python.org/')
@@ -136,7 +155,7 @@
Content-Type: text/html; charset=iso-8859-1
''')
try:
- self.assertRaises(IOError, urllib.urlopen, "http://python.org/")
+ self.assertRaises(IOError, urlopen, "http://python.org/")
finally:
self.unfakehttp()
@@ -145,7 +164,7 @@
# data. (#1680230)
self.fakehttp(b'')
try:
- self.assertRaises(IOError, urllib.urlopen, "http://something")
+ self.assertRaises(IOError, urlopen, "http://something")
finally:
self.unfakehttp()
@@ -180,7 +199,8 @@
except: pass
def constructLocalFileUrl(self, filePath):
- return "file://%s" % urllib.pathname2url(os.path.abspath(filePath))
+ return "file://%s" % urllib.request.pathname2url(
+ os.path.abspath(filePath))
def createNewTempFile(self, data=b""):
"""Creates a new temporary file containing the specified data,
@@ -204,7 +224,7 @@
def test_basic(self):
# Make sure that a local file just gets its own location returned and
# a headers value is returned.
- result = urllib.urlretrieve("file:%s" % support.TESTFN)
+ result = urllib.request.urlretrieve("file:%s" % support.TESTFN)
self.assertEqual(result[0], support.TESTFN)
self.assert_(isinstance(result[1], email.message.Message),
"did not get a email.message.Message instance as second "
@@ -214,7 +234,7 @@
# Test that setting the filename argument works.
second_temp = "%s.2" % support.TESTFN
self.registerFileForCleanUp(second_temp)
- result = urllib.urlretrieve(self.constructLocalFileUrl(
+ result = urllib.request.urlretrieve(self.constructLocalFileUrl(
support.TESTFN), second_temp)
self.assertEqual(second_temp, result[0])
self.assert_(os.path.exists(second_temp), "copy of the file was not "
@@ -238,7 +258,8 @@
count_holder[0] = count_holder[0] + 1
second_temp = "%s.2" % support.TESTFN
self.registerFileForCleanUp(second_temp)
- urllib.urlretrieve(self.constructLocalFileUrl(support.TESTFN),
+ urllib.request.urlretrieve(
+ self.constructLocalFileUrl(support.TESTFN),
second_temp, hooktester)
def test_reporthook_0_bytes(self):
@@ -247,7 +268,7 @@
def hooktester(count, block_size, total_size, _report=report):
_report.append((count, block_size, total_size))
srcFileName = self.createNewTempFile()
- urllib.urlretrieve(self.constructLocalFileUrl(srcFileName),
+ urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName),
support.TESTFN, hooktester)
self.assertEqual(len(report), 1)
self.assertEqual(report[0][2], 0)
@@ -261,7 +282,7 @@
def hooktester(count, block_size, total_size, _report=report):
_report.append((count, block_size, total_size))
srcFileName = self.createNewTempFile(b"x" * 5)
- urllib.urlretrieve(self.constructLocalFileUrl(srcFileName),
+ urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName),
support.TESTFN, hooktester)
self.assertEqual(len(report), 2)
self.assertEqual(report[0][1], 8192)
@@ -275,7 +296,7 @@
def hooktester(count, block_size, total_size, _report=report):
_report.append((count, block_size, total_size))
srcFileName = self.createNewTempFile(b"x" * 8193)
- urllib.urlretrieve(self.constructLocalFileUrl(srcFileName),
+ urllib.request.urlretrieve(self.constructLocalFileUrl(srcFileName),
support.TESTFN, hooktester)
self.assertEqual(len(report), 3)
self.assertEqual(report[0][1], 8192)
@@ -284,10 +305,10 @@
class QuotingTests(unittest.TestCase):
"""Tests for urllib.quote() and urllib.quote_plus()
- According to RFC 2396 ("Uniform Resource Identifiers), to escape a
- character you write it as '%' + <2 character US-ASCII hex value>. The Python
- code of ``'%' + hex(ord(<character>))[2:]`` escapes a character properly.
- Case does not matter on the hex letters.
+ According to RFC 2396 (Uniform Resource Identifiers), to escape a
+ character you write it as '%' + <2 character US-ASCII hex value>.
+ The Python code of ``'%' + hex(ord(<character>))[2:]`` escapes a
+ character properly. Case does not matter on the hex letters.
The various character sets specified are:
@@ -313,24 +334,24 @@
"abcdefghijklmnopqrstuvwxyz",
"0123456789",
"_.-"])
- result = urllib.quote(do_not_quote)
+ result = urllib.parse.quote(do_not_quote)
self.assertEqual(do_not_quote, result,
"using quote(): %s != %s" % (do_not_quote, result))
- result = urllib.quote_plus(do_not_quote)
+ result = urllib.parse.quote_plus(do_not_quote)
self.assertEqual(do_not_quote, result,
"using quote_plus(): %s != %s" % (do_not_quote, result))
def test_default_safe(self):
# Test '/' is default value for 'safe' parameter
- self.assertEqual(urllib.quote.__defaults__[0], '/')
+ self.assertEqual(urllib.parse.quote.__defaults__[0], '/')
def test_safe(self):
# Test setting 'safe' parameter does what it should do
quote_by_default = "<>"
- result = urllib.quote(quote_by_default, safe=quote_by_default)
+ result = urllib.parse.quote(quote_by_default, safe=quote_by_default)
self.assertEqual(quote_by_default, result,
"using quote(): %s != %s" % (quote_by_default, result))
- result = urllib.quote_plus(quote_by_default, safe=quote_by_default)
+ result = urllib.parse.quote_plus(quote_by_default, safe=quote_by_default)
self.assertEqual(quote_by_default, result,
"using quote_plus(): %s != %s" %
(quote_by_default, result))
@@ -343,11 +364,11 @@
should_quote.append(chr(127)) # For 0x7F
should_quote = ''.join(should_quote)
for char in should_quote:
- result = urllib.quote(char)
+ result = urllib.parse.quote(char)
self.assertEqual(hexescape(char), result,
"using quote(): %s should be escaped to %s, not %s" %
(char, hexescape(char), result))
- result = urllib.quote_plus(char)
+ result = urllib.parse.quote_plus(char)
self.assertEqual(hexescape(char), result,
"using quote_plus(): "
"%s should be escapes to %s, not %s" %
@@ -355,7 +376,7 @@
del should_quote
partial_quote = "ab[]cd"
expected = "ab%5B%5Dcd"
- result = urllib.quote(partial_quote)
+ result = urllib.parse.quote(partial_quote)
self.assertEqual(expected, result,
"using quote(): %s != %s" % (expected, result))
self.assertEqual(expected, result,
@@ -364,26 +385,26 @@
def test_quoting_space(self):
# Make sure quote() and quote_plus() handle spaces as specified in
# their unique way
- result = urllib.quote(' ')
+ result = urllib.parse.quote(' ')
self.assertEqual(result, hexescape(' '),
"using quote(): %s != %s" % (result, hexescape(' ')))
- result = urllib.quote_plus(' ')
+ result = urllib.parse.quote_plus(' ')
self.assertEqual(result, '+',
"using quote_plus(): %s != +" % result)
given = "a b cd e f"
expect = given.replace(' ', hexescape(' '))
- result = urllib.quote(given)
+ result = urllib.parse.quote(given)
self.assertEqual(expect, result,
"using quote(): %s != %s" % (expect, result))
expect = given.replace(' ', '+')
- result = urllib.quote_plus(given)
+ result = urllib.parse.quote_plus(given)
self.assertEqual(expect, result,
"using quote_plus(): %s != %s" % (expect, result))
def test_quoting_plus(self):
- self.assertEqual(urllib.quote_plus('alpha+beta gamma'),
+ self.assertEqual(urllib.parse.quote_plus('alpha+beta gamma'),
'alpha%2Bbeta+gamma')
- self.assertEqual(urllib.quote_plus('alpha+beta gamma', '+'),
+ self.assertEqual(urllib.parse.quote_plus('alpha+beta gamma', '+'),
'alpha+beta+gamma')
class UnquotingTests(unittest.TestCase):
@@ -399,21 +420,21 @@
for num in range(128):
given = hexescape(chr(num))
expect = chr(num)
- result = urllib.unquote(given)
+ result = urllib.parse.unquote(given)
self.assertEqual(expect, result,
"using unquote(): %s != %s" % (expect, result))
- result = urllib.unquote_plus(given)
+ result = urllib.parse.unquote_plus(given)
self.assertEqual(expect, result,
"using unquote_plus(): %s != %s" %
(expect, result))
escape_list.append(given)
escape_string = ''.join(escape_list)
del escape_list
- result = urllib.unquote(escape_string)
+ result = urllib.parse.unquote(escape_string)
self.assertEqual(result.count('%'), 1,
"using quote(): not all characters escaped; %s" %
result)
- result = urllib.unquote(escape_string)
+ result = urllib.parse.unquote(escape_string)
self.assertEqual(result.count('%'), 1,
"using unquote(): not all characters escaped: "
"%s" % result)
@@ -423,10 +444,10 @@
# interspersed
given = 'ab%sd' % hexescape('c')
expect = "abcd"
- result = urllib.unquote(given)
+ result = urllib.parse.unquote(given)
self.assertEqual(expect, result,
"using quote(): %s != %s" % (expect, result))
- result = urllib.unquote_plus(given)
+ result = urllib.parse.unquote_plus(given)
self.assertEqual(expect, result,
"using unquote_plus(): %s != %s" % (expect, result))
@@ -434,16 +455,16 @@
# Test difference between unquote() and unquote_plus()
given = "are+there+spaces..."
expect = given
- result = urllib.unquote(given)
+ result = urllib.parse.unquote(given)
self.assertEqual(expect, result,
"using unquote(): %s != %s" % (expect, result))
expect = given.replace('+', ' ')
- result = urllib.unquote_plus(given)
+ result = urllib.parse.unquote_plus(given)
self.assertEqual(expect, result,
"using unquote_plus(): %s != %s" % (expect, result))
def test_unquote_with_unicode(self):
- r = urllib.unquote('br%C3%BCckner_sapporo_20050930.doc')
+ r = urllib.parse.unquote('br%C3%BCckner_sapporo_20050930.doc')
self.assertEqual(r, 'br\xc3\xbcckner_sapporo_20050930.doc')
class urlencode_Tests(unittest.TestCase):
@@ -462,7 +483,7 @@
"""
expect_somewhere = ["1st=1", "2nd=2", "3rd=3"]
- result = urllib.urlencode(given)
+ result = urllib.parse.urlencode(given)
for expected in expect_somewhere:
self.assert_(expected in result,
"testing %s: %s not found in %s" %
@@ -495,20 +516,20 @@
# Make sure keys and values are quoted using quote_plus()
given = {"&":"="}
expect = "%s=%s" % (hexescape('&'), hexescape('='))
- result = urllib.urlencode(given)
+ result = urllib.parse.urlencode(given)
self.assertEqual(expect, result)
given = {"key name":"A bunch of pluses"}
expect = "key+name=A+bunch+of+pluses"
- result = urllib.urlencode(given)
+ result = urllib.parse.urlencode(given)
self.assertEqual(expect, result)
def test_doseq(self):
# Test that passing True for 'doseq' parameter works correctly
given = {'sequence':['1', '2', '3']}
- expect = "sequence=%s" % urllib.quote_plus(str(['1', '2', '3']))
- result = urllib.urlencode(given)
+ expect = "sequence=%s" % urllib.parse.quote_plus(str(['1', '2', '3']))
+ result = urllib.parse.urlencode(given)
self.assertEqual(expect, result)
- result = urllib.urlencode(given, True)
+ result = urllib.parse.urlencode(given, True)
for value in given["sequence"]:
expect = "sequence=%s" % value
self.assert_(expect in result,
@@ -523,11 +544,11 @@
# Make sure simple tests pass
expected_path = os.path.join("parts", "of", "a", "path")
expected_url = "parts/of/a/path"
- result = urllib.pathname2url(expected_path)
+ result = urllib.request.pathname2url(expected_path)
self.assertEqual(expected_url, result,
"pathname2url() failed; %s != %s" %
(result, expected_url))
- result = urllib.url2pathname(expected_url)
+ result = urllib.request.url2pathname(expected_url)
self.assertEqual(expected_path, result,
"url2pathame() failed; %s != %s" %
(result, expected_path))
@@ -536,25 +557,25 @@
# Test automatic quoting and unquoting works for pathnam2url() and
# url2pathname() respectively
given = os.path.join("needs", "quot=ing", "here")
- expect = "needs/%s/here" % urllib.quote("quot=ing")
- result = urllib.pathname2url(given)
+ expect = "needs/%s/here" % urllib.parse.quote("quot=ing")
+ result = urllib.request.pathname2url(given)
self.assertEqual(expect, result,
"pathname2url() failed; %s != %s" %
(expect, result))
expect = given
- result = urllib.url2pathname(result)
+ result = urllib.request.url2pathname(result)
self.assertEqual(expect, result,
"url2pathname() failed; %s != %s" %
(expect, result))
given = os.path.join("make sure", "using_quote")
- expect = "%s/using_quote" % urllib.quote("make sure")
- result = urllib.pathname2url(given)
+ expect = "%s/using_quote" % urllib.parse.quote("make sure")
+ result = urllib.request.pathname2url(given)
self.assertEqual(expect, result,
"pathname2url() failed; %s != %s" %
(expect, result))
given = "make+sure/using_unquote"
expect = os.path.join("make+sure", "using_unquote")
- result = urllib.url2pathname(given)
+ result = urllib.request.url2pathname(given)
self.assertEqual(expect, result,
"url2pathname() failed; %s != %s" %
(expect, result))