Merged revisions 83209 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/py3k
........
r83209 | senthil.kumaran | 2010-07-28 21:57:56 +0530 (Wed, 28 Jul 2010) | 3 lines
Fix Issue6325 - robotparse to honor urls with query strings.
........
diff --git a/Lib/robotparser.py b/Lib/robotparser.py
index 447563f..730426f 100644
--- a/Lib/robotparser.py
+++ b/Lib/robotparser.py
@@ -131,7 +131,12 @@
return True
# search for given user agent matches
# the first match counts
- url = urllib.quote(urlparse.urlparse(urllib.unquote(url))[2]) or "/"
+ parsed_url = urlparse.urlparse(urllib.unquote(url))
+ url = urlparse.urlunparse(('', '', parsed_url.path,
+ parsed_url.params, parsed_url.query, parsed_url.fragment))
+ url = urllib.quote(url)
+ if not url:
+ url = "/"
for entry in self.entries:
if entry.applies_to(useragent):
return entry.allowance(url)