Merged revisions 83449 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/release27-maint
................
r83449 | georg.brandl | 2010-08-01 22:59:03 +0200 (So, 01 Aug 2010) | 9 lines
Merged revisions 83238 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/py3k
........
r83238 | georg.brandl | 2010-07-29 19:55:01 +0200 (Do, 29 Jul 2010) | 1 line
#4108: the first default entry (User-agent: *) wins.
........
................
diff --git a/Lib/robotparser.py b/Lib/robotparser.py
index 447563f..726854b 100644
--- a/Lib/robotparser.py
+++ b/Lib/robotparser.py
@@ -68,7 +68,9 @@
def _add_entry(self, entry):
if "*" in entry.useragents:
# the default entry is considered last
- self.default_entry = entry
+ if self.default_entry is None:
+ # the first default entry wins
+ self.default_entry = entry
else:
self.entries.append(entry)
@@ -120,7 +122,7 @@
entry.rulelines.append(RuleLine(line[1], True))
state = 2
if state == 2:
- self.entries.append(entry)
+ self._add_entry(entry)
def can_fetch(self, useragent, url):
diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index 431b8ff..650b603 100644
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -202,6 +202,20 @@
RobotTest(13, doc, good, bad, agent="googlebot")
+# 14. For issue #4108 (obey first * entry)
+doc = """
+User-agent: *
+Disallow: /some/path
+
+User-agent: *
+Disallow: /another/path
+"""
+
+good = ['/another/path']
+bad = ['/some/path']
+
+RobotTest(14, doc, good, bad)
+
class TestCase(unittest.TestCase):
def runTest(self):