bpo-35922: Fix RobotFileParser when robots.txt has no relevant crawl delay or request rate (GH-11791)
Co-Authored-By: Tal Einat <taleinat+github@gmail.com>
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index 7089916..c58565e 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -186,7 +186,9 @@
for entry in self.entries:
if entry.applies_to(useragent):
return entry.delay
- return self.default_entry.delay
+ if self.default_entry:
+ return self.default_entry.delay
+ return None
def request_rate(self, useragent):
if not self.mtime():
@@ -194,7 +196,9 @@
for entry in self.entries:
if entry.applies_to(useragent):
return entry.req_rate
- return self.default_entry.req_rate
+ if self.default_entry:
+ return self.default_entry.req_rate
+ return None
def site_maps(self):
if not self.sitemaps: