Guido van Rossum | 34d1928 | 2007-08-09 01:03:29 +0000 | [diff] [blame] | 1 | import io |
Berker Peksag | 2a8d7f1 | 2016-09-18 11:21:57 +0300 | [diff] [blame] | 2 | import os |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 3 | import unittest |
| 4 | import urllib.robotparser |
Berker Peksag | 960e848 | 2015-10-08 12:27:06 +0300 | [diff] [blame] | 5 | from collections import namedtuple |
Benjamin Peterson | ee8712c | 2008-05-20 21:35:26 +0000 | [diff] [blame] | 6 | from test import support |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 7 | from http.server import BaseHTTPRequestHandler, HTTPServer |
Berker Peksag | ad324f6 | 2014-06-29 15:54:56 +0300 | [diff] [blame] | 8 | try: |
| 9 | import threading |
| 10 | except ImportError: |
| 11 | threading = None |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 12 | |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 13 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 14 | class BaseRobotTest: |
| 15 | robots_txt = '' |
| 16 | agent = 'test_robotparser' |
| 17 | good = [] |
| 18 | bad = [] |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 19 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 20 | def setUp(self): |
| 21 | lines = io.StringIO(self.robots_txt).readlines() |
| 22 | self.parser = urllib.robotparser.RobotFileParser() |
| 23 | self.parser.parse(lines) |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 24 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 25 | def get_agent_and_url(self, url): |
| 26 | if isinstance(url, tuple): |
| 27 | agent, url = url |
| 28 | return agent, url |
| 29 | return self.agent, url |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 30 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 31 | def test_good_urls(self): |
| 32 | for url in self.good: |
| 33 | agent, url = self.get_agent_and_url(url) |
| 34 | with self.subTest(url=url, agent=agent): |
| 35 | self.assertTrue(self.parser.can_fetch(agent, url)) |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 36 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 37 | def test_bad_urls(self): |
| 38 | for url in self.bad: |
| 39 | agent, url = self.get_agent_and_url(url) |
| 40 | with self.subTest(url=url, agent=agent): |
| 41 | self.assertFalse(self.parser.can_fetch(agent, url)) |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 42 | |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 43 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 44 | class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): |
| 45 | robots_txt = """\ |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 46 | User-agent: * |
| 47 | Disallow: /cyberworld/map/ # This is an infinite virtual URL space |
| 48 | Disallow: /tmp/ # these will soon disappear |
| 49 | Disallow: /foo.html |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 50 | """ |
| 51 | good = ['/', '/test.html'] |
| 52 | bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html'] |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 53 | |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 54 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 55 | class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase): |
| 56 | robots_txt = """\ |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 57 | # robots.txt for http://www.example.com/ |
| 58 | |
| 59 | User-agent: * |
Berker Peksag | 960e848 | 2015-10-08 12:27:06 +0300 | [diff] [blame] | 60 | Crawl-delay: 1 |
| 61 | Request-rate: 3/15 |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 62 | Disallow: /cyberworld/map/ # This is an infinite virtual URL space |
| 63 | |
| 64 | # Cybermapper knows where to go. |
| 65 | User-agent: cybermapper |
| 66 | Disallow: |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 67 | """ |
| 68 | good = ['/', '/test.html', ('cybermapper', '/cyberworld/map/index.html')] |
| 69 | bad = ['/cyberworld/map/index.html'] |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 70 | |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 71 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 72 | class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): |
| 73 | robots_txt = """\ |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 74 | # go away |
| 75 | User-agent: * |
| 76 | Disallow: / |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 77 | """ |
| 78 | good = [] |
| 79 | bad = ['/cyberworld/map/index.html', '/', '/tmp/'] |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 80 | |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 81 | |
Berker Peksag | 9a7bbb2 | 2016-09-18 20:17:58 +0300 | [diff] [blame] | 82 | class BaseRequestRateTest(BaseRobotTest): |
| 83 | |
| 84 | def test_request_rate(self): |
| 85 | for url in self.good + self.bad: |
| 86 | agent, url = self.get_agent_and_url(url) |
| 87 | with self.subTest(url=url, agent=agent): |
| 88 | if self.crawl_delay: |
| 89 | self.assertEqual( |
| 90 | self.parser.crawl_delay(agent), self.crawl_delay |
| 91 | ) |
| 92 | if self.request_rate: |
| 93 | self.assertEqual( |
| 94 | self.parser.request_rate(agent).requests, |
| 95 | self.request_rate.requests |
| 96 | ) |
| 97 | self.assertEqual( |
| 98 | self.parser.request_rate(agent).seconds, |
| 99 | self.request_rate.seconds |
| 100 | ) |
| 101 | |
| 102 | |
| 103 | class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 104 | robots_txt = """\ |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 105 | User-agent: figtree |
Berker Peksag | 960e848 | 2015-10-08 12:27:06 +0300 | [diff] [blame] | 106 | Crawl-delay: 3 |
| 107 | Request-rate: 9/30 |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 108 | Disallow: /tmp |
| 109 | Disallow: /a%3cd.html |
| 110 | Disallow: /a%2fb.html |
| 111 | Disallow: /%7ejoe/index.html |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 112 | """ |
| 113 | agent = 'figtree' |
| 114 | request_rate = namedtuple('req_rate', 'requests seconds')(9, 30) |
| 115 | crawl_delay = 3 |
| 116 | good = [('figtree', '/foo.html')] |
| 117 | bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', |
| 118 | '/a%2fb.html', '/~joe/index.html'] |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 119 | |
Berker Peksag | 960e848 | 2015-10-08 12:27:06 +0300 | [diff] [blame] | 120 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 121 | class DifferentAgentTest(CrawlDelayAndRequestRateTest): |
| 122 | agent = 'FigTree Robot libwww-perl/5.04' |
| 123 | # these are not actually tested, but we still need to parse it |
| 124 | # in order to accommodate the input parameters |
| 125 | request_rate = None |
| 126 | crawl_delay = None |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 127 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 128 | |
| 129 | class InvalidRequestRateTest(BaseRobotTest, unittest.TestCase): |
| 130 | robots_txt = """\ |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 131 | User-agent: * |
| 132 | Disallow: /tmp/ |
| 133 | Disallow: /a%3Cd.html |
| 134 | Disallow: /a/b.html |
| 135 | Disallow: /%7ejoe/index.html |
Berker Peksag | 960e848 | 2015-10-08 12:27:06 +0300 | [diff] [blame] | 136 | Crawl-delay: 3 |
| 137 | Request-rate: 9/banana |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 138 | """ |
| 139 | good = ['/tmp'] |
| 140 | bad = ['/tmp/', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', '/a/b.html', |
| 141 | '/%7Ejoe/index.html'] |
| 142 | crawl_delay = 3 |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 143 | |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 144 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 145 | class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): |
| 146 | # From bug report #523041 |
| 147 | robots_txt = """\ |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 148 | User-Agent: * |
| 149 | Disallow: /. |
Berker Peksag | 960e848 | 2015-10-08 12:27:06 +0300 | [diff] [blame] | 150 | Crawl-delay: pears |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 151 | """ |
| 152 | good = ['/foo.html'] |
| 153 | # bug report says "/" should be denied, but that is not in the RFC |
| 154 | bad = [] |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 155 | |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 156 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 157 | class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): |
| 158 | # also test that Allow and Diasallow works well with each other |
| 159 | robots_txt = """\ |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 160 | User-agent: Googlebot |
| 161 | Allow: /folder1/myfile.html |
| 162 | Disallow: /folder1/ |
Berker Peksag | 960e848 | 2015-10-08 12:27:06 +0300 | [diff] [blame] | 163 | Request-rate: whale/banana |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 164 | """ |
| 165 | agent = 'Googlebot' |
| 166 | good = ['/folder1/myfile.html'] |
| 167 | bad = ['/folder1/anotherfile.html'] |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 168 | |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 169 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 170 | class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase): |
| 171 | # the order of User-agent should be correct. note |
| 172 | # that this file is incorrect because "Googlebot" is a |
| 173 | # substring of "Googlebot-Mobile" |
| 174 | robots_txt = """\ |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 175 | User-agent: Googlebot |
| 176 | Disallow: / |
| 177 | |
| 178 | User-agent: Googlebot-Mobile |
| 179 | Allow: / |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 180 | """ |
| 181 | agent = 'Googlebot' |
| 182 | bad = ['/something.jpg'] |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 183 | |
| 184 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 185 | class UserAgentGoogleMobileTest(UserAgentOrderingTest): |
| 186 | agent = 'Googlebot-Mobile' |
| 187 | |
| 188 | |
| 189 | class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase): |
| 190 | # Google also got the order wrong. You need |
| 191 | # to specify the URLs from more specific to more general |
| 192 | robots_txt = """\ |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 193 | User-agent: Googlebot |
| 194 | Allow: /folder1/myfile.html |
| 195 | Disallow: /folder1/ |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 196 | """ |
| 197 | agent = 'googlebot' |
| 198 | good = ['/folder1/myfile.html'] |
| 199 | bad = ['/folder1/anotherfile.html'] |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 200 | |
| 201 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 202 | class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): |
| 203 | # see issue #6325 for details |
| 204 | robots_txt = """\ |
Senthil Kumaran | 3f8ab96 | 2010-07-28 16:27:56 +0000 | [diff] [blame] | 205 | User-agent: * |
| 206 | Disallow: /some/path?name=value |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 207 | """ |
| 208 | good = ['/some/path'] |
| 209 | bad = ['/some/path?name=value'] |
Senthil Kumaran | 3f8ab96 | 2010-07-28 16:27:56 +0000 | [diff] [blame] | 210 | |
Senthil Kumaran | 3f8ab96 | 2010-07-28 16:27:56 +0000 | [diff] [blame] | 211 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 212 | class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): |
| 213 | # obey first * entry (#4108) |
| 214 | robots_txt = """\ |
Georg Brandl | 0a0fc07 | 2010-07-29 17:55:01 +0000 | [diff] [blame] | 215 | User-agent: * |
| 216 | Disallow: /some/path |
| 217 | |
| 218 | User-agent: * |
| 219 | Disallow: /another/path |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 220 | """ |
| 221 | good = ['/another/path'] |
| 222 | bad = ['/some/path'] |
Georg Brandl | 0a0fc07 | 2010-07-29 17:55:01 +0000 | [diff] [blame] | 223 | |
Georg Brandl | 0a0fc07 | 2010-07-29 17:55:01 +0000 | [diff] [blame] | 224 | |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 225 | class EmptyQueryStringTest(BaseRobotTest, unittest.TestCase): |
| 226 | # normalize the URL first (#17403) |
| 227 | robots_txt = """\ |
Senthil Kumaran | c70a6ae | 2013-05-29 05:54:31 -0700 | [diff] [blame] | 228 | User-agent: * |
| 229 | Allow: /some/path? |
| 230 | Disallow: /another/path? |
Berker Peksag | 4da0fd0 | 2016-09-11 14:53:16 +0300 | [diff] [blame] | 231 | """ |
| 232 | good = ['/some/path?'] |
| 233 | bad = ['/another/path?'] |
Senthil Kumaran | c70a6ae | 2013-05-29 05:54:31 -0700 | [diff] [blame] | 234 | |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 235 | |
Berker Peksag | 9a7bbb2 | 2016-09-18 20:17:58 +0300 | [diff] [blame] | 236 | class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): |
| 237 | robots_txt = """\ |
| 238 | User-agent: * |
| 239 | Crawl-delay: 1 |
| 240 | Request-rate: 3/15 |
| 241 | Disallow: /cyberworld/map/ |
| 242 | """ |
| 243 | request_rate = namedtuple('req_rate', 'requests seconds')(3, 15) |
| 244 | crawl_delay = 1 |
| 245 | good = ['/', '/test.html'] |
| 246 | bad = ['/cyberworld/map/index.html'] |
| 247 | |
| 248 | |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 249 | class RobotHandler(BaseHTTPRequestHandler): |
| 250 | |
| 251 | def do_GET(self): |
| 252 | self.send_error(403, "Forbidden access") |
| 253 | |
| 254 | def log_message(self, format, *args): |
| 255 | pass |
| 256 | |
| 257 | |
Berker Peksag | ad324f6 | 2014-06-29 15:54:56 +0300 | [diff] [blame] | 258 | @unittest.skipUnless(threading, 'threading required for this test') |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 259 | class PasswordProtectedSiteTestCase(unittest.TestCase): |
| 260 | |
| 261 | def setUp(self): |
| 262 | self.server = HTTPServer((support.HOST, 0), RobotHandler) |
| 263 | |
| 264 | self.t = threading.Thread( |
| 265 | name='HTTPServer serving', |
| 266 | target=self.server.serve_forever, |
| 267 | # Short poll interval to make the test finish quickly. |
| 268 | # Time between requests is short enough that we won't wake |
| 269 | # up spuriously too many times. |
| 270 | kwargs={'poll_interval':0.01}) |
| 271 | self.t.daemon = True # In case this function raises. |
| 272 | self.t.start() |
| 273 | |
| 274 | def tearDown(self): |
| 275 | self.server.shutdown() |
| 276 | self.t.join() |
| 277 | self.server.server_close() |
| 278 | |
Berker Peksag | 2a9f5ed | 2016-09-11 15:17:53 +0300 | [diff] [blame] | 279 | @support.reap_threads |
Jeremy Hylton | 73fd46d | 2008-07-18 20:59:44 +0000 | [diff] [blame] | 280 | def testPasswordProtectedSite(self): |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 281 | addr = self.server.server_address |
| 282 | url = 'http://' + support.HOST + ':' + str(addr[1]) |
| 283 | robots_url = url + "/robots.txt" |
| 284 | parser = urllib.robotparser.RobotFileParser() |
| 285 | parser.set_url(url) |
| 286 | parser.read() |
| 287 | self.assertFalse(parser.can_fetch("*", robots_url)) |
| 288 | |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 289 | |
| 290 | class NetworkTestCase(unittest.TestCase): |
Thomas Wouters | 47b49bf | 2007-08-30 22:15:33 +0000 | [diff] [blame] | 291 | |
Berker Peksag | 2a8d7f1 | 2016-09-18 11:21:57 +0300 | [diff] [blame] | 292 | base_url = 'http://www.pythontest.net/' |
| 293 | robots_txt = '{}elsewhere/robots.txt'.format(base_url) |
| 294 | |
| 295 | @classmethod |
| 296 | def setUpClass(cls): |
Florent Xicluna | 41fe615 | 2010-04-02 18:52:12 +0000 | [diff] [blame] | 297 | support.requires('network') |
Berker Peksag | 2a8d7f1 | 2016-09-18 11:21:57 +0300 | [diff] [blame] | 298 | with support.transient_internet(cls.base_url): |
| 299 | cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt) |
| 300 | cls.parser.read() |
| 301 | |
| 302 | def url(self, path): |
| 303 | return '{}{}{}'.format( |
| 304 | self.base_url, path, '/' if not os.path.splitext(path)[1] else '' |
| 305 | ) |
| 306 | |
| 307 | def test_basic(self): |
| 308 | self.assertFalse(self.parser.disallow_all) |
| 309 | self.assertFalse(self.parser.allow_all) |
| 310 | self.assertGreater(self.parser.mtime(), 0) |
| 311 | self.assertFalse(self.parser.crawl_delay('*')) |
| 312 | self.assertFalse(self.parser.request_rate('*')) |
| 313 | |
| 314 | def test_can_fetch(self): |
| 315 | self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere'))) |
| 316 | self.assertFalse(self.parser.can_fetch('Nutch', self.base_url)) |
| 317 | self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian'))) |
| 318 | self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats'))) |
| 319 | self.assertFalse(self.parser.can_fetch('*', self.url('webstats'))) |
| 320 | self.assertTrue(self.parser.can_fetch('*', self.base_url)) |
| 321 | |
| 322 | def test_read_404(self): |
| 323 | parser = urllib.robotparser.RobotFileParser(self.url('i-robot.txt')) |
| 324 | parser.read() |
| 325 | self.assertTrue(parser.allow_all) |
| 326 | self.assertFalse(parser.disallow_all) |
| 327 | self.assertEqual(parser.mtime(), 0) |
Berker Peksag | 9a7bbb2 | 2016-09-18 20:17:58 +0300 | [diff] [blame] | 328 | self.assertIsNone(parser.crawl_delay('*')) |
| 329 | self.assertIsNone(parser.request_rate('*')) |
Jeremy Hylton | 73fd46d | 2008-07-18 20:59:44 +0000 | [diff] [blame] | 330 | |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 331 | if __name__=='__main__': |
Ezio Melotti | 0fb37ea | 2013-03-12 07:49:12 +0200 | [diff] [blame] | 332 | unittest.main() |