Guido van Rossum | 34d1928 | 2007-08-09 01:03:29 +0000 | [diff] [blame] | 1 | import io |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 2 | import unittest |
| 3 | import urllib.robotparser |
Antoine Pitrou | 95531ea | 2011-07-08 19:43:51 +0200 | [diff] [blame] | 4 | from urllib.error import URLError, HTTPError |
| 5 | from urllib.request import urlopen |
Benjamin Peterson | ee8712c | 2008-05-20 21:35:26 +0000 | [diff] [blame] | 6 | from test import support |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 7 | from http.server import BaseHTTPRequestHandler, HTTPServer |
Berker Peksag | ad324f6 | 2014-06-29 15:54:56 +0300 | [diff] [blame] | 8 | try: |
| 9 | import threading |
| 10 | except ImportError: |
| 11 | threading = None |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 12 | |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 13 | |
| 14 | class RobotTestCase(unittest.TestCase): |
Ezio Melotti | 0fb37ea | 2013-03-12 07:49:12 +0200 | [diff] [blame] | 15 | def __init__(self, index=None, parser=None, url=None, good=None, agent=None): |
| 16 | # workaround to make unittest discovery work (see #17066) |
| 17 | if not isinstance(index, int): |
| 18 | return |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 19 | unittest.TestCase.__init__(self) |
| 20 | if good: |
| 21 | self.str = "RobotTest(%d, good, %s)" % (index, url) |
| 22 | else: |
| 23 | self.str = "RobotTest(%d, bad, %s)" % (index, url) |
| 24 | self.parser = parser |
| 25 | self.url = url |
| 26 | self.good = good |
| 27 | self.agent = agent |
| 28 | |
| 29 | def runTest(self): |
| 30 | if isinstance(self.url, tuple): |
| 31 | agent, url = self.url |
| 32 | else: |
| 33 | url = self.url |
| 34 | agent = self.agent |
| 35 | if self.good: |
Benjamin Peterson | c9c0f20 | 2009-06-30 23:06:06 +0000 | [diff] [blame] | 36 | self.assertTrue(self.parser.can_fetch(agent, url)) |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 37 | else: |
Benjamin Peterson | c9c0f20 | 2009-06-30 23:06:06 +0000 | [diff] [blame] | 38 | self.assertFalse(self.parser.can_fetch(agent, url)) |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 39 | |
| 40 | def __str__(self): |
| 41 | return self.str |
| 42 | |
| 43 | tests = unittest.TestSuite() |
| 44 | |
| 45 | def RobotTest(index, robots_txt, good_urls, bad_urls, |
| 46 | agent="test_robotparser"): |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 47 | |
Guido van Rossum | 34d1928 | 2007-08-09 01:03:29 +0000 | [diff] [blame] | 48 | lines = io.StringIO(robots_txt).readlines() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 49 | parser = urllib.robotparser.RobotFileParser() |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 50 | parser.parse(lines) |
| 51 | for url in good_urls: |
| 52 | tests.addTest(RobotTestCase(index, parser, url, 1, agent)) |
| 53 | for url in bad_urls: |
| 54 | tests.addTest(RobotTestCase(index, parser, url, 0, agent)) |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 55 | |
| 56 | # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002) |
| 57 | |
| 58 | # 1. |
| 59 | doc = """ |
| 60 | User-agent: * |
| 61 | Disallow: /cyberworld/map/ # This is an infinite virtual URL space |
| 62 | Disallow: /tmp/ # these will soon disappear |
| 63 | Disallow: /foo.html |
| 64 | """ |
| 65 | |
| 66 | good = ['/','/test.html'] |
| 67 | bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html'] |
| 68 | |
| 69 | RobotTest(1, doc, good, bad) |
| 70 | |
| 71 | # 2. |
| 72 | doc = """ |
| 73 | # robots.txt for http://www.example.com/ |
| 74 | |
| 75 | User-agent: * |
| 76 | Disallow: /cyberworld/map/ # This is an infinite virtual URL space |
| 77 | |
| 78 | # Cybermapper knows where to go. |
| 79 | User-agent: cybermapper |
| 80 | Disallow: |
| 81 | |
| 82 | """ |
| 83 | |
| 84 | good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')] |
| 85 | bad = ['/cyberworld/map/index.html'] |
| 86 | |
| 87 | RobotTest(2, doc, good, bad) |
| 88 | |
| 89 | # 3. |
| 90 | doc = """ |
| 91 | # go away |
| 92 | User-agent: * |
| 93 | Disallow: / |
| 94 | """ |
| 95 | |
| 96 | good = [] |
| 97 | bad = ['/cyberworld/map/index.html','/','/tmp/'] |
| 98 | |
| 99 | RobotTest(3, doc, good, bad) |
| 100 | |
| 101 | # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002) |
| 102 | |
| 103 | # 4. |
| 104 | doc = """ |
| 105 | User-agent: figtree |
| 106 | Disallow: /tmp |
| 107 | Disallow: /a%3cd.html |
| 108 | Disallow: /a%2fb.html |
| 109 | Disallow: /%7ejoe/index.html |
| 110 | """ |
| 111 | |
| 112 | good = [] # XFAIL '/a/b.html' |
| 113 | bad = ['/tmp','/tmp.html','/tmp/a.html', |
| 114 | '/a%3cd.html','/a%3Cd.html','/a%2fb.html', |
| 115 | '/~joe/index.html' |
| 116 | ] |
| 117 | |
| 118 | RobotTest(4, doc, good, bad, 'figtree') |
| 119 | RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04') |
| 120 | |
| 121 | # 6. |
| 122 | doc = """ |
| 123 | User-agent: * |
| 124 | Disallow: /tmp/ |
| 125 | Disallow: /a%3Cd.html |
| 126 | Disallow: /a/b.html |
| 127 | Disallow: /%7ejoe/index.html |
| 128 | """ |
| 129 | |
| 130 | good = ['/tmp',] # XFAIL: '/a%2fb.html' |
| 131 | bad = ['/tmp/','/tmp/a.html', |
| 132 | '/a%3cd.html','/a%3Cd.html',"/a/b.html", |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 133 | '/%7Ejoe/index.html'] |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 134 | |
| 135 | RobotTest(6, doc, good, bad) |
| 136 | |
| 137 | # From bug report #523041 |
| 138 | |
| 139 | # 7. |
| 140 | doc = """ |
| 141 | User-Agent: * |
| 142 | Disallow: /. |
| 143 | """ |
| 144 | |
| 145 | good = ['/foo.html'] |
| 146 | bad = [] # Bug report says "/" should be denied, but that is not in the RFC |
| 147 | |
| 148 | RobotTest(7, doc, good, bad) |
| 149 | |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 150 | # From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364 |
| 151 | |
| 152 | # 8. |
| 153 | doc = """ |
| 154 | User-agent: Googlebot |
| 155 | Allow: /folder1/myfile.html |
| 156 | Disallow: /folder1/ |
| 157 | """ |
| 158 | |
| 159 | good = ['/folder1/myfile.html'] |
| 160 | bad = ['/folder1/anotherfile.html'] |
| 161 | |
| 162 | RobotTest(8, doc, good, bad, agent="Googlebot") |
| 163 | |
| 164 | # 9. This file is incorrect because "Googlebot" is a substring of |
| 165 | # "Googlebot-Mobile", so test 10 works just like test 9. |
| 166 | doc = """ |
| 167 | User-agent: Googlebot |
| 168 | Disallow: / |
| 169 | |
| 170 | User-agent: Googlebot-Mobile |
| 171 | Allow: / |
| 172 | """ |
| 173 | |
| 174 | good = [] |
| 175 | bad = ['/something.jpg'] |
| 176 | |
| 177 | RobotTest(9, doc, good, bad, agent="Googlebot") |
| 178 | |
| 179 | good = [] |
| 180 | bad = ['/something.jpg'] |
| 181 | |
| 182 | RobotTest(10, doc, good, bad, agent="Googlebot-Mobile") |
| 183 | |
| 184 | # 11. Get the order correct. |
| 185 | doc = """ |
| 186 | User-agent: Googlebot-Mobile |
| 187 | Allow: / |
| 188 | |
| 189 | User-agent: Googlebot |
| 190 | Disallow: / |
| 191 | """ |
| 192 | |
| 193 | good = [] |
| 194 | bad = ['/something.jpg'] |
| 195 | |
| 196 | RobotTest(11, doc, good, bad, agent="Googlebot") |
| 197 | |
| 198 | good = ['/something.jpg'] |
| 199 | bad = [] |
| 200 | |
| 201 | RobotTest(12, doc, good, bad, agent="Googlebot-Mobile") |
| 202 | |
| 203 | |
| 204 | # 13. Google also got the order wrong in #8. You need to specify the |
| 205 | # URLs from more specific to more general. |
| 206 | doc = """ |
| 207 | User-agent: Googlebot |
| 208 | Allow: /folder1/myfile.html |
| 209 | Disallow: /folder1/ |
| 210 | """ |
| 211 | |
| 212 | good = ['/folder1/myfile.html'] |
| 213 | bad = ['/folder1/anotherfile.html'] |
| 214 | |
| 215 | RobotTest(13, doc, good, bad, agent="googlebot") |
| 216 | |
| 217 | |
Senthil Kumaran | 3f8ab96 | 2010-07-28 16:27:56 +0000 | [diff] [blame] | 218 | # 14. For issue #6325 (query string support) |
| 219 | doc = """ |
| 220 | User-agent: * |
| 221 | Disallow: /some/path?name=value |
| 222 | """ |
| 223 | |
| 224 | good = ['/some/path'] |
| 225 | bad = ['/some/path?name=value'] |
| 226 | |
| 227 | RobotTest(14, doc, good, bad) |
| 228 | |
Georg Brandl | 0a0fc07 | 2010-07-29 17:55:01 +0000 | [diff] [blame] | 229 | # 15. For issue #4108 (obey first * entry) |
| 230 | doc = """ |
| 231 | User-agent: * |
| 232 | Disallow: /some/path |
| 233 | |
| 234 | User-agent: * |
| 235 | Disallow: /another/path |
| 236 | """ |
| 237 | |
| 238 | good = ['/another/path'] |
| 239 | bad = ['/some/path'] |
| 240 | |
| 241 | RobotTest(15, doc, good, bad) |
| 242 | |
Senthil Kumaran | c70a6ae | 2013-05-29 05:54:31 -0700 | [diff] [blame] | 243 | # 16. Empty query (issue #17403). Normalizing the url first. |
| 244 | doc = """ |
| 245 | User-agent: * |
| 246 | Allow: /some/path? |
| 247 | Disallow: /another/path? |
| 248 | """ |
| 249 | |
| 250 | good = ['/some/path?'] |
| 251 | bad = ['/another/path?'] |
| 252 | |
| 253 | RobotTest(16, doc, good, bad) |
| 254 | |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 255 | |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 256 | class RobotHandler(BaseHTTPRequestHandler): |
| 257 | |
| 258 | def do_GET(self): |
| 259 | self.send_error(403, "Forbidden access") |
| 260 | |
| 261 | def log_message(self, format, *args): |
| 262 | pass |
| 263 | |
| 264 | |
Berker Peksag | ad324f6 | 2014-06-29 15:54:56 +0300 | [diff] [blame] | 265 | @unittest.skipUnless(threading, 'threading required for this test') |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 266 | class PasswordProtectedSiteTestCase(unittest.TestCase): |
| 267 | |
| 268 | def setUp(self): |
| 269 | self.server = HTTPServer((support.HOST, 0), RobotHandler) |
| 270 | |
| 271 | self.t = threading.Thread( |
| 272 | name='HTTPServer serving', |
| 273 | target=self.server.serve_forever, |
| 274 | # Short poll interval to make the test finish quickly. |
| 275 | # Time between requests is short enough that we won't wake |
| 276 | # up spuriously too many times. |
| 277 | kwargs={'poll_interval':0.01}) |
| 278 | self.t.daemon = True # In case this function raises. |
| 279 | self.t.start() |
| 280 | |
| 281 | def tearDown(self): |
| 282 | self.server.shutdown() |
| 283 | self.t.join() |
| 284 | self.server.server_close() |
| 285 | |
| 286 | def runTest(self): |
| 287 | self.testPasswordProtectedSite() |
Jeremy Hylton | 73fd46d | 2008-07-18 20:59:44 +0000 | [diff] [blame] | 288 | |
| 289 | def testPasswordProtectedSite(self): |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 290 | addr = self.server.server_address |
| 291 | url = 'http://' + support.HOST + ':' + str(addr[1]) |
| 292 | robots_url = url + "/robots.txt" |
| 293 | parser = urllib.robotparser.RobotFileParser() |
| 294 | parser.set_url(url) |
| 295 | parser.read() |
| 296 | self.assertFalse(parser.can_fetch("*", robots_url)) |
| 297 | |
| 298 | def __str__(self): |
| 299 | return '%s' % self.__class__.__name__ |
| 300 | |
| 301 | class NetworkTestCase(unittest.TestCase): |
Thomas Wouters | 47b49bf | 2007-08-30 22:15:33 +0000 | [diff] [blame] | 302 | |
Georg Brandl | 89e5671 | 2014-02-23 08:45:15 +0100 | [diff] [blame] | 303 | @unittest.skip('does not handle the gzip encoding delivered by pydotorg') |
Jeremy Hylton | 73fd46d | 2008-07-18 20:59:44 +0000 | [diff] [blame] | 304 | def testPythonOrg(self): |
Florent Xicluna | 41fe615 | 2010-04-02 18:52:12 +0000 | [diff] [blame] | 305 | support.requires('network') |
Antoine Pitrou | 8bc0903 | 2010-09-07 21:09:09 +0000 | [diff] [blame] | 306 | with support.transient_internet('www.python.org'): |
| 307 | parser = urllib.robotparser.RobotFileParser( |
| 308 | "http://www.python.org/robots.txt") |
| 309 | parser.read() |
| 310 | self.assertTrue( |
| 311 | parser.can_fetch("*", "http://www.python.org/robots.txt")) |
Jeremy Hylton | 73fd46d | 2008-07-18 20:59:44 +0000 | [diff] [blame] | 312 | |
Ezio Melotti | 0fb37ea | 2013-03-12 07:49:12 +0200 | [diff] [blame] | 313 | def load_tests(loader, suite, pattern): |
| 314 | suite = unittest.makeSuite(NetworkTestCase) |
| 315 | suite.addTest(tests) |
Senthil Kumaran | 601d6ec | 2014-06-25 02:58:15 -0700 | [diff] [blame] | 316 | suite.addTest(PasswordProtectedSiteTestCase()) |
Ezio Melotti | 0fb37ea | 2013-03-12 07:49:12 +0200 | [diff] [blame] | 317 | return suite |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 318 | |
| 319 | if __name__=='__main__': |
Ezio Melotti | 0fb37ea | 2013-03-12 07:49:12 +0200 | [diff] [blame] | 320 | unittest.main() |