Barry Warsaw | 04f357c | 2002-07-23 19:04:11 +0000 | [diff] [blame] | 1 | import unittest, StringIO, robotparser |
| 2 | from test import test_support |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 3 | |
| 4 | class RobotTestCase(unittest.TestCase): |
| 5 | def __init__(self, index, parser, url, good, agent): |
| 6 | unittest.TestCase.__init__(self) |
| 7 | if good: |
| 8 | self.str = "RobotTest(%d, good, %s)" % (index, url) |
| 9 | else: |
| 10 | self.str = "RobotTest(%d, bad, %s)" % (index, url) |
| 11 | self.parser = parser |
| 12 | self.url = url |
| 13 | self.good = good |
| 14 | self.agent = agent |
| 15 | |
| 16 | def runTest(self): |
| 17 | if isinstance(self.url, tuple): |
| 18 | agent, url = self.url |
| 19 | else: |
| 20 | url = self.url |
| 21 | agent = self.agent |
| 22 | if self.good: |
| 23 | self.failUnless(self.parser.can_fetch(agent, url)) |
| 24 | else: |
| 25 | self.failIf(self.parser.can_fetch(agent, url)) |
| 26 | |
| 27 | def __str__(self): |
| 28 | return self.str |
| 29 | |
| 30 | tests = unittest.TestSuite() |
| 31 | |
| 32 | def RobotTest(index, robots_txt, good_urls, bad_urls, |
| 33 | agent="test_robotparser"): |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 34 | |
| 35 | lines = StringIO.StringIO(robots_txt).readlines() |
| 36 | parser = robotparser.RobotFileParser() |
| 37 | parser.parse(lines) |
| 38 | for url in good_urls: |
| 39 | tests.addTest(RobotTestCase(index, parser, url, 1, agent)) |
| 40 | for url in bad_urls: |
| 41 | tests.addTest(RobotTestCase(index, parser, url, 0, agent)) |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 42 | |
| 43 | # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002) |
| 44 | |
| 45 | # 1. |
| 46 | doc = """ |
| 47 | User-agent: * |
| 48 | Disallow: /cyberworld/map/ # This is an infinite virtual URL space |
| 49 | Disallow: /tmp/ # these will soon disappear |
| 50 | Disallow: /foo.html |
| 51 | """ |
| 52 | |
| 53 | good = ['/','/test.html'] |
| 54 | bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html'] |
| 55 | |
| 56 | RobotTest(1, doc, good, bad) |
| 57 | |
| 58 | # 2. |
| 59 | doc = """ |
| 60 | # robots.txt for http://www.example.com/ |
| 61 | |
| 62 | User-agent: * |
| 63 | Disallow: /cyberworld/map/ # This is an infinite virtual URL space |
| 64 | |
| 65 | # Cybermapper knows where to go. |
| 66 | User-agent: cybermapper |
| 67 | Disallow: |
| 68 | |
| 69 | """ |
| 70 | |
| 71 | good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')] |
| 72 | bad = ['/cyberworld/map/index.html'] |
| 73 | |
| 74 | RobotTest(2, doc, good, bad) |
| 75 | |
| 76 | # 3. |
| 77 | doc = """ |
| 78 | # go away |
| 79 | User-agent: * |
| 80 | Disallow: / |
| 81 | """ |
| 82 | |
| 83 | good = [] |
| 84 | bad = ['/cyberworld/map/index.html','/','/tmp/'] |
| 85 | |
| 86 | RobotTest(3, doc, good, bad) |
| 87 | |
| 88 | # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002) |
| 89 | |
| 90 | # 4. |
| 91 | doc = """ |
| 92 | User-agent: figtree |
| 93 | Disallow: /tmp |
| 94 | Disallow: /a%3cd.html |
| 95 | Disallow: /a%2fb.html |
| 96 | Disallow: /%7ejoe/index.html |
| 97 | """ |
| 98 | |
| 99 | good = [] # XFAIL '/a/b.html' |
| 100 | bad = ['/tmp','/tmp.html','/tmp/a.html', |
| 101 | '/a%3cd.html','/a%3Cd.html','/a%2fb.html', |
| 102 | '/~joe/index.html' |
| 103 | ] |
| 104 | |
| 105 | RobotTest(4, doc, good, bad, 'figtree') |
| 106 | RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04') |
| 107 | |
| 108 | # 6. |
| 109 | doc = """ |
| 110 | User-agent: * |
| 111 | Disallow: /tmp/ |
| 112 | Disallow: /a%3Cd.html |
| 113 | Disallow: /a/b.html |
| 114 | Disallow: /%7ejoe/index.html |
| 115 | """ |
| 116 | |
| 117 | good = ['/tmp',] # XFAIL: '/a%2fb.html' |
| 118 | bad = ['/tmp/','/tmp/a.html', |
| 119 | '/a%3cd.html','/a%3Cd.html',"/a/b.html", |
Tim Peters | 863ac44 | 2002-04-16 01:38:40 +0000 | [diff] [blame] | 120 | '/%7Ejoe/index.html'] |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 121 | |
| 122 | RobotTest(6, doc, good, bad) |
| 123 | |
| 124 | # From bug report #523041 |
| 125 | |
| 126 | # 7. |
| 127 | doc = """ |
| 128 | User-Agent: * |
| 129 | Disallow: /. |
| 130 | """ |
| 131 | |
| 132 | good = ['/foo.html'] |
| 133 | bad = [] # Bug report says "/" should be denied, but that is not in the RFC |
| 134 | |
| 135 | RobotTest(7, doc, good, bad) |
| 136 | |
| 137 | def test_main(): |
| 138 | test_support.run_suite(tests) |
| 139 | |
| 140 | if __name__=='__main__': |
| 141 | test_support.Verbose = 1 |
| 142 | test_support.run_suite(tests) |