Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 1 | """ robotparser.py |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 2 | |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 3 | Copyright (C) 2000 Bastian Kleineidam |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 4 | |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 5 | You can choose between two licenses when using this package: |
| 6 | 1) GNU GPLv2 |
Martin v. Löwis | d22368f | 2002-03-18 10:41:20 +0000 | [diff] [blame] | 7 | 2) PSF license for Python 2.2 |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 8 | |
| 9 | The robots.txt Exclusion Protocol is implemented as specified in |
Raymond Hettinger | 122541b | 2014-05-12 21:56:33 -0700 | [diff] [blame] | 10 | http://www.robotstxt.org/norobots-rfc.txt |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 11 | """ |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 12 | |
| 13 | import urllib.parse, urllib.request |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 14 | |
Skip Montanaro | e99d5ea | 2001-01-20 19:54:20 +0000 | [diff] [blame] | 15 | __all__ = ["RobotFileParser"] |
| 16 | |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 17 | class RobotFileParser: |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 18 | """ This class provides a set of methods to read, parse and answer |
| 19 | questions about a single robots.txt file. |
| 20 | |
| 21 | """ |
| 22 | |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 23 | def __init__(self, url=''): |
| 24 | self.entries = [] |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 25 | self.default_entry = None |
Martin v. Löwis | 31bd529 | 2004-08-23 20:42:35 +0000 | [diff] [blame] | 26 | self.disallow_all = False |
| 27 | self.allow_all = False |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 28 | self.set_url(url) |
Guido van Rossum | 986abac | 1998-04-06 14:29:28 +0000 | [diff] [blame] | 29 | self.last_checked = 0 |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 30 | |
| 31 | def mtime(self): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 32 | """Returns the time the robots.txt file was last fetched. |
| 33 | |
| 34 | This is useful for long-running web spiders that need to |
| 35 | check for new robots.txt files periodically. |
| 36 | |
| 37 | """ |
Guido van Rossum | 986abac | 1998-04-06 14:29:28 +0000 | [diff] [blame] | 38 | return self.last_checked |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 39 | |
| 40 | def modified(self): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 41 | """Sets the time the robots.txt file was last fetched to the |
| 42 | current time. |
| 43 | |
| 44 | """ |
Guido van Rossum | 986abac | 1998-04-06 14:29:28 +0000 | [diff] [blame] | 45 | import time |
| 46 | self.last_checked = time.time() |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 47 | |
| 48 | def set_url(self, url): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 49 | """Sets the URL referring to a robots.txt file.""" |
Guido van Rossum | 986abac | 1998-04-06 14:29:28 +0000 | [diff] [blame] | 50 | self.url = url |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 51 | self.host, self.path = urllib.parse.urlparse(url)[1:3] |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 52 | |
| 53 | def read(self): |
Raymond Hettinger | aef22fb | 2002-05-29 16:18:42 +0000 | [diff] [blame] | 54 | """Reads the robots.txt URL and feeds it to the parser.""" |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 55 | try: |
| 56 | f = urllib.request.urlopen(self.url) |
| 57 | except urllib.error.HTTPError as err: |
| 58 | if err.code in (401, 403): |
| 59 | self.disallow_all = True |
Raymond Hettinger | 122541b | 2014-05-12 21:56:33 -0700 | [diff] [blame] | 60 | elif err.code >= 400 and err.code < 500: |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 61 | self.allow_all = True |
| 62 | else: |
Jeremy Hylton | 73fd46d | 2008-07-18 20:59:44 +0000 | [diff] [blame] | 63 | raw = f.read() |
| 64 | self.parse(raw.decode("utf-8").splitlines()) |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 65 | |
Martin v. Löwis | 73f570b | 2002-03-18 10:43:18 +0000 | [diff] [blame] | 66 | def _add_entry(self, entry): |
| 67 | if "*" in entry.useragents: |
| 68 | # the default entry is considered last |
Georg Brandl | 0a0fc07 | 2010-07-29 17:55:01 +0000 | [diff] [blame] | 69 | if self.default_entry is None: |
| 70 | # the first default entry wins |
| 71 | self.default_entry = entry |
Martin v. Löwis | 73f570b | 2002-03-18 10:43:18 +0000 | [diff] [blame] | 72 | else: |
| 73 | self.entries.append(entry) |
| 74 | |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 75 | def parse(self, lines): |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 76 | """Parse the input lines from a robots.txt file. |
| 77 | |
| 78 | We allow that a user-agent: line is not preceded by |
| 79 | one or more blank lines. |
| 80 | """ |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 81 | # states: |
| 82 | # 0: start state |
| 83 | # 1: saw user-agent line |
| 84 | # 2: saw an allow or disallow line |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 85 | state = 0 |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 86 | entry = Entry() |
Tim Peters | dfc538a | 2001-01-21 04:49:16 +0000 | [diff] [blame] | 87 | |
Raymond Hettinger | 122541b | 2014-05-12 21:56:33 -0700 | [diff] [blame] | 88 | self.modified() |
Guido van Rossum | 986abac | 1998-04-06 14:29:28 +0000 | [diff] [blame] | 89 | for line in lines: |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 90 | if not line: |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 91 | if state == 1: |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 92 | entry = Entry() |
| 93 | state = 0 |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 94 | elif state == 2: |
Martin v. Löwis | 73f570b | 2002-03-18 10:43:18 +0000 | [diff] [blame] | 95 | self._add_entry(entry) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 96 | entry = Entry() |
| 97 | state = 0 |
Guido van Rossum | 986abac | 1998-04-06 14:29:28 +0000 | [diff] [blame] | 98 | # remove optional comment and strip line |
Eric S. Raymond | 141971f | 2001-02-09 08:40:40 +0000 | [diff] [blame] | 99 | i = line.find('#') |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 100 | if i >= 0: |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 101 | line = line[:i] |
Eric S. Raymond | 141971f | 2001-02-09 08:40:40 +0000 | [diff] [blame] | 102 | line = line.strip() |
Guido van Rossum | 986abac | 1998-04-06 14:29:28 +0000 | [diff] [blame] | 103 | if not line: |
| 104 | continue |
Eric S. Raymond | 141971f | 2001-02-09 08:40:40 +0000 | [diff] [blame] | 105 | line = line.split(':', 1) |
Guido van Rossum | 986abac | 1998-04-06 14:29:28 +0000 | [diff] [blame] | 106 | if len(line) == 2: |
Eric S. Raymond | 141971f | 2001-02-09 08:40:40 +0000 | [diff] [blame] | 107 | line[0] = line[0].strip().lower() |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 108 | line[1] = urllib.parse.unquote(line[1].strip()) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 109 | if line[0] == "user-agent": |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 110 | if state == 2: |
Martin v. Löwis | 73f570b | 2002-03-18 10:43:18 +0000 | [diff] [blame] | 111 | self._add_entry(entry) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 112 | entry = Entry() |
| 113 | entry.useragents.append(line[1]) |
| 114 | state = 1 |
| 115 | elif line[0] == "disallow": |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 116 | if state != 0: |
Martin v. Löwis | 31bd529 | 2004-08-23 20:42:35 +0000 | [diff] [blame] | 117 | entry.rulelines.append(RuleLine(line[1], False)) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 118 | state = 2 |
| 119 | elif line[0] == "allow": |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 120 | if state != 0: |
Martin v. Löwis | 31bd529 | 2004-08-23 20:42:35 +0000 | [diff] [blame] | 121 | entry.rulelines.append(RuleLine(line[1], True)) |
Benjamin Peterson | d631371 | 2008-07-31 16:23:04 +0000 | [diff] [blame] | 122 | state = 2 |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 123 | if state == 2: |
Georg Brandl | 0a0fc07 | 2010-07-29 17:55:01 +0000 | [diff] [blame] | 124 | self._add_entry(entry) |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 125 | |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 126 | |
Guido van Rossum | dc8b798 | 2000-03-27 19:29:31 +0000 | [diff] [blame] | 127 | def can_fetch(self, useragent, url): |
| 128 | """using the parsed robots.txt decide if useragent can fetch url""" |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 129 | if self.disallow_all: |
Tim Peters | bc0e910 | 2002-04-04 22:55:58 +0000 | [diff] [blame] | 130 | return False |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 131 | if self.allow_all: |
Tim Peters | bc0e910 | 2002-04-04 22:55:58 +0000 | [diff] [blame] | 132 | return True |
Raymond Hettinger | 122541b | 2014-05-12 21:56:33 -0700 | [diff] [blame] | 133 | # Until the robots.txt file has been read or found not |
| 134 | # to exist, we must assume that no url is allowable. |
| 135 | # This prevents false positives when a user erronenously |
| 136 | # calls can_fetch() before calling read(). |
| 137 | if not self.last_checked: |
| 138 | return False |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 139 | # search for given user agent matches |
| 140 | # the first match counts |
Senthil Kumaran | 3f8ab96 | 2010-07-28 16:27:56 +0000 | [diff] [blame] | 141 | parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url)) |
| 142 | url = urllib.parse.urlunparse(('','',parsed_url.path, |
| 143 | parsed_url.params,parsed_url.query, parsed_url.fragment)) |
| 144 | url = urllib.parse.quote(url) |
Jeremy Hylton | 73fd46d | 2008-07-18 20:59:44 +0000 | [diff] [blame] | 145 | if not url: |
| 146 | url = "/" |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 147 | for entry in self.entries: |
| 148 | if entry.applies_to(useragent): |
| 149 | return entry.allowance(url) |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 150 | # try the default entry last |
| 151 | if self.default_entry: |
| 152 | return self.default_entry.allowance(url) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 153 | # agent not found ==> access granted |
Tim Peters | bc0e910 | 2002-04-04 22:55:58 +0000 | [diff] [blame] | 154 | return True |
Guido van Rossum | bbf8c2f | 1997-01-30 03:18:23 +0000 | [diff] [blame] | 155 | |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 156 | def __str__(self): |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 157 | return ''.join([str(entry) + "\n" for entry in self.entries]) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 158 | |
| 159 | |
| 160 | class RuleLine: |
Martin v. Löwis | 31bd529 | 2004-08-23 20:42:35 +0000 | [diff] [blame] | 161 | """A rule line is a single "Allow:" (allowance==True) or "Disallow:" |
| 162 | (allowance==False) followed by a path.""" |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 163 | def __init__(self, path, allowance): |
Martin v. Löwis | 1c63f6e | 2002-02-28 15:24:47 +0000 | [diff] [blame] | 164 | if path == '' and not allowance: |
| 165 | # an empty value means allow all |
Martin v. Löwis | 31bd529 | 2004-08-23 20:42:35 +0000 | [diff] [blame] | 166 | allowance = True |
Senthil Kumaran | c70a6ae | 2013-05-29 05:54:31 -0700 | [diff] [blame] | 167 | path = urllib.parse.urlunparse(urllib.parse.urlparse(path)) |
Jeremy Hylton | 1afc169 | 2008-06-18 20:49:58 +0000 | [diff] [blame] | 168 | self.path = urllib.parse.quote(path) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 169 | self.allowance = allowance |
| 170 | |
| 171 | def applies_to(self, filename): |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 172 | return self.path == "*" or filename.startswith(self.path) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 173 | |
| 174 | def __str__(self): |
Raymond Hettinger | 38acd4c | 2014-05-12 22:22:46 -0700 | [diff] [blame] | 175 | return ("Allow" if self.allowance else "Disallow") + ": " + self.path |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 176 | |
| 177 | |
| 178 | class Entry: |
| 179 | """An entry has one or more user-agents and zero or more rulelines""" |
| 180 | def __init__(self): |
| 181 | self.useragents = [] |
| 182 | self.rulelines = [] |
| 183 | |
| 184 | def __str__(self): |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 185 | ret = [] |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 186 | for agent in self.useragents: |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 187 | ret.extend(["User-agent: ", agent, "\n"]) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 188 | for line in self.rulelines: |
Guido van Rossum | d8faa36 | 2007-04-27 19:54:29 +0000 | [diff] [blame] | 189 | ret.extend([str(line), "\n"]) |
| 190 | return ''.join(ret) |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 191 | |
| 192 | def applies_to(self, useragent): |
Skip Montanaro | 5bba231 | 2001-02-12 20:58:30 +0000 | [diff] [blame] | 193 | """check if this entry applies to the specified agent""" |
| 194 | # split the name token and make it lower case |
| 195 | useragent = useragent.split("/")[0].lower() |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 196 | for agent in self.useragents: |
Christian Heimes | 81ee3ef | 2008-05-04 22:42:01 +0000 | [diff] [blame] | 197 | if agent == '*': |
Skip Montanaro | 5bba231 | 2001-02-12 20:58:30 +0000 | [diff] [blame] | 198 | # we have the catch-all agent |
Tim Peters | bc0e910 | 2002-04-04 22:55:58 +0000 | [diff] [blame] | 199 | return True |
Skip Montanaro | 5bba231 | 2001-02-12 20:58:30 +0000 | [diff] [blame] | 200 | agent = agent.lower() |
Raymond Hettinger | bac788a | 2004-05-04 09:21:43 +0000 | [diff] [blame] | 201 | if agent in useragent: |
Tim Peters | bc0e910 | 2002-04-04 22:55:58 +0000 | [diff] [blame] | 202 | return True |
| 203 | return False |
Skip Montanaro | 663f6c2 | 2001-01-20 15:59:25 +0000 | [diff] [blame] | 204 | |
| 205 | def allowance(self, filename): |
| 206 | """Preconditions: |
| 207 | - our agent applies to this entry |
| 208 | - filename is URL decoded""" |
| 209 | for line in self.rulelines: |
| 210 | if line.applies_to(filename): |
| 211 | return line.allowance |
Martin v. Löwis | 31bd529 | 2004-08-23 20:42:35 +0000 | [diff] [blame] | 212 | return True |