Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 1 | '''Parse a Python file and retrieve classes and methods. |
| 2 | |
| 3 | Parse enough of a Python file to recognize class and method |
| 4 | definitions and to find out the superclasses of a class. |
| 5 | |
| 6 | The interface consists of a single function: |
| 7 | readmodule(module, path) |
| 8 | module is the name of a Python module, path is an optional list of |
| 9 | directories where the module is to be searched. If present, path is |
| 10 | prepended to the system search path sys.path. |
| 11 | The return value is a dictionary. The keys of the dictionary are |
| 12 | the names of the classes defined in the module (including classes |
| 13 | that are defined via the from XXX import YYY construct). The values |
| 14 | are class instances of the class Class defined here. |
| 15 | |
| 16 | A class is described by the class Class in this module. Instances |
| 17 | of this class have the following instance variables: |
| 18 | name -- the name of the class |
| 19 | super -- a list of super classes (Class instances) |
| 20 | methods -- a dictionary of methods |
| 21 | file -- the file in which the class was defined |
| 22 | lineno -- the line in the file on which the class statement occurred |
| 23 | The dictionary of methods uses the method names as keys and the line |
| 24 | numbers on which the method was defined as values. |
| 25 | If the name of a super class is not recognized, the corresponding |
| 26 | entry in the list of super classes is not a class instance but a |
| 27 | string giving the name of the super class. Since import statements |
| 28 | are recognized and imported modules are scanned as well, this |
| 29 | shouldn't happen often. |
| 30 | |
| 31 | BUGS |
Guido van Rossum | b269302 | 1999-06-10 19:05:54 +0000 | [diff] [blame^] | 32 | - Continuation lines are not dealt with at all. |
| 33 | - While triple-quoted strings won't confuse it, lines that look like |
| 34 | def, class, import or "from ... import" stmts inside backslash-continued |
| 35 | single-quoted strings are treated like code. The expense of stopping |
| 36 | that isn't worth it. |
| 37 | - Code that doesn't pass tabnanny or python -t will confuse it, unless |
| 38 | you set the module TABWIDTH vrbl (default 8) to the correct tab width |
| 39 | for the file. |
| 40 | |
| 41 | PACKAGE RELATED BUGS |
| 42 | - If you have a package and a module inside that or another package |
| 43 | with the same name, module caching doesn't work properly since the |
| 44 | key is the base name of the module/package. |
| 45 | - The only entry that is returned when you readmodule a package is a |
| 46 | __path__ whose value is a list which confuses certain class browsers. |
| 47 | - When code does: |
| 48 | from package import subpackage |
| 49 | class MyClass(subpackage.SuperClass): |
| 50 | ... |
| 51 | It can't locate the parent. It probably needs to have the same |
| 52 | hairy logic that the import locator already does. (This logic |
| 53 | exists coded in Python in the freeze package.) |
| 54 | ''' # ' <-- bow to font lock |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 55 | |
| 56 | import os |
| 57 | import sys |
| 58 | import imp |
Guido van Rossum | 31626bc | 1997-10-24 14:46:16 +0000 | [diff] [blame] | 59 | import re |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 60 | import string |
| 61 | |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 62 | TABWIDTH = 8 |
| 63 | |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 64 | _getnext = re.compile(r""" |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 65 | (?P<String> |
| 66 | \""" [^"\\]* (?: |
| 67 | (?: \\. | "(?!"") ) |
| 68 | [^"\\]* |
| 69 | )* |
| 70 | \""" |
| 71 | |
| 72 | | ''' [^'\\]* (?: |
| 73 | (?: \\. | '(?!'') ) |
| 74 | [^'\\]* |
| 75 | )* |
| 76 | ''' |
| 77 | ) |
| 78 | |
| 79 | | (?P<Method> |
| 80 | ^ |
| 81 | (?P<MethodIndent> [ \t]* ) |
| 82 | def [ \t]+ |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 83 | (?P<MethodName> [a-zA-Z_] \w* ) |
| 84 | [ \t]* \( |
| 85 | ) |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 86 | |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 87 | | (?P<Class> |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 88 | ^ |
| 89 | (?P<ClassIndent> [ \t]* ) |
| 90 | class [ \t]+ |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 91 | (?P<ClassName> [a-zA-Z_] \w* ) |
| 92 | [ \t]* |
| 93 | (?P<ClassSupers> \( [^)\n]* \) )? |
| 94 | [ \t]* : |
| 95 | ) |
| 96 | |
| 97 | | (?P<Import> |
| 98 | ^ import [ \t]+ |
| 99 | (?P<ImportList> [^#;\n]+ ) |
| 100 | ) |
| 101 | |
| 102 | | (?P<ImportFrom> |
| 103 | ^ from [ \t]+ |
| 104 | (?P<ImportFromPath> |
| 105 | [a-zA-Z_] \w* |
| 106 | (?: |
| 107 | [ \t]* \. [ \t]* [a-zA-Z_] \w* |
| 108 | )* |
| 109 | ) |
| 110 | [ \t]+ |
| 111 | import [ \t]+ |
| 112 | (?P<ImportFromList> [^#;\n]+ ) |
| 113 | ) |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 114 | """, re.VERBOSE | re.DOTALL | re.MULTILINE).search |
| 115 | |
| 116 | _modules = {} # cache of modules we've seen |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 117 | |
| 118 | # each Python class is represented by an instance of this class |
| 119 | class Class: |
| 120 | '''Class to represent a Python class.''' |
Sjoerd Mullender | 825bae7 | 1995-11-02 17:21:33 +0000 | [diff] [blame] | 121 | def __init__(self, module, name, super, file, lineno): |
| 122 | self.module = module |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 123 | self.name = name |
| 124 | if super is None: |
| 125 | super = [] |
| 126 | self.super = super |
| 127 | self.methods = {} |
| 128 | self.file = file |
| 129 | self.lineno = lineno |
| 130 | |
| 131 | def _addmethod(self, name, lineno): |
| 132 | self.methods[name] = lineno |
| 133 | |
Guido van Rossum | a3b4a33 | 1999-06-10 14:39:39 +0000 | [diff] [blame] | 134 | class Function(Class): |
| 135 | '''Class to represent a top-level Python function''' |
| 136 | def __init__(self, module, name, file, lineno): |
| 137 | Class.__init__(self, module, name, None, file, lineno) |
| 138 | def _addmethod(self, name, lineno): |
| 139 | assert 0, "Function._addmethod() shouldn't be called" |
| 140 | |
Guido van Rossum | 7a840e8 | 1998-10-12 15:21:38 +0000 | [diff] [blame] | 141 | def readmodule(module, path=[], inpackage=0): |
Guido van Rossum | a3b4a33 | 1999-06-10 14:39:39 +0000 | [diff] [blame] | 142 | '''Backwards compatible interface. |
| 143 | |
| 144 | Like readmodule_ex() but strips Function objects from the |
| 145 | resulting dictionary.''' |
| 146 | |
| 147 | dict = readmodule_ex(module, path, inpackage) |
| 148 | res = {} |
| 149 | for key, value in dict.items(): |
| 150 | if not isinstance(value, Function): |
| 151 | res[key] = value |
| 152 | return res |
| 153 | |
| 154 | def readmodule_ex(module, path=[], inpackage=0): |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 155 | '''Read a module file and return a dictionary of classes. |
| 156 | |
| 157 | Search for MODULE in PATH and sys.path, read and parse the |
| 158 | module and return a dictionary with one entry for each class |
| 159 | found in the module.''' |
| 160 | |
Guido van Rossum | 3d54871 | 1999-06-09 15:49:09 +0000 | [diff] [blame] | 161 | dict = {} |
| 162 | |
Guido van Rossum | 7a840e8 | 1998-10-12 15:21:38 +0000 | [diff] [blame] | 163 | i = string.rfind(module, '.') |
| 164 | if i >= 0: |
| 165 | # Dotted module name |
Guido van Rossum | 0688436 | 1998-10-12 15:23:04 +0000 | [diff] [blame] | 166 | package = string.strip(module[:i]) |
| 167 | submodule = string.strip(module[i+1:]) |
Guido van Rossum | 7a840e8 | 1998-10-12 15:21:38 +0000 | [diff] [blame] | 168 | parent = readmodule(package, path, inpackage) |
| 169 | child = readmodule(submodule, parent['__path__'], 1) |
| 170 | return child |
| 171 | |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 172 | if _modules.has_key(module): |
| 173 | # we've seen this module before... |
| 174 | return _modules[module] |
| 175 | if module in sys.builtin_module_names: |
| 176 | # this is a built-in module |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 177 | _modules[module] = dict |
| 178 | return dict |
| 179 | |
| 180 | # search the path for the module |
| 181 | f = None |
Guido van Rossum | 7a840e8 | 1998-10-12 15:21:38 +0000 | [diff] [blame] | 182 | if inpackage: |
| 183 | try: |
| 184 | f, file, (suff, mode, type) = \ |
| 185 | imp.find_module(module, path) |
| 186 | except ImportError: |
| 187 | f = None |
| 188 | if f is None: |
Fred Drake | 3d199af | 1999-02-18 20:51:50 +0000 | [diff] [blame] | 189 | fullpath = list(path) + sys.path |
Guido van Rossum | 7a840e8 | 1998-10-12 15:21:38 +0000 | [diff] [blame] | 190 | f, file, (suff, mode, type) = imp.find_module(module, fullpath) |
| 191 | if type == imp.PKG_DIRECTORY: |
Guido van Rossum | 3d54871 | 1999-06-09 15:49:09 +0000 | [diff] [blame] | 192 | dict['__path__'] = [file] |
Guido van Rossum | 7a840e8 | 1998-10-12 15:21:38 +0000 | [diff] [blame] | 193 | _modules[module] = dict |
Guido van Rossum | 3d54871 | 1999-06-09 15:49:09 +0000 | [diff] [blame] | 194 | path = [file] + path |
| 195 | f, file, (suff, mode, type) = \ |
| 196 | imp.find_module('__init__', [file]) |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 197 | if type != imp.PY_SOURCE: |
| 198 | # not Python source, can't do anything with this module |
| 199 | f.close() |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 200 | _modules[module] = dict |
| 201 | return dict |
| 202 | |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 203 | _modules[module] = dict |
| 204 | imports = [] |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 205 | classstack = [] # stack of (class, indent) pairs |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 206 | src = f.read() |
| 207 | f.close() |
| 208 | |
| 209 | # To avoid having to stop the regexp at each newline, instead |
| 210 | # when we need a line number we simply string.count the number of |
| 211 | # newlines in the string since the last time we did this; i.e., |
| 212 | # lineno = lineno + \ |
| 213 | # string.count(src, '\n', last_lineno_pos, here) |
| 214 | # last_lineno_pos = here |
| 215 | countnl = string.count |
| 216 | lineno, last_lineno_pos = 1, 0 |
| 217 | i = 0 |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 218 | while 1: |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 219 | m = _getnext(src, i) |
| 220 | if not m: |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 221 | break |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 222 | start, i = m.span() |
| 223 | |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 224 | if m.start("Method") >= 0: |
| 225 | # found a method definition or function |
| 226 | thisindent = _indent(m.group("MethodIndent")) |
Guido van Rossum | a3b4a33 | 1999-06-10 14:39:39 +0000 | [diff] [blame] | 227 | meth_name = m.group("MethodName") |
| 228 | lineno = lineno + \ |
| 229 | countnl(src, '\n', |
| 230 | last_lineno_pos, start) |
| 231 | last_lineno_pos = start |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 232 | # close all classes indented at least as much |
| 233 | while classstack and \ |
| 234 | classstack[-1][1] >= thisindent: |
| 235 | del classstack[-1] |
| 236 | if classstack: |
Guido van Rossum | a3b4a33 | 1999-06-10 14:39:39 +0000 | [diff] [blame] | 237 | # it's a class method |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 238 | cur_class = classstack[-1][0] |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 239 | cur_class._addmethod(meth_name, lineno) |
Guido van Rossum | a3b4a33 | 1999-06-10 14:39:39 +0000 | [diff] [blame] | 240 | else: |
| 241 | # it's a function |
| 242 | f = Function(module, meth_name, |
| 243 | file, lineno) |
| 244 | dict[meth_name] = f |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 245 | |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 246 | elif m.start("String") >= 0: |
| 247 | pass |
| 248 | |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 249 | elif m.start("Class") >= 0: |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 250 | # we found a class definition |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 251 | thisindent = _indent(m.group("ClassIndent")) |
| 252 | # close all classes indented at least as much |
| 253 | while classstack and \ |
| 254 | classstack[-1][1] >= thisindent: |
| 255 | del classstack[-1] |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 256 | lineno = lineno + \ |
| 257 | countnl(src, '\n', last_lineno_pos, start) |
| 258 | last_lineno_pos = start |
| 259 | class_name = m.group("ClassName") |
| 260 | inherit = m.group("ClassSupers") |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 261 | if inherit: |
| 262 | # the class inherits from other classes |
| 263 | inherit = string.strip(inherit[1:-1]) |
| 264 | names = [] |
| 265 | for n in string.splitfields(inherit, ','): |
| 266 | n = string.strip(n) |
| 267 | if dict.has_key(n): |
| 268 | # we know this super class |
| 269 | n = dict[n] |
| 270 | else: |
| 271 | c = string.splitfields(n, '.') |
| 272 | if len(c) > 1: |
| 273 | # super class |
| 274 | # is of the |
| 275 | # form module.class: |
| 276 | # look in |
| 277 | # module for class |
| 278 | m = c[-2] |
| 279 | c = c[-1] |
| 280 | if _modules.has_key(m): |
| 281 | d = _modules[m] |
| 282 | if d.has_key(c): |
| 283 | n = d[c] |
| 284 | names.append(n) |
| 285 | inherit = names |
| 286 | # remember this class |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 287 | cur_class = Class(module, class_name, inherit, |
| 288 | file, lineno) |
Sjoerd Mullender | 8cb4b1f | 1995-07-28 09:30:01 +0000 | [diff] [blame] | 289 | dict[class_name] = cur_class |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 290 | classstack.append((cur_class, thisindent)) |
Guido van Rossum | 31626bc | 1997-10-24 14:46:16 +0000 | [diff] [blame] | 291 | |
Guido van Rossum | ad38055 | 1999-06-07 15:25:18 +0000 | [diff] [blame] | 292 | elif m.start("Import") >= 0: |
| 293 | # import module |
| 294 | for n in string.split(m.group("ImportList"), ','): |
| 295 | n = string.strip(n) |
| 296 | try: |
| 297 | # recursively read the imported module |
| 298 | d = readmodule(n, path, inpackage) |
| 299 | except: |
| 300 | print 'module', n, 'not found' |
| 301 | |
| 302 | elif m.start("ImportFrom") >= 0: |
| 303 | # from module import stuff |
| 304 | mod = m.group("ImportFromPath") |
| 305 | names = string.split(m.group("ImportFromList"), ',') |
| 306 | try: |
| 307 | # recursively read the imported module |
| 308 | d = readmodule(mod, path, inpackage) |
| 309 | except: |
| 310 | print 'module', mod, 'not found' |
| 311 | continue |
| 312 | # add any classes that were defined in the |
| 313 | # imported module to our name space if they |
| 314 | # were mentioned in the list |
| 315 | for n in names: |
| 316 | n = string.strip(n) |
| 317 | if d.has_key(n): |
| 318 | dict[n] = d[n] |
| 319 | elif n == '*': |
| 320 | # only add a name if not |
| 321 | # already there (to mimic what |
| 322 | # Python does internally) |
| 323 | # also don't add names that |
| 324 | # start with _ |
| 325 | for n in d.keys(): |
| 326 | if n[0] != '_' and \ |
| 327 | not dict.has_key(n): |
| 328 | dict[n] = d[n] |
| 329 | else: |
| 330 | assert 0, "regexp _getnext found something unexpected" |
| 331 | |
| 332 | return dict |
Guido van Rossum | df9f7a3 | 1999-06-08 12:53:21 +0000 | [diff] [blame] | 333 | |
| 334 | def _indent(ws, _expandtabs=string.expandtabs): |
| 335 | return len(_expandtabs(ws, TABWIDTH)) |