blob: 52cbdd53ab3d106ec53496079c8fd924526f4bc6 [file] [log] [blame]
Guido van Rossum0a6f9542002-12-03 08:14:35 +00001"""Parse a Python module and describe its classes and methods.
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +00002
Guido van Rossum0a6f9542002-12-03 08:14:35 +00003Parse enough of a Python file to recognize imports and class and
4method definitions, and to find out the superclasses of a class.
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +00005
6The interface consists of a single function:
Guido van Rossum0a6f9542002-12-03 08:14:35 +00007 readmodule_ex(module [, path])
8where module is the name of a Python module, and path is an optional
9list of directories where the module is to be searched. If present,
10path is prepended to the system search path sys.path. The return
11value is a dictionary. The keys of the dictionary are the names of
12the classes defined in the module (including classes that are defined
13via the from XXX import YYY construct). The values are class
14instances of the class Class defined here. One special key/value pair
15is present for packages: the key '__path__' has a list as its value
16which contains the package search path.
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000017
18A class is described by the class Class in this module. Instances
19of this class have the following instance variables:
Guido van Rossum0ed7aa12002-12-02 14:54:20 +000020 module -- the module name
Tim Peters2344fae2001-01-15 00:50:52 +000021 name -- the name of the class
22 super -- a list of super classes (Class instances)
23 methods -- a dictionary of methods
24 file -- the file in which the class was defined
25 lineno -- the line in the file on which the class statement occurred
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000026The dictionary of methods uses the method names as keys and the line
27numbers on which the method was defined as values.
28If the name of a super class is not recognized, the corresponding
29entry in the list of super classes is not a class instance but a
30string giving the name of the super class. Since import statements
31are recognized and imported modules are scanned as well, this
32shouldn't happen often.
33
Guido van Rossum0ed7aa12002-12-02 14:54:20 +000034A function is described by the class Function in this module.
35Instances of this class have the following instance variables:
36 module -- the module name
37 name -- the name of the class
38 file -- the file in which the class was defined
39 lineno -- the line in the file on which the class statement occurred
Guido van Rossum4b8c6ea2000-02-04 15:39:30 +000040"""
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000041
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000042import sys
43import imp
Christian Heimes81ee3ef2008-05-04 22:42:01 +000044import tokenize
45from token import NAME, DEDENT, OP
Raymond Hettinger3375fc52003-12-01 20:12:15 +000046from operator import itemgetter
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000047
Guido van Rossum0ed7aa12002-12-02 14:54:20 +000048__all__ = ["readmodule", "readmodule_ex", "Class", "Function"]
Skip Montanaroc62c81e2001-02-12 02:00:42 +000049
Guido van Rossumad380551999-06-07 15:25:18 +000050_modules = {} # cache of modules we've seen
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000051
52# each Python class is represented by an instance of this class
53class Class:
Tim Peters2344fae2001-01-15 00:50:52 +000054 '''Class to represent a Python class.'''
55 def __init__(self, module, name, super, file, lineno):
56 self.module = module
57 self.name = name
58 if super is None:
59 super = []
60 self.super = super
61 self.methods = {}
62 self.file = file
63 self.lineno = lineno
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000064
Tim Peters2344fae2001-01-15 00:50:52 +000065 def _addmethod(self, name, lineno):
66 self.methods[name] = lineno
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000067
Guido van Rossum0ed7aa12002-12-02 14:54:20 +000068class Function:
Tim Peters2344fae2001-01-15 00:50:52 +000069 '''Class to represent a top-level Python function'''
70 def __init__(self, module, name, file, lineno):
Guido van Rossum0ed7aa12002-12-02 14:54:20 +000071 self.module = module
72 self.name = name
73 self.file = file
74 self.lineno = lineno
Guido van Rossuma3b4a331999-06-10 14:39:39 +000075
Christian Heimes81ee3ef2008-05-04 22:42:01 +000076def readmodule(module, path=None):
Tim Peters2344fae2001-01-15 00:50:52 +000077 '''Backwards compatible interface.
Guido van Rossuma3b4a331999-06-10 14:39:39 +000078
Guido van Rossum0ed7aa12002-12-02 14:54:20 +000079 Call readmodule_ex() and then only keep Class objects from the
Tim Peters2344fae2001-01-15 00:50:52 +000080 resulting dictionary.'''
Guido van Rossuma3b4a331999-06-10 14:39:39 +000081
Tim Peters2344fae2001-01-15 00:50:52 +000082 res = {}
Christian Heimes81ee3ef2008-05-04 22:42:01 +000083 for key, value in _readmodule(module, path or []).items():
Guido van Rossum0ed7aa12002-12-02 14:54:20 +000084 if isinstance(value, Class):
Tim Peters2344fae2001-01-15 00:50:52 +000085 res[key] = value
86 return res
Guido van Rossuma3b4a331999-06-10 14:39:39 +000087
Christian Heimes81ee3ef2008-05-04 22:42:01 +000088def readmodule_ex(module, path=None):
Tim Peters2344fae2001-01-15 00:50:52 +000089 '''Read a module file and return a dictionary of classes.
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000090
Tim Peters2344fae2001-01-15 00:50:52 +000091 Search for MODULE in PATH and sys.path, read and parse the
92 module and return a dictionary with one entry for each class
Guido van Rossum0ed7aa12002-12-02 14:54:20 +000093 found in the module.
Christian Heimes81ee3ef2008-05-04 22:42:01 +000094 '''
95 return _readmodule(module, path or [])
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +000096
Christian Heimes81ee3ef2008-05-04 22:42:01 +000097def _readmodule(module, path, inpackage=None):
98 '''Do the hard work for readmodule[_ex].
99
100 If INPACKAGE is given, it must be the dotted name of the package in
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000101 which we are searching for a submodule, and then PATH must be the
102 package search path; otherwise, we are searching for a top-level
103 module, and PATH is combined with sys.path.
104 '''
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000105 # Compute the full module name (prepending inpackage if set)
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000106 if inpackage is not None:
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000107 fullmodule = "%s.%s" % (inpackage, module)
108 else:
109 fullmodule = module
110
111 # Check in the cache
112 if fullmodule in _modules:
113 return _modules[fullmodule]
114
115 # Initialize the dict for this module's contents
Tim Peters2344fae2001-01-15 00:50:52 +0000116 dict = {}
Guido van Rossum3d548711999-06-09 15:49:09 +0000117
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000118 # Check if it is a built-in module; we don't do much for these
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000119 if module in sys.builtin_module_names and inpackage is None:
Tim Peters2344fae2001-01-15 00:50:52 +0000120 _modules[module] = dict
121 return dict
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +0000122
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000123 # Check for a dotted module name
124 i = module.rfind('.')
125 if i >= 0:
126 package = module[:i]
127 submodule = module[i+1:]
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000128 parent = _readmodule(package, path, inpackage)
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000129 if inpackage is not None:
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000130 package = "%s.%s" % (inpackage, package)
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000131 return _readmodule(submodule, parent['__path__'], package)
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000132
133 # Search the path for the module
Tim Peters2344fae2001-01-15 00:50:52 +0000134 f = None
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000135 if inpackage is not None:
136 f, fname, (_s, _m, ty) = imp.find_module(module, path)
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000137 else:
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000138 f, fname, (_s, _m, ty) = imp.find_module(module, path + sys.path)
139 if ty == imp.PKG_DIRECTORY:
140 dict['__path__'] = [fname]
141 path = [fname] + path
142 f, fname, (_s, _m, ty) = imp.find_module('__init__', [fname])
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000143 _modules[fullmodule] = dict
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000144 if ty != imp.PY_SOURCE:
Tim Peters2344fae2001-01-15 00:50:52 +0000145 # not Python source, can't do anything with this module
146 f.close()
Tim Peters2344fae2001-01-15 00:50:52 +0000147 return dict
Sjoerd Mullender8cb4b1f1995-07-28 09:30:01 +0000148
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000149 stack = [] # stack of (class, indent) pairs
Guido van Rossumad380551999-06-07 15:25:18 +0000150
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000151 g = tokenize.generate_tokens(f.readline)
152 try:
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000153 for tokentype, token, start, _end, _line in g:
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000154 if tokentype == DEDENT:
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000155 lineno, thisindent = start
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000156 # close nested classes and defs
157 while stack and stack[-1][1] >= thisindent:
158 del stack[-1]
159 elif token == 'def':
160 lineno, thisindent = start
161 # close previous nested classes and defs
162 while stack and stack[-1][1] >= thisindent:
163 del stack[-1]
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000164 tokentype, meth_name, start = next(g)[0:3]
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000165 if tokentype != NAME:
166 continue # Syntax error
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000167 if stack:
168 cur_class = stack[-1][0]
169 if isinstance(cur_class, Class):
170 # it's a method
171 cur_class._addmethod(meth_name, lineno)
172 # else it's a nested def
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000173 else:
174 # it's a function
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000175 dict[meth_name] = Function(fullmodule, meth_name,
176 fname, lineno)
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000177 stack.append((None, thisindent)) # Marker for nested fns
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000178 elif token == 'class':
179 lineno, thisindent = start
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000180 # close previous nested classes and defs
181 while stack and stack[-1][1] >= thisindent:
182 del stack[-1]
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000183 tokentype, class_name, start = next(g)[0:3]
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000184 if tokentype != NAME:
185 continue # Syntax error
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000186 # parse what follows the class name
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000187 tokentype, token, start = next(g)[0:3]
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000188 inherit = None
189 if token == '(':
190 names = [] # List of superclasses
191 # there's a list of superclasses
192 level = 1
193 super = [] # Tokens making up current superclass
194 while True:
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000195 tokentype, token, start = next(g)[0:3]
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000196 if token in (')', ',') and level == 1:
197 n = "".join(super)
198 if n in dict:
199 # we know this super class
200 n = dict[n]
201 else:
202 c = n.split('.')
203 if len(c) > 1:
204 # super class is of the form
205 # module.class: look in module for
206 # class
207 m = c[-2]
208 c = c[-1]
209 if m in _modules:
210 d = _modules[m]
211 if c in d:
212 n = d[c]
213 names.append(n)
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000214 super = []
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000215 if token == '(':
216 level += 1
217 elif token == ')':
218 level -= 1
219 if level == 0:
220 break
221 elif token == ',' and level == 1:
222 pass
Thomas Wouters4d70c3d2006-06-08 14:42:34 +0000223 # only use NAME and OP (== dot) tokens for type name
224 elif tokentype in (NAME, OP) and level == 1:
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000225 super.append(token)
Thomas Wouters4d70c3d2006-06-08 14:42:34 +0000226 # expressions in the base list are not supported
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000227 inherit = names
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000228 cur_class = Class(fullmodule, class_name, inherit,
229 fname, lineno)
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000230 if not stack:
231 dict[class_name] = cur_class
232 stack.append((cur_class, thisindent))
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000233 elif token == 'import' and start[1] == 0:
234 modules = _getnamelist(g)
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000235 for mod, _mod2 in modules:
Guido van Rossum258cba82002-09-16 16:36:02 +0000236 try:
237 # Recursively read the imported module
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000238 if inpackage is None:
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000239 _readmodule(mod, path)
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000240 else:
241 try:
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000242 _readmodule(mod, path, inpackage)
Guido van Rossum0ed7aa12002-12-02 14:54:20 +0000243 except ImportError:
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000244 _readmodule(mod, [])
Guido van Rossum258cba82002-09-16 16:36:02 +0000245 except:
246 # If we can't find or parse the imported module,
247 # too bad -- don't die here.
248 pass
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000249 elif token == 'from' and start[1] == 0:
250 mod, token = _getname(g)
251 if not mod or token != "import":
252 continue
253 names = _getnamelist(g)
Tim Peters2344fae2001-01-15 00:50:52 +0000254 try:
Guido van Rossum258cba82002-09-16 16:36:02 +0000255 # Recursively read the imported module
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000256 d = _readmodule(mod, path, inpackage)
Tim Peters2344fae2001-01-15 00:50:52 +0000257 except:
Guido van Rossum258cba82002-09-16 16:36:02 +0000258 # If we can't find or parse the imported module,
259 # too bad -- don't die here.
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000260 continue
261 # add any classes that were defined in the imported module
262 # to our name space if they were mentioned in the list
263 for n, n2 in names:
264 if n in d:
265 dict[n2 or n] = d[n]
266 elif n == '*':
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000267 # don't add names that start with _
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000268 for n in d:
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000269 if n[0] != '_':
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000270 dict[n] = d[n]
271 except StopIteration:
272 pass
Guido van Rossumad380551999-06-07 15:25:18 +0000273
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000274 f.close()
Tim Peters2344fae2001-01-15 00:50:52 +0000275 return dict
Guido van Rossumdf9f7a31999-06-08 12:53:21 +0000276
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000277def _getnamelist(g):
278 # Helper to get a comma-separated list of dotted names plus 'as'
279 # clauses. Return a list of pairs (name, name2) where name2 is
280 # the 'as' name, or None if there is no 'as' clause.
281 names = []
282 while True:
283 name, token = _getname(g)
284 if not name:
285 break
286 if token == 'as':
287 name2, token = _getname(g)
288 else:
289 name2 = None
290 names.append((name, name2))
291 while token != "," and "\n" not in token:
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000292 token = next(g)[1]
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000293 if token != ",":
294 break
295 return names
296
297def _getname(g):
298 # Helper to get a dotted name, return a pair (name, token) where
299 # name is the dotted name, or None if there was no dotted name,
300 # and token is the next input token.
301 parts = []
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000302 tokentype, token = next(g)[0:2]
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000303 if tokentype != NAME and token != '*':
304 return (None, token)
305 parts.append(token)
306 while True:
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000307 tokentype, token = next(g)[0:2]
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000308 if token != '.':
309 break
Christian Heimes81ee3ef2008-05-04 22:42:01 +0000310 tokentype, token = next(g)[0:2]
Guido van Rossum040d7ca2002-08-23 01:36:01 +0000311 if tokentype != NAME:
312 break
313 parts.append(token)
314 return (".".join(parts), token)
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000315
316def _main():
317 # Main program for testing.
318 import os
319 mod = sys.argv[1]
320 if os.path.exists(mod):
321 path = [os.path.dirname(mod)]
322 mod = os.path.basename(mod)
323 if mod.lower().endswith(".py"):
324 mod = mod[:-3]
325 else:
326 path = []
327 dict = readmodule_ex(mod, path)
Raymond Hettinger8b5eb2f2011-01-27 00:06:54 +0000328 objs = list(dict.values())
Raymond Hettingerd4cb56d2008-01-30 02:55:10 +0000329 objs.sort(key=lambda a: getattr(a, 'lineno', 0))
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000330 for obj in objs:
331 if isinstance(obj, Class):
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000332 print("class", obj.name, obj.super, obj.lineno)
Guido van Rossumcc2b0162007-02-11 06:12:03 +0000333 methods = sorted(obj.methods.items(), key=itemgetter(1))
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000334 for name, lineno in methods:
335 if name != "__path__":
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000336 print(" def", name, lineno)
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000337 elif isinstance(obj, Function):
Guido van Rossumbe19ed72007-02-09 05:37:30 +0000338 print("def", obj.name, obj.lineno)
Guido van Rossum0a6f9542002-12-03 08:14:35 +0000339
340if __name__ == "__main__":
341 _main()