Merged revisions 65715,65724,65726,65732,65736-65739,65775 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r65715 | benjamin.peterson | 2008-08-16 16:04:16 -0500 (Sat, 16 Aug 2008) | 1 line
add some documentation for symtable
........
r65724 | benjamin.peterson | 2008-08-16 17:11:33 -0500 (Sat, 16 Aug 2008) | 2 lines
include filename and line number in SyntaxError
........
r65726 | georg.brandl | 2008-08-16 17:37:05 -0500 (Sat, 16 Aug 2008) | 2 lines
Review symtable docs.
........
r65732 | benjamin.peterson | 2008-08-16 18:29:40 -0500 (Sat, 16 Aug 2008) | 1 line
PySTEntry's constructor is static; there's no point in a fancy API name
........
r65736 | benjamin.peterson | 2008-08-16 20:09:17 -0500 (Sat, 16 Aug 2008) | 1 line
expose PySTEntry.nested so the symtable module will work
........
r65737 | benjamin.peterson | 2008-08-16 20:17:15 -0500 (Sat, 16 Aug 2008) | 1 line
a few improvements
........
r65738 | benjamin.peterson | 2008-08-16 20:27:30 -0500 (Sat, 16 Aug 2008) | 1 line
fix compile errors
........
r65739 | benjamin.peterson | 2008-08-16 21:23:43 -0500 (Sat, 16 Aug 2008) | 1 line
uhh PySTEntry->ste_unoptimized has to be exposed too
........
r65775 | benjamin.peterson | 2008-08-17 12:13:26 -0500 (Sun, 17 Aug 2008) | 5 lines
get the symtable module back in working order
- Fix broken functions
- Add (hopefully) extensive tests
- Modernize a little
........
diff --git a/Doc/library/language.rst b/Doc/library/language.rst
index bcf9ac0..3cc4b82 100644
--- a/Doc/library/language.rst
+++ b/Doc/library/language.rst
@@ -16,6 +16,7 @@
parser.rst
ast.rst
+ symtable.rst
symbol.rst
token.rst
keyword.rst
diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst
new file mode 100644
index 0000000..0082287
--- /dev/null
+++ b/Doc/library/symtable.rst
@@ -0,0 +1,187 @@
+:mod:`symtable` --- Access to the compiler's symbol tables
+==========================================================
+
+.. module:: symtable
+ :synopsis: Interface to the compiler's internal symbol tables.
+
+.. moduleauthor:: Jeremy Hylton <jeremy@alum.mit.edu>
+.. sectionauthor:: Benjamin Peterson
+
+
+Symbol tables are generated by the compiler from AST just before bytecode is
+generated. The symbol table is responsible for calculating the scope of every
+identifier in the code. :mod:`symtable` provides an interface to examine these
+tables.
+
+
+Generating Symbol Tables
+------------------------
+
+.. function:: symtable(code, filename, compile_type)
+
+ Return the toplevel :class:`SymbolTable` for the Python source *code*.
+ *filename* is the name of the file containing the code. *compile_type* is
+ like the *mode* argument to :func:`compile`.
+
+
+Examining Symbol Tables
+-----------------------
+
+.. class:: SymbolTable
+
+ A namespace table for a block. The constructor is not public.
+
+ .. method:: get_type()
+
+ Return the type of the symbol table. Possible values are ``'class'``,
+ ``'module'``, and ``'function'``.
+
+ .. method:: get_id()
+
+ Return the table's identifier.
+
+ .. method:: get_name()
+
+ Return the table's name. This is the name of the class if the table is
+ for a class, the name of the function if the table is for a function, or
+ ``'top'`` if the table is global (:meth:`get_type` returns ``'module'``).
+
+ .. method:: get_lineno()
+
+ Return the number of the first line in the block this table represents.
+
+ .. method:: is_optimized()
+
+ Return ``True`` if the locals in this table can be optimized.
+
+ .. method:: is_nested()
+
+ Return ``True`` if the block is a nested class or function.
+
+ .. method:: has_children()
+
+ Return ``True`` if the block has nested namespaces within it. These can
+ be obtained with :meth:`get_children`.
+
+ .. method:: has_exec()
+
+ Return ``True`` if the block uses ``exec``.
+
+ .. method:: has_import_start()
+
+ Return ``True`` if the block uses a starred from-import.
+
+ .. method:: get_identifiers()
+
+ Return a list of names of symbols in this table.
+
+ .. method:: lookup(name)
+
+ Lookup *name* in the table and return a :class:`Symbol` instance.
+
+ .. method:: get_symbols()
+
+ Return a list of :class:`Symbol` instances for names in the table.
+
+ .. method:: get_children()
+
+ Return a list of the nested symbol tables.
+
+
+.. class:: Function
+
+ A namespace for a function or method. This class inherits
+ :class:`SymbolTable`.
+
+ .. method:: get_parameters()
+
+ Return a tuple containing names of parameters to this function.
+
+ .. method:: get_locals()
+
+ Return a tuple containing names of locals in this function.
+
+ .. method:: get_globals()
+
+ Return a tuple containing names of globals in this function.
+
+ .. method:: get_frees()
+
+ Return a tuple containing names of free variables in this function.
+
+
+.. class:: Class
+
+ A namespace of a class. This class inherits :class:`SymbolTable`.
+
+ .. method:: get_methods()
+
+ Return a tuple containing the names of methods declared in the class.
+
+
+.. class:: Symbol
+
+ An entry in a :class:`SymbolTable` corresponding to an identifier in the
+ source. The constructor is not public.
+
+ .. method:: get_name()
+
+ Return the symbol's name.
+
+ .. method:: is_referenced()
+
+ Return ``True`` if the symbol is used in its block.
+
+ .. method:: is_imported()
+
+ Return ``True`` if the symbol is created from an import statement.
+
+ .. method:: is_parameter()
+
+ Return ``True`` if the symbol is a parameter.
+
+ .. method:: is_global()
+
+ Return ``True`` if the symbol is global.
+
+ .. method:: is_vararg()
+
+ Return ``True`` if the symbol is a star arg (receives varargs).
+
+ .. method:: is_kewordarg()
+
+ Return ``True`` if the symbol is a two-star arg (receives keyword
+ arguments).
+
+ .. method:: is_local()
+
+ Return ``True`` if the symbol is local to its block.
+
+ .. method:: is_free()
+
+ Return ``True`` if the symbol is referenced in its block, but not assigned
+ to.
+
+ .. method:: is_assigned()
+
+ Return ``True`` if the symbol is assigned to in its block.
+
+ .. method:: is_namespace()
+
+ Return ``True`` if name binding introduces new namespace.
+
+ If the name is used as the target of a function or class statement, this
+ will be true.
+
+ Note that a single name can be bound to multiple objects. If the result
+ is ``True``, the name may also be bound to other objects, like an int or
+ list, that does not introduce a new namespace.
+
+ .. method:: get_namespaces()
+
+ Return a list of namespaces bound to this name.
+
+ .. method:: get_namespace()
+
+ Return the namespace bound to this name. If more than one namespace is
+ bound, a :exc:`ValueError` is raised.
diff --git a/Include/symtable.h b/Include/symtable.h
index 8c1dc41..b07d77c 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -36,7 +36,7 @@
PyObject *ste_children; /* list of child blocks */
_Py_block_ty ste_type; /* module, class, or function */
int ste_unoptimized; /* false if namespace is optimized */
- unsigned ste_nested : 1; /* true if block is nested */
+ int ste_nested; /* true if block is nested */
unsigned ste_free : 1; /* true if block has free variables */
unsigned ste_child_free : 1; /* true if a child block has free vars,
including free refs to globals */
diff --git a/Lib/symtable.py b/Lib/symtable.py
index 089c8dc..ff332f2 100644
--- a/Lib/symtable.py
+++ b/Lib/symtable.py
@@ -1,10 +1,11 @@
"""Interface to the compiler's internal symbol tables"""
import _symtable
-from _symtable import USE, DEF_GLOBAL, DEF_LOCAL, DEF_PARAM, \
- DEF_STAR, DEF_DOUBLESTAR, DEF_INTUPLE, DEF_FREE, \
- DEF_FREE_GLOBAL, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND, \
- OPT_IMPORT_STAR
+from _symtable import (USE, DEF_GLOBAL, DEF_LOCAL, DEF_PARAM,
+ DEF_STAR, DEF_DOUBLESTAR, DEF_INTUPLE, DEF_FREE,
+ DEF_FREE_GLOBAL, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND,
+ OPT_IMPORT_STAR, SCOPE_OFF, SCOPE_MASK, FREE,
+ GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
import weakref
@@ -38,15 +39,9 @@
newSymbolTable = SymbolTableFactory()
-def is_free(flags):
- if (flags & (USE | DEF_FREE)) \
- and (flags & (DEF_LOCAL | DEF_PARAM | DEF_GLOBAL)):
- return True
- if flags & DEF_FREE_CLASS:
- return True
- return False
-class SymbolTable:
+class SymbolTable(object):
+
def __init__(self, raw_table, filename):
self._table = raw_table
self._filename = filename
@@ -59,10 +54,11 @@
kind = "%s " % self.__class__.__name__
if self._table.name == "global":
- return "<%sSymbolTable for module %s>" % (kind, self._filename)
+ return "<{0}SymbolTable for module {1}>".format(kind, self._filename)
else:
- return "<%sSymbolTable for %s in %s>" % (kind, self._table.name,
- self._filename)
+ return "<{0}SymbolTable for {1} in {2}>".format(kind,
+ self._table.name,
+ self._filename)
def get_type(self):
if self._table.type == _symtable.TYPE_MODULE:
@@ -72,7 +68,7 @@
if self._table.type == _symtable.TYPE_CLASS:
return "class"
assert self._table.type in (1, 2, 3), \
- "unexpected type: %s" % self._table.type
+ "unexpected type: {0}".format(self._table.type)
def get_id(self):
return self._table.id
@@ -124,6 +120,7 @@
return [newSymbolTable(st, self._filename)
for st in self._table.children]
+
class Function(SymbolTable):
# Default values for instance variables
@@ -148,15 +145,18 @@
def get_globals(self):
if self.__globals is None:
- glob = DEF_GLOBAL | DEF_FREE_GLOBAL
- self.__globals = self.__idents_matching(lambda x:x & glob)
+ glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
+ test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob
+ self.__globals = self.__idents_matching(test)
return self.__globals
def get_frees(self):
if self.__frees is None:
+ is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE
self.__frees = self.__idents_matching(is_free)
return self.__frees
+
class Class(SymbolTable):
__methods = None
@@ -169,14 +169,17 @@
self.__methods = tuple(d)
return self.__methods
-class Symbol:
+
+class Symbol(object):
+
def __init__(self, name, flags, namespaces=None):
self.__name = name
self.__flags = flags
+ self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope()
self.__namespaces = namespaces or ()
def __repr__(self):
- return "<symbol '%s'>" % self.__name
+ return "<symbol {0!r}>".format(self.__name)
def get_name(self):
return self.__name
@@ -188,8 +191,7 @@
return bool(self.__flags & DEF_PARAM)
def is_global(self):
- return bool((self.__flags & DEF_GLOBAL)
- or (self.__flags & DEF_FREE_GLOBAL))
+ return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT))
def is_vararg(self):
return bool(self.__flags & DEF_STAR)
@@ -201,12 +203,7 @@
return bool(self.__flags & DEF_BOUND)
def is_free(self):
- if (self.__flags & (USE | DEF_FREE)) \
- and (self.__flags & (DEF_LOCAL | DEF_PARAM | DEF_GLOBAL)):
- return True
- if self.__flags & DEF_FREE_CLASS:
- return True
- return False
+ return bool(self.__scope == FREE)
def is_imported(self):
return bool(self.__flags & DEF_IMPORT)
diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py
index 77115e0..7e0206d 100644
--- a/Lib/test/test_symtable.py
+++ b/Lib/test/test_symtable.py
@@ -1,31 +1,161 @@
-from test import support
-
+"""
+Test the API of the symtable module.
+"""
import symtable
import unittest
+from test import support
-## XXX
-## Test disabled because symtable module needs to be rewritten for new compiler
-##vereq(symbols[0].name, "global")
-##vereq(len([ste for ste in symbols.values() if ste.name == "f"]), 1)
+TEST_CODE = """
+import sys
-### Bug tickler: SyntaxError file name correct whether error raised
-### while parsing or building symbol table.
-##def checkfilename(brokencode):
-## try:
-## _symtable.symtable(brokencode, "spam", "exec")
-## except SyntaxError, e:
-## vereq(e.filename, "spam")
-## else:
-## raise TestFailed("no SyntaxError for %r" % (brokencode,))
-##checkfilename("def f(x): foo)(") # parse-time
-##checkfilename("def f(x): global x") # symtable-build-time
+glob = 42
+
+class Mine:
+ instance_var = 24
+ def a_method(p1, p2):
+ pass
+
+def spam(a, b, *var, **kw):
+ global bar
+ bar = 47
+ x = 23
+ glob
+ def internal():
+ return x
+ return internal
+
+def foo():
+ pass
+
+def namespace_test(): pass
+def namespace_test(): pass
+"""
+
+
+def find_block(block, name):
+ for ch in block.get_children():
+ if ch.get_name() == name:
+ return ch
+
class SymtableTest(unittest.TestCase):
- def test_invalid_args(self):
- self.assertRaises(TypeError, symtable.symtable, "42")
- self.assertRaises(ValueError, symtable.symtable, "42", "?", "")
+
+ top = symtable.symtable(TEST_CODE, "?", "exec")
+ # These correspond to scopes in TEST_CODE
+ Mine = find_block(top, "Mine")
+ a_method = find_block(Mine, "a_method")
+ spam = find_block(top, "spam")
+ internal = find_block(spam, "internal")
+ foo = find_block(top, "foo")
+
+ def test_type(self):
+ self.assertEqual(self.top.get_type(), "module")
+ self.assertEqual(self.Mine.get_type(), "class")
+ self.assertEqual(self.a_method.get_type(), "function")
+ self.assertEqual(self.spam.get_type(), "function")
+ self.assertEqual(self.internal.get_type(), "function")
+
+ def test_optimized(self):
+ self.assertFalse(self.top.is_optimized())
+ self.assertFalse(self.top.has_exec())
+
+ self.assertTrue(self.spam.is_optimized())
+
+ def test_nested(self):
+ self.assertFalse(self.top.is_nested())
+ self.assertFalse(self.Mine.is_nested())
+ self.assertFalse(self.spam.is_nested())
+ self.assertTrue(self.internal.is_nested())
+
+ def test_children(self):
+ self.assertTrue(self.top.has_children())
+ self.assertTrue(self.Mine.has_children())
+ self.assertFalse(self.foo.has_children())
+
+ def test_lineno(self):
+ self.assertEqual(self.top.get_lineno(), 0)
+ self.assertEqual(self.spam.get_lineno(), 11)
+
+ def test_function_info(self):
+ func = self.spam
+ self.assertEqual(func.get_parameters(), ("a", "b", "kw", "var"))
+ self.assertEqual(func.get_locals(),
+ ("a", "b", "bar", "glob", "internal", "kw", "var", "x"))
+ self.assertEqual(func.get_globals(), ("bar", "glob"))
+ self.assertEqual(self.internal.get_frees(), ("x",))
+
+ def test_globals(self):
+ self.assertTrue(self.spam.lookup("glob").is_global())
+ self.assertTrue(self.spam.lookup("bar").is_global())
+ self.assertFalse(self.internal.lookup("x").is_global())
+ self.assertFalse(self.Mine.lookup("instance_var").is_global())
+
+ def test_local(self):
+ self.assertTrue(self.spam.lookup("x").is_local())
+ self.assertFalse(self.internal.lookup("x").is_local())
+
+ def test_referenced(self):
+ self.assertTrue(self.internal.lookup("x").is_referenced())
+ self.assertTrue(self.spam.lookup("internal").is_referenced())
+ self.assertFalse(self.spam.lookup("x").is_referenced())
+
+ def test_parameters(self):
+ for sym in ("a", "var", "kw"):
+ self.assertTrue(self.spam.lookup(sym).is_parameter())
+ self.assertFalse(self.spam.lookup("x").is_parameter())
+
+ def test_symbol_lookup(self):
+ self.assertEqual(len(self.top.get_identifiers()),
+ len(self.top.get_symbols()))
+
+ self.assertRaises(KeyError, self.top.lookup, "not_here")
+
+ def test_namespaces(self):
+ self.assertTrue(self.top.lookup("Mine").is_namespace())
+ self.assertTrue(self.Mine.lookup("a_method").is_namespace())
+ self.assertTrue(self.top.lookup("spam").is_namespace())
+ self.assertTrue(self.spam.lookup("internal").is_namespace())
+ self.assertTrue(self.top.lookup("namespace_test").is_namespace())
+ self.assertFalse(self.spam.lookup("x").is_namespace())
+
+ self.assert_(self.top.lookup("spam").get_namespace() is self.spam)
+ ns_test = self.top.lookup("namespace_test")
+ self.assertEqual(len(ns_test.get_namespaces()), 2)
+ self.assertRaises(ValueError, ns_test.get_namespace)
+
+ def test_assigned(self):
+ self.assertTrue(self.spam.lookup("x").is_assigned())
+ self.assertTrue(self.spam.lookup("bar").is_assigned())
+ self.assertTrue(self.top.lookup("spam").is_assigned())
+ self.assertTrue(self.Mine.lookup("a_method").is_assigned())
+ self.assertFalse(self.internal.lookup("x").is_assigned())
+
+ def test_imported(self):
+ self.assertTrue(self.top.lookup("sys").is_imported())
+
+ def test_name(self):
+ self.assertEqual(self.top.get_name(), "top")
+ self.assertEqual(self.spam.get_name(), "spam")
+ self.assertEqual(self.spam.lookup("x").get_name(), "x")
+ self.assertEqual(self.Mine.get_name(), "Mine")
+
+ def test_class_info(self):
+ self.assertEqual(self.Mine.get_methods(), ('a_method',))
+
+ def test_filename_correct(self):
+ ### Bug tickler: SyntaxError file name correct whether error raised
+ ### while parsing or building symbol table.
+ def checkfilename(brokencode):
+ try:
+ symtable.symtable(brokencode, "spam", "exec")
+ except SyntaxError as e:
+ self.assertEqual(e.filename, "spam")
+ else:
+ self.fail("no SyntaxError for %r" % (brokencode,))
+ checkfilename("def f(x): foo)(") # parse-time
+ checkfilename("def f(x): global x") # symtable-build-time
def test_eval(self):
symbols = symtable.symtable("42", "?", "eval")
diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c
index e3c9e51..3388d63 100644
--- a/Modules/symtablemodule.c
+++ b/Modules/symtablemodule.c
@@ -92,6 +92,10 @@
PyModule_AddIntConstant(m, "GLOBAL_IMPLICIT", GLOBAL_IMPLICIT);
PyModule_AddIntConstant(m, "FREE", FREE);
PyModule_AddIntConstant(m, "CELL", CELL);
+
+ PyModule_AddIntConstant(m, "SCOPE_OFF", SCOPE_OFFSET);
+ PyModule_AddIntConstant(m, "SCOPE_MASK", SCOPE_MASK);
+
if (PyErr_Occurred()) {
Py_DECREF(m);
m = 0;
diff --git a/Python/symtable.c b/Python/symtable.c
index 92432ab..e3bb270 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -22,9 +22,9 @@
#define RETURN_VAL_IN_GENERATOR \
"'return' with argument inside generator"
-/* XXX(nnorwitz): change name since static? */
+
static PySTEntryObject *
-PySTEntry_New(struct symtable *st, identifier name, _Py_block_ty block,
+ste_new(struct symtable *st, identifier name, _Py_block_ty block,
void *key, int lineno)
{
PySTEntryObject *ste = NULL;
@@ -114,6 +114,8 @@
{"symbols", T_OBJECT, OFF(ste_symbols), READONLY},
{"varnames", T_OBJECT, OFF(ste_varnames), READONLY},
{"children", T_OBJECT, OFF(ste_children), READONLY},
+ {"optimized",T_INT, OFF(ste_unoptimized), READONLY},
+ {"nested", T_INT, OFF(ste_nested), READONLY},
{"type", T_INT, OFF(ste_type), READONLY},
{"lineno", T_INT, OFF(ste_lineno), READONLY},
{NULL}
@@ -388,6 +390,9 @@
PyErr_Format(PyExc_SyntaxError,
"name '%U' is parameter and global",
name);
+ PyErr_SyntaxLocation(ste->ste_table->st_filename,
+ ste->ste_lineno);
+
return 0;
}
if (flags & DEF_NONLOCAL) {
@@ -788,7 +793,7 @@
return 1;
}
-/* symtable_enter_block() gets a reference via PySTEntry_New().
+/* symtable_enter_block() gets a reference via ste_new.
This reference is released when the block is exited, via the DECREF
in symtable_exit_block().
*/
@@ -825,7 +830,7 @@
}
Py_DECREF(st->st_cur);
}
- st->st_cur = PySTEntry_New(st, name, block, ast, lineno);
+ st->st_cur = ste_new(st, name, block, ast, lineno);
if (st->st_cur == NULL)
return 0;
if (name == GET_IDENTIFIER(top))