PEP 3107 - Function Annotations thanks to Tony Lownds
diff --git a/Lib/compiler/ast.py b/Lib/compiler/ast.py
index d21905f..ac48710 100644
--- a/Lib/compiler/ast.py
+++ b/Lib/compiler/ast.py
@@ -33,7 +33,10 @@
pass # implemented by subclasses
class EmptyNode(Node):
- pass
+ def getChildNodes(self):
+ return ()
+ def getChildren(self):
+ return ()
class Expression(Node):
# Expression is an artificial node class to support "eval"
@@ -487,12 +490,13 @@
return "From(%s, %s, %s)" % (repr(self.modname), repr(self.names), repr(self.level))
class Function(Node):
- def __init__(self, decorators, name, argnames, defaults, kwonlyargs, flags, doc, code, lineno=None):
+ def __init__(self, decorators, name, arguments, defaults, kwonlyargs, returns, flags, doc, code, lineno=None):
self.decorators = decorators
self.name = name
- self.argnames = argnames
+ self.arguments = arguments
self.defaults = defaults
self.kwonlyargs = kwonlyargs
+ self.returns = returns
self.flags = flags
self.doc = doc
self.code = code
@@ -508,9 +512,10 @@
children = []
children.append(self.decorators)
children.append(self.name)
- children.append(self.argnames)
+ children.extend(flatten(self.arguments))
children.extend(flatten(self.defaults))
- children.append(self.kwonlyargs)
+ children.extend(flatten(self.kwonlyargs))
+ children.append(self.returns)
children.append(self.flags)
children.append(self.doc)
children.append(self.code)
@@ -520,18 +525,22 @@
nodelist = []
if self.decorators is not None:
nodelist.append(self.decorators)
+ nodelist.extend(flatten_nodes(self.arguments))
nodelist.extend(flatten_nodes(self.defaults))
+ nodelist.extend(flatten_nodes(self.kwonlyargs))
+ if self.returns is not None:
+ nodelist.append(self.returns)
nodelist.append(self.code)
return tuple(nodelist)
def __repr__(self):
- return "Function(%s, %s, %s, %s, %s, %s, %s, %s)" % (repr(self.decorators), repr(self.name), repr(self.argnames), repr(self.defaults), repr(self.kwonlyargs), repr(self.flags), repr(self.doc), repr(self.code))
+ return "Function(%s, %s, %s, %s, %s, %s, %s, %s, %s)" % (repr(self.decorators), repr(self.name), repr(self.arguments), repr(self.defaults), repr(self.kwonlyargs), repr(self.returns), repr(self.flags), repr(self.doc), repr(self.code))
class GenExpr(Node):
def __init__(self, code, lineno=None):
self.code = code
self.lineno = lineno
- self.argnames = ['.0']
+ self.arguments = [SimpleArg('.0', None)]
self.varargs = self.kwargs = None
self.kwonlyargs = ()
@@ -715,9 +724,24 @@
def __repr__(self):
return "Keyword(%s, %s)" % (repr(self.name), repr(self.expr))
+class Kwarg(Node):
+ def __init__(self, arg, expr, lineno=None):
+ self.arg = arg
+ self.expr = expr
+ self.lineno = lineno
+
+ def getChildren(self):
+ return self.arg, self.expr
+
+ def getChildNodes(self):
+ return self.arg, self.expr
+
+ def __repr__(self):
+ return "Kwarg(%s, %s)" % (repr(self.arg), repr(self.expr))
+
class Lambda(Node):
- def __init__(self, argnames, defaults, kwonlyargs, flags, code, lineno=None):
- self.argnames = argnames
+ def __init__(self, arguments, defaults, kwonlyargs, flags, code, lineno=None):
+ self.arguments = arguments
self.defaults = defaults
self.kwonlyargs = kwonlyargs
self.flags = flags
@@ -728,25 +752,28 @@
self.varargs = 1
if flags & CO_VARKEYWORDS:
self.kwargs = 1
+ self.returns = None
def getChildren(self):
children = []
- children.append(self.argnames)
+ children.extend(flatten(self.arguments))
children.extend(flatten(self.defaults))
- children.append(self.kwonlyargs)
+ children.extend(flatten(self.kwonlyargs))
children.append(self.flags)
children.append(self.code)
return tuple(children)
def getChildNodes(self):
nodelist = []
+ nodelist.extend(flatten_nodes(self.arguments))
nodelist.extend(flatten_nodes(self.defaults))
+ nodelist.extend(flatten_nodes(self.kwonlyargs))
nodelist.append(self.code)
return tuple(nodelist)
def __repr__(self):
- return "Lambda(%s, %s, %s, %s, %s)" % (repr(self.argnames), repr(self.defaults), repr(self.kwonlyargs), repr(self.flags), repr(self.code))
+ return "Lambda(%s, %s, %s, %s, %s)" % (repr(self.arguments), repr(self.defaults), repr(self.kwonlyargs), repr(self.flags), repr(self.code))
class LeftShift(Node):
def __init__(self, (left, right), lineno=None):
@@ -897,6 +924,22 @@
def __repr__(self):
return "Name(%s)" % (repr(self.name),)
+class NestedArgs(Node):
+ def __init__(self, args, lineno=None):
+ self.args = args
+ self.lineno = lineno
+
+ def getChildren(self):
+ return tuple(flatten(self.args))
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.args))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "NestedArgs(%s)" % (repr(self.args),)
+
class Not(Node):
def __init__(self, expr, lineno=None):
self.expr = expr
@@ -1071,6 +1114,27 @@
def __repr__(self):
return "Set(%s)" % (repr(self.items),)
+class SimpleArg(Node):
+ def __init__(self, name, annotation, lineno=None):
+ self.name = name
+ self.annotation = annotation
+ self.lineno = lineno
+
+ def getChildren(self):
+ children = []
+ children.append(self.name)
+ children.append(self.annotation)
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ if self.annotation is not None:
+ nodelist.append(self.annotation)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "SimpleArg(%s, %s)" % (repr(self.name), repr(self.annotation))
+
class Slice(Node):
def __init__(self, expr, flags, lower, upper, lineno=None):
self.expr = expr
diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py
index 7f44d46..86e2c49 100644
--- a/Lib/compiler/pyassem.py
+++ b/Lib/compiler/pyassem.py
@@ -314,7 +314,7 @@
super_init = FlowGraph.__init__
def __init__(self, name, filename,
- args=(), kwonlyargs={}, optimized=0, klass=None):
+ args=(), kwonlyargs=(), optimized=0, klass=None):
self.super_init()
self.name = name
self.filename = filename
@@ -338,24 +338,40 @@
# The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both
# kinds of variables.
self.closure = []
- self.varnames = list(args) or []
- for i in range(len(self.varnames)):
+ # The varnames list needs to be computed after flags have been set
+ self.varnames = []
+ self.stage = RAW
+
+ def computeVarnames(self):
+ # self.args is positional, vararg, kwarg, kwonly, unpacked. This
+ # order is due to the visit order in symbol module and could change.
+ # argcount is # len(self.args) - len(unpacked). We want
+ # self.varnames to be positional, kwonly, vararg, kwarg, unpacked
+ # and argcount to be len(positional).
+
+ # determine starting index of unpacked, kwonly, vararg
+ u = self.argcount # starting index of unpacked
+ k = u - len(self.kwonlyargs) # starting index of kwonly
+ v = k - self.checkFlag(CO_VARARGS) - self.checkFlag(CO_VARKEYWORDS)
+
+ vars = list(self.args)
+ self.varnames = vars[:v] + vars[k:u] + vars[v:k] + vars[u:]
+ self.argcount = v
+
+ # replace TupleArgs with calculated var name
+ for i in range(self.argcount):
var = self.varnames[i]
if isinstance(var, TupleArg):
self.varnames[i] = var.getName()
- self.stage = RAW
def setDocstring(self, doc):
self.docstring = doc
def setFlag(self, flag):
self.flags = self.flags | flag
- if flag == CO_VARARGS:
- self.argcount = self.argcount - 1
def checkFlag(self, flag):
- if self.flags & flag:
- return 1
+ return (self.flags & flag) == flag
def setFreeVars(self, names):
self.freevars = list(names)
@@ -366,6 +382,7 @@
def getCode(self):
"""Get a Python code object"""
assert self.stage == RAW
+ self.computeVarnames()
self.computeStackDepth()
self.flattenGraph()
assert self.stage == FLAT
@@ -575,6 +592,12 @@
lnotab.nextLine(oparg)
continue
hi, lo = twobyte(oparg)
+
+ extended, hi = twobyte(hi)
+ if extended:
+ ehi, elo = twobyte(extended)
+ lnotab.addCode(self.opnum['EXTENDED_ARG'], elo, ehi)
+
try:
lnotab.addCode(self.opnum[opname], lo, hi)
except ValueError:
@@ -595,8 +618,6 @@
else:
nlocals = len(self.varnames)
argcount = self.argcount
- if self.flags & CO_VARKEYWORDS:
- argcount = argcount - 1
kwonlyargcount = len(self.kwonlyargs)
return new.code(argcount, kwonlyargcount,
nlocals, self.stacksize, self.flags,
@@ -809,7 +830,8 @@
return self.CALL_FUNCTION(argc)-2
def MAKE_FUNCTION(self, argc):
hi, lo = divmod(argc, 256)
- return -(lo + hi * 2)
+ ehi, hi = divmod(hi, 256)
+ return -(lo + hi * 2 + ehi)
def MAKE_CLOSURE(self, argc):
# XXX need to account for free variables too!
return -argc
diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py
index 353c2c9..325ca06 100644
--- a/Lib/compiler/pycodegen.py
+++ b/Lib/compiler/pycodegen.py
@@ -378,18 +378,57 @@
walk(node.code, gen)
gen.finish()
self.set_lineno(node)
+ num_kwargs = 0
for keyword in node.kwonlyargs:
default = keyword.expr
if isinstance(default, ast.EmptyNode):
continue
- self.emit('LOAD_CONST', keyword.name)
+ self.emit('LOAD_CONST', keyword.arg.name)
self.visit(default)
+ num_kwargs += 1
for default in node.defaults:
self.visit(default)
- self._makeClosure(gen, len(node.defaults))
+
+ num_annotations = self._visit_annotations(node)
+
+ oparg = len(node.defaults)
+ oparg |= num_kwargs << 8
+ oparg |= num_annotations << 16
+
+ self._makeClosure(gen, oparg)
for i in range(ndecorators):
self.emit('CALL_FUNCTION', 1)
+ def _visit_annotations(self, node):
+ # emit code, return num_annotations
+ annotations = []
+ annotations.extend(self._visit_argument_annotations(node.arguments))
+ annotations.extend(self._visit_kwarg_annotations(node.kwonlyargs))
+ if node.returns:
+ self.visit(node.returns)
+ annotations.append('return')
+ if not annotations:
+ return 0
+ self.emit('LOAD_CONST', tuple(annotations))
+ return len(annotations) + 1
+
+ def _visit_argument_annotations(self, arguments):
+ for arg in arguments:
+ if isinstance(arg, ast.SimpleArg):
+ if arg.annotation:
+ self.visit(arg.annotation)
+ yield arg.name
+ else:
+ for name in self._visit_argument_annotations(arg.args):
+ yield name
+
+ def _visit_kwarg_annotations(self, kwargs):
+ for kwarg in kwargs:
+ arg = kwarg.arg
+ if arg.annotation:
+ self.visit(arg.annotation)
+ yield arg.name
+
def visitClass(self, node):
gen = self.ClassGen(node, self.scopes,
self.get_module())
@@ -1323,7 +1362,7 @@
else:
name = func.name
- args, hasTupleArg = generateArgList(func.argnames)
+ args, hasTupleArg = generateArgList(func.arguments)
kwonlyargs = generateKwonlyArgList(func.kwonlyargs)
self.graph = pyassem.PyFlowGraph(name, func.filename, args,
kwonlyargs=kwonlyargs,
@@ -1334,7 +1373,7 @@
if not isLambda and func.doc:
self.setDocstring(func.doc)
- lnf = walk(func.code, self.NameFinder(args), verbose=0)
+ lnf = walk(func.code, self.NameFinder(args+kwonlyargs), verbose=0)
self.locals.push(lnf.getLocals())
if func.varargs:
self.graph.setFlag(CO_VARARGS)
@@ -1342,7 +1381,7 @@
self.graph.setFlag(CO_VARKEYWORDS)
self.set_lineno(func)
if hasTupleArg:
- self.generateArgUnpack(func.argnames)
+ self.generateArgUnpack(func.arguments)
def get_module(self):
return self.module
@@ -1356,9 +1395,9 @@
def generateArgUnpack(self, args):
for i in range(len(args)):
arg = args[i]
- if isinstance(arg, tuple):
+ if isinstance(arg, ast.NestedArgs):
self.emit('LOAD_FAST', '.%d' % (i * 2))
- self.unpackSequence(arg)
+ self.unpackSequence(tuple(_nested_names(arg)))
def unpackSequence(self, tup):
if VERSION > 1:
@@ -1452,21 +1491,29 @@
count = 0
for i in range(len(arglist)):
elt = arglist[i]
- if isinstance(elt, str):
- args.append(elt)
- elif isinstance(elt, tuple):
- args.append(TupleArg(i * 2, elt))
- extra.extend(misc.flatten(elt))
+ if isinstance(elt, ast.SimpleArg):
+ args.append(elt.name)
+ elif isinstance(elt, ast.NestedArgs):
+ t = tuple(_nested_names(elt))
+ args.append(TupleArg(i * 2, t))
+ extra.extend(misc.flatten(t))
count = count + 1
else:
raise ValueError, "unexpect argument type:", elt
return args + extra, count
+def _nested_names(elt):
+ for arg in elt.args:
+ if isinstance(arg, ast.SimpleArg):
+ yield arg.name
+ elif isinstance(arg, ast.NestedArgs):
+ yield tuple(_nested_names(arg))
+
def generateKwonlyArgList(keywordOnlyArgs):
- kwonlyargs = {}
+ kwonlyargs = []
for elt in keywordOnlyArgs:
- assert isinstance(elt, ast.Keyword)
- kwonlyargs[elt.name] = elt.expr
+ assert isinstance(elt, ast.Kwarg)
+ kwonlyargs.append(elt.arg.name)
return kwonlyargs
def findOp(node):
diff --git a/Lib/compiler/symbols.py b/Lib/compiler/symbols.py
index 7ddf42c..3585efc 100644
--- a/Lib/compiler/symbols.py
+++ b/Lib/compiler/symbols.py
@@ -233,7 +233,12 @@
if parent.nested or isinstance(parent, FunctionScope):
scope.nested = 1
self.scopes[node] = scope
- self._do_args(scope, node.argnames)
+
+ args = node.arguments
+ for kwonly in node.kwonlyargs:
+ args.append(kwonly.arg)
+ self._do_arguments(scope, args)
+
self.visit(node.code, scope)
self.handle_free_vars(scope, parent)
@@ -275,16 +280,18 @@
if parent.nested or isinstance(parent, FunctionScope):
scope.nested = 1
self.scopes[node] = scope
- self._do_args(scope, node.argnames)
+ self._do_arguments(scope, node.arguments)
self.visit(node.code, scope)
self.handle_free_vars(scope, parent)
- def _do_args(self, scope, args):
- for name in args:
- if type(name) == types.TupleType:
- self._do_args(scope, name)
+ def _do_arguments(self, scope, arguments):
+ for node in arguments:
+ if isinstance(node, ast.SimpleArg):
+ scope.add_param(node.name)
+ if node.annotation:
+ self.visit(node.annotation, scope)
else:
- scope.add_param(name)
+ self._do_arguments(scope, node.args)
def handle_free_vars(self, scope, parent):
parent.add_child(scope)
diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py
index dc88222..4a8e623 100644
--- a/Lib/compiler/transformer.py
+++ b/Lib/compiler/transformer.py
@@ -234,25 +234,24 @@
return Decorators(items)
def funcdef(self, nodelist):
- # -6 -5 -4 -3 -2 -1
- # funcdef: [decorators] 'def' NAME parameters ':' suite
- # parameters: '(' [varargslist] ')'
-
- if len(nodelist) == 6:
- assert nodelist[0][0] == symbol.decorators
+ # 0 1 2 4 -1
+ # funcdef: [decorators] 'def' NAME parameters ['->' test] ':' suite
+ # parameters: '(' [typedargslist] ')'
+ if nodelist[0][0] == symbol.decorators:
decorators = self.decorators(nodelist[0][1:])
+ nodelist = nodelist[1:]
else:
- assert len(nodelist) == 5
decorators = None
+ assert len(nodelist) in (5, 7)
- lineno = nodelist[-4][2]
- name = nodelist[-4][1]
- args = nodelist[-3][2]
+ lineno = nodelist[0][2]
+ name = nodelist[1][1]
+ args = nodelist[2][2]
- if args[0] == symbol.varargslist:
- names, defaults, kwonlyargs, flags = self.com_arglist(args[1:])
+ if args[0] == symbol.varargslist or args[0] == symbol.typedargslist:
+ arguments, defaults, kwonly, flags = self.com_arglist(args[1:])
else:
- names = defaults = kwonlyargs = ()
+ arguments = defaults = kwonly = ()
flags = 0
doc = self.get_docstring(nodelist[-1])
@@ -263,22 +262,28 @@
assert isinstance(code, Stmt)
assert isinstance(code.nodes[0], Discard)
del code.nodes[0]
- return Function(decorators, name, names, defaults,
- kwonlyargs, flags, doc, code, lineno=lineno)
+
+ if len(nodelist) == 7:
+ returns = self.com_node(nodelist[4])
+ else:
+ returns = None
+
+ return Function(decorators, name, arguments, defaults,
+ kwonly, returns, flags, doc, code, lineno=lineno)
def lambdef(self, nodelist):
# lambdef: 'lambda' [varargslist] ':' test
if nodelist[2][0] == symbol.varargslist:
- names, defaults, kwonlyargs, flags = \
+ arguments, defaults, kwonlyargs, flags = \
self.com_arglist(nodelist[2][1:])
else:
- names = defaults = kwonlyargs = ()
+ arguments = defaults = kwonlyargs = ()
flags = 0
# code for lambda
code = self.com_node(nodelist[-1])
- return Lambda(names, defaults, kwonlyargs,
+ return Lambda(arguments, defaults, kwonlyargs,
flags, code, lineno=nodelist[1][2])
old_lambdef = lambdef
@@ -324,10 +329,25 @@
def varargslist(self, nodelist):
raise WalkerError
- def fpdef(self, nodelist):
+ def vfpdef(self, nodelist):
raise WalkerError
- def fplist(self, nodelist):
+ def vfplist(self, nodelist):
+ raise WalkerError
+
+ def vname(self, nodelist):
+ raise WalkerError
+
+ def typedargslist(self, nodelist):
+ raise WalkerError
+
+ def tfpdef(self, nodelist):
+ raise WalkerError
+
+ def tfplist(self, nodelist):
+ raise WalkerError
+
+ def tname(self, nodelist):
raise WalkerError
def dotted_name(self, nodelist):
@@ -786,9 +806,10 @@
return Discard(Const(None))
def keywordonlyargs(self, nodelist):
- # (',' NAME ['=' test])*
+ # (',' tname ['=' test])*
# ^^^
# ------+
+ # tname and vname are handled.
kwonlyargs = []
i = 0
while i < len(nodelist):
@@ -802,10 +823,25 @@
i += 2
if node[0] == token.DOUBLESTAR:
return kwonlyargs, i
- elif node[0] == token.NAME:
- kwonlyargs.append(Keyword(node[1], default, lineno=node[2]))
+ elif node[0] in (symbol.vname, symbol.tname):
+ lineno = extractLineNo(node)
+ kwarg = Kwarg(self._simplearg(node), default, lineno=lineno)
+ kwonlyargs.append(kwarg)
i += 2
return kwonlyargs, i
+
+ def _simplearg(self, node):
+ # tname: NAME [':' test]
+ # vname: NAME
+ assert node[0] == symbol.vname or node[0] == symbol.tname
+ name = node[1][1]
+ lineno = node[1][2]
+ assert isinstance(name, str)
+ if len(node) > 2:
+ annotation = self.com_node(node[3])
+ else:
+ annotation = None
+ return SimpleArg(name, annotation, lineno)
def com_arglist(self, nodelist):
# varargslist:
@@ -814,7 +850,7 @@
# | fpdef ['=' test] (',' fpdef ['=' test])* [',']
# fpdef: NAME | '(' fplist ')'
# fplist: fpdef (',' fpdef)* [',']
- names = []
+ arguments = []
kwonlyargs = []
defaults = []
flags = 0
@@ -825,14 +861,15 @@
if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
if node[0] == token.STAR:
node = nodelist[i+1]
- if node[0] == token.NAME: # vararg
- names.append(node[1])
+ if node[0] in (symbol.tname, symbol.vname): # vararg
+ arguments.append(self._simplearg(node))
flags = flags | CO_VARARGS
i = i + 3
else: # no vararg
assert node[0] == token.COMMA
i += 2
- if i < len(nodelist) and nodelist[i][0] == token.NAME:
+ if i < len(nodelist) and \
+ nodelist[i][0] in (symbol.tname, symbol.vname):
kwonlyargs, skip = self.keywordonlyargs(nodelist[i:])
i += skip
@@ -843,13 +880,13 @@
node = nodelist[i+1]
else:
raise ValueError, "unexpected token: %s" % t
- names.append(node[1])
+ arguments.append(self._simplearg(node))
flags = flags | CO_VARKEYWORDS
break
- # fpdef: NAME | '(' fplist ')'
- names.append(self.com_fpdef(node))
+ # tfpdef: tname | '(' tfplist ')'
+ arguments.append(self.com_tfpdef(node))
i = i + 1
if i < len(nodelist) and nodelist[i][0] == token.EQUAL:
@@ -863,21 +900,24 @@
# skip the comma
i = i + 1
- return names, defaults, kwonlyargs, flags
+ return arguments, defaults, kwonlyargs, flags
- def com_fpdef(self, node):
- # fpdef: NAME | '(' fplist ')'
+ def com_tfpdef(self, node):
+ # tfpdef: tname | '(' tfplist ')'
+ # def f((x)): -- x is not nested
+ while node[1][0] == token.LPAR and len(node[2]) == 2:
+ node = node[2][1]
if node[1][0] == token.LPAR:
- return self.com_fplist(node[2])
- return node[1][1]
+ return NestedArgs(self.com_tfplist(node[2]))
+ return self._simplearg(node[1])
- def com_fplist(self, node):
- # fplist: fpdef (',' fpdef)* [',']
+ def com_tfplist(self, node):
+ # tfplist: tfpdef (',' tfpdef)* [',']
if len(node) == 2:
- return self.com_fpdef(node[1])
+ return self.com_tfpdef(node[1]),
list = []
for i in range(1, len(node), 2):
- list.append(self.com_fpdef(node[i]))
+ list.append(self.com_tfpdef(node[i]))
return tuple(list)
def com_dotted_name(self, node):
diff --git a/Lib/symbol.py b/Lib/symbol.py
index a7f7a85..658974c 100755
--- a/Lib/symbol.py
+++ b/Lib/symbol.py
@@ -17,82 +17,87 @@
decorators = 260
funcdef = 261
parameters = 262
-varargslist = 263
-fpdef = 264
-fplist = 265
-stmt = 266
-simple_stmt = 267
-small_stmt = 268
-expr_stmt = 269
-augassign = 270
-print_stmt = 271
-del_stmt = 272
-pass_stmt = 273
-flow_stmt = 274
-break_stmt = 275
-continue_stmt = 276
-return_stmt = 277
-yield_stmt = 278
-raise_stmt = 279
-import_stmt = 280
-import_name = 281
-import_from = 282
-import_as_name = 283
-dotted_as_name = 284
-import_as_names = 285
-dotted_as_names = 286
-dotted_name = 287
-global_stmt = 288
-assert_stmt = 289
-compound_stmt = 290
-if_stmt = 291
-while_stmt = 292
-for_stmt = 293
-try_stmt = 294
-with_stmt = 295
-with_var = 296
-except_clause = 297
-suite = 298
-testlist_safe = 299
-old_test = 300
-old_lambdef = 301
-test = 302
-or_test = 303
-and_test = 304
-not_test = 305
-comparison = 306
-comp_op = 307
-expr = 308
-xor_expr = 309
-and_expr = 310
-shift_expr = 311
-arith_expr = 312
-term = 313
-factor = 314
-power = 315
-atom = 316
-listmaker = 317
-testlist_gexp = 318
-lambdef = 319
-trailer = 320
-subscriptlist = 321
-subscript = 322
-sliceop = 323
-exprlist = 324
-testlist = 325
-dictsetmaker = 326
-classdef = 327
-arglist = 328
-argument = 329
-list_iter = 330
-list_for = 331
-list_if = 332
-gen_iter = 333
-gen_for = 334
-gen_if = 335
-testlist1 = 336
-encoding_decl = 337
-yield_expr = 338
+typedargslist = 263
+tname = 264
+tfpdef = 265
+tfplist = 266
+varargslist = 267
+vname = 268
+vfpdef = 269
+vfplist = 270
+stmt = 271
+simple_stmt = 272
+small_stmt = 273
+expr_stmt = 274
+augassign = 275
+print_stmt = 276
+del_stmt = 277
+pass_stmt = 278
+flow_stmt = 279
+break_stmt = 280
+continue_stmt = 281
+return_stmt = 282
+yield_stmt = 283
+raise_stmt = 284
+import_stmt = 285
+import_name = 286
+import_from = 287
+import_as_name = 288
+dotted_as_name = 289
+import_as_names = 290
+dotted_as_names = 291
+dotted_name = 292
+global_stmt = 293
+assert_stmt = 294
+compound_stmt = 295
+if_stmt = 296
+while_stmt = 297
+for_stmt = 298
+try_stmt = 299
+with_stmt = 300
+with_var = 301
+except_clause = 302
+suite = 303
+testlist_safe = 304
+old_test = 305
+old_lambdef = 306
+test = 307
+or_test = 308
+and_test = 309
+not_test = 310
+comparison = 311
+comp_op = 312
+expr = 313
+xor_expr = 314
+and_expr = 315
+shift_expr = 316
+arith_expr = 317
+term = 318
+factor = 319
+power = 320
+atom = 321
+listmaker = 322
+testlist_gexp = 323
+lambdef = 324
+trailer = 325
+subscriptlist = 326
+subscript = 327
+sliceop = 328
+exprlist = 329
+testlist = 330
+dictsetmaker = 331
+classdef = 332
+arglist = 333
+argument = 334
+list_iter = 335
+list_for = 336
+list_if = 337
+gen_iter = 338
+gen_for = 339
+gen_if = 340
+testlist1 = 341
+encoding_decl = 342
+yield_expr = 343
#--end constants--
sym_name = {}
diff --git a/Lib/test/output/test_tokenize b/Lib/test/output/test_tokenize
index 4a3d58c..a46824d 100644
--- a/Lib/test/output/test_tokenize
+++ b/Lib/test/output/test_tokenize
@@ -682,4 +682,20 @@
177,11-177,15: NAME 'pass'
177,15-177,16: NEWLINE '\n'
178,0-178,1: NL '\n'
-179,0-179,0: ENDMARKER ''
+179,0-179,1: OP '@'
+179,1-179,13: NAME 'staticmethod'
+179,13-179,14: NEWLINE '\n'
+180,0-180,3: NAME 'def'
+180,4-180,7: NAME 'foo'
+180,7-180,8: OP '('
+180,8-180,9: NAME 'x'
+180,9-180,10: OP ':'
+180,10-180,11: NUMBER '1'
+180,11-180,12: OP ')'
+180,12-180,14: OP '->'
+180,14-180,15: NUMBER '1'
+180,15-180,16: OP ':'
+180,17-180,21: NAME 'pass'
+180,21-180,22: NEWLINE '\n'
+181,0-181,1: NL '\n'
+182,0-182,0: ENDMARKER ''
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 11623ec..914f1d9 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -151,9 +151,9 @@
#### EVERYTHING BELOW IS GENERATED #####
exec_results = [
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], None, [], []), [('Pass', (1, 9))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]),
('Module', [('ClassDef', (1, 0), 'C', [], [('Pass', (1, 8))])]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [])]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]),
@@ -180,13 +180,13 @@
('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
-('Expression', ('Lambda', (1, 0), ('arguments', [], None, [], None, [], []), ('Name', (1, 7), 'None', ('Load',)))),
+('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Name', (1, 7), 'None', ('Load',)))),
('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
-('Expression', ('Num', (1, 0), 10L)),
+('Expression', ('Num', (1, 0), 10)),
('Expression', ('Str', (1, 0), 'string')),
('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))),
diff --git a/Lib/test/test_compiler.py b/Lib/test/test_compiler.py
index 783a34c..2ecd093 100644
--- a/Lib/test/test_compiler.py
+++ b/Lib/test/test_compiler.py
@@ -115,6 +115,24 @@
dct = {}
exec(c, dct)
self.assertEquals(dct.get('result'), 3)
+ c = compiler.compile('def g(a):\n'
+ ' def f(): return a + 2\n'
+ ' return f()\n'
+ 'result = g(1)',
+ '<string>',
+ 'exec')
+ dct = {}
+ exec(c, dct)
+ self.assertEquals(dct.get('result'), 3)
+ c = compiler.compile('def g((a, b)):\n'
+ ' def f(): return a + b\n'
+ ' return f()\n'
+ 'result = g((1, 2))',
+ '<string>',
+ 'exec')
+ dct = {}
+ exec(c, dct)
+ self.assertEquals(dct.get('result'), 3)
def testGenExp(self):
c = compiler.compile('list((i,j) for i in range(3) if i < 3'
@@ -123,6 +141,22 @@
'eval')
self.assertEquals(eval(c), [(0, 3), (1, 3), (2, 3)])
+ def testFuncAnnotations(self):
+ testdata = [
+ ('def f(a: 1): pass', {'a': 1}),
+ ('''def f(a, (b:1, c:2, d), e:3=4, f=5,
+ *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass
+ ''', {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
+ 'k': 11, 'return': 12}),
+ ]
+ for sourcecode, expected in testdata:
+ # avoid IndentationError: unexpected indent from trailing lines
+ sourcecode = sourcecode.rstrip()+'\n'
+ c = compiler.compile(sourcecode, '<string>', 'exec')
+ dct = {}
+ exec(c, dct)
+ self.assertEquals(dct['f'].func_annotations, expected)
+
NOLINENO = (compiler.ast.Module, compiler.ast.Stmt, compiler.ast.Discard)
@@ -167,10 +201,11 @@
###############################################################################
-def test_main():
+def test_main(all=False):
global TEST_ALL
- TEST_ALL = test.test_support.is_resource_enabled("compiler")
+ TEST_ALL = all or test.test_support.is_resource_enabled("compiler")
test.test_support.run_unittest(CompilerTest)
if __name__ == "__main__":
- test_main()
+ import sys
+ test_main('all' in sys.argv)
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index f4a0478..34c550e 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -138,16 +138,22 @@
x = eval('1, 0 or 1')
def testFuncdef(self):
- ### 'def' NAME parameters ':' suite
- ### parameters: '(' [varargslist] ')'
- ### varargslist: (fpdef ['=' test] ',')*
- ### ('*' (NAME|',' fpdef ['=' test]) [',' ('**'|'*' '*') NAME]
- ### | ('**'|'*' '*') NAME)
- ### | fpdef ['=' test] (',' fpdef ['=' test])* [',']
- ### fpdef: NAME | '(' fplist ')'
- ### fplist: fpdef (',' fpdef)* [',']
- ### arglist: (argument ',')* (argument | *' test [',' '**' test] | '**' test)
- ### argument: [test '='] test # Really [keyword '='] test
+ ### [decorators] 'def' NAME parameters ['->' test] ':' suite
+ ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+ ### decorators: decorator+
+ ### parameters: '(' [typedargslist] ')'
+ ### typedargslist: ((tfpdef ['=' test] ',')*
+ ### ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname)
+ ### | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])
+ ### tname: NAME [':' test]
+ ### tfpdef: tname | '(' tfplist ')'
+ ### tfplist: tfpdef (',' tfpdef)* [',']
+ ### varargslist: ((vfpdef ['=' test] ',')*
+ ### ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname)
+ ### | vfpdef ['=' test] (',' vfpdef ['=' test])* [','])
+ ### vname: NAME
+ ### vfpdef: vname | '(' vfplist ')'
+ ### vfplist: vfpdef (',' vfpdef)* [',']
def f1(): pass
f1()
f1(*())
@@ -294,6 +300,28 @@
pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
+ # argument annotation tests
+ def f(x) -> list: pass
+ self.assertEquals(f.func_annotations, {'return': list})
+ def f(x:int): pass
+ self.assertEquals(f.func_annotations, {'x': int})
+ def f(*x:str): pass
+ self.assertEquals(f.func_annotations, {'x': str})
+ def f(**x:float): pass
+ self.assertEquals(f.func_annotations, {'x': float})
+ def f(x, y:1+2): pass
+ self.assertEquals(f.func_annotations, {'y': 3})
+ def f(a, (b:1, c:2, d)): pass
+ self.assertEquals(f.func_annotations, {'b': 1, 'c': 2})
+ def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass
+ self.assertEquals(f.func_annotations,
+ {'b': 1, 'c': 2, 'e': 3, 'g': 6})
+ def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6, h:7, i=8, j:9=10,
+ **k:11) -> 12: pass
+ self.assertEquals(f.func_annotations,
+ {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
+ 'k': 11, 'return': 12})
+
def testLambdef(self):
### lambdef: 'lambda' [varargslist] ':' test
l1 = lambda : 0
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index be6c18d..de6e888 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -219,5 +219,15 @@
if verbose:
print 'finished'
+def test_rarrow():
+ """
+ This function exists solely to test the tokenization of the RARROW
+ operator.
+
+ >>> tokenize(iter(['->']).next) #doctest: +NORMALIZE_WHITESPACE
+ 1,0-1,2:\tOP\t'->'
+ 2,0-2,0:\tENDMARKER\t''
+ """
+
if __name__ == "__main__":
test_main()
diff --git a/Lib/test/tokenize_tests.txt b/Lib/test/tokenize_tests.txt
index 1facfc1..b1aa020 100644
--- a/Lib/test/tokenize_tests.txt
+++ b/Lib/test/tokenize_tests.txt
@@ -176,3 +176,6 @@
@staticmethod
def foo(): pass
+@staticmethod
+def foo(x:1)->1: pass
+
diff --git a/Lib/token.py b/Lib/token.py
index 5f8d53a..2770cfd 100755
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -60,9 +60,10 @@
DOUBLESLASH = 48
DOUBLESLASHEQUAL = 49
AT = 50
-OP = 51
-ERRORTOKEN = 52
-N_TOKENS = 53
+RARROW = 51
+OP = 52
+ERRORTOKEN = 53
+N_TOKENS = 54
NT_OFFSET = 256
#--end constants--
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1253822..152bfdb 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -78,7 +78,7 @@
# longest operators first (e.g., if = came before ==, == would get
# recognized as two instances of =).
Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
- r"//=?",
+ r"//=?", r"->",
r"[+\-*/%&|^=<>]=?",
r"~")