bpo-30465: Fix lineno and col_offset in fstring AST nodes (GH-1800) (gh-3409)
For f-string ast nodes, fix the line and columns so that tools such as flake8 can identify them correctly.
(cherry picked from commit e7c566caf177afe43b57f0b2723e723d880368e8)
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 3d762b5..5e7efe2 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -70,6 +70,253 @@
# Make sure x was called.
self.assertTrue(x.called)
+ def test_ast_line_numbers(self):
+ expr = """
+a = 10
+f'{a * x()}'"""
+ t = ast.parse(expr)
+ self.assertEqual(type(t), ast.Module)
+ self.assertEqual(len(t.body), 2)
+ # check `a = 10`
+ self.assertEqual(type(t.body[0]), ast.Assign)
+ self.assertEqual(t.body[0].lineno, 2)
+ # check `f'...'`
+ self.assertEqual(type(t.body[1]), ast.Expr)
+ self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+ self.assertEqual(len(t.body[1].value.values), 1)
+ self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue)
+ self.assertEqual(t.body[1].lineno, 3)
+ self.assertEqual(t.body[1].value.lineno, 3)
+ self.assertEqual(t.body[1].value.values[0].lineno, 3)
+ # check the binop location
+ binop = t.body[1].value.values[0].value
+ self.assertEqual(type(binop), ast.BinOp)
+ self.assertEqual(type(binop.left), ast.Name)
+ self.assertEqual(type(binop.op), ast.Mult)
+ self.assertEqual(type(binop.right), ast.Call)
+ self.assertEqual(binop.lineno, 3)
+ self.assertEqual(binop.left.lineno, 3)
+ self.assertEqual(binop.right.lineno, 3)
+ self.assertEqual(binop.col_offset, 3)
+ self.assertEqual(binop.left.col_offset, 3)
+ self.assertEqual(binop.right.col_offset, 7)
+
+ def test_ast_line_numbers_multiple_formattedvalues(self):
+ expr = """
+f'no formatted values'
+f'eggs {a * x()} spam {b + y()}'"""
+ t = ast.parse(expr)
+ self.assertEqual(type(t), ast.Module)
+ self.assertEqual(len(t.body), 2)
+ # check `f'no formatted value'`
+ self.assertEqual(type(t.body[0]), ast.Expr)
+ self.assertEqual(type(t.body[0].value), ast.JoinedStr)
+ self.assertEqual(t.body[0].lineno, 2)
+ # check `f'...'`
+ self.assertEqual(type(t.body[1]), ast.Expr)
+ self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+ self.assertEqual(len(t.body[1].value.values), 4)
+ self.assertEqual(type(t.body[1].value.values[0]), ast.Str)
+ self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue)
+ self.assertEqual(type(t.body[1].value.values[2]), ast.Str)
+ self.assertEqual(type(t.body[1].value.values[3]), ast.FormattedValue)
+ self.assertEqual(t.body[1].lineno, 3)
+ self.assertEqual(t.body[1].value.lineno, 3)
+ self.assertEqual(t.body[1].value.values[0].lineno, 3)
+ self.assertEqual(t.body[1].value.values[1].lineno, 3)
+ self.assertEqual(t.body[1].value.values[2].lineno, 3)
+ self.assertEqual(t.body[1].value.values[3].lineno, 3)
+ # check the first binop location
+ binop1 = t.body[1].value.values[1].value
+ self.assertEqual(type(binop1), ast.BinOp)
+ self.assertEqual(type(binop1.left), ast.Name)
+ self.assertEqual(type(binop1.op), ast.Mult)
+ self.assertEqual(type(binop1.right), ast.Call)
+ self.assertEqual(binop1.lineno, 3)
+ self.assertEqual(binop1.left.lineno, 3)
+ self.assertEqual(binop1.right.lineno, 3)
+ self.assertEqual(binop1.col_offset, 8)
+ self.assertEqual(binop1.left.col_offset, 8)
+ self.assertEqual(binop1.right.col_offset, 12)
+ # check the second binop location
+ binop2 = t.body[1].value.values[3].value
+ self.assertEqual(type(binop2), ast.BinOp)
+ self.assertEqual(type(binop2.left), ast.Name)
+ self.assertEqual(type(binop2.op), ast.Add)
+ self.assertEqual(type(binop2.right), ast.Call)
+ self.assertEqual(binop2.lineno, 3)
+ self.assertEqual(binop2.left.lineno, 3)
+ self.assertEqual(binop2.right.lineno, 3)
+ self.assertEqual(binop2.col_offset, 23)
+ self.assertEqual(binop2.left.col_offset, 23)
+ self.assertEqual(binop2.right.col_offset, 27)
+
+ def test_ast_line_numbers_nested(self):
+ expr = """
+a = 10
+f'{a * f"-{x()}-"}'"""
+ t = ast.parse(expr)
+ self.assertEqual(type(t), ast.Module)
+ self.assertEqual(len(t.body), 2)
+ # check `a = 10`
+ self.assertEqual(type(t.body[0]), ast.Assign)
+ self.assertEqual(t.body[0].lineno, 2)
+ # check `f'...'`
+ self.assertEqual(type(t.body[1]), ast.Expr)
+ self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+ self.assertEqual(len(t.body[1].value.values), 1)
+ self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue)
+ self.assertEqual(t.body[1].lineno, 3)
+ self.assertEqual(t.body[1].value.lineno, 3)
+ self.assertEqual(t.body[1].value.values[0].lineno, 3)
+ # check the binop location
+ binop = t.body[1].value.values[0].value
+ self.assertEqual(type(binop), ast.BinOp)
+ self.assertEqual(type(binop.left), ast.Name)
+ self.assertEqual(type(binop.op), ast.Mult)
+ self.assertEqual(type(binop.right), ast.JoinedStr)
+ self.assertEqual(binop.lineno, 3)
+ self.assertEqual(binop.left.lineno, 3)
+ self.assertEqual(binop.right.lineno, 3)
+ self.assertEqual(binop.col_offset, 3)
+ self.assertEqual(binop.left.col_offset, 3)
+ self.assertEqual(binop.right.col_offset, 7)
+ # check the nested call location
+ self.assertEqual(len(binop.right.values), 3)
+ self.assertEqual(type(binop.right.values[0]), ast.Str)
+ self.assertEqual(type(binop.right.values[1]), ast.FormattedValue)
+ self.assertEqual(type(binop.right.values[2]), ast.Str)
+ self.assertEqual(binop.right.values[0].lineno, 3)
+ self.assertEqual(binop.right.values[1].lineno, 3)
+ self.assertEqual(binop.right.values[2].lineno, 3)
+ call = binop.right.values[1].value
+ self.assertEqual(type(call), ast.Call)
+ self.assertEqual(call.lineno, 3)
+ self.assertEqual(call.col_offset, 11)
+
+ def test_ast_line_numbers_duplicate_expression(self):
+ """Duplicate expression
+
+ NOTE: this is currently broken, always sets location of the first
+ expression.
+ """
+ expr = """
+a = 10
+f'{a * x()} {a * x()} {a * x()}'
+"""
+ t = ast.parse(expr)
+ self.assertEqual(type(t), ast.Module)
+ self.assertEqual(len(t.body), 2)
+ # check `a = 10`
+ self.assertEqual(type(t.body[0]), ast.Assign)
+ self.assertEqual(t.body[0].lineno, 2)
+ # check `f'...'`
+ self.assertEqual(type(t.body[1]), ast.Expr)
+ self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+ self.assertEqual(len(t.body[1].value.values), 5)
+ self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue)
+ self.assertEqual(type(t.body[1].value.values[1]), ast.Str)
+ self.assertEqual(type(t.body[1].value.values[2]), ast.FormattedValue)
+ self.assertEqual(type(t.body[1].value.values[3]), ast.Str)
+ self.assertEqual(type(t.body[1].value.values[4]), ast.FormattedValue)
+ self.assertEqual(t.body[1].lineno, 3)
+ self.assertEqual(t.body[1].value.lineno, 3)
+ self.assertEqual(t.body[1].value.values[0].lineno, 3)
+ self.assertEqual(t.body[1].value.values[1].lineno, 3)
+ self.assertEqual(t.body[1].value.values[2].lineno, 3)
+ self.assertEqual(t.body[1].value.values[3].lineno, 3)
+ self.assertEqual(t.body[1].value.values[4].lineno, 3)
+ # check the first binop location
+ binop = t.body[1].value.values[0].value
+ self.assertEqual(type(binop), ast.BinOp)
+ self.assertEqual(type(binop.left), ast.Name)
+ self.assertEqual(type(binop.op), ast.Mult)
+ self.assertEqual(type(binop.right), ast.Call)
+ self.assertEqual(binop.lineno, 3)
+ self.assertEqual(binop.left.lineno, 3)
+ self.assertEqual(binop.right.lineno, 3)
+ self.assertEqual(binop.col_offset, 3)
+ self.assertEqual(binop.left.col_offset, 3)
+ self.assertEqual(binop.right.col_offset, 7)
+ # check the second binop location
+ binop = t.body[1].value.values[2].value
+ self.assertEqual(type(binop), ast.BinOp)
+ self.assertEqual(type(binop.left), ast.Name)
+ self.assertEqual(type(binop.op), ast.Mult)
+ self.assertEqual(type(binop.right), ast.Call)
+ self.assertEqual(binop.lineno, 3)
+ self.assertEqual(binop.left.lineno, 3)
+ self.assertEqual(binop.right.lineno, 3)
+ self.assertEqual(binop.col_offset, 3) # FIXME: this is wrong
+ self.assertEqual(binop.left.col_offset, 3) # FIXME: this is wrong
+ self.assertEqual(binop.right.col_offset, 7) # FIXME: this is wrong
+ # check the third binop location
+ binop = t.body[1].value.values[4].value
+ self.assertEqual(type(binop), ast.BinOp)
+ self.assertEqual(type(binop.left), ast.Name)
+ self.assertEqual(type(binop.op), ast.Mult)
+ self.assertEqual(type(binop.right), ast.Call)
+ self.assertEqual(binop.lineno, 3)
+ self.assertEqual(binop.left.lineno, 3)
+ self.assertEqual(binop.right.lineno, 3)
+ self.assertEqual(binop.col_offset, 3) # FIXME: this is wrong
+ self.assertEqual(binop.left.col_offset, 3) # FIXME: this is wrong
+ self.assertEqual(binop.right.col_offset, 7) # FIXME: this is wrong
+
+ def test_ast_line_numbers_multiline_fstring(self):
+ # FIXME: This test demonstrates invalid behavior due to JoinedStr's
+ # immediate child nodes containing the wrong lineno. The enclosed
+ # expressions have valid line information and column offsets.
+ # See bpo-16806 and bpo-30465 for details.
+ expr = """
+a = 10
+f'''
+ {a
+ *
+ x()}
+non-important content
+'''
+"""
+ t = ast.parse(expr)
+ self.assertEqual(type(t), ast.Module)
+ self.assertEqual(len(t.body), 2)
+ # check `a = 10`
+ self.assertEqual(type(t.body[0]), ast.Assign)
+ self.assertEqual(t.body[0].lineno, 2)
+ # check `f'...'`
+ self.assertEqual(type(t.body[1]), ast.Expr)
+ self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+ self.assertEqual(len(t.body[1].value.values), 3)
+ self.assertEqual(type(t.body[1].value.values[0]), ast.Str)
+ self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue)
+ self.assertEqual(type(t.body[1].value.values[2]), ast.Str)
+ # NOTE: the following invalid behavior is described in bpo-16806.
+ # - line number should be the *first* line (3), not the *last* (8)
+ # - column offset should not be -1
+ self.assertEqual(t.body[1].lineno, 8)
+ self.assertEqual(t.body[1].value.lineno, 8)
+ self.assertEqual(t.body[1].value.values[0].lineno, 8)
+ self.assertEqual(t.body[1].value.values[1].lineno, 8)
+ self.assertEqual(t.body[1].value.values[2].lineno, 8)
+ self.assertEqual(t.body[1].col_offset, -1)
+ self.assertEqual(t.body[1].value.col_offset, -1)
+ self.assertEqual(t.body[1].value.values[0].col_offset, -1)
+ self.assertEqual(t.body[1].value.values[1].col_offset, -1)
+ self.assertEqual(t.body[1].value.values[2].col_offset, -1)
+ # NOTE: the following lineno information and col_offset is correct for
+ # expressions within FormattedValues.
+ binop = t.body[1].value.values[1].value
+ self.assertEqual(type(binop), ast.BinOp)
+ self.assertEqual(type(binop.left), ast.Name)
+ self.assertEqual(type(binop.op), ast.Mult)
+ self.assertEqual(type(binop.right), ast.Call)
+ self.assertEqual(binop.lineno, 4)
+ self.assertEqual(binop.left.lineno, 4)
+ self.assertEqual(binop.right.lineno, 6)
+ self.assertEqual(binop.col_offset, 3)
+ self.assertEqual(binop.left.col_offset, 3)
+ self.assertEqual(binop.right.col_offset, 7)
+
def test_docstring(self):
def f():
f'''Not a docstring'''
@@ -786,5 +1033,6 @@
self.assertEqual(eval('f"\\\n"'), '')
self.assertEqual(eval('f"\\\r"'), '')
+
if __name__ == '__main__':
unittest.main()