bpo-25324: copy tok_name before changing it (#1608)
* add test to check if were modifying token
* copy list so import tokenize doesnt have side effects on token
* shorten line
* add tokenize tokens to token.h to get them to show up in token
* move ERRORTOKEN back to its previous location, and fix nitpick
* copy comments from token.h automatically
* fix whitespace and make more pythonic
* change to fix comments from @haypo
* update token.rst and Misc/NEWS
* change wording
* some more wording changes
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index dcaf58f..538612c 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1343,13 +1343,13 @@
tokens = list(tokenize(BytesIO(opstr.encode('utf-8')).readline))
num_optypes = len(optypes)
self.assertEqual(len(tokens), 2 + num_optypes)
- self.assertEqual(token.tok_name[tokens[0].exact_type],
- token.tok_name[ENCODING])
+ self.assertEqual(tok_name[tokens[0].exact_type],
+ tok_name[ENCODING])
for i in range(num_optypes):
- self.assertEqual(token.tok_name[tokens[i + 1].exact_type],
- token.tok_name[optypes[i]])
- self.assertEqual(token.tok_name[tokens[1 + num_optypes].exact_type],
- token.tok_name[token.ENDMARKER])
+ self.assertEqual(tok_name[tokens[i + 1].exact_type],
+ tok_name[optypes[i]])
+ self.assertEqual(tok_name[tokens[1 + num_optypes].exact_type],
+ tok_name[token.ENDMARKER])
def test_exact_type(self):
self.assertExactTypeEqual('()', token.LPAR, token.RPAR)