bpo-44667: Treat correctly lines ending with comments and no newlines in the Python tokenizer (GH-27499) (GH-27500)
(cherry picked from commit b6bde9fc42aecad5be0457198d17cfe7b481ad79)
Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 681f2c7..4bce1ca 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1458,6 +1458,16 @@ def test_pathological_trailing_whitespace(self):
# See http://bugs.python.org/issue16152
self.assertExactTypeEqual('@ ', token.AT)
+ def test_comment_at_the_end_of_the_source_without_newline(self):
+ # See http://bugs.python.org/issue44667
+ source = 'b = 1\n\n#test'
+ expected_tokens = [token.NAME, token.EQUAL, token.NUMBER, token.NEWLINE, token.NL, token.COMMENT]
+
+ tokens = list(tokenize(BytesIO(source.encode('utf-8')).readline))
+ self.assertEqual(tok_name[tokens[0].exact_type], tok_name[ENCODING])
+ for i in range(6):
+ self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]])
+ self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER])
class UntokenizeTest(TestCase):