bpo-44667: Treat correctly lines ending with comments and no newlines in the Python tokenizer (GH-27499) (GH-27500)
(cherry picked from commit b6bde9fc42aecad5be0457198d17cfe7b481ad79)
Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 42c1f10..7d7736f 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -604,7 +604,7 @@ def _tokenize(readline, encoding):
pos += 1
# Add an implicit NEWLINE if the input doesn't end in one
- if last_line and last_line[-1] not in '\r\n':
+ if last_line and last_line[-1] not in '\r\n' and not last_line.strip().startswith("#"):
yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
for indent in indents[1:]: # pop remaining indent levels
yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')