Issue #14629: Raise SyntaxError in tokenizer.detect_encoding
if the first two lines have non-UTF-8 characters without an encoding declaration.
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index f575e9b..f283c6d 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -292,9 +292,12 @@
def find_cookie(line):
try:
- line_string = line.decode('ascii')
+ # Decode as UTF-8. Either the line is an encoding declaration,
+ # in which case it should be pure ASCII, or it must be UTF-8
+ # per default encoding.
+ line_string = line.decode('utf-8')
except UnicodeDecodeError:
- return None
+ raise SyntaxError("invalid or missing encoding declaration")
matches = cookie_re.findall(line_string)
if not matches: