Merged revisions 79077,79137,79304-79305 via svnmerge from
svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3
........
r79077 | benjamin.peterson | 2010-03-18 18:05:29 -0500 (Thu, 18 Mar 2010) | 1 line
port detect_encoding improvements from py3k
........
r79137 | benjamin.peterson | 2010-03-20 11:12:53 -0500 (Sat, 20 Mar 2010) | 1 line
add a fixer for setting sys.exitfunc #2356
........
r79304 | benjamin.peterson | 2010-03-22 17:20:22 -0500 (Mon, 22 Mar 2010) | 1 line
fix test_parser when it's run in a path with spaces #7666
........
r79305 | benjamin.peterson | 2010-03-22 17:27:07 -0500 (Mon, 22 Mar 2010) | 1 line
normalize whitespace
........
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index e0f45f9..521e7b7 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -253,14 +253,16 @@
in.
It detects the encoding from the presence of a utf-8 bom or an encoding
- cookie as specified in pep-0263. If both a bom and a cookie are present,
- but disagree, a SyntaxError will be raised. If the encoding cookie is an
- invalid charset, raise a SyntaxError.
+ cookie as specified in pep-0263. If both a bom and a cookie are present, but
+ disagree, a SyntaxError will be raised. If the encoding cookie is an invalid
+ charset, raise a SyntaxError. Note that if a utf-8 bom is found,
+ 'utf-8-sig' is returned.
If no encoding is specified, then the default of 'utf-8' will be returned.
"""
bom_found = False
encoding = None
+ default = 'utf-8'
def read_or_stop():
try:
return readline()
@@ -287,17 +289,16 @@
if codec.name != 'utf-8':
# This behaviour mimics the Python interpreter
raise SyntaxError('encoding problem: utf-8')
- else:
- # Allow it to be properly encoded and decoded.
- encoding = 'utf-8-sig'
+ encoding += '-sig'
return encoding
first = read_or_stop()
if first.startswith(BOM_UTF8):
bom_found = True
first = first[3:]
+ default = 'utf-8-sig'
if not first:
- return 'utf-8', []
+ return default, []
encoding = find_cookie(first)
if encoding:
@@ -305,13 +306,13 @@
second = read_or_stop()
if not second:
- return 'utf-8', [first]
+ return default, [first]
encoding = find_cookie(second)
if encoding:
return encoding, [first, second]
- return 'utf-8', [first, second]
+ return default, [first, second]
def untokenize(iterable):
"""Transform tokens back into Python source code.