fix for files with coding cookies and BOMs
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 8972137..519dfa5 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -333,9 +333,11 @@
# This behaviour mimics the Python interpreter
raise SyntaxError("unknown encoding: " + encoding)
- if bom_found and codec.name != 'utf-8':
- # This behaviour mimics the Python interpreter
- raise SyntaxError('encoding problem: utf-8')
+ if bom_found:
+ if codec.name != 'utf-8':
+ # This behaviour mimics the Python interpreter
+ raise SyntaxError('encoding problem: utf-8')
+ encoding += '-sig'
return encoding
first = read_or_stop()