Added PEP 3127 support to tokenize (with tests); added PEP 3127 to NEWS.
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1c93944..76ab430 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -51,9 +51,10 @@
Name = r'[a-zA-Z_]\w*'
Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
-Octnumber = r'0[0-7]*[lL]?'
+Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
+Binnumber = r'0[bB][01]+[lL]?'
Decnumber = r'[1-9]\d*[lL]?'
-Intnumber = group(Hexnumber, Octnumber, Decnumber)
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
Exponent = r'[eE][-+]?\d+'
Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
Expfloat = r'\d+' + Exponent