Added PEP 3127 support to tokenize (with tests); added PEP 3127 to NEWS.
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index d6cfb65..cbfafa8 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -4,7 +4,7 @@
>>> import glob, random, sys
The tests can be really simple. Given a small fragment of source
-code, print out a table with thokens. The ENDMARK is omitted for
+code, print out a table with tokens. The ENDMARK is omitted for
brevity.
>>> dump_tokens("1 + 1")
@@ -106,7 +106,7 @@
... "else: print 'Loaded'\\n")
True
-Balancing contunuation
+Balancing continuation
>>> roundtrip("a = (3,4, \\n"
... "5,6)\\n"
@@ -126,6 +126,14 @@
NUMBER '0xff' (1, 0) (1, 4)
OP '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
+ >>> dump_tokens("0b10 <= 255")
+ NUMBER '0b10' (1, 0) (1, 4)
+ OP '<=' (1, 5) (1, 7)
+ NUMBER '255' (1, 8) (1, 11)
+ >>> dump_tokens("0o123 <= 0123")
+ NUMBER '0o123' (1, 0) (1, 5)
+ OP '<=' (1, 6) (1, 8)
+ NUMBER '0123' (1, 9) (1, 13)
>>> dump_tokens("01234567 > ~0x15")
NUMBER '01234567' (1, 0) (1, 8)
OP '>' (1, 9) (1, 10)
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1c93944..76ab430 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -51,9 +51,10 @@
Name = r'[a-zA-Z_]\w*'
Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
-Octnumber = r'0[0-7]*[lL]?'
+Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
+Binnumber = r'0[bB][01]+[lL]?'
Decnumber = r'[1-9]\d*[lL]?'
-Intnumber = group(Hexnumber, Octnumber, Decnumber)
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
Exponent = r'[eE][-+]?\d+'
Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
Expfloat = r'\d+' + Exponent
diff --git a/Misc/NEWS b/Misc/NEWS
index 1963ad5..1fc6691 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@
Core and builtins
-----------------
+- PEP 3127: octal literals now start with "0o". Old-style octal literals
+ are still valid. There are binary literals with a prefix of "0b".
+ This also affects int(x, 0).
+
- Issue #1779871: Gnu gcc can now build Python on OS X because the
flags -Wno-long-double, -no-cpp-precomp, and -mno-fused-madd are no
longer passed.