commit 8d5c0b8c198374d0b88f30f04dd29d1f19c1c913
author Meador Inge <meadori@gmail.com> Sat Jun 16 21:49:08 2012 -0500
committer Meador Inge <meadori@gmail.com> Sat Jun 16 21:49:08 2012 -0500
tree 82c76c228a2074c074e58249dc57b05f0d2776ef
parent 2d9db1dfceeb4e9b2c9572bd1abdbc3bc2b663e5

Issue #15054: Fix incorrect tokenization of 'b' string literals.

Patch by Serhiy Storchaka.

Lib/tokenize.py

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index e4c9d3c..e41cd6e 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -127,7 +127,7 @@
 Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
 Number = group(Imagnumber, Floatnumber, Intnumber)
 
-StringPrefix = r'(?:[uU][rR]?|[bB][rR]|[rR][bB]|[rR]|[uU])?'
+StringPrefix = r'(?:[uUbB][rR]?|[rR][bB]?)?'
 
 # Tail end of ' string.
 Single = r"[^'\\]*(?:\\.[^'\\]*)*'"