port simplejson upgrade from the trunk #4136
json also now works only with unicode strings
Patch by Antoine Pitrou; updated by me
diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py
index 4b065ab..b4f3561 100644
--- a/Lib/json/scanner.py
+++ b/Lib/json/scanner.py
@@ -1,69 +1,65 @@
-"""Iterator based sre token scanner
-
+"""JSON token scanner
"""
-
import re
-import sre_parse
-import sre_compile
-import sre_constants
+try:
+ from _json import make_scanner as c_make_scanner
+except ImportError:
+ c_make_scanner = None
-from re import VERBOSE, MULTILINE, DOTALL
-from sre_constants import BRANCH, SUBPATTERN
+__all__ = ['make_scanner']
-__all__ = ['Scanner', 'pattern']
+NUMBER_RE = re.compile(
+ r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
+ (re.VERBOSE | re.MULTILINE | re.DOTALL))
-FLAGS = (VERBOSE | MULTILINE | DOTALL)
+def py_make_scanner(context):
+ parse_object = context.parse_object
+ parse_array = context.parse_array
+ parse_string = context.parse_string
+ match_number = NUMBER_RE.match
+ strict = context.strict
+ parse_float = context.parse_float
+ parse_int = context.parse_int
+ parse_constant = context.parse_constant
+ object_hook = context.object_hook
-class Scanner(object):
- def __init__(self, lexicon, flags=FLAGS):
- self.actions = [None]
- # Combine phrases into a compound pattern
- s = sre_parse.Pattern()
- s.flags = flags
- p = []
- for idx, token in enumerate(lexicon):
- phrase = token.pattern
- try:
- subpattern = sre_parse.SubPattern(s,
- [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))])
- except sre_constants.error:
- raise
- p.append(subpattern)
- self.actions.append(token)
+ def _scan_once(string, idx):
+ try:
+ nextchar = string[idx]
+ except IndexError:
+ raise StopIteration
- s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work
- p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
- self.scanner = sre_compile.compile(p)
+ if nextchar == '"':
+ return parse_string(string, idx + 1, strict)
+ elif nextchar == '{':
+ return parse_object((string, idx + 1), strict,
+ _scan_once, object_hook, object_pairs_hook)
+ elif nextchar == '[':
+ return parse_array((string, idx + 1), _scan_once)
+ elif nextchar == 'n' and string[idx:idx + 4] == 'null':
+ return None, idx + 4
+ elif nextchar == 't' and string[idx:idx + 4] == 'true':
+ return True, idx + 4
+ elif nextchar == 'f' and string[idx:idx + 5] == 'false':
+ return False, idx + 5
- def iterscan(self, string, idx=0, context=None):
- """Yield match, end_idx for each match
+ m = match_number(string, idx)
+ if m is not None:
+ integer, frac, exp = m.groups()
+ if frac or exp:
+ res = parse_float(integer + (frac or '') + (exp or ''))
+ else:
+ res = parse_int(integer)
+ return res, m.end()
+ elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
+ return parse_constant('NaN'), idx + 3
+ elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
+ return parse_constant('Infinity'), idx + 8
+ elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
+ return parse_constant('-Infinity'), idx + 9
+ else:
+ raise StopIteration
- """
- match = self.scanner.scanner(string, idx).match
- actions = self.actions
- lastend = idx
- end = len(string)
- while True:
- m = match()
- if m is None:
- break
- matchbegin, matchend = m.span()
- if lastend == matchend:
- break
- action = actions[m.lastindex]
- if action is not None:
- rval, next_pos = action(m, context)
- if next_pos is not None and next_pos != matchend:
- # "fast forward" the scanner
- matchend = next_pos
- match = self.scanner.scanner(string, matchend).match
- yield rval, matchend
- lastend = matchend
+ return _scan_once
-
-def pattern(pattern, flags=FLAGS):
- def decorator(fn):
- fn.pattern = pattern
- fn.regex = re.compile(pattern, flags)
- return fn
- return decorator
+make_scanner = c_make_scanner or py_make_scanner