Make the example a little more interesting and useful.
diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index c627f1a..423540c 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -1300,6 +1300,7 @@
Token = collections.namedtuple('Token', 'typ value line column')
def tokenize(s):
+ keywords = {'IF', 'THEN', 'FOR', 'NEXT', 'GOSUB', 'RETURN'}
tok_spec = [
('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number
('ASSIGN', r':='), # Assignment operator
@@ -1320,6 +1321,8 @@
line_start = pos
line += 1
elif typ != 'SKIP':
+ if typ == 'ID' and val in keywords:
+ typ = val
yield Token(typ, mo.group(typ), line, mo.start()-line_start)
pos = mo.end()
mo = gettok(s, pos)