| Pablo Galindo | 1f24a71 | 2019-03-01 15:34:44 -0800 | [diff] [blame] | 1 | import itertools |
| 2 | |
| Pablo Galindo | 8bc401a | 2019-03-04 07:26:13 +0000 | [diff] [blame] | 3 | |
| Pablo Galindo | 1f24a71 | 2019-03-01 15:34:44 -0800 | [diff] [blame] | 4 | def generate_tokens(tokens): |
| 5 | numbers = itertools.count(0) |
| 6 | for line in tokens: |
| 7 | line = line.strip() |
| 8 | |
| Hansraj Das | e018dc5 | 2019-07-25 02:01:19 +0530 | [diff] [blame] | 9 | if not line or line.startswith('#'): |
| Pablo Galindo | 1f24a71 | 2019-03-01 15:34:44 -0800 | [diff] [blame] | 10 | continue |
| 11 | |
| 12 | name = line.split()[0] |
| 13 | yield (name, next(numbers)) |
| 14 | |
| 15 | yield ('N_TOKENS', next(numbers)) |
| 16 | yield ('NT_OFFSET', 256) |
| 17 | |
| Pablo Galindo | 8bc401a | 2019-03-04 07:26:13 +0000 | [diff] [blame] | 18 | |
| Pablo Galindo | 1f24a71 | 2019-03-01 15:34:44 -0800 | [diff] [blame] | 19 | def generate_opmap(tokens): |
| 20 | for line in tokens: |
| 21 | line = line.strip() |
| 22 | |
| Hansraj Das | e018dc5 | 2019-07-25 02:01:19 +0530 | [diff] [blame] | 23 | if not line or line.startswith('#'): |
| Pablo Galindo | 1f24a71 | 2019-03-01 15:34:44 -0800 | [diff] [blame] | 24 | continue |
| 25 | |
| 26 | pieces = line.split() |
| 27 | |
| 28 | if len(pieces) != 2: |
| 29 | continue |
| 30 | |
| 31 | name, op = pieces |
| 32 | yield (op.strip("'"), name) |
| 33 | |
| 34 | # Yield independently <>. This is needed so it does not collide |
| 35 | # with the token generation in "generate_tokens" because if this |
| 36 | # symbol is included in Grammar/Tokens, it will collide with != |
| 37 | # as it has the same name (NOTEQUAL). |
| 38 | yield ('<>', 'NOTEQUAL') |