Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. |
| 2 | # Licensed to PSF under a Contributor Agreement. |
| 3 | |
| 4 | # Modifications: |
| 5 | # Copyright 2006 Google, Inc. All Rights Reserved. |
| 6 | # Licensed to PSF under a Contributor Agreement. |
| 7 | |
| 8 | """Parser driver. |
| 9 | |
| 10 | This provides a high-level interface to parse a file into a syntax tree. |
| 11 | |
| 12 | """ |
| 13 | |
| 14 | __author__ = "Guido van Rossum <guido@python.org>" |
| 15 | |
| 16 | __all__ = ["Driver", "load_grammar"] |
| 17 | |
| 18 | # Python imports |
Benjamin Peterson | d481e3d | 2009-05-09 19:42:23 +0000 | [diff] [blame] | 19 | import codecs |
Benjamin Peterson | 2e7965e | 2011-02-26 22:12:10 +0000 | [diff] [blame] | 20 | import io |
Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 21 | import os |
| 22 | import logging |
| 23 | import sys |
| 24 | |
| 25 | # Pgen imports |
Martin v. Löwis | 3faa84f | 2008-03-22 00:07:09 +0000 | [diff] [blame] | 26 | from . import grammar, parse, token, tokenize, pgen |
Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 27 | |
| 28 | |
| 29 | class Driver(object): |
| 30 | |
| 31 | def __init__(self, grammar, convert=None, logger=None): |
| 32 | self.grammar = grammar |
| 33 | if logger is None: |
| 34 | logger = logging.getLogger() |
| 35 | self.logger = logger |
| 36 | self.convert = convert |
| 37 | |
| 38 | def parse_tokens(self, tokens, debug=False): |
| 39 | """Parse a series of tokens and return the syntax tree.""" |
| 40 | # XXX Move the prefix computation into a wrapper around tokenize. |
| 41 | p = parse.Parser(self.grammar, self.convert) |
| 42 | p.setup() |
| 43 | lineno = 1 |
| 44 | column = 0 |
| 45 | type = value = start = end = line_text = None |
| 46 | prefix = "" |
| 47 | for quintuple in tokens: |
| 48 | type, value, start, end, line_text = quintuple |
| 49 | if start != (lineno, column): |
| 50 | assert (lineno, column) <= start, ((lineno, column), start) |
| 51 | s_lineno, s_column = start |
| 52 | if lineno < s_lineno: |
| 53 | prefix += "\n" * (s_lineno - lineno) |
| 54 | lineno = s_lineno |
| 55 | column = 0 |
| 56 | if column < s_column: |
| 57 | prefix += line_text[column:s_column] |
| 58 | column = s_column |
| 59 | if type in (tokenize.COMMENT, tokenize.NL): |
| 60 | prefix += value |
| 61 | lineno, column = end |
| 62 | if value.endswith("\n"): |
| 63 | lineno += 1 |
| 64 | column = 0 |
| 65 | continue |
| 66 | if type == token.OP: |
| 67 | type = grammar.opmap[value] |
| 68 | if debug: |
| 69 | self.logger.debug("%s %r (prefix=%r)", |
| 70 | token.tok_name[type], value, prefix) |
| 71 | if p.addtoken(type, value, (prefix, start)): |
| 72 | if debug: |
| 73 | self.logger.debug("Stop.") |
| 74 | break |
| 75 | prefix = "" |
| 76 | lineno, column = end |
| 77 | if value.endswith("\n"): |
| 78 | lineno += 1 |
| 79 | column = 0 |
| 80 | else: |
| 81 | # We never broke out -- EOF is too soon (how can this happen???) |
Benjamin Peterson | 28d88b4 | 2009-01-09 03:03:23 +0000 | [diff] [blame] | 82 | raise parse.ParseError("incomplete input", |
| 83 | type, value, (prefix, start)) |
Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 84 | return p.rootnode |
| 85 | |
| 86 | def parse_stream_raw(self, stream, debug=False): |
| 87 | """Parse a stream and return the syntax tree.""" |
| 88 | tokens = tokenize.generate_tokens(stream.readline) |
| 89 | return self.parse_tokens(tokens, debug) |
| 90 | |
| 91 | def parse_stream(self, stream, debug=False): |
| 92 | """Parse a stream and return the syntax tree.""" |
| 93 | return self.parse_stream_raw(stream, debug) |
| 94 | |
Benjamin Peterson | d481e3d | 2009-05-09 19:42:23 +0000 | [diff] [blame] | 95 | def parse_file(self, filename, encoding=None, debug=False): |
Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 96 | """Parse a file and return the syntax tree.""" |
Benjamin Peterson | d481e3d | 2009-05-09 19:42:23 +0000 | [diff] [blame] | 97 | stream = codecs.open(filename, "r", encoding) |
Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 98 | try: |
| 99 | return self.parse_stream(stream, debug) |
| 100 | finally: |
| 101 | stream.close() |
| 102 | |
| 103 | def parse_string(self, text, debug=False): |
| 104 | """Parse a string and return the syntax tree.""" |
Benjamin Peterson | 2e7965e | 2011-02-26 22:12:10 +0000 | [diff] [blame] | 105 | tokens = tokenize.generate_tokens(io.StringIO(text).readline) |
Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 106 | return self.parse_tokens(tokens, debug) |
| 107 | |
| 108 | |
Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 109 | def load_grammar(gt="Grammar.txt", gp=None, |
| 110 | save=True, force=False, logger=None): |
| 111 | """Load the grammar (maybe from a pickle).""" |
| 112 | if logger is None: |
| 113 | logger = logging.getLogger() |
| 114 | if gp is None: |
| 115 | head, tail = os.path.splitext(gt) |
| 116 | if tail == ".txt": |
| 117 | tail = "" |
| 118 | gp = head + tail + ".".join(map(str, sys.version_info)) + ".pickle" |
| 119 | if force or not _newer(gp, gt): |
| 120 | logger.info("Generating grammar tables from %s", gt) |
Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 121 | g = pgen.generate_grammar(gt) |
| 122 | if save: |
| 123 | logger.info("Writing grammar tables to %s", gp) |
Martin v. Löwis | 346c921 | 2008-05-25 17:22:03 +0000 | [diff] [blame] | 124 | try: |
| 125 | g.dump(gp) |
Andrew Svetlov | f7a17b4 | 2012-12-25 16:47:37 +0200 | [diff] [blame^] | 126 | except OSError as e: |
Martin v. Löwis | 346c921 | 2008-05-25 17:22:03 +0000 | [diff] [blame] | 127 | logger.info("Writing failed:"+str(e)) |
Martin v. Löwis | ef04c44 | 2008-03-19 05:04:44 +0000 | [diff] [blame] | 128 | else: |
| 129 | g = grammar.Grammar() |
| 130 | g.load(gp) |
| 131 | return g |
| 132 | |
| 133 | |
| 134 | def _newer(a, b): |
| 135 | """Inquire whether file a was written since file b.""" |
| 136 | if not os.path.exists(a): |
| 137 | return False |
| 138 | if not os.path.exists(b): |
| 139 | return True |
| 140 | return os.path.getmtime(a) >= os.path.getmtime(b) |
Ned Deily | 24ec054 | 2012-09-08 19:04:47 -0700 | [diff] [blame] | 141 | |
| 142 | |
| 143 | def main(*args): |
| 144 | """Main program, when run as a script: produce grammar pickle files. |
| 145 | |
| 146 | Calls load_grammar for each argument, a path to a grammar text file. |
| 147 | """ |
| 148 | if not args: |
| 149 | args = sys.argv[1:] |
| 150 | logging.basicConfig(level=logging.INFO, stream=sys.stdout, |
| 151 | format='%(message)s') |
| 152 | for gt in args: |
| 153 | load_grammar(gt, save=True, force=True) |
| 154 | return True |
| 155 | |
| 156 | if __name__ == "__main__": |
| 157 | sys.exit(int(not main())) |