Guido van Rossum | e7b146f | 2000-02-04 15:28:42 +0000 | [diff] [blame] | 1 | """A lexical analyzer class for simple shell-like syntaxes.""" |
| 2 | |
Tim Peters | 70c4378 | 2001-01-17 08:48:39 +0000 | [diff] [blame] | 3 | # Module and documentation by Eric S. Raymond, 21 Dec 1998 |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 4 | # Input stacking and error message cleanup added by ESR, March 2000 |
Tim Peters | 70c4378 | 2001-01-17 08:48:39 +0000 | [diff] [blame] | 5 | # push_source() and pop_source() made explicit by ESR, January 2001. |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 6 | # Posix compliance, split(), string arguments, and |
| 7 | # iterator interface by Gustavo Niemeyer, April 2003. |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 8 | # changes to tokenize more like Posix shells by Vinay Sajip, July 2016. |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 9 | |
Éric Araujo | 9bce311 | 2011-07-27 18:29:31 +0200 | [diff] [blame] | 10 | import os |
| 11 | import re |
Guido van Rossum | 73898c7 | 1999-05-03 18:14:16 +0000 | [diff] [blame] | 12 | import sys |
Raymond Hettinger | 756b3f3 | 2004-01-29 06:37:52 +0000 | [diff] [blame] | 13 | from collections import deque |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 14 | |
Guido van Rossum | 68937b4 | 2007-05-18 00:51:22 +0000 | [diff] [blame] | 15 | from io import StringIO |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 16 | |
Bo Bayles | ca80495 | 2019-05-29 03:06:12 -0500 | [diff] [blame] | 17 | __all__ = ["shlex", "split", "quote", "join"] |
Skip Montanaro | 0de6580 | 2001-02-15 22:15:14 +0000 | [diff] [blame] | 18 | |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 19 | class shlex: |
Tim Peters | 70c4378 | 2001-01-17 08:48:39 +0000 | [diff] [blame] | 20 | "A lexical analyzer class for simple shell-like syntaxes." |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 21 | def __init__(self, instream=None, infile=None, posix=False, |
| 22 | punctuation_chars=False): |
Guido van Rossum | 3172c5d | 2007-10-16 18:12:55 +0000 | [diff] [blame] | 23 | if isinstance(instream, str): |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 24 | instream = StringIO(instream) |
Raymond Hettinger | f13eb55 | 2002-06-02 00:40:05 +0000 | [diff] [blame] | 25 | if instream is not None: |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 26 | self.instream = instream |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 27 | self.infile = infile |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 28 | else: |
| 29 | self.instream = sys.stdin |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 30 | self.infile = None |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 31 | self.posix = posix |
| 32 | if posix: |
| 33 | self.eof = None |
| 34 | else: |
| 35 | self.eof = '' |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 36 | self.commenters = '#' |
Fred Drake | dbbf76b | 2000-07-09 16:44:26 +0000 | [diff] [blame] | 37 | self.wordchars = ('abcdfeghijklmnopqrstuvwxyz' |
| 38 | 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_') |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 39 | if self.posix: |
Antoine Pitrou | d72402e | 2010-10-27 18:52:48 +0000 | [diff] [blame] | 40 | self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ' |
| 41 | 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ') |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 42 | self.whitespace = ' \t\r\n' |
Fred Drake | 2431523 | 2003-04-17 22:01:17 +0000 | [diff] [blame] | 43 | self.whitespace_split = False |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 44 | self.quotes = '\'"' |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 45 | self.escape = '\\' |
| 46 | self.escapedquotes = '"' |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 47 | self.state = ' ' |
Raymond Hettinger | 756b3f3 | 2004-01-29 06:37:52 +0000 | [diff] [blame] | 48 | self.pushback = deque() |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 49 | self.lineno = 1 |
| 50 | self.debug = 0 |
| 51 | self.token = '' |
Raymond Hettinger | 756b3f3 | 2004-01-29 06:37:52 +0000 | [diff] [blame] | 52 | self.filestack = deque() |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 53 | self.source = None |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 54 | if not punctuation_chars: |
| 55 | punctuation_chars = '' |
| 56 | elif punctuation_chars is True: |
| 57 | punctuation_chars = '();<>|&' |
Alex | 972cf5c | 2019-09-11 14:04:04 +0300 | [diff] [blame] | 58 | self._punctuation_chars = punctuation_chars |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 59 | if punctuation_chars: |
| 60 | # _pushback_chars is a push back queue used by lookahead logic |
| 61 | self._pushback_chars = deque() |
| 62 | # these chars added because allowed in file names, args, wildcards |
| 63 | self.wordchars += '~-./*?=' |
| 64 | #remove any punctuation chars from wordchars |
| 65 | t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars)) |
| 66 | self.wordchars = self.wordchars.translate(t) |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 67 | |
Alex | 972cf5c | 2019-09-11 14:04:04 +0300 | [diff] [blame] | 68 | @property |
| 69 | def punctuation_chars(self): |
| 70 | return self._punctuation_chars |
| 71 | |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 72 | def push_token(self, tok): |
| 73 | "Push a token onto the stack popped by the get_token method" |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 74 | if self.debug >= 1: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 75 | print("shlex: pushing token " + repr(tok)) |
Raymond Hettinger | 756b3f3 | 2004-01-29 06:37:52 +0000 | [diff] [blame] | 76 | self.pushback.appendleft(tok) |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 77 | |
Eric S. Raymond | bddbaf7 | 2001-01-16 15:19:13 +0000 | [diff] [blame] | 78 | def push_source(self, newstream, newfile=None): |
| 79 | "Push an input source onto the lexer's input source stack." |
Guido van Rossum | 3172c5d | 2007-10-16 18:12:55 +0000 | [diff] [blame] | 80 | if isinstance(newstream, str): |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 81 | newstream = StringIO(newstream) |
Raymond Hettinger | 756b3f3 | 2004-01-29 06:37:52 +0000 | [diff] [blame] | 82 | self.filestack.appendleft((self.infile, self.instream, self.lineno)) |
Eric S. Raymond | bddbaf7 | 2001-01-16 15:19:13 +0000 | [diff] [blame] | 83 | self.infile = newfile |
| 84 | self.instream = newstream |
| 85 | self.lineno = 1 |
| 86 | if self.debug: |
Raymond Hettinger | f13eb55 | 2002-06-02 00:40:05 +0000 | [diff] [blame] | 87 | if newfile is not None: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 88 | print('shlex: pushing to file %s' % (self.infile,)) |
Eric S. Raymond | bddbaf7 | 2001-01-16 15:19:13 +0000 | [diff] [blame] | 89 | else: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 90 | print('shlex: pushing to stream %s' % (self.instream,)) |
Eric S. Raymond | bddbaf7 | 2001-01-16 15:19:13 +0000 | [diff] [blame] | 91 | |
| 92 | def pop_source(self): |
| 93 | "Pop the input source stack." |
| 94 | self.instream.close() |
Raymond Hettinger | 756b3f3 | 2004-01-29 06:37:52 +0000 | [diff] [blame] | 95 | (self.infile, self.instream, self.lineno) = self.filestack.popleft() |
Eric S. Raymond | bddbaf7 | 2001-01-16 15:19:13 +0000 | [diff] [blame] | 96 | if self.debug: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 97 | print('shlex: popping to %s, line %d' \ |
| 98 | % (self.instream, self.lineno)) |
Eric S. Raymond | bddbaf7 | 2001-01-16 15:19:13 +0000 | [diff] [blame] | 99 | self.state = ' ' |
| 100 | |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 101 | def get_token(self): |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 102 | "Get a token from the input stream (or from stack if it's nonempty)" |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 103 | if self.pushback: |
Raymond Hettinger | 756b3f3 | 2004-01-29 06:37:52 +0000 | [diff] [blame] | 104 | tok = self.pushback.popleft() |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 105 | if self.debug >= 1: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 106 | print("shlex: popping token " + repr(tok)) |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 107 | return tok |
Fred Drake | dbbf76b | 2000-07-09 16:44:26 +0000 | [diff] [blame] | 108 | # No pushback. Get a token. |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 109 | raw = self.read_token() |
| 110 | # Handle inclusions |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 111 | if self.source is not None: |
| 112 | while raw == self.source: |
| 113 | spec = self.sourcehook(self.read_token()) |
| 114 | if spec: |
| 115 | (newfile, newstream) = spec |
| 116 | self.push_source(newstream, newfile) |
| 117 | raw = self.get_token() |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 118 | # Maybe we got EOF instead? |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 119 | while raw == self.eof: |
Fred Drake | 2431523 | 2003-04-17 22:01:17 +0000 | [diff] [blame] | 120 | if not self.filestack: |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 121 | return self.eof |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 122 | else: |
Eric S. Raymond | bddbaf7 | 2001-01-16 15:19:13 +0000 | [diff] [blame] | 123 | self.pop_source() |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 124 | raw = self.get_token() |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 125 | # Neither inclusion nor EOF |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 126 | if self.debug >= 1: |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 127 | if raw != self.eof: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 128 | print("shlex: token=" + repr(raw)) |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 129 | else: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 130 | print("shlex: token=EOF") |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 131 | return raw |
| 132 | |
| 133 | def read_token(self): |
Fred Drake | 2431523 | 2003-04-17 22:01:17 +0000 | [diff] [blame] | 134 | quoted = False |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 135 | escapedstate = ' ' |
Neal Norwitz | 10cf218 | 2003-04-17 23:09:08 +0000 | [diff] [blame] | 136 | while True: |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 137 | if self.punctuation_chars and self._pushback_chars: |
| 138 | nextchar = self._pushback_chars.pop() |
| 139 | else: |
| 140 | nextchar = self.instream.read(1) |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 141 | if nextchar == '\n': |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 142 | self.lineno += 1 |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 143 | if self.debug >= 3: |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 144 | print("shlex: in state %r I see character: %r" % (self.state, |
| 145 | nextchar)) |
Fred Drake | dbbf76b | 2000-07-09 16:44:26 +0000 | [diff] [blame] | 146 | if self.state is None: |
Eric S. Raymond | bddbaf7 | 2001-01-16 15:19:13 +0000 | [diff] [blame] | 147 | self.token = '' # past end of file |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 148 | break |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 149 | elif self.state == ' ': |
| 150 | if not nextchar: |
Eric S. Raymond | bddbaf7 | 2001-01-16 15:19:13 +0000 | [diff] [blame] | 151 | self.state = None # end of file |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 152 | break |
| 153 | elif nextchar in self.whitespace: |
| 154 | if self.debug >= 2: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 155 | print("shlex: I see whitespace in whitespace state") |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 156 | if self.token or (self.posix and quoted): |
Fred Drake | dbbf76b | 2000-07-09 16:44:26 +0000 | [diff] [blame] | 157 | break # emit current token |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 158 | else: |
| 159 | continue |
| 160 | elif nextchar in self.commenters: |
| 161 | self.instream.readline() |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 162 | self.lineno += 1 |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 163 | elif self.posix and nextchar in self.escape: |
| 164 | escapedstate = 'a' |
| 165 | self.state = nextchar |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 166 | elif nextchar in self.wordchars: |
| 167 | self.token = nextchar |
| 168 | self.state = 'a' |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 169 | elif nextchar in self.punctuation_chars: |
| 170 | self.token = nextchar |
| 171 | self.state = 'c' |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 172 | elif nextchar in self.quotes: |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 173 | if not self.posix: |
| 174 | self.token = nextchar |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 175 | self.state = nextchar |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 176 | elif self.whitespace_split: |
| 177 | self.token = nextchar |
| 178 | self.state = 'a' |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 179 | else: |
| 180 | self.token = nextchar |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 181 | if self.token or (self.posix and quoted): |
Fred Drake | dbbf76b | 2000-07-09 16:44:26 +0000 | [diff] [blame] | 182 | break # emit current token |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 183 | else: |
| 184 | continue |
| 185 | elif self.state in self.quotes: |
Fred Drake | 2431523 | 2003-04-17 22:01:17 +0000 | [diff] [blame] | 186 | quoted = True |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 187 | if not nextchar: # end of file |
Andrew M. Kuchling | 9d56cd1 | 2001-01-09 03:01:15 +0000 | [diff] [blame] | 188 | if self.debug >= 2: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 189 | print("shlex: I see EOF in quotes state") |
Andrew M. Kuchling | 9d56cd1 | 2001-01-09 03:01:15 +0000 | [diff] [blame] | 190 | # XXX what error should be raised here? |
Collin Winter | ce36ad8 | 2007-08-30 01:19:48 +0000 | [diff] [blame] | 191 | raise ValueError("No closing quotation") |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 192 | if nextchar == self.state: |
| 193 | if not self.posix: |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 194 | self.token += nextchar |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 195 | self.state = ' ' |
| 196 | break |
| 197 | else: |
| 198 | self.state = 'a' |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 199 | elif (self.posix and nextchar in self.escape and self.state |
| 200 | in self.escapedquotes): |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 201 | escapedstate = self.state |
| 202 | self.state = nextchar |
| 203 | else: |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 204 | self.token += nextchar |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 205 | elif self.state in self.escape: |
| 206 | if not nextchar: # end of file |
| 207 | if self.debug >= 2: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 208 | print("shlex: I see EOF in escape state") |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 209 | # XXX what error should be raised here? |
Collin Winter | ce36ad8 | 2007-08-30 01:19:48 +0000 | [diff] [blame] | 210 | raise ValueError("No escaped character") |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 211 | # In posix shells, only the quote itself or the escape |
| 212 | # character may be escaped within quotes. |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 213 | if (escapedstate in self.quotes and |
| 214 | nextchar != self.state and nextchar != escapedstate): |
| 215 | self.token += self.state |
| 216 | self.token += nextchar |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 217 | self.state = escapedstate |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 218 | elif self.state in ('a', 'c'): |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 219 | if not nextchar: |
Tim Peters | 70c4378 | 2001-01-17 08:48:39 +0000 | [diff] [blame] | 220 | self.state = None # end of file |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 221 | break |
| 222 | elif nextchar in self.whitespace: |
| 223 | if self.debug >= 2: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 224 | print("shlex: I see whitespace in word state") |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 225 | self.state = ' ' |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 226 | if self.token or (self.posix and quoted): |
Fred Drake | dbbf76b | 2000-07-09 16:44:26 +0000 | [diff] [blame] | 227 | break # emit current token |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 228 | else: |
| 229 | continue |
| 230 | elif nextchar in self.commenters: |
| 231 | self.instream.readline() |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 232 | self.lineno += 1 |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 233 | if self.posix: |
| 234 | self.state = ' ' |
| 235 | if self.token or (self.posix and quoted): |
| 236 | break # emit current token |
| 237 | else: |
| 238 | continue |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 239 | elif self.state == 'c': |
| 240 | if nextchar in self.punctuation_chars: |
| 241 | self.token += nextchar |
| 242 | else: |
| 243 | if nextchar not in self.whitespace: |
| 244 | self._pushback_chars.append(nextchar) |
| 245 | self.state = ' ' |
| 246 | break |
Vinay Sajip | 61eda72 | 2017-01-15 10:06:52 +0000 | [diff] [blame] | 247 | elif self.posix and nextchar in self.quotes: |
| 248 | self.state = nextchar |
| 249 | elif self.posix and nextchar in self.escape: |
| 250 | escapedstate = 'a' |
| 251 | self.state = nextchar |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 252 | elif (nextchar in self.wordchars or nextchar in self.quotes |
Evan | 56624a9 | 2019-06-02 05:09:22 +1000 | [diff] [blame] | 253 | or (self.whitespace_split and |
| 254 | nextchar not in self.punctuation_chars)): |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 255 | self.token += nextchar |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 256 | else: |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 257 | if self.punctuation_chars: |
| 258 | self._pushback_chars.append(nextchar) |
| 259 | else: |
| 260 | self.pushback.appendleft(nextchar) |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 261 | if self.debug >= 2: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 262 | print("shlex: I see punctuation in word state") |
Guido van Rossum | f247d75 | 1999-03-22 15:28:08 +0000 | [diff] [blame] | 263 | self.state = ' ' |
Vinay Sajip | c1f974c | 2016-07-29 22:35:03 +0100 | [diff] [blame] | 264 | if self.token or (self.posix and quoted): |
Fred Drake | dbbf76b | 2000-07-09 16:44:26 +0000 | [diff] [blame] | 265 | break # emit current token |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 266 | else: |
| 267 | continue |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 268 | result = self.token |
| 269 | self.token = '' |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 270 | if self.posix and not quoted and result == '': |
| 271 | result = None |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 272 | if self.debug > 1: |
| 273 | if result: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 274 | print("shlex: raw token=" + repr(result)) |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 275 | else: |
Guido van Rossum | be19ed7 | 2007-02-09 05:37:30 +0000 | [diff] [blame] | 276 | print("shlex: raw token=EOF") |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 277 | return result |
| 278 | |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 279 | def sourcehook(self, newfile): |
| 280 | "Hook called on a filename to be sourced." |
| 281 | if newfile[0] == '"': |
| 282 | newfile = newfile[1:-1] |
Fred Drake | 52dc76c | 2000-07-03 09:56:23 +0000 | [diff] [blame] | 283 | # This implements cpp-like semantics for relative-path inclusion. |
Guido van Rossum | 3172c5d | 2007-10-16 18:12:55 +0000 | [diff] [blame] | 284 | if isinstance(self.infile, str) and not os.path.isabs(newfile): |
Fred Drake | 52dc76c | 2000-07-03 09:56:23 +0000 | [diff] [blame] | 285 | newfile = os.path.join(os.path.dirname(self.infile), newfile) |
Guido van Rossum | eb4e11a | 2000-05-01 20:08:46 +0000 | [diff] [blame] | 286 | return (newfile, open(newfile, "r")) |
| 287 | |
Guido van Rossum | 4b83ecb | 2000-05-01 20:14:12 +0000 | [diff] [blame] | 288 | def error_leader(self, infile=None, lineno=None): |
| 289 | "Emit a C-compiler-like, Emacs-friendly error-message leader." |
Raymond Hettinger | f13eb55 | 2002-06-02 00:40:05 +0000 | [diff] [blame] | 290 | if infile is None: |
Guido van Rossum | 4b83ecb | 2000-05-01 20:14:12 +0000 | [diff] [blame] | 291 | infile = self.infile |
Raymond Hettinger | f13eb55 | 2002-06-02 00:40:05 +0000 | [diff] [blame] | 292 | if lineno is None: |
Guido van Rossum | 4b83ecb | 2000-05-01 20:14:12 +0000 | [diff] [blame] | 293 | lineno = self.lineno |
| 294 | return "\"%s\", line %d: " % (infile, lineno) |
| 295 | |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 296 | def __iter__(self): |
| 297 | return self |
| 298 | |
Georg Brandl | a18af4e | 2007-04-21 15:47:16 +0000 | [diff] [blame] | 299 | def __next__(self): |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 300 | token = self.get_token() |
| 301 | if token == self.eof: |
| 302 | raise StopIteration |
| 303 | return token |
| 304 | |
Guido van Rossum | e7ba495 | 2007-06-06 23:52:48 +0000 | [diff] [blame] | 305 | def split(s, comments=False, posix=True): |
MaT1g3R | 65c7382 | 2019-10-31 06:23:20 -0400 | [diff] [blame] | 306 | """Split the string *s* using shell-like syntax.""" |
Zackery Spytz | 975ac32 | 2020-04-01 07:58:55 -0600 | [diff] [blame] | 307 | if s is None: |
| 308 | import warnings |
| 309 | warnings.warn("Passing None for 's' to shlex.split() is deprecated.", |
| 310 | DeprecationWarning, stacklevel=2) |
Guido van Rossum | e7ba495 | 2007-06-06 23:52:48 +0000 | [diff] [blame] | 311 | lex = shlex(s, posix=posix) |
Gustavo Niemeyer | 48f3dcc | 2003-04-20 01:57:03 +0000 | [diff] [blame] | 312 | lex.whitespace_split = True |
| 313 | if not comments: |
| 314 | lex.commenters = '' |
Gustavo Niemeyer | 68d8cef | 2003-04-17 21:31:33 +0000 | [diff] [blame] | 315 | return list(lex) |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 316 | |
Éric Araujo | 9bce311 | 2011-07-27 18:29:31 +0200 | [diff] [blame] | 317 | |
Bo Bayles | ca80495 | 2019-05-29 03:06:12 -0500 | [diff] [blame] | 318 | def join(split_command): |
| 319 | """Return a shell-escaped string from *split_command*.""" |
| 320 | return ' '.join(quote(arg) for arg in split_command) |
| 321 | |
| 322 | |
Ezio Melotti | 67321cc | 2011-08-16 19:03:41 +0300 | [diff] [blame] | 323 | _find_unsafe = re.compile(r'[^\w@%+=:,./-]', re.ASCII).search |
Éric Araujo | 9bce311 | 2011-07-27 18:29:31 +0200 | [diff] [blame] | 324 | |
| 325 | def quote(s): |
| 326 | """Return a shell-escaped version of the string *s*.""" |
| 327 | if not s: |
| 328 | return "''" |
| 329 | if _find_unsafe(s) is None: |
| 330 | return s |
| 331 | |
| 332 | # use single quotes, and put single quotes into double quotes |
| 333 | # the string $'b is then quoted as '$'"'"'b' |
| 334 | return "'" + s.replace("'", "'\"'\"'") + "'" |
| 335 | |
| 336 | |
R David Murray | 838f2c4 | 2014-10-17 20:28:47 -0400 | [diff] [blame] | 337 | def _print_tokens(lexer): |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 338 | while 1: |
| 339 | tt = lexer.get_token() |
R David Murray | 838f2c4 | 2014-10-17 20:28:47 -0400 | [diff] [blame] | 340 | if not tt: |
Guido van Rossum | 9c30c24 | 1998-12-22 05:19:29 +0000 | [diff] [blame] | 341 | break |
R David Murray | 838f2c4 | 2014-10-17 20:28:47 -0400 | [diff] [blame] | 342 | print("Token: " + repr(tt)) |
| 343 | |
| 344 | if __name__ == '__main__': |
| 345 | if len(sys.argv) == 1: |
| 346 | _print_tokens(shlex()) |
| 347 | else: |
| 348 | fn = sys.argv[1] |
| 349 | with open(fn) as f: |
| 350 | _print_tokens(shlex(f, fn)) |